Merge branch 'master' of https://gitlink.org.cn/cloudream/storage

# Conflicts:
#	agent/internal/grpc/io.go
#	agent/internal/grpc/service.go
#	agent/internal/mq/agent.go
#	agent/internal/mq/cache.go
#	agent/internal/mq/io.go
#	agent/internal/mq/object.go
#	agent/internal/mq/service.go
#	agent/internal/mq/storage.go
#	agent/internal/task/cache_move_package.go
#	agent/internal/task/execute_io_plan.go
#	agent/internal/task/ipfs_pin.go
#	agent/internal/task/ipfs_read.go
#	agent/internal/task/storage_load_package.go
#	agent/main.go
#	client/internal/cmdline/bucket.go
#	client/internal/cmdline/cache.go
#	client/internal/cmdline/commandline.go
#	client/internal/cmdline/scanner.go
#	client/internal/http/bucket.go
#	client/internal/http/cache.go
#	client/internal/http/node.go
#	client/internal/http/object.go
#	client/internal/http/server.go
#	client/internal/http/storage.go
#	client/internal/services/cache.go
#	client/internal/services/storage.go
#	client/main.go
#	common/pkgs/cmd/upload_objects.go
#	common/pkgs/db/object.go
#	coordinator/internal/mq/package.go
#	coordinator/main.go
#	go.mod
#	go.sum
#	scanner/internal/event/agent_check_state.go
#	scanner/internal/event/agent_check_storage.go
#	scanner/internal/event/agent_storage_gc.go
#	scanner/main.go

.gitignore

@@ -1,2 +1,3 @@
 build
-*.tmp.*
+*.tmp.*
+.git.local

agent/internal/config/config.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"gitlink.org.cn/cloudream/common/pkgs/distlock"
-	"gitlink.org.cn/cloudream/common/pkgs/ipfs"
 	log "gitlink.org.cn/cloudream/common/pkgs/logger"
 	c "gitlink.org.cn/cloudream/common/utils/config"
 	stgmodels "gitlink.org.cn/cloudream/storage/common/models"
@@ -13,16 +12,15 @@ import (
 )
 
 type Config struct {
-	ID               int64                      `json:"id"`
-	Local            stgmodels.LocalMachineInfo `json:"local"`
-	GRPC             *grpc.Config               `json:"grpc"`
-	TempFileLifetime int                        `json:"tempFileLifetime"` // temp状态的副本最多能保持多久时间，单位：秒
-	Logger           log.Config                 `json:"logger"`
-	RabbitMQ         stgmq.Config               `json:"rabbitMQ"`
-	IPFS             ipfs.Config                `json:"ipfs"`
-	DistLock         distlock.Config            `json:"distlock"`
-	Connectivity     connectivity.Config        `json:"connectivity"`
-	Downloader       downloader.Config          `json:"downloader"`
+	ID           int64                      `json:"id"`
+	ListenAddr   string                     `json:"listenAddr"`
+	Local        stgmodels.LocalMachineInfo `json:"local"`
+	GRPC         *grpc.Config               `json:"grpc"`
+	Logger       log.Config                 `json:"logger"`
+	RabbitMQ     stgmq.Config               `json:"rabbitMQ"`
+	DistLock     distlock.Config            `json:"distlock"`
+	Connectivity connectivity.Config        `json:"connectivity"`
+	Downloader   downloader.Config          `json:"downloader"`
 }
 
 var cfg Config

agent/internal/grpc/io.go

@@ -1,55 +1,95 @@
 package grpc
 
 import (
+	"context"
 	"fmt"
 	"io"
+	"time"
 
+	"github.com/inhies/go-bytesize"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	"gitlink.org.cn/cloudream/common/utils/io2"
-	agentserver "gitlink.org.cn/cloudream/storage/common/pkgs/grpc/agent"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
+	"gitlink.org.cn/cloudream/common/utils/serder"
+	agtrpc "gitlink.org.cn/cloudream/storage/common/pkgs/grpc/agent"
 )
 
-// SendStream 接收客户端通过流式传输发送的文件数据。
-//
-// server: 代表服务端发送流的接口，用于接收和响应客户端请求。
-// 返回值: 返回错误信息，如果处理成功则返回nil。
-func (s *Service) SendStream(server agentserver.Agent_SendStreamServer) error {
-	// 接收流式传输的初始化信息包
+func (s *Service) ExecuteIOPlan(ctx context.Context, req *agtrpc.ExecuteIOPlanReq) (*agtrpc.ExecuteIOPlanResp, error) {
+	plan, err := serder.JSONToObjectEx[exec.Plan]([]byte(req.Plan))
+	if err != nil {
+		return nil, fmt.Errorf("deserializing plan: %w", err)
+	}
+
+	logger.WithField("PlanID", plan.ID).Infof("begin execute io plan")
+	defer logger.WithField("PlanID", plan.ID).Infof("plan finished")
+
+	sw := exec.NewExecutor(plan)
+
+	s.swWorker.Add(sw)
+	defer s.swWorker.Remove(sw)
+
+	execCtx := exec.NewWithContext(ctx)
+
+	// TODO2 注入依赖
+
+	_, err = sw.Run(execCtx)
+	if err != nil {
+		return nil, fmt.Errorf("running io plan: %w", err)
+	}
+
+	return &agtrpc.ExecuteIOPlanResp{}, nil
+}
+
+func (s *Service) SendStream(server agtrpc.Agent_SendStreamServer) error {
 	msg, err := server.Recv()
 	if err != nil {
 		return fmt.Errorf("recving stream id packet: %w", err)
 	}
-	// 校验初始化信息包类型
-	if msg.Type != agentserver.StreamDataPacketType_SendArgs {
+	if msg.Type != agtrpc.StreamDataPacketType_SendArgs {
 		return fmt.Errorf("first packet must be a SendArgs packet")
 	}
 
 	logger.
 		WithField("PlanID", msg.PlanID).
-		WithField("StreamID", msg.StreamID).
-		Debugf("receive stream from grpc")
+		WithField("VarID", msg.VarID).
+		Debugf("stream input")
+
+	// 同一批Plan中每个节点的Plan的启动时间有先后，但最多不应该超过30秒
+	ctx, cancel := context.WithTimeout(server.Context(), time.Second*30)
+	defer cancel()
+
+	sw := s.swWorker.FindByIDContexted(ctx, exec.PlanID(msg.PlanID))
+	if sw == nil {
+		return fmt.Errorf("plan not found")
+	}
 
 	pr, pw := io.Pipe()
 
-	// 通知系统，流式传输已准备就绪
-	s.sw.StreamReady(ioswitch.PlanID(msg.PlanID), ioswitch.NewStream(ioswitch.StreamID(msg.StreamID), pr))
+	varID := exec.VarID(msg.VarID)
+	sw.PutVars(&exec.StreamVar{
+		ID:     varID,
+		Stream: pr,
+	})
 
-	// 循环接收客户端发送的文件数据
+	// 然后读取文件数据
 	var recvSize int64
 	for {
 		msg, err := server.Recv()
 
-		// 处理接收数据错误
+		// 读取客户端数据失败
+		// 即使err是io.EOF，只要没有收到客户端包含EOF数据包就被断开了连接，就认为接收失败
 		if err != nil {
+			// 关闭文件写入
 			pw.CloseWithError(io.ErrClosedPipe)
 			logger.WithField("ReceiveSize", recvSize).
+				WithField("VarID", varID).
 				Warnf("recv message failed, err: %s", err.Error())
 			return fmt.Errorf("recv message failed, err: %w", err)
 		}
 
 		err = io2.WriteAll(pw, msg.Data)
 		if err != nil {
+			// 关闭文件写入
 			pw.CloseWithError(io.ErrClosedPipe)
 			logger.Warnf("write data to file failed, err: %s", err.Error())
 			return fmt.Errorf("write data to file failed, err: %w", err)
@@ -57,16 +97,16 @@ func (s *Service) SendStream(server agentserver.Agent_SendStreamServer) error {
 
 		recvSize += int64(len(msg.Data))
 
-		// 当接收到EOF信息时，结束写入并返回
-		if msg.Type == agentserver.StreamDataPacketType_EOF {
+		if msg.Type == agtrpc.StreamDataPacketType_EOF {
+			// 客户端明确说明文件传输已经结束，那么结束写入，获得文件Hash
 			err := pw.Close()
 			if err != nil {
 				logger.Warnf("finish writing failed, err: %s", err.Error())
 				return fmt.Errorf("finish writing failed, err: %w", err)
 			}
 
-			// 向客户端发送传输完成的响应
-			err = server.SendAndClose(&agentserver.SendStreamResp{})
+			// 并将结果返回到客户端
+			err = server.SendAndClose(&agtrpc.SendStreamResp{})
 			if err != nil {
 				logger.Warnf("send response failed, err: %s", err.Error())
 				return fmt.Errorf("send response failed, err: %w", err)
@@ -77,71 +117,135 @@ func (s *Service) SendStream(server agentserver.Agent_SendStreamServer) error {
 	}
 }
 
-// FetchStream 从服务端获取流式数据并发送给客户端。
-//
-// req: 包含获取流式数据所需的计划ID和流ID的请求信息。
-// server: 用于向客户端发送流数据的服务器接口。
-// 返回值: 返回处理过程中出现的任何错误。
-func (s *Service) FetchStream(req *agentserver.FetchStreamReq, server agentserver.Agent_FetchStreamServer) error {
+func (s *Service) GetStream(req *agtrpc.GetStreamReq, server agtrpc.Agent_GetStreamServer) error {
 	logger.
 		WithField("PlanID", req.PlanID).
-		WithField("StreamID", req.StreamID).
-		Debugf("send stream by grpc")
+		WithField("VarID", req.VarID).
+		Debugf("stream output")
+
+	// 同上
+	ctx, cancel := context.WithTimeout(server.Context(), time.Second*30)
+	defer cancel()
+
+	sw := s.swWorker.FindByIDContexted(ctx, exec.PlanID(req.PlanID))
+	if sw == nil {
+		return fmt.Errorf("plan not found")
+	}
 
-	// 等待对应的流数据准备就绪
-	strs, err := s.sw.WaitStreams(ioswitch.PlanID(req.PlanID), ioswitch.StreamID(req.StreamID))
+	signal, err := serder.JSONToObjectEx[*exec.SignalVar]([]byte(req.Signal))
 	if err != nil {
-		logger.
-			WithField("PlanID", req.PlanID).
-			WithField("StreamID", req.StreamID).
-			Warnf("watting stream: %s", err.Error())
-		return fmt.Errorf("watting stream: %w", err)
+		return fmt.Errorf("deserializing var: %w", err)
 	}
 
-	reader := strs[0].Stream
+	sw.PutVars(signal)
+
+	strVar := &exec.StreamVar{
+		ID: exec.VarID(req.VarID),
+	}
+	err = sw.BindVars(server.Context(), strVar)
+	if err != nil {
+		return fmt.Errorf("binding vars: %w", err)
+	}
+
+	reader := strVar.Stream
 	defer reader.Close()
 
-	// 读取流数据并发送给客户端
-	buf := make([]byte, 4096)
+	buf := make([]byte, 1024*64)
 	readAllCnt := 0
+	startTime := time.Now()
 	for {
 		readCnt, err := reader.Read(buf)
 
 		if readCnt > 0 {
 			readAllCnt += readCnt
-			err = server.Send(&agentserver.StreamDataPacket{
-				Type: agentserver.StreamDataPacketType_Data,
+			err = server.Send(&agtrpc.StreamDataPacket{
+				Type: agtrpc.StreamDataPacketType_Data,
 				Data: buf[:readCnt],
 			})
 			if err != nil {
 				logger.
 					WithField("PlanID", req.PlanID).
-					WithField("StreamID", req.StreamID).
+					WithField("VarID", req.VarID).
 					Warnf("send stream data failed, err: %s", err.Error())
 				return fmt.Errorf("send stream data failed, err: %w", err)
 			}
 		}
 
-		// 当读取完毕或遇到EOF时返回
+		// 文件读取完毕
 		if err == io.EOF {
+			dt := time.Since(startTime)
 			logger.
 				WithField("PlanID", req.PlanID).
-				WithField("StreamID", req.StreamID).
-				Debugf("send data size %d", readAllCnt)
-			// 发送EOF消息通知客户端数据传输完成
-			server.Send(&agentserver.StreamDataPacket{
-				Type: agentserver.StreamDataPacketType_EOF,
+				WithField("VarID", req.VarID).
+				Debugf("send data size %d in %v, speed %v/s", readAllCnt, dt, bytesize.New(float64(readAllCnt)/dt.Seconds()))
+			// 发送EOF消息
+			server.Send(&agtrpc.StreamDataPacket{
+				Type: agtrpc.StreamDataPacketType_EOF,
 			})
 			return nil
 		}
 
-		// 处理除EOF和io.ErrUnexpectedEOF之外的读取错误
+		// io.ErrUnexpectedEOF没有读满整个buf就遇到了EOF，此时正常发送剩余数据即可。除了这两个错误之外，其他错误都中断操作
 		if err != nil && err != io.ErrUnexpectedEOF {
 			logger.
 				WithField("PlanID", req.PlanID).
-				WithField("StreamID", req.StreamID).
+				WithField("VarID", req.VarID).
 				Warnf("reading stream data: %s", err.Error())
 			return fmt.Errorf("reading stream data: %w", err)
 		}
 	}
 }
+
+func (s *Service) SendVar(ctx context.Context, req *agtrpc.SendVarReq) (*agtrpc.SendVarResp, error) {
+	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+
+	sw := s.swWorker.FindByIDContexted(ctx, exec.PlanID(req.PlanID))
+	if sw == nil {
+		return nil, fmt.Errorf("plan not found")
+	}
+
+	v, err := serder.JSONToObjectEx[exec.Var]([]byte(req.Var))
+	if err != nil {
+		return nil, fmt.Errorf("deserializing var: %w", err)
+	}
+
+	sw.PutVars(v)
+	return &agtrpc.SendVarResp{}, nil
+}
+
+func (s *Service) GetVar(ctx context.Context, req *agtrpc.GetVarReq) (*agtrpc.GetVarResp, error) {
+	ctx2, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+
+	sw := s.swWorker.FindByIDContexted(ctx2, exec.PlanID(req.PlanID))
+	if sw == nil {
+		return nil, fmt.Errorf("plan not found")
+	}
+
+	v, err := serder.JSONToObjectEx[exec.Var]([]byte(req.Var))
+	if err != nil {
+		return nil, fmt.Errorf("deserializing var: %w", err)
+	}
+
+	signal, err := serder.JSONToObjectEx[*exec.SignalVar]([]byte(req.Signal))
+	if err != nil {
+		return nil, fmt.Errorf("deserializing var: %w", err)
+	}
+
+	sw.PutVars(signal)
+
+	err = sw.BindVars(ctx, v)
+	if err != nil {
+		return nil, fmt.Errorf("binding vars: %w", err)
+	}
+
+	vd, err := serder.ObjectToJSONEx(v)
+	if err != nil {
+		return nil, fmt.Errorf("serializing var: %w", err)
+	}
+
+	return &agtrpc.GetVarResp{
+		Var: string(vd),
+	}, nil
+}

agent/internal/grpc/service.go

@@ -1,134 +1,17 @@
 package grpc
 
 import (
-	"fmt"
-	"io"
-
-	log "gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
 	agentserver "gitlink.org.cn/cloudream/storage/common/pkgs/grpc/agent"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
 )
 
-// Service 类定义了与agent服务相关的操作
 type Service struct {
 	agentserver.AgentServer
-	sw *ioswitch.Switch
+	swWorker *exec.Worker
 }
 
-// NewService 创建并返回一个新的Service实例
-func NewService(sw *ioswitch.Switch) *Service {
+func NewService(swWorker *exec.Worker) *Service {
 	return &Service{
-		sw: sw,
-	}
-}
-
-// SendIPFSFile 处理客户端上传文件到IPFS的请求
-func (s *Service) SendIPFSFile(server agentserver.Agent_SendIPFSFileServer) error {
-	log.Debugf("client upload file")
-
-	ipfsCli, err := stgglb.IPFSPool.Acquire() // 获取一个IPFS客户端实例
-	if err != nil {
-		log.Warnf("new ipfs client: %s", err.Error())
-		return fmt.Errorf("new ipfs client: %w", err)
-	}
-	defer ipfsCli.Close()
-
-	writer, err := ipfsCli.CreateFileStream() // 在IPFS上创建一个文件流
-	if err != nil {
-		log.Warnf("create file failed, err: %s", err.Error())
-		return fmt.Errorf("create file failed, err: %w", err)
-	}
-
-	var recvSize int64
-	for {
-		msg, err := server.Recv() // 接收客户端发送的文件数据
-
-		if err != nil {
-			writer.Abort(io.ErrClosedPipe) // 出错时关闭文件写入
-			log.WithField("ReceiveSize", recvSize).
-				Warnf("recv message failed, err: %s", err.Error())
-			return fmt.Errorf("recv message failed, err: %w", err)
-		}
-
-		err = io2.WriteAll(writer, msg.Data)
-		if err != nil {
-			writer.Abort(io.ErrClosedPipe) // 写入出错时关闭文件写入
-			log.Warnf("write data to file failed, err: %s", err.Error())
-			return fmt.Errorf("write data to file failed, err: %w", err)
-		}
-
-		recvSize += int64(len(msg.Data))
-
-		if msg.Type == agentserver.StreamDataPacketType_EOF { // 当接收到EOF标志时，结束文件写入并返回文件Hash
-			hash, err := writer.Finish()
-			if err != nil {
-				log.Warnf("finish writing failed, err: %s", err.Error())
-				return fmt.Errorf("finish writing failed, err: %w", err)
-			}
-
-			err = server.SendAndClose(&agentserver.SendIPFSFileResp{
-				FileHash: hash,
-			})
-			if err != nil {
-				log.Warnf("send response failed, err: %s", err.Error())
-				return fmt.Errorf("send response failed, err: %w", err)
-			}
-
-			log.Debugf("%d bytes received ", recvSize)
-			return nil
-		}
-	}
-}
-
-// GetIPFSFile 处理客户端从IPFS下载文件的请求
-func (s *Service) GetIPFSFile(req *agentserver.GetIPFSFileReq, server agentserver.Agent_GetIPFSFileServer) error {
-	log.WithField("FileHash", req.FileHash).Debugf("client download file")
-
-	ipfsCli, err := stgglb.IPFSPool.Acquire() // 获取一个IPFS客户端实例
-	if err != nil {
-		log.Warnf("new ipfs client: %s", err.Error())
-		return fmt.Errorf("new ipfs client: %w", err)
-	}
-	defer ipfsCli.Close()
-
-	reader, err := ipfsCli.OpenRead(req.FileHash) // 通过文件Hash打开一个读取流
-	if err != nil {
-		log.Warnf("open file %s to read failed, err: %s", req.FileHash, err.Error())
-		return fmt.Errorf("open file to read failed, err: %w", err)
-	}
-	defer reader.Close()
-
-	buf := make([]byte, 1024)
-	readAllCnt := 0
-	for {
-		readCnt, err := reader.Read(buf) // 从IPFS读取数据
-
-		if readCnt > 0 {
-			readAllCnt += readCnt
-			err = server.Send(&agentserver.FileDataPacket{
-				Type: agentserver.StreamDataPacketType_Data,
-				Data: buf[:readCnt],
-			})
-			if err != nil {
-				log.WithField("FileHash", req.FileHash).
-					Warnf("send file data failed, err: %s", err.Error())
-				return fmt.Errorf("send file data failed, err: %w", err)
-			}
-		}
-
-		if err == io.EOF { // 当读取完毕时，发送EOF标志并返回
-			log.WithField("FileHash", req.FileHash).Debugf("send data size %d", readAllCnt)
-			server.Send(&agentserver.FileDataPacket{
-				Type: agentserver.StreamDataPacketType_EOF,
-			})
-			return nil
-		}
-
-		if err != nil && err != io.ErrUnexpectedEOF { // 遇到除EOF和ErrUnexpectedEOF外的其他错误，中断操作
-			log.Warnf("read file %s data failed, err: %s", req.FileHash, err.Error())
-			return fmt.Errorf("read file data failed, err: %w", err)
-		}
+		swWorker: swWorker,
 	}
 }

agent/internal/http/http.go

@@ -0,0 +1,24 @@
+package http
+
+import "gitlink.org.cn/cloudream/common/consts/errorcode"
+
+type Response struct {
+	Code    string `json:"code"`
+	Message string `json:"message"`
+	Data    any    `json:"data"`
+}
+
+func OK(data any) Response {
+	return Response{
+		Code:    errorcode.OK,
+		Message: "",
+		Data:    data,
+	}
+}
+
+func Failed(code string, msg string) Response {
+	return Response{
+		Code:    code,
+		Message: msg,
+	}
+}

agent/internal/http/hub_io.go

@@ -0,0 +1,301 @@
+package http
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/inhies/go-bytesize"
+	"gitlink.org.cn/cloudream/common/consts/errorcode"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
+	"gitlink.org.cn/cloudream/common/utils/serder"
+)
+
+type IOService struct {
+	*Server
+}
+
+func (s *Server) IOSvc() *IOService {
+	return &IOService{
+		Server: s,
+	}
+}
+
+func (s *IOService) GetStream(ctx *gin.Context) {
+	var req cdsapi.GetStreamReq
+	if err := ctx.ShouldBindJSON(&req); err != nil {
+		logger.Warnf("binding body: %s", err.Error())
+		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
+		return
+	}
+
+	logger.
+		WithField("PlanID", req.PlanID).
+		WithField("VarID", req.VarID).
+		Debugf("stream output")
+
+	// 设置超时
+	c, cancel := context.WithTimeout(ctx.Request.Context(), time.Second*30)
+	defer cancel()
+
+	sw := s.svc.swWorker.FindByIDContexted(c, req.PlanID)
+	if sw == nil {
+		ctx.JSON(http.StatusNotFound, gin.H{"error": "plan not found"})
+		return
+	}
+
+	signalBytes, err := serder.ObjectToJSON(req.Signal)
+	if err != nil {
+		logger.Warnf("serializing SignalVar: %s", err)
+		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "serializing SignalVar fail"))
+		return
+	}
+	signal, err := serder.JSONToObjectEx[*exec.SignalVar](signalBytes)
+	if err != nil {
+		ctx.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("deserializing var: %v", err)})
+		return
+	}
+
+	sw.PutVars(signal)
+
+	strVar := &exec.StreamVar{
+		ID: req.VarID,
+	}
+	err = sw.BindVars(ctx.Request.Context(), strVar)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("binding vars: %v", err)})
+		return
+	}
+
+	reader := strVar.Stream
+	defer reader.Close()
+
+	ctx.Header("Content-Type", "application/octet-stream")
+	ctx.Status(http.StatusOK)
+
+	buf := make([]byte, 1024*64)
+	readAllCnt := 0
+	startTime := time.Now()
+	for {
+		readCnt, err := reader.Read(buf)
+
+		if readCnt > 0 {
+			readAllCnt += readCnt
+			_, err := ctx.Writer.Write(buf[:readCnt])
+			if err != nil {
+				logger.
+					WithField("PlanID", req.PlanID).
+					WithField("VarID", req.VarID).
+					Warnf("send stream data failed, err: %s", err.Error())
+				ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("send stream data failed, err: %v", err)})
+				return
+			}
+			// 刷新缓冲区，确保数据立即发送
+			ctx.Writer.Flush()
+		}
+
+		// 文件读取完毕
+		if err == io.EOF {
+			dt := time.Since(startTime)
+			logger.
+				WithField("PlanID", req.PlanID).
+				WithField("VarID", req.VarID).
+				Debugf("send data size %d in %v, speed %v/s", readAllCnt, dt, bytesize.New(float64(readAllCnt)/dt.Seconds()))
+			return
+		}
+
+		// io.ErrUnexpectedEOF 没有读满整个 buf 就遇到了 EOF，此时正常发送剩余数据即可
+		if err != nil && err != io.ErrUnexpectedEOF {
+			logger.
+				WithField("PlanID", req.PlanID).
+				WithField("VarID", req.VarID).
+				Warnf("reading stream data: %s", err.Error())
+			ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("reading stream data: %v", err)})
+			return
+		}
+	}
+}
+
+func (s *IOService) SendStream(ctx *gin.Context) {
+	var req cdsapi.SendStreamReq
+	if err := ctx.ShouldBindJSON(&req); err != nil {
+		logger.Warnf("binding body: %s", err.Error())
+		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
+		return
+	}
+
+	logger.
+		WithField("PlanID", req.PlanID).
+		WithField("VarID", req.VarID).
+		Debugf("stream input")
+
+	// 超时设置
+	c, cancel := context.WithTimeout(ctx.Request.Context(), time.Second*30)
+	defer cancel()
+
+	sw := s.svc.swWorker.FindByIDContexted(c, req.PlanID)
+	if sw == nil {
+		ctx.JSON(http.StatusNotFound, gin.H{"error": "plan not found"})
+		return
+	}
+
+	pr, pw := io.Pipe()
+	defer pr.Close()
+
+	streamVar := &exec.StreamVar{
+		ID:     req.VarID,
+		Stream: pr,
+	}
+	sw.PutVars(streamVar)
+
+	var recvSize int64
+
+	go func() {
+		defer pw.Close()
+		_, err := io.Copy(pw, ctx.Request.Body)
+		if err != nil {
+			logger.Warnf("write data to file failed, err: %s", err.Error())
+			pw.CloseWithError(fmt.Errorf("write data to file failed: %w", err))
+		}
+	}()
+
+	for {
+		buf := make([]byte, 1024*64)
+		n, err := pr.Read(buf)
+		if err != nil {
+			if err == io.EOF {
+				logger.WithField("ReceiveSize", recvSize).
+					WithField("VarID", req.VarID).
+					Debugf("file transmission completed")
+
+				// 将结果返回给客户端
+				ctx.JSON(http.StatusOK, gin.H{"message": "file transmission completed"})
+				return
+			}
+			logger.Warnf("read stream failed, err: %s", err.Error())
+			ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("read stream failed: %v", err)})
+			return
+		}
+
+		if n > 0 {
+			recvSize += int64(n)
+			// 处理接收到的数据，例如写入文件或进行其他操作
+		}
+	}
+}
+
+func (s *IOService) ExecuteIOPlan(ctx *gin.Context) {
+	data, err := io.ReadAll(ctx.Request.Body)
+	if err != nil {
+		logger.Warnf("reading body: %s", err.Error())
+		ctx.JSON(http.StatusInternalServerError, Failed("400", "internal error"))
+		return
+	}
+
+	plan, err := serder.JSONToObjectEx[exec.Plan](data)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("deserializing plan: %v", err)})
+		return
+	}
+
+	logger.WithField("PlanID", plan.ID).Infof("begin execute io plan")
+	defer logger.WithField("PlanID", plan.ID).Infof("plan finished")
+
+	sw := exec.NewExecutor(plan)
+
+	s.svc.swWorker.Add(sw)
+	defer s.svc.swWorker.Remove(sw)
+
+	execCtx := exec.NewWithContext(ctx.Request.Context())
+
+	// TODO 注入依赖
+
+	_, err = sw.Run(execCtx)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("running io plan: %v", err)})
+		return
+	}
+
+	ctx.JSON(http.StatusOK, gin.H{"message": "plan executed successfully"})
+}
+
+func (s *IOService) SendVar(ctx *gin.Context) {
+	var req cdsapi.SendVarReq
+	if err := ctx.ShouldBindJSON(&req); err != nil {
+		logger.Warnf("binding body: %s", err.Error())
+		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
+		return
+	}
+
+	c, cancel := context.WithTimeout(ctx.Request.Context(), time.Second*30)
+	defer cancel()
+
+	sw := s.svc.swWorker.FindByIDContexted(c, req.PlanID)
+	if sw == nil {
+		ctx.JSON(http.StatusNotFound, gin.H{"error": "plan not found"})
+		return
+	}
+
+	VarBytes, err := serder.ObjectToJSON(req.Var)
+	v, err := serder.JSONToObjectEx[exec.Var](VarBytes)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("deserializing var: %v", err)})
+		return
+	}
+
+	sw.PutVars(v)
+	ctx.JSON(http.StatusOK, gin.H{"message": "var sent successfully"})
+}
+
+func (s *IOService) GetVar(ctx *gin.Context) {
+	var req cdsapi.GetVarReq
+	if err := ctx.ShouldBindJSON(&req); err != nil {
+		logger.Warnf("binding body: %s", err.Error())
+		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
+		return
+	}
+
+	c, cancel := context.WithTimeout(ctx.Request.Context(), time.Second*30)
+	defer cancel()
+
+	sw := s.svc.swWorker.FindByIDContexted(c, req.PlanID)
+	if sw == nil {
+		ctx.JSON(http.StatusNotFound, gin.H{"error": "plan not found"})
+		return
+	}
+
+	VarBytes, err := serder.ObjectToJSON(req.Var)
+	v, err := serder.JSONToObjectEx[exec.Var](VarBytes)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("deserializing var: %v", err)})
+		return
+	}
+
+	SignalBytes, err := serder.ObjectToJSON(req.Signal)
+	signal, err := serder.JSONToObjectEx[*exec.SignalVar](SignalBytes)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("deserializing signal: %v", err)})
+		return
+	}
+
+	sw.PutVars(signal)
+
+	err = sw.BindVars(c, v)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("binding vars: %v", err)})
+		return
+	}
+
+	vd, err := serder.ObjectToJSONEx(v)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("serializing var: %v", err)})
+		return
+	}
+
+	ctx.JSON(http.StatusOK, gin.H{"var": string(vd)})
+}

agent/internal/http/server.go

@@ -0,0 +1,46 @@
+package http
+
+import (
+	"github.com/gin-gonic/gin"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
+)
+
+type Server struct {
+	engine     *gin.Engine
+	listenAddr string
+	svc        *Service
+}
+
+func NewServer(listenAddr string, svc *Service) (*Server, error) {
+	engine := gin.New()
+
+	return &Server{
+		engine:     engine,
+		listenAddr: listenAddr,
+		svc:        svc,
+	}, nil
+}
+
+func (s *Server) Serve() error {
+	s.initRouters()
+
+	logger.Infof("start serving http at: %s", s.listenAddr)
+	err := s.engine.Run(s.listenAddr)
+
+	if err != nil {
+		logger.Infof("http stopped with error: %s", err.Error())
+		return err
+	}
+
+	logger.Infof("http stopped")
+	return nil
+}
+
+func (s *Server) initRouters() {
+	s.engine.GET(cdsapi.GetStreamPath, s.IOSvc().GetStream)
+	s.engine.POST(cdsapi.SendStreamPath, s.IOSvc().SendStream)
+	s.engine.POST(cdsapi.ExecuteIOPlanPath, s.IOSvc().ExecuteIOPlan)
+	s.engine.POST(cdsapi.SendVarPath, s.IOSvc().SendVar)
+	s.engine.GET(cdsapi.GetVarPath, s.IOSvc().GetVar)
+}

agent/internal/http/service.go

@@ -0,0 +1,13 @@
+package http
+
+import "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
+
+type Service struct {
+	swWorker *exec.Worker
+}
+
+func NewService(swWorker *exec.Worker) *Service {
+	return &Service{
+		swWorker: swWorker,
+	}
+}

agent/internal/mq/agent.go

@@ -1,41 +1,10 @@
 package mq
 
 import (
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	"gitlink.org.cn/cloudream/common/pkgs/mq"
-	"gitlink.org.cn/cloudream/storage/common/consts"
-	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
 	agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent"
 )
 
-// GetState 用于获取IPFS节点的状态
-// 参数:
-// msg: 包含请求信息的GetState消息结构体
-// 返回值:
-// *agtmq.GetStateResp: 包含响应信息的GetStateResp消息结构体
-// *mq.CodeMessage: 错误代码和消息
 func (svc *Service) GetState(msg *agtmq.GetState) (*agtmq.GetStateResp, *mq.CodeMessage) {
-	var ipfsState string
-
-	// 尝试从IPFS池中获取一个客户端实例
-	ipfsCli, err := stgglb.IPFSPool.Acquire()
-	if err != nil {
-		// 如果获取失败，记录警告信息，并设置IPFS状态为不可用
-		logger.Warnf("new ipfs client: %s", err.Error())
-		ipfsState = consts.IPFSStateUnavailable
-
-	} else {
-		// 如果获取成功，检查IPFS节点是否正常
-		if ipfsCli.IsUp() {
-			ipfsState = consts.IPFSStateOK
-		} else {
-			// 如果节点不正常，设置IPFS状态为不可用
-			ipfsState = consts.IPFSStateUnavailable
-		}
-		// 释放IPFS客户端实例
-		ipfsCli.Close()
-	}
-
-	// 构造并返回响应
-	return mq.ReplyOK(agtmq.NewGetStateResp(ipfsState))
+	return mq.ReplyOK(agtmq.NewGetStateResp())
 }

agent/internal/mq/cache.go

@@ -1,104 +1,81 @@
 package mq
 
 import (
+	"fmt"
 	"time"
 
-	"github.com/samber/lo"
 	"gitlink.org.cn/cloudream/common/consts/errorcode"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	"gitlink.org.cn/cloudream/common/pkgs/mq"
 	mytask "gitlink.org.cn/cloudream/storage/agent/internal/task"
-	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
 	agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/storage/shard/types"
 )
 
-// CheckCache 检查IPFS缓存
-// 参数 msg: 包含检查缓存请求信息的结构体
-// 返回值: 检查缓存响应结构体和错误信息
 func (svc *Service) CheckCache(msg *agtmq.CheckCache) (*agtmq.CheckCacheResp, *mq.CodeMessage) {
-	ipfsCli, err := stgglb.IPFSPool.Acquire() // 尝试从IPFS池获取客户端
+	store, err := svc.shardStorePool.Get(msg.StorageID)
 	if err != nil {
-		logger.Warnf("new ipfs client: %s", err.Error())
-		return nil, mq.Failed(errorcode.OperationFailed, "new ipfs client failed")
+		return nil, mq.Failed(errorcode.OperationFailed, fmt.Sprintf("finding shard store: %v", err))
 	}
-	defer ipfsCli.Close() // 确保IPFS客户端被正确关闭
 
-	files, err := ipfsCli.GetPinnedFiles() // 获取IPFS上被固定的文件列表
+	infos, err := store.ListAll()
 	if err != nil {
-		logger.Warnf("get pinned files from ipfs failed, err: %s", err.Error())
-		return nil, mq.Failed(errorcode.OperationFailed, "get pinned files from ipfs failed")
+		return nil, mq.Failed(errorcode.OperationFailed, fmt.Sprintf("listting file in shard store: %v", err))
 	}
 
-	return mq.ReplyOK(agtmq.NewCheckCacheResp(lo.Keys(files))) // 返回文件列表的键
+	var fileHashes []types.FileHash
+	for _, info := range infos {
+		fileHashes = append(fileHashes, info.Hash)
+	}
+
+	return mq.ReplyOK(agtmq.NewCheckCacheResp(fileHashes))
 }
 
-// CacheGC 执行缓存垃圾回收
-// 参数 msg: 包含垃圾回收请求信息的结构体
-// 返回值: 垃圾回收响应结构体和错误信息
 func (svc *Service) CacheGC(msg *agtmq.CacheGC) (*agtmq.CacheGCResp, *mq.CodeMessage) {
-	ipfsCli, err := stgglb.IPFSPool.Acquire() // 尝试从IPFS池获取客户端
+	store, err := svc.shardStorePool.Get(msg.StorageID)
 	if err != nil {
-		logger.Warnf("new ipfs client: %s", err.Error())
-		return nil, mq.Failed(errorcode.OperationFailed, "new ipfs client failed")
+		return nil, mq.Failed(errorcode.OperationFailed, fmt.Sprintf("finding shard store: %v", err))
 	}
-	defer ipfsCli.Close() // 确保IPFS客户端被正确关闭
 
-	files, err := ipfsCli.GetPinnedFiles() // 获取IPFS上被固定的文件列表
+	err = store.Purge(msg.Avaiables)
 	if err != nil {
-		logger.Warnf("get pinned files from ipfs failed, err: %s", err.Error())
-		return nil, mq.Failed(errorcode.OperationFailed, "get pinned files from ipfs failed")
-	}
-
-	// 根据请求对比当前被固定的文件，将未记录到元数据的文件取消固定
-	shouldPinnedFiles := lo.SliceToMap(msg.PinnedFileHashes, func(hash string) (string, bool) { return hash, true })
-	for hash := range files {
-		if !shouldPinnedFiles[hash] {
-			ipfsCli.Unpin(hash) // 取消固定文件
-			logger.WithField("FileHash", hash).Debugf("unpinned by gc")
-		}
+		return nil, mq.Failed(errorcode.OperationFailed, fmt.Sprintf("purging cache: %v", err))
 	}
 
-	return mq.ReplyOK(agtmq.RespCacheGC()) // 返回垃圾回收完成的响应
+	return mq.ReplyOK(agtmq.RespCacheGC())
 }
 
-// StartCacheMovePackage 开始缓存移动包
-// 参数 msg: 包含启动缓存移动请求信息的结构体
-// 返回值: 启动缓存移动响应结构体和错误信息
 func (svc *Service) StartCacheMovePackage(msg *agtmq.StartCacheMovePackage) (*agtmq.StartCacheMovePackageResp, *mq.CodeMessage) {
-	tsk := svc.taskManager.StartNew(mytask.NewCacheMovePackage(msg.UserID, msg.PackageID)) // 启动新的缓存移动任务
-	return mq.ReplyOK(agtmq.NewStartCacheMovePackageResp(tsk.ID()))                        // 返回任务ID
+	tsk := svc.taskManager.StartNew(mytask.NewCacheMovePackage(msg.UserID, msg.PackageID, msg.StorageID))
+	return mq.ReplyOK(agtmq.NewStartCacheMovePackageResp(tsk.ID()))
 }
 
-// WaitCacheMovePackage 等待缓存移动包完成
-// 参数 msg: 包含等待缓存移动请求信息的结构体
-// 返回值: 等待缓存移动响应结构体和错误信息
 func (svc *Service) WaitCacheMovePackage(msg *agtmq.WaitCacheMovePackage) (*agtmq.WaitCacheMovePackageResp, *mq.CodeMessage) {
-	tsk := svc.taskManager.FindByID(msg.TaskID) // 根据任务ID查找任务
+	tsk := svc.taskManager.FindByID(msg.TaskID)
 	if tsk == nil {
-		return nil, mq.Failed(errorcode.TaskNotFound, "task not found") // 如果任务不存在，返回错误
+		return nil, mq.Failed(errorcode.TaskNotFound, "task not found")
 	}
 
 	if msg.WaitTimeoutMs == 0 {
-		tsk.Wait() // 等待任务完成
+		tsk.Wait()
 
 		errMsg := ""
 		if tsk.Error() != nil {
-			errMsg = tsk.Error().Error() // 获取任务错误信息
+			errMsg = tsk.Error().Error()
 		}
 
-		return mq.ReplyOK(agtmq.NewWaitCacheMovePackageResp(true, errMsg)) // 返回任务完成状态和错误信息
+		return mq.ReplyOK(agtmq.NewWaitCacheMovePackageResp(true, errMsg))
 
 	} else {
-		if tsk.WaitTimeout(time.Duration(msg.WaitTimeoutMs) * time.Millisecond) { // 设置等待超时
+		if tsk.WaitTimeout(time.Duration(msg.WaitTimeoutMs) * time.Millisecond) {
 
 			errMsg := ""
 			if tsk.Error() != nil {
-				errMsg = tsk.Error().Error() // 获取任务错误信息
+				errMsg = tsk.Error().Error()
 			}
 
-			return mq.ReplyOK(agtmq.NewWaitCacheMovePackageResp(true, errMsg)) // 返回任务完成状态和错误信息
+			return mq.ReplyOK(agtmq.NewWaitCacheMovePackageResp(true, errMsg))
 		}
 
-		return mq.ReplyOK(agtmq.NewWaitCacheMovePackageResp(false, "")) // 返回等待超时状态
+		return mq.ReplyOK(agtmq.NewWaitCacheMovePackageResp(false, ""))
 	}
 }

agent/internal/mq/io.go

@@ -1,77 +0,0 @@
-package mq
-
-import (
-	"time"
-
-	"gitlink.org.cn/cloudream/common/consts/errorcode"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/pkgs/mq"
-	mytask "gitlink.org.cn/cloudream/storage/agent/internal/task"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-	agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent"
-)
-
-// SetupIOPlan 设置I/O计划。
-// msg: 包含I/O计划信息的消息体。
-// 返回值: 成功时返回响应消息和成功标志，失败时返回错误代码和消息。
-func (svc *Service) SetupIOPlan(msg *agtmq.SetupIOPlan) (*agtmq.SetupIOPlanResp, *mq.CodeMessage) {
-	err := svc.sw.SetupPlan(msg.Plan)
-	if err != nil {
-		logger.WithField("PlanID", msg.Plan.ID).Warnf("adding plan: %s", err.Error())
-		return nil, mq.Failed(errorcode.OperationFailed, "adding plan failed")
-	}
-
-	return mq.ReplyOK(agtmq.NewSetupIOPlanResp())
-}
-
-// StartIOPlan 启动I/O计划。
-// msg: 包含I/O计划ID的消息体。
-// 返回值: 成功时返回任务ID和成功标志，失败时返回错误代码和消息。
-func (svc *Service) StartIOPlan(msg *agtmq.StartIOPlan) (*agtmq.StartIOPlanResp, *mq.CodeMessage) {
-	tsk := svc.taskManager.StartNew(mytask.NewExecuteIOPlan(msg.PlanID))
-	return mq.ReplyOK(agtmq.NewStartIOPlanResp(tsk.ID()))
-}
-
-// WaitIOPlan 等待I/O计划完成。
-// msg: 包含任务ID和等待超时时间的消息体。
-// 返回值: 成功时返回任务完成状态、错误消息和结果，失败时返回错误代码和消息。
-func (svc *Service) WaitIOPlan(msg *agtmq.WaitIOPlan) (*agtmq.WaitIOPlanResp, *mq.CodeMessage) {
-	tsk := svc.taskManager.FindByID(msg.TaskID)
-	if tsk == nil {
-		return nil, mq.Failed(errorcode.TaskNotFound, "task not found")
-	}
-
-	if msg.WaitTimeoutMs == 0 {
-		tsk.Wait()
-
-		errMsg := ""
-		if tsk.Error() != nil {
-			errMsg = tsk.Error().Error()
-		}
-
-		planTsk := tsk.Body().(*mytask.ExecuteIOPlan)
-		return mq.ReplyOK(agtmq.NewWaitIOPlanResp(true, errMsg, planTsk.Result))
-
-	} else {
-		if tsk.WaitTimeout(time.Duration(msg.WaitTimeoutMs) * time.Millisecond) {
-
-			errMsg := ""
-			if tsk.Error() != nil {
-				errMsg = tsk.Error().Error()
-			}
-
-			planTsk := tsk.Body().(*mytask.ExecuteIOPlan)
-			return mq.ReplyOK(agtmq.NewWaitIOPlanResp(true, errMsg, planTsk.Result))
-		}
-
-		return mq.ReplyOK(agtmq.NewWaitIOPlanResp(false, "", ioswitch.PlanResult{}))
-	}
-}
-
-// CancelIOPlan 取消I/O计划。
-// msg: 包含要取消的I/O计划ID的消息体。
-// 返回值: 成功时返回响应消息和成功标志，失败时返回错误代码和消息。
-func (svc *Service) CancelIOPlan(msg *agtmq.CancelIOPlan) (*agtmq.CancelIOPlanResp, *mq.CodeMessage) {
-	svc.sw.CancelPlan(msg.PlanID)
-	return mq.ReplyOK(agtmq.NewCancelIOPlanResp())
-}

agent/internal/mq/object.go

@@ -1,37 +0,0 @@
-package mq
-
-import (
-	"gitlink.org.cn/cloudream/common/consts/errorcode"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/pkgs/mq"
-	"gitlink.org.cn/cloudream/storage/agent/internal/task"
-	agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent"
-)
-
-// PinObject 用于处理对象固定（pin）的请求。
-// msg: 包含要固定的对象的文件哈希和是否为后台任务的标志。
-// 返回值1: 成功时返回固定操作的响应信息。
-// 返回值2: 操作失败时返回错误码和错误信息。
-func (svc *Service) PinObject(msg *agtmq.PinObject) (*agtmq.PinObjectResp, *mq.CodeMessage) {
-	// 开始记录固定对象操作的日志
-	logger.WithField("FileHash", msg.FileHashes).Debugf("pin object")
-
-	// 启动一个新的任务来处理IPFS固定操作
-	tsk := svc.taskManager.StartNew(task.NewIPFSPin(msg.FileHashes))
-
-	// 检查任务是否出错，若有错误则记录日志并返回操作失败的信息
-	if tsk.Error() != nil {
-		logger.WithField("FileHash", msg.FileHashes).
-			Warnf("pin object failed, err: %s", tsk.Error().Error())
-		return nil, mq.Failed(errorcode.OperationFailed, "pin object failed")
-	}
-
-	// 如果是后台任务，则直接返回成功响应，不等待任务完成
-	if msg.IsBackground {
-		return mq.ReplyOK(agtmq.RespPinObject())
-	}
-
-	// 等待任务完成
-	tsk.Wait()
-	return mq.ReplyOK(agtmq.RespPinObject())
-}

agent/internal/mq/service.go

@@ -2,25 +2,17 @@ package mq
 
 import (
 	"gitlink.org.cn/cloudream/storage/agent/internal/task"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/storage/shard/pool"
 )
 
-// Service 表示一个消息队列服务
-// 它包含了任务管理和IO切换器两个核心组件
 type Service struct {
-	taskManager *task.Manager    // taskManager 用于管理和调度任务
-	sw          *ioswitch.Switch // sw 用于控制IO切换
+	taskManager    *task.Manager
+	shardStorePool *pool.ShardStorePool
 }
 
-// NewService 创建一个新的消息队列服务实例
-// 参数：
-// - taskMgr：任务管理器，负责任务的调度和管理
-// - sw：IO切换器，用于控制数据的输入输出
-// 返回值：
-// - *Service：指向创建的消息队列服务实例的指针
-func NewService(taskMgr *task.Manager, sw *ioswitch.Switch) *Service {
+func NewService(taskMgr *task.Manager, shardStorePool *pool.ShardStorePool) *Service {
 	return &Service{
-		taskManager: taskMgr,
-		sw:          sw,
+		taskManager:    taskMgr,
+		shardStorePool: shardStorePool,
 	}
 }

agent/internal/mq/storage.go

@@ -23,77 +23,69 @@ import (
 	"gitlink.org.cn/cloudream/storage/common/utils"
 )
 
-// StartStorageLoadPackage 启动存储加载包任务
-// 参数:
-// - msg: 包含启动存储加载包任务所需信息的消息对象，包括用户ID、包ID和存储ID
-// 返回值:
-// - *agtmq.StartStorageLoadPackageResp: 任务启动成功的响应对象，包含任务ID
-// - *mq.CodeMessage: 任务启动失败时的错误信息对象，包含错误码和错误消息
 func (svc *Service) StartStorageLoadPackage(msg *agtmq.StartStorageLoadPackage) (*agtmq.StartStorageLoadPackageResp, *mq.CodeMessage) {
-	// 在任务管理器中启动一个新的存储加载包任务，并获取任务ID
 	tsk := svc.taskManager.StartNew(mytask.NewStorageLoadPackage(msg.UserID, msg.PackageID, msg.StorageID))
-	// 构造并返回任务启动成功的响应消息，包含任务ID
 	return mq.ReplyOK(agtmq.NewStartStorageLoadPackageResp(tsk.ID()))
 }
 
-// WaitStorageLoadPackage 等待存储加载包的任务完成。
-//
-// 参数:
-//
-//	msg *agtmq.WaitStorageLoadPackage: 包含任务ID和可选的等待超时时间的消息。
-//
-// 返回值:
-//
-//	*agtmq.WaitStorageLoadPackageResp: 如果任务找到且已完成（或超时），则返回任务的响应信息，包括是否成功、错误信息和完整输出路径。
-//	*mq.CodeMessage: 如果任务未找到，则返回错误代码和消息。
 func (svc *Service) WaitStorageLoadPackage(msg *agtmq.WaitStorageLoadPackage) (*agtmq.WaitStorageLoadPackageResp, *mq.CodeMessage) {
-	logger.WithField("TaskID", msg.TaskID).Debugf("wait loading package") // 记录等待加载包任务的debug信息
+	logger.WithField("TaskID", msg.TaskID).Debugf("wait loading package")
 
-	tsk := svc.taskManager.FindByID(msg.TaskID) // 根据任务ID查找任务
+	tsk := svc.taskManager.FindByID(msg.TaskID)
 	if tsk == nil {
-		return nil, mq.Failed(errorcode.TaskNotFound, "task not found") // 如果任务未找到，返回任务未找到的错误信息
+		return nil, mq.Failed(errorcode.TaskNotFound, "task not found")
 	}
 
 	if msg.WaitTimeoutMs == 0 {
-		tsk.Wait() // 如果没有设置等待超时，那么就无限等待任务完成
+		tsk.Wait()
 
-		errMsg := "" // 初始化错误信息为空
+		errMsg := ""
 		if tsk.Error() != nil {
-			errMsg = tsk.Error().Error() // 如果任务有错误，记录错误信息
+			errMsg = tsk.Error().Error()
 		}
 
-		loadTsk := tsk.Body().(*mytask.StorageLoadPackage) // 将任务体转换为存储加载包类型
+		loadTsk := tsk.Body().(*mytask.StorageLoadPackage)
+
+		return mq.ReplyOK(agtmq.NewWaitStorageLoadPackageResp(true, errMsg, loadTsk.PackagePath, loadTsk.LocalBase, loadTsk.RemoteBase))
 
-		return mq.ReplyOK(agtmq.NewWaitStorageLoadPackageResp(true, errMsg, loadTsk.FullOutputPath)) // 返回任务完成的状态，错误信息和完整输出路径
 	} else {
-		// 如果设置了等待超时，就设置超时时间等待任务完成
 		if tsk.WaitTimeout(time.Duration(msg.WaitTimeoutMs) * time.Millisecond) {
 
-			errMsg := "" // 初始化错误信息为空
+			errMsg := ""
 			if tsk.Error() != nil {
-				errMsg = tsk.Error().Error() // 如果任务有错误，记录错误信息
+				errMsg = tsk.Error().Error()
 			}
 
-			loadTsk := tsk.Body().(*mytask.StorageLoadPackage) // 将任务体转换为存储加载包类型
+			loadTsk := tsk.Body().(*mytask.StorageLoadPackage)
 
-			return mq.ReplyOK(agtmq.NewWaitStorageLoadPackageResp(true, errMsg, loadTsk.FullOutputPath)) // 返回任务完成的状态，错误信息和完整输出路径
+			return mq.ReplyOK(agtmq.NewWaitStorageLoadPackageResp(true, errMsg, loadTsk.PackagePath, loadTsk.LocalBase, loadTsk.RemoteBase))
 		}
 
-		return mq.ReplyOK(agtmq.NewWaitStorageLoadPackageResp(false, "", "")) // 如果等待超时，返回任务未完成的状态
+		return mq.ReplyOK(agtmq.NewWaitStorageLoadPackageResp(false, "", "", "", ""))
 	}
 }
 
-// StorageCheck 对指定目录进行存储检查
-// 参数:
-// - msg: 包含需要检查的存储目录信息
-// 返回值:
-// - *agtmq.StorageCheckResp: 存储检查响应，包含检查结果和存储包信息
-// - *mq.CodeMessage: 错误信息，如果操作成功，则为nil
 func (svc *Service) StorageCheck(msg *agtmq.StorageCheck) (*agtmq.StorageCheckResp, *mq.CodeMessage) {
-	// 尝试读取指定的目录
-	infos, err := os.ReadDir(msg.Directory)
+	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+	if err != nil {
+		return mq.ReplyOK(agtmq.NewStorageCheckResp(
+			err.Error(),
+			nil,
+		))
+	}
+	defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+	// TODO UserID。应该设计两种接口，一种需要UserID，一种不需要。
+	getStg, err := coorCli.GetStorage(coormq.ReqGetStorage(cdssdk.UserID(1), msg.StorageID))
+	if err != nil {
+		return mq.ReplyOK(agtmq.NewStorageCheckResp(
+			err.Error(),
+			nil,
+		))
+	}
+
+	entries, err := os.ReadDir(utils.MakeStorageLoadDirectory(getStg.Storage.LocalBase))
 	if err != nil {
-		// 如果读取目录失败，记录警告信息，并返回错误信息和空的存储包列表
 		logger.Warnf("list storage directory failed, err: %s", err.Error())
 		return mq.ReplyOK(agtmq.NewStorageCheckResp(
 			err.Error(),
@@ -103,31 +95,24 @@ func (svc *Service) StorageCheck(msg *agtmq.StorageCheck) (*agtmq.StorageCheckRe
 
 	var stgPkgs []model.StoragePackage
 
-	// 过滤出目录中的子目录（用户目录）
-	userDirs := lo.Filter(infos, func(info fs.DirEntry, index int) bool { return info.IsDir() })
+	userDirs := lo.Filter(entries, func(info fs.DirEntry, index int) bool { return info.IsDir() })
 	for _, dir := range userDirs {
-		// 尝试将子目录名称解析为用户ID
 		userIDInt, err := strconv.ParseInt(dir.Name(), 10, 64)
 		if err != nil {
-			// 如果解析失败，记录警告信息，并继续处理下一个目录
 			logger.Warnf("parsing user id %s: %s", dir.Name(), err.Error())
 			continue
 		}
 
-		// 构造存储包目录路径，并读取该目录
-		pkgDir := utils.MakeStorageLoadDirectory(msg.Directory, dir.Name())
+		pkgDir := filepath.Join(utils.MakeStorageLoadDirectory(getStg.Storage.LocalBase), dir.Name())
 		pkgDirs, err := os.ReadDir(pkgDir)
 		if err != nil {
-			// 如果读取目录失败，记录警告信息，并继续处理下一个用户目录
 			logger.Warnf("reading package dir %s: %s", pkgDir, err.Error())
 			continue
 		}
 
-		// 遍历存储包目录中的包，解析包ID，并添加到存储包列表中
 		for _, pkg := range pkgDirs {
 			pkgIDInt, err := strconv.ParseInt(pkg.Name(), 10, 64)
 			if err != nil {
-				// 如果解析失败，记录警告信息，并继续处理下一个包
 				logger.Warnf("parsing package dir %s: %s", pkg.Name(), err.Error())
 				continue
 			}
@@ -140,31 +125,29 @@ func (svc *Service) StorageCheck(msg *agtmq.StorageCheck) (*agtmq.StorageCheckRe
 		}
 	}
 
-	// 返回存储检查成功的响应，包含存储包列表
 	return mq.ReplyOK(agtmq.NewStorageCheckResp(consts.StorageDirectoryStateOK, stgPkgs))
 }
 
-// StorageGC 执行存储垃圾回收
-// 根据提供的目录和包信息，清理不再需要的文件和目录。
-//
-// 参数:
-//
-//	msg *agtmq.StorageGC: 包含需要进行垃圾回收的目录和包信息。
-//
-// 返回值:
-//
-//	*agtmq.StorageGCResp: 垃圾回收操作的响应信息。
-//	*mq.CodeMessage: 如果操作失败，返回错误代码和消息。
 func (svc *Service) StorageGC(msg *agtmq.StorageGC) (*agtmq.StorageGCResp, *mq.CodeMessage) {
-	// 尝试列出指定目录下的所有文件和目录
-	infos, err := os.ReadDir(msg.Directory)
+	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+	if err != nil {
+		return nil, mq.Failed(errorcode.OperationFailed, err.Error())
+	}
+	defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+	// TODO UserID。应该设计两种接口，一种需要UserID，一种不需要。
+	getStg, err := coorCli.GetStorage(coormq.ReqGetStorage(cdssdk.UserID(1), msg.StorageID))
+	if err != nil {
+		return nil, mq.Failed(errorcode.OperationFailed, err.Error())
+	}
+
+	entries, err := os.ReadDir(utils.MakeStorageLoadDirectory(getStg.Storage.LocalBase))
 	if err != nil {
-		// 如果列出失败，记录日志并返回操作失败信息
 		logger.Warnf("list storage directory failed, err: %s", err.Error())
 		return nil, mq.Failed(errorcode.OperationFailed, "list directory files failed")
 	}
 
-	// 构建用户ID到包ID的映射，以便知道哪些包是需要保留的
+	// userID->pkgID->pkg
 	userPkgs := make(map[string]map[string]bool)
 	for _, pkg := range msg.Packages {
 		userIDStr := fmt.Sprintf("%d", pkg.UserID)
@@ -179,13 +162,12 @@ func (svc *Service) StorageGC(msg *agtmq.StorageGC) (*agtmq.StorageGCResp, *mq.C
 		pkgs[pkgIDStr] = true
 	}
 
-	// 过滤出目录条目，并遍历这些目录
-	userDirs := lo.Filter(infos, func(info fs.DirEntry, index int) bool { return info.IsDir() })
+	userDirs := lo.Filter(entries, func(info fs.DirEntry, index int) bool { return info.IsDir() })
 	for _, dir := range userDirs {
 		pkgMap, ok := userPkgs[dir.Name()]
-		// 如果当前目录在需要保留的包映射中不存在，则删除该目录
+		// 第一级目录名是UserID，先删除UserID在StoragePackage表里没出现过的文件夹
 		if !ok {
-			rmPath := filepath.Join(msg.Directory, dir.Name())
+			rmPath := filepath.Join(utils.MakeStorageLoadDirectory(getStg.Storage.LocalBase), dir.Name())
 			err := os.RemoveAll(rmPath)
 			if err != nil {
 				logger.Warnf("removing user dir %s: %s", rmPath, err.Error())
@@ -195,8 +177,8 @@ func (svc *Service) StorageGC(msg *agtmq.StorageGC) (*agtmq.StorageGCResp, *mq.C
 			continue
 		}
 
-		// 遍历每个用户目录下的packages目录，删除不在保留包映射中的包
-		pkgDir := utils.MakeStorageLoadDirectory(msg.Directory, dir.Name())
+		pkgDir := filepath.Join(utils.MakeStorageLoadDirectory(getStg.Storage.LocalBase), dir.Name())
+		// 遍历每个UserID目录的packages目录里的内容
 		pkgs, err := os.ReadDir(pkgDir)
 		if err != nil {
 			logger.Warnf("reading package dir %s: %s", pkgDir, err.Error())
@@ -216,47 +198,28 @@ func (svc *Service) StorageGC(msg *agtmq.StorageGC) (*agtmq.StorageGCResp, *mq.C
 		}
 	}
 
-	// 垃圾回收完成，返回成功响应
 	return mq.ReplyOK(agtmq.RespStorageGC())
 }
 
-// StartStorageCreatePackage 开始创建存储包的任务。
-// 接收一个启动存储创建包的消息，并返回任务响应或错误消息。
-//
-// 参数:
-//
-//	msg *agtmq.StartStorageCreatePackage - 包含创建存储包所需信息的消息。
-//
-// 返回值:
-//
-//	*agtmq.StartStorageCreatePackageResp - 创建任务成功的响应，包含任务ID。
-//	*mq.CodeMessage - 创建任务失败时返回的错误信息。
 func (svc *Service) StartStorageCreatePackage(msg *agtmq.StartStorageCreatePackage) (*agtmq.StartStorageCreatePackageResp, *mq.CodeMessage) {
-	// 从协调器MQ池获取客户端
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
-		// 如果获取客户端失败，记录警告并返回错误消息
 		logger.Warnf("new coordinator client: %s", err.Error())
 
 		return nil, mq.Failed(errorcode.OperationFailed, "new coordinator client failed")
 	}
-	// 确保在函数结束时释放协调器MQ客户端
 	defer stgglb.CoordinatorMQPool.Release(coorCli)
 
-	// 获取存储信息
-	getStgResp, err := coorCli.GetStorageInfo(coormq.NewGetStorageInfo(msg.UserID, msg.StorageID))
+	getStgResp, err := coorCli.GetStorage(coormq.ReqGetStorage(msg.UserID, msg.StorageID))
 	if err != nil {
-		// 如果获取存储信息失败，记录警告并返回错误消息
 		logger.WithField("StorageID", msg.StorageID).
 			Warnf("getting storage info: %s", err.Error())
 
 		return nil, mq.Failed(errorcode.OperationFailed, "get storage info failed")
 	}
 
-	// 计算打包文件的完整路径
-	fullPath := filepath.Clean(filepath.Join(getStgResp.Directory, msg.Path))
+	fullPath := filepath.Clean(filepath.Join(getStgResp.Storage.LocalBase, msg.Path))
 
-	// 遍历目录，收集所有需要上传的文件路径
 	var uploadFilePathes []string
 	err = filepath.WalkDir(fullPath, func(fname string, fi os.DirEntry, err error) error {
 		if err != nil {
@@ -270,52 +233,32 @@ func (svc *Service) StartStorageCreatePackage(msg *agtmq.StartStorageCreatePacka
 		return nil
 	})
 	if err != nil {
-		// 如果目录读取失败，记录警告并返回错误消息
 		logger.Warnf("opening directory %s: %s", fullPath, err.Error())
 
 		return nil, mq.Failed(errorcode.OperationFailed, "read directory failed")
 	}
 
-	// 创建上传对象的迭代器
 	objIter := iterator.NewUploadingObjectIterator(fullPath, uploadFilePathes)
-	// 启动新任务来创建存储包
 	tsk := svc.taskManager.StartNew(mytask.NewCreatePackage(msg.UserID, msg.BucketID, msg.Name, objIter, msg.NodeAffinity))
-	// 返回任务成功的响应
 	return mq.ReplyOK(agtmq.NewStartStorageCreatePackageResp(tsk.ID()))
 }
 
-// WaitStorageCreatePackage 等待存储创建包的处理函数。
-//
-// 参数:
-// msg: 包含任务ID和等待超时时间的消息对象。
-//
-// 返回值:
-// 返回一个任务响应对象和一个错误消息对象。如果任务找到且未超时，将返回任务的结果；如果任务未找到或超时，将返回相应的错误信息。
 func (svc *Service) WaitStorageCreatePackage(msg *agtmq.WaitStorageCreatePackage) (*agtmq.WaitStorageCreatePackageResp, *mq.CodeMessage) {
-	// 根据任务ID查找任务
 	tsk := svc.taskManager.FindByID(msg.TaskID)
 	if tsk == nil {
-		// 如果任务未找到，返回任务未找到错误
 		return nil, mq.Failed(errorcode.TaskNotFound, "task not found")
 	}
 
-	// 根据等待超时时间进行等待处理
 	if msg.WaitTimeoutMs == 0 {
-		// 如果没有设置超时时间，无限等待
 		tsk.Wait()
 	} else if !tsk.WaitTimeout(time.Duration(msg.WaitTimeoutMs) * time.Millisecond) {
-		// 如果设置了超时时间，且超时未完成，返回超时处理结果
 		return mq.ReplyOK(agtmq.NewWaitStorageCreatePackageResp(false, "", 0))
 	}
 
-	// 检查任务是否有错误
 	if tsk.Error() != nil {
-		// 如果任务有错误，返回错误信息
 		return mq.ReplyOK(agtmq.NewWaitStorageCreatePackageResp(true, tsk.Error().Error(), 0))
 	}
 
-	// 获取任务结果
 	taskBody := tsk.Body().(*mytask.CreatePackage)
-	// 返回任务成功处理结果
 	return mq.ReplyOK(agtmq.NewWaitStorageCreatePackageResp(true, "", taskBody.Result.PackageID))
 }

agent/internal/task/cache_move_package.go

@@ -2,6 +2,7 @@ package task
 
 import (
 	"fmt"
+	"io"
 	"time"
 
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
@@ -13,24 +14,20 @@ import (
 	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
 )
 
-// CacheMovePackage 代表缓存移动包的任务实体。
 type CacheMovePackage struct {
-	userID    cdssdk.UserID    // 用户ID
-	packageID cdssdk.PackageID // 包ID
+	userID    cdssdk.UserID
+	packageID cdssdk.PackageID
+	storageID cdssdk.StorageID
 }
 
-// NewCacheMovePackage 创建一个新的缓存移动包任务实例。
-func NewCacheMovePackage(userID cdssdk.UserID, packageID cdssdk.PackageID) *CacheMovePackage {
+func NewCacheMovePackage(userID cdssdk.UserID, packageID cdssdk.PackageID, storageID cdssdk.StorageID) *CacheMovePackage {
 	return &CacheMovePackage{
 		userID:    userID,
 		packageID: packageID,
+		storageID: storageID,
 	}
 }
 
-// Execute 执行缓存移动包的任务。
-// task: 任务实例。
-// ctx: 任务上下文。
-// complete: 任务完成的回调函数。
 func (t *CacheMovePackage) Execute(task *task.Task[TaskContext], ctx TaskContext, complete CompleteFn) {
 	err := t.do(ctx)
 	complete(err, CompleteOption{
@@ -38,14 +35,18 @@ func (t *CacheMovePackage) Execute(task *task.Task[TaskContext], ctx TaskContext
 	})
 }
 
-// do 实际执行缓存移动的逻辑。
 func (t *CacheMovePackage) do(ctx TaskContext) error {
 	log := logger.WithType[CacheMovePackage]("Task")
 	log.Debugf("begin with %v", logger.FormatStruct(t))
 	defer log.Debugf("end")
 
-	// 获取分布式锁以保护操作
+	store, err := ctx.shardStorePool.Get(t.storageID)
+	if err != nil {
+		return fmt.Errorf("getting shard store: %w", err)
+	}
+
 	mutex, err := reqbuilder.NewBuilder().
+		// 保护解码出来的Object数据
 		IPFS().Buzy(*stgglb.Local.NodeID).
 		MutexLock(ctx.distlock)
 	if err != nil {
@@ -53,19 +54,12 @@ func (t *CacheMovePackage) do(ctx TaskContext) error {
 	}
 	defer mutex.Unlock()
 
-	// 获取协调器MQ客户端
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
 		return fmt.Errorf("new coordinator client: %w", err)
 	}
 	defer stgglb.CoordinatorMQPool.Release(coorCli)
 
-	ipfsCli, err := stgglb.IPFSPool.Acquire()
-	if err != nil {
-		return fmt.Errorf("new ipfs client: %w", err)
-	}
-	defer ipfsCli.Close()
-
 	// TODO 可以考虑优化，比如rep类型的直接pin就可以
 	objIter := ctx.downloader.DownloadPackage(t.packageID)
 	defer objIter.Close()
@@ -80,15 +74,20 @@ func (t *CacheMovePackage) do(ctx TaskContext) error {
 		}
 		defer obj.File.Close()
 
-		// 将对象文件添加到IPFS
-		_, err = ipfsCli.CreateFile(obj.File)
+		writer := store.New()
+		_, err = io.Copy(writer, obj.File)
+		if err != nil {
+			return fmt.Errorf("writing to store: %w", err)
+		}
+		_, err = writer.Finish()
 		if err != nil {
-			return fmt.Errorf("creating ipfs file: %w", err)
+			return fmt.Errorf("finishing store: %w", err)
 		}
+
+		ctx.accessStat.AddAccessCounter(obj.Object.ObjectID, t.packageID, *stgglb.Local.NodeID, 1)
 	}
 
-	// 通知协调器缓存已移动
-	_, err = coorCli.CachePackageMoved(coormq.NewCachePackageMoved(t.packageID, *stgglb.Local.NodeID))
+	_, err = coorCli.CachePackageMoved(coormq.NewCachePackageMoved(t.packageID, t.storageID))
 	if err != nil {
 		return fmt.Errorf("request to coordinator: %w", err)
 	}

agent/internal/task/create_package.go

@@ -28,7 +28,7 @@ type CreatePackage struct {
 	name         string
 	objIter      iterator.UploadingObjectIterator
 	nodeAffinity *cdssdk.NodeID
-	Result       *CreatePackageResult
+	Result       CreatePackageResult
 }
 
 // NewCreatePackage 创建一个新的CreatePackage实例

agent/internal/task/execute_io_plan.go

@@ -1,68 +0,0 @@
-package task
-
-import (
-	"fmt"
-	"time"
-
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/pkgs/task"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-)
-
-// ExecuteIOPlan 用于执行I/O计划的任务结构体
-// 临时使用Task来等待Plan执行进度
-type ExecuteIOPlan struct {
-	PlanID ioswitch.PlanID     // 计划ID
-	Result ioswitch.PlanResult // 执行结果
-}
-
-// NewExecuteIOPlan 创建一个新的ExecuteIOPlan实例
-// 参数:
-//
-//	planID: 要执行的I/O计划的ID
-//
-// 返回值:
-//
-//	*ExecuteIOPlan: 新创建的ExecuteIOPlan实例的指针
-func NewExecuteIOPlan(planID ioswitch.PlanID) *ExecuteIOPlan {
-	return &ExecuteIOPlan{
-		PlanID: planID,
-	}
-}
-
-// Execute 执行I/O计划
-// 参数:
-//
-//	task: 任务实例
-//	ctx: 任务执行上下文
-//	complete: 完成回调函数
-//
-// 说明:
-//
-//	此函数开始执行指定的I/O计划，并通过回调函数报告完成状态
-func (t *ExecuteIOPlan) Execute(task *task.Task[TaskContext], ctx TaskContext, complete CompleteFn) {
-	// 记录任务日志
-	log := logger.WithType[ExecuteIOPlan]("Task")
-	log.Debugf("begin with %v", logger.FormatStruct(t))
-	defer log.Debugf("end") // 确保日志记录任务结束
-
-	// 执行I/O计划
-	ret, err := ctx.sw.ExecutePlan(t.PlanID)
-	if err != nil {
-		// 执行计划失败，记录警告日志并调用完成回调函数
-		err := fmt.Errorf("executing io plan: %w", err)
-		log.WithField("PlanID", t.PlanID).Warn(err.Error())
-
-		complete(err, CompleteOption{
-			RemovingDelay: time.Minute, // 设置延迟删除选项
-		})
-		return
-	}
-
-	// 计划执行成功，更新结果并调用完成回调函数
-	t.Result = ret
-
-	complete(nil, CompleteOption{
-		RemovingDelay: time.Minute, // 设置延迟删除选项
-	})
-}

agent/internal/task/ipfs_pin.go

@@ -1,70 +0,0 @@
-package task
-
-import (
-	"fmt"
-	"time"
-
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/pkgs/task"
-	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
-)
-
-// IPFSPin 定义了一个结构体，用于IPFS的pin操作任务。
-type IPFSPin struct {
-	FileHashes []string // FileHashes 存储需要pin的文件的hash列表。
-}
-
-// NewIPFSPin 创建一个新的IPFSPin实例。
-// fileHashes 是一个包含需要pin的文件hash的字符串切片。
-// 返回一个指向IPFSPin实例的指针。
-func NewIPFSPin(fileHashes []string) *IPFSPin {
-	return &IPFSPin{
-		FileHashes: fileHashes,
-	}
-}
-
-// Execute 执行IPFSPin任务。
-// 该函数负责获取IPFS客户端，然后对FileHashes中的每个文件hash执行pin操作。
-// task 是一个指向task.Task[TaskContext]的指针，代表当前的任务实例。
-// ctx 是当前任务的上下文信息。
-// complete 是一个完成回调函数，用于在任务结束时（成功或失败）进行一些清理工作。
-func (t *IPFSPin) Execute(task *task.Task[TaskContext], ctx TaskContext, complete CompleteFn) {
-	// 使用logger记录任务开始的信息。
-	log := logger.WithType[IPFSPin]("Task")
-	log.Debugf("begin with %v", logger.FormatStruct(t))
-	defer log.Debugf("end") // 确保记录任务结束的信息。
-
-	// 尝试从IPFS池中获取一个客户端实例。
-	ipfsCli, err := stgglb.IPFSPool.Acquire()
-	if err != nil {
-		// 如果获取客户端失败，则使用complete函数通知任务失败，并设置移除延迟。
-		err := fmt.Errorf("new ipfs client: %w", err)
-		log.Warn(err.Error())
-
-		complete(err, CompleteOption{
-			RemovingDelay: time.Minute,
-		})
-		return
-	}
-	defer ipfsCli.Close() // 确保在函数返回前释放IPFS客户端实例。
-
-	// 遍历文件hash列表，并尝试对每个hash执行pin操作。
-	for _, fileHash := range t.FileHashes {
-		err = ipfsCli.Pin(fileHash)
-		if err != nil {
-			// 如果pin操作失败，则使用complete函数通知任务失败，并设置移除延迟。
-			err := fmt.Errorf("pin file failed, err: %w", err)
-			log.WithField("FileHash", fileHash).Warn(err.Error())
-
-			complete(err, CompleteOption{
-				RemovingDelay: time.Minute,
-			})
-			return
-		}
-	}
-
-	// 所有文件的pin操作成功，使用complete函数通知任务成功完成，并设置移除延迟。
-	complete(nil, CompleteOption{
-		RemovingDelay: time.Minute,
-	})
-}

agent/internal/task/ipfs_read.go

@@ -1,125 +0,0 @@
-package task
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"time"
-
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/pkgs/task"
-	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
-)
-
-// IPFSRead 代表从IPFS读取文件的任务
-type IPFSRead struct {
-	FileHash  string // 文件的IPFS哈希值
-	LocalPath string // 本地存储路径
-}
-
-// NewIPFSRead 创建一个新的IPFS读取任务实例
-func NewIPFSRead(fileHash string, localPath string) *IPFSRead {
-	return &IPFSRead{
-		FileHash:  fileHash,
-		LocalPath: localPath,
-	}
-}
-
-// Compare 比较当前任务与另一个任务是否相同
-// other: 要比较的另一个任务
-// 返回值: 如果两个任务相同返回true，否则返回false
-func (t *IPFSRead) Compare(other *Task) bool {
-	tsk, ok := other.Body().(*IPFSRead)
-	if !ok {
-		return false
-	}
-
-	return t.FileHash == tsk.FileHash && t.LocalPath == tsk.LocalPath
-}
-
-// Execute 执行从IPFS读取文件并存储到本地的任务
-// task: 任务实例
-// ctx: 任务上下文
-// complete: 任务完成的回调函数
-func (t *IPFSRead) Execute(task *task.Task[TaskContext], ctx TaskContext, complete CompleteFn) {
-	// 初始化日志
-	log := logger.WithType[IPFSRead]("Task")
-	log.Debugf("begin with %v", logger.FormatStruct(t))
-	defer log.Debugf("end")
-
-	// 获取输出文件的目录并创建该目录
-	outputFileDir := filepath.Dir(t.LocalPath)
-
-	// 创建输出文件的目录
-	err := os.MkdirAll(outputFileDir, os.ModePerm)
-	if err != nil {
-		// 目录创建失败的处理
-		err := fmt.Errorf("create output file directory %s failed, err: %w", outputFileDir, err)
-		log.WithField("LocalPath", t.LocalPath).Warn(err.Error())
-
-		complete(err, CompleteOption{
-			RemovingDelay: time.Minute,
-		})
-		return
-	}
-
-	// 创建输出文件
-	outputFile, err := os.Create(t.LocalPath)
-	if err != nil {
-		// 输出文件创建失败的处理
-		err := fmt.Errorf("create output file %s failed, err: %w", t.LocalPath, err)
-		log.WithField("LocalPath", t.LocalPath).Warn(err.Error())
-
-		complete(err, CompleteOption{
-			RemovingDelay: time.Minute,
-		})
-		return
-	}
-	defer outputFile.Close()
-
-	// 获取IPFS客户端
-	ipfsCli, err := stgglb.IPFSPool.Acquire()
-	if err != nil {
-		// 获取IPFS客户端失败的处理
-		err := fmt.Errorf("new ipfs client: %w", err)
-		log.Warn(err.Error())
-
-		complete(err, CompleteOption{
-			RemovingDelay: time.Minute,
-		})
-		return
-	}
-	defer ipfsCli.Close()
-
-	// 打开IPFS中的文件进行读取
-	rd, err := ipfsCli.OpenRead(t.FileHash)
-	if err != nil {
-		// 打开IPFS文件失败的处理
-		err := fmt.Errorf("read ipfs file failed, err: %w", err)
-		log.WithField("FileHash", t.FileHash).Warn(err.Error())
-
-		complete(err, CompleteOption{
-			RemovingDelay: time.Minute,
-		})
-		return
-	}
-
-	// 将IPFS文件内容复制到本地文件
-	_, err = io.Copy(outputFile, rd)
-	if err != nil {
-		// 文件复制失败的处理
-		err := fmt.Errorf("copy ipfs file to local file failed, err: %w", err)
-		log.WithField("LocalPath", t.LocalPath).Warn(err.Error())
-
-		complete(err, CompleteOption{
-			RemovingDelay: time.Minute,
-		})
-		return
-	}
-
-	// 任务完成，调用回调函数
-	complete(nil, CompleteOption{
-		RemovingDelay: time.Minute,
-	})
-}

agent/internal/task/storage_load_package.go

@@ -11,10 +11,11 @@ import (
 	"github.com/samber/lo"
 	"gitlink.org.cn/cloudream/common/pkgs/bitmap"
 	"gitlink.org.cn/cloudream/common/pkgs/ipfs"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	"gitlink.org.cn/cloudream/common/pkgs/task"
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 	"gitlink.org.cn/cloudream/common/utils/io2"
-	myref "gitlink.org.cn/cloudream/common/utils/reflect"
+	"gitlink.org.cn/cloudream/common/utils/reflect2"
 	"gitlink.org.cn/cloudream/common/utils/sort2"
 	"gitlink.org.cn/cloudream/storage/common/consts"
 	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
@@ -25,9 +26,10 @@ import (
 	"gitlink.org.cn/cloudream/storage/common/utils"
 )
 
-// StorageLoadPackage 定义了存储加载包的结构体，包含完整的输出路径和与存储、包、用户相关的ID。
 type StorageLoadPackage struct {
-	FullOutputPath string
+	PackagePath string
+	LocalBase   string
+	RemoteBase  string
 
 	userID       cdssdk.UserID
 	packageID    cdssdk.PackageID
@@ -35,11 +37,6 @@ type StorageLoadPackage struct {
 	pinnedBlocks []stgmod.ObjectBlock
 }
 
-// NewStorageLoadPackage 创建一个新的StorageLoadPackage实例。
-// userID: 用户ID。
-// packageID: 包ID。
-// storageID: 存储ID。
-// 返回一个新的StorageLoadPackage指针。
 func NewStorageLoadPackage(userID cdssdk.UserID, packageID cdssdk.PackageID, storageID cdssdk.StorageID) *StorageLoadPackage {
 	return &StorageLoadPackage{
 		userID:    userID,
@@ -47,80 +44,77 @@ func NewStorageLoadPackage(userID cdssdk.UserID, packageID cdssdk.PackageID, sto
 		storageID: storageID,
 	}
 }
-
-// Execute 执行存储加载任务。
-// task: 任务实例。
-// ctx: 任务上下文。
-// complete: 完成回调函数。
-// 无返回值。
 func (t *StorageLoadPackage) Execute(task *task.Task[TaskContext], ctx TaskContext, complete CompleteFn) {
+	startTime := time.Now()
+	log := logger.WithType[StorageLoadPackage]("Task")
+	log.WithField("TaskID", task.ID()).
+		Infof("begin to load package %v to %v", t.packageID, t.storageID)
+
 	err := t.do(task, ctx)
+	if err == nil {
+		log.WithField("TaskID", task.ID()).
+			Infof("loading success, cost: %v", time.Since(startTime))
+	} else {
+		log.WithField("TaskID", task.ID()).
+			Warnf("loading package: %v, cost: %v", err, time.Since(startTime))
+	}
 
 	complete(err, CompleteOption{
 		RemovingDelay: time.Minute,
 	})
 }
 
-// do 实际执行存储加载的过程。
-// task: 任务实例。
-// ctx: 任务上下文。
-// 返回执行过程中可能出现的错误。
 func (t *StorageLoadPackage) do(task *task.Task[TaskContext], ctx TaskContext) error {
-	// 获取协调器客户端
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
 		return fmt.Errorf("new coordinator client: %w", err)
 	}
 	defer stgglb.CoordinatorMQPool.Release(coorCli)
 
-	// 获取IPFS客户端
 	ipfsCli, err := stgglb.IPFSPool.Acquire()
 	if err != nil {
 		return fmt.Errorf("new IPFS client: %w", err)
 	}
 	defer stgglb.IPFSPool.Release(ipfsCli)
 
-	// 从协调器获取存储信息
-	getStgResp, err := coorCli.GetStorageInfo(coormq.NewGetStorageInfo(t.userID, t.storageID))
+	getStgResp, err := coorCli.GetStorage(coormq.ReqGetStorage(t.userID, t.storageID))
 	if err != nil {
 		return fmt.Errorf("request to coordinator: %w", err)
 	}
 
-	// 构造输出目录路径并创建该目录
-	outputDirPath := utils.MakeStorageLoadPackagePath(getStgResp.Directory, t.userID, t.packageID)
-	if err = os.MkdirAll(outputDirPath, 0755); err != nil {
+	t.PackagePath = utils.MakeLoadedPackagePath(t.userID, t.packageID)
+	fullLocalPath := filepath.Join(getStgResp.Storage.LocalBase, t.PackagePath)
+
+	if err = os.MkdirAll(fullLocalPath, 0755); err != nil {
 		return fmt.Errorf("creating output directory: %w", err)
 	}
-	t.FullOutputPath = outputDirPath
 
 	getObjectDetails, err := coorCli.GetPackageObjectDetails(coormq.ReqGetPackageObjectDetails(t.packageID))
 	if err != nil {
 		return fmt.Errorf("getting package object details: %w", err)
 	}
 
-	// 获取互斥锁以确保并发安全
 	mutex, err := reqbuilder.NewBuilder().
 		// 提前占位
 		Metadata().StoragePackage().CreateOne(t.userID, t.storageID, t.packageID).
 		// 保护在storage目录中下载的文件
 		Storage().Buzy(t.storageID).
 		// 保护下载文件时同时保存到IPFS的文件
-		IPFS().Buzy(getStgResp.NodeID).
+		IPFS().Buzy(getStgResp.Storage.NodeID).
 		MutexLock(ctx.distlock)
 	if err != nil {
 		return fmt.Errorf("acquire locks failed, err: %w", err)
 	}
 	defer mutex.Unlock()
 
-	// 下载每个对象
 	for _, obj := range getObjectDetails.Objects {
-		err := t.downloadOne(coorCli, ipfsCli, outputDirPath, obj)
+		err := t.downloadOne(coorCli, ipfsCli, fullLocalPath, obj)
 		if err != nil {
 			return err
 		}
+		ctx.accessStat.AddAccessCounter(obj.Object.ObjectID, t.packageID, *stgglb.Local.NodeID, 1)
 	}
 
-	// 通知协调器包已加载到存储
 	_, err = coorCli.StoragePackageLoaded(coormq.NewStoragePackageLoaded(t.userID, t.storageID, t.packageID, t.pinnedBlocks))
 	if err != nil {
 		return fmt.Errorf("loading package to storage: %w", err)
@@ -130,23 +124,11 @@ func (t *StorageLoadPackage) do(task *task.Task[TaskContext], ctx TaskContext) e
 	return err
 }
 
-// downloadOne 用于下载一种特定冗余类型的对象。
-//
-// 参数:
-// - coorCli: 协调客户端，用于与CDN协调器进行通信。
-// - ipfsCli: IPFS池客户端，用于与IPFS网络进行交互。
-// - dir: 下载对象的目标目录。
-// - obj: 要下载的对象详细信息，包括对象路径和冗余类型等。
-//
-// 返回值:
-// - error: 下载过程中遇到的任何错误。
 func (t *StorageLoadPackage) downloadOne(coorCli *coormq.Client, ipfsCli *ipfs.PoolClient, dir string, obj stgmod.ObjectDetail) error {
 	var file io.ReadCloser
 
-	// 根据对象的冗余类型选择不同的下载策略。
 	switch red := obj.Object.Redundancy.(type) {
 	case *cdssdk.NoneRedundancy:
-		// 无冗余或复制冗余对象的下载处理。
 		reader, err := t.downloadNoneOrRepObject(ipfsCli, obj)
 		if err != nil {
 			return fmt.Errorf("downloading object: %w", err)
@@ -154,7 +136,6 @@ func (t *StorageLoadPackage) downloadOne(coorCli *coormq.Client, ipfsCli *ipfs.P
 		file = reader
 
 	case *cdssdk.RepRedundancy:
-		// 复制冗余对象的下载处理。
 		reader, err := t.downloadNoneOrRepObject(ipfsCli, obj)
 		if err != nil {
 			return fmt.Errorf("downloading rep object: %w", err)
@@ -162,7 +143,6 @@ func (t *StorageLoadPackage) downloadOne(coorCli *coormq.Client, ipfsCli *ipfs.P
 		file = reader
 
 	case *cdssdk.ECRedundancy:
-		// 前向纠错冗余对象的下载处理。
 		reader, pinnedBlocks, err := t.downloadECObject(coorCli, ipfsCli, obj, red)
 		if err != nil {
 			return fmt.Errorf("downloading ec object: %w", err)
@@ -171,12 +151,10 @@ func (t *StorageLoadPackage) downloadOne(coorCli *coormq.Client, ipfsCli *ipfs.P
 		t.pinnedBlocks = append(t.pinnedBlocks, pinnedBlocks...)
 
 	default:
-		// 遇到未知的冗余类型返回错误。
-		return fmt.Errorf("unknow redundancy type: %v", myref.TypeOfValue(obj.Object.Redundancy))
+		return fmt.Errorf("unknow redundancy type: %v", reflect2.TypeOfValue(obj.Object.Redundancy))
 	}
-	defer file.Close() // 确保文件在函数返回前被关闭。
+	defer file.Close()
 
-	// 拼接完整的文件路径，并创建包含该文件的目录。
 	fullPath := filepath.Join(dir, obj.Object.Path)
 
 	lastDirPath := filepath.Dir(fullPath)
@@ -184,14 +162,12 @@ func (t *StorageLoadPackage) downloadOne(coorCli *coormq.Client, ipfsCli *ipfs.P
 		return fmt.Errorf("creating object last dir: %w", err)
 	}
 
-	// 创建输出文件。
 	outputFile, err := os.Create(fullPath)
 	if err != nil {
 		return fmt.Errorf("creating object file: %w", err)
 	}
-	defer outputFile.Close() // 确保文件在函数返回前被关闭。
+	defer outputFile.Close()
 
-	// 将下载的内容写入本地文件。
 	if _, err := io.Copy(outputFile, file); err != nil {
 		return fmt.Errorf("writting object to file: %w", err)
 	}
@@ -199,25 +175,14 @@ func (t *StorageLoadPackage) downloadOne(coorCli *coormq.Client, ipfsCli *ipfs.P
 	return nil
 }
 
-// downloadNoneOrRepObject 用于下载没有冗余或需要从IPFS网络中检索的对象。
-// 如果对象不存在于任何节点上，则返回错误。
-//
-// 参数:
-// - ipfsCli: IPFS客户端池的指针，用于与IPFS网络交互。
-// - obj: 要下载的对象的详细信息。
-//
-// 返回值:
-// - io.ReadCloser: 下载文件的读取器。
-// - error: 如果下载过程中出现错误，则返回错误信息。
 func (t *StorageLoadPackage) downloadNoneOrRepObject(ipfsCli *ipfs.PoolClient, obj stgmod.ObjectDetail) (io.ReadCloser, error) {
 	if len(obj.Blocks) == 0 && len(obj.PinnedAt) == 0 {
 		return nil, fmt.Errorf("no node has this object")
 	}
 
-	// 将对象文件哈希添加到本地Pin列表，无论是否真正需要
+	// 不管实际有没有成功
 	ipfsCli.Pin(obj.Object.FileHash)
 
-	// 尝试打开并读取对象文件
 	file, err := ipfsCli.OpenRead(obj.Object.FileHash)
 	if err != nil {
 		return nil, err
@@ -226,31 +191,13 @@ func (t *StorageLoadPackage) downloadNoneOrRepObject(ipfsCli *ipfs.PoolClient, o
 	return file, nil
 }
 
-// downloadECObject 用于下载采用EC（Erasure Coding）编码的对象。
-// 该方法会根据对象的块信息和EC冗余策略，从网络中下载必要的数据块并恢复整个对象。
-//
-// 参数:
-// - coorCli: 协调器客户端的指针，用于节点间的协调与通信。
-// - ipfsCli: IPFS客户端池的指针，用于与IPFS网络交互。
-// - obj: 要下载的对象的详细信息。
-// - ecRed: EC冗余策略的详细配置。
-//
-// 返回值:
-// - io.ReadCloser: 恢复后的对象文件的读取器。
-// - []stgmod.ObjectBlock: 被Pin住的对象块列表。
-// - error: 如果下载或恢复过程中出现错误，则返回错误信息。
 func (t *StorageLoadPackage) downloadECObject(coorCli *coormq.Client, ipfsCli *ipfs.PoolClient, obj stgmod.ObjectDetail, ecRed *cdssdk.ECRedundancy) (io.ReadCloser, []stgmod.ObjectBlock, error) {
-	// 根据对象信息和节点状态，排序选择最优的下载节点
 	allNodes, err := t.sortDownloadNodes(coorCli, obj)
 	if err != nil {
 		return nil, nil, err
 	}
-
-	// 计算最小读取块解决方案和最小读取对象解决方案
 	bsc, blocks := t.getMinReadingBlockSolution(allNodes, ecRed.K)
 	osc, _ := t.getMinReadingObjectSolution(allNodes, ecRed.K)
-
-	// 如果通过块恢复更高效，则执行块恢复流程
 	if bsc < osc {
 		var fileStrs []io.ReadCloser
 
@@ -259,8 +206,8 @@ func (t *StorageLoadPackage) downloadECObject(coorCli *coormq.Client, ipfsCli *i
 			return nil, nil, fmt.Errorf("new rs: %w", err)
 		}
 
-		// 为每个需要读取的块执行Pin操作和打开读取流
 		for i := range blocks {
+			// 不管实际有没有成功
 			ipfsCli.Pin(blocks[i].Block.FileHash)
 
 			str, err := ipfsCli.OpenRead(blocks[i].Block.FileHash)
@@ -276,7 +223,6 @@ func (t *StorageLoadPackage) downloadECObject(coorCli *coormq.Client, ipfsCli *i
 
 		fileReaders, filesCloser := io2.ToReaders(fileStrs)
 
-		// 准备恢复数据所需的信息和变量
 		var indexes []int
 		var pinnedBlocks []stgmod.ObjectBlock
 		for _, b := range blocks {
@@ -296,11 +242,12 @@ func (t *StorageLoadPackage) downloadECObject(coorCli *coormq.Client, ipfsCli *i
 		}), pinnedBlocks, nil
 	}
 
-	// 如果通过对象恢复更高效或没有足够的块来恢复文件，则直接尝试读取对象文件
+	// bsc >= osc，如果osc是MaxFloat64，那么bsc也一定是，也就意味着没有足够块来恢复文件
 	if osc == math.MaxFloat64 {
 		return nil, nil, fmt.Errorf("no enough blocks to reconstruct the file, want %d, get only %d", ecRed.K, len(blocks))
 	}
 
+	// 如果是直接读取的文件，那么就不需要Pin文件块
 	str, err := ipfsCli.OpenRead(obj.Object.FileHash)
 	return str, nil, err
 }
@@ -312,15 +259,7 @@ type downloadNodeInfo struct {
 	Distance     float64
 }
 
-// sortDownloadNodes 对存储对象的下载节点进行排序
-// 参数:
-// - coorCli *coormq.Client: 协调器客户端，用于获取节点信息
-// - obj stgmod.ObjectDetail: 存储对象的详细信息，包含固定存储节点和数据块信息
-// 返回值:
-// - []*downloadNodeInfo: 排序后的下载节点信息数组
-// - error: 如果过程中发生错误，则返回错误信息
 func (t *StorageLoadPackage) sortDownloadNodes(coorCli *coormq.Client, obj stgmod.ObjectDetail) ([]*downloadNodeInfo, error) {
-	// 收集对象的固定存储节点ID和数据块所在节点ID
 	var nodeIDs []cdssdk.NodeID
 	for _, id := range obj.PinnedAt {
 		if !lo.Contains(nodeIDs, id) {
@@ -333,13 +272,11 @@ func (t *StorageLoadPackage) sortDownloadNodes(coorCli *coormq.Client, obj stgmo
 		}
 	}
 
-	// 获取节点信息
 	getNodes, err := coorCli.GetNodes(coormq.NewGetNodes(nodeIDs))
 	if err != nil {
 		return nil, fmt.Errorf("getting nodes: %w", err)
 	}
 
-	// 建立下载节点信息的映射表
 	downloadNodeMap := make(map[cdssdk.NodeID]*downloadNodeInfo)
 	for _, id := range obj.PinnedAt {
 		node, ok := downloadNodeMap[id]
@@ -353,10 +290,9 @@ func (t *StorageLoadPackage) sortDownloadNodes(coorCli *coormq.Client, obj stgmo
 			downloadNodeMap[id] = node
 		}
 
-		node.ObjectPinned = true // 标记为固定存储对象
+		node.ObjectPinned = true
 	}
 
-	// 为每个数据块所在节点填充信息，并收集到映射表中
 	for _, b := range obj.Blocks {
 		node, ok := downloadNodeMap[b.NodeID]
 		if !ok {
@@ -368,10 +304,9 @@ func (t *StorageLoadPackage) sortDownloadNodes(coorCli *coormq.Client, obj stgmo
 			downloadNodeMap[b.NodeID] = node
 		}
 
-		node.Blocks = append(node.Blocks, b) // 添加数据块信息
+		node.Blocks = append(node.Blocks, b)
 	}
 
-	// 根据节点与存储对象的距离进行排序
 	return sort2.Sort(lo.Values(downloadNodeMap), func(left, right *downloadNodeInfo) int {
 		return sort2.Cmp(left.Distance, right.Distance)
 	}), nil
@@ -382,19 +317,12 @@ type downloadBlock struct {
 	Block stgmod.ObjectBlock
 }
 
-// getMinReadingBlockSolution 获取最小读取区块解决方案
-// sortedNodes: 已排序的节点信息列表，每个节点包含多个区块信息
-// k: 需要获取的区块数量
-// 返回值: 返回获取到的区块的总距离和区块列表
 func (t *StorageLoadPackage) getMinReadingBlockSolution(sortedNodes []*downloadNodeInfo, k int) (float64, []downloadBlock) {
-	// 初始化已获取区块的bitmap和距离
 	gotBlocksMap := bitmap.Bitmap64(0)
 	var gotBlocks []downloadBlock
 	dist := float64(0.0)
-	// 遍历所有节点及其区块，直到获取到k个不同的区块
 	for _, n := range sortedNodes {
 		for _, b := range n.Blocks {
-			// 如果区块未被获取，则添加到列表中，并更新距离
 			if !gotBlocksMap.Get(b.Index) {
 				gotBlocks = append(gotBlocks, downloadBlock{
 					Node:  n.Node,
@@ -404,51 +332,39 @@ func (t *StorageLoadPackage) getMinReadingBlockSolution(sortedNodes []*downloadN
 				dist += n.Distance
 			}
 
-			// 如果已获取的区块数量达到k，返回结果
 			if len(gotBlocks) >= k {
 				return dist, gotBlocks
 			}
 		}
 	}
 
-	// 如果无法获取到k个不同的区块，返回最大距离和空的区块列表
 	return math.MaxFloat64, gotBlocks
 }
 
-// getMinReadingObjectSolution 获取最小读取对象解决方案
-// sortedNodes: 已排序的节点信息列表，每个节点包含一个对象是否被固定的信息
-// k: 需要获取的对象数量
-// 返回值: 返回获取对象的最小距离和对应的节点
 func (t *StorageLoadPackage) getMinReadingObjectSolution(sortedNodes []*downloadNodeInfo, k int) (float64, *cdssdk.Node) {
 	dist := math.MaxFloat64
 	var downloadNode *cdssdk.Node
-	// 遍历节点，寻找距离最小且对象被固定的节点
 	for _, n := range sortedNodes {
 		if n.ObjectPinned && float64(k)*n.Distance < dist {
 			dist = float64(k) * n.Distance
-			downloadNode = &n.Node
+			node := n.Node
+			downloadNode = &node
 		}
 	}
 
 	return dist, downloadNode
 }
 
-// getNodeDistance 获取节点距离
-// node: 需要计算距离的节点
-// 返回值: 返回节点与当前节点或位置的距离
 func (t *StorageLoadPackage) getNodeDistance(node cdssdk.Node) float64 {
-	// 如果有本地节点ID且与目标节点ID相同，返回同一节点距离
 	if stgglb.Local.NodeID != nil {
 		if node.NodeID == *stgglb.Local.NodeID {
 			return consts.NodeDistanceSameNode
 		}
 	}
 
-	// 如果节点位置与本地位置相同，返回同一位置距离
 	if node.LocationID == stgglb.Local.LocationID {
 		return consts.NodeDistanceSameLocation
 	}
 
-	// 默认返回其他距离
 	return consts.NodeDistanceOther
 }

agent/internal/task/task.go

@@ -3,17 +3,19 @@ package task
 import (
 	"gitlink.org.cn/cloudream/common/pkgs/distlock"
 	"gitlink.org.cn/cloudream/common/pkgs/task"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/accessstat"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/connectivity"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/downloader"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/storage/shard/pool"
 )
 
 // TaskContext 定义了任务执行的上下文环境，包含分布式锁服务、IO开关和网络连接状态收集器
 type TaskContext struct {
-	distlock     *distlock.Service
-	sw           *ioswitch.Switch
-	connectivity *connectivity.Collector
-	downloader   *downloader.Downloader
+	distlock       *distlock.Service
+	connectivity   *connectivity.Collector
+	downloader     *downloader.Downloader
+	accessStat     *accessstat.AccessStat
+	shardStorePool *pool.ShardStorePool
 }
 
 // CompleteFn 类型定义了任务完成时需要执行的函数，用于设置任务的执行结果
@@ -31,11 +33,12 @@ type Task = task.Task[TaskContext]
 // CompleteOption 类型定义了任务完成时的选项，可用于定制化任务完成的处理方式
 type CompleteOption = task.CompleteOption
 
-func NewManager(distlock *distlock.Service, sw *ioswitch.Switch, connectivity *connectivity.Collector, downloader *downloader.Downloader) Manager {
+func NewManager(distlock *distlock.Service, connectivity *connectivity.Collector, downloader *downloader.Downloader, accessStat *accessstat.AccessStat, shardStorePool *pool.ShardStorePool) Manager {
 	return task.NewManager(TaskContext{
-		distlock:     distlock,
-		sw:           sw,
-		connectivity: connectivity,
-		downloader:   downloader,
+		distlock:       distlock,
+		connectivity:   connectivity,
+		downloader:     downloader,
+		accessStat:     accessStat,
+		shardStorePool: shardStorePool,
 	})
 }

agent/main.go

@@ -4,21 +4,22 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"sync"
+	"time"
 
-	log "gitlink.org.cn/cloudream/common/pkgs/logger"
+	"gitlink.org.cn/cloudream/storage/agent/internal/http"
+
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 	"gitlink.org.cn/cloudream/storage/agent/internal/config"
 	"gitlink.org.cn/cloudream/storage/agent/internal/task"
 	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/accessstat"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/connectivity"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/distlock"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/downloader"
 	agtrpc "gitlink.org.cn/cloudream/storage/common/pkgs/grpc/agent"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-
-	// TODO 注册OpUnion，但在mq包中注册会造成循环依赖，所以只能放到这里
-	_ "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch/ops"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/storage/shard/pool"
 
 	"google.golang.org/grpc"
 
@@ -32,46 +33,49 @@ import (
 // TODO 此数据是否在运行时会发生变化？
 var AgentIpList []string
 
-// 主程序入口
 func main() {
-	// TODO: 将Agent的IP列表放到配置文件中读取
+	// TODO 放到配置里读取
 	AgentIpList = []string{"pcm01", "pcm1", "pcm2"}
 
-	// 初始化配置
 	err := config.Init()
 	if err != nil {
 		fmt.Printf("init config failed, err: %s", err.Error())
 		os.Exit(1)
 	}
 
-	// 初始化日志系统
-	err = log.Init(&config.Cfg().Logger)
+	err = logger.Init(&config.Cfg().Logger)
 	if err != nil {
 		fmt.Printf("init logger failed, err: %s", err.Error())
 		os.Exit(1)
 	}
 
-	// 初始化全局变量和连接池
 	stgglb.InitLocal(&config.Cfg().Local)
 	stgglb.InitMQPool(&config.Cfg().RabbitMQ)
 	stgglb.InitAgentRPCPool(&agtrpc.PoolConfig{})
-	stgglb.InitIPFSPool(&config.Cfg().IPFS)
 
-	// 启动网络连通性检测，并进行一次就地检测
+	sw := exec.NewWorker()
+
+	svc := http.NewService(&sw)
+	if err != nil {
+		logger.Fatalf("new http service failed, err: %s", err.Error())
+	}
+	server, err := http.NewServer(config.Cfg().ListenAddr, svc)
+	err = server.Serve()
+	if err != nil {
+		logger.Fatalf("http server stopped with error: %s", err.Error())
+	}
+
+	// 启动网络连通性检测，并就地检测一次
 	conCol := connectivity.NewCollector(&config.Cfg().Connectivity, func(collector *connectivity.Collector) {
-		log := log.WithField("Connectivity", "")
+		log := logger.WithField("Connectivity", "")
 
-		// 从协调器MQ连接池获取客户端
 		coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 		if err != nil {
 			log.Warnf("acquire coordinator mq failed, err: %s", err.Error())
 			return
 		}
-
-		// 确保在函数返回前释放客户端
 		defer stgglb.CoordinatorMQPool.Release(coorCli)
 
-		// 处理网络连通性数据，并更新到协调器
 		cons := collector.GetAll()
 		nodeCons := make([]cdssdk.NodeConnectivity, 0, len(cons))
 		for _, con := range cons {
@@ -96,97 +100,116 @@ func main() {
 	})
 	conCol.CollectInPlace()
 
-	// 初始化分布式锁服务
+	acStat := accessstat.NewAccessStat(accessstat.Config{
+		// TODO 考虑放到配置里
+		ReportInterval: time.Second * 10,
+	})
+	go serveAccessStat(acStat)
+
+	// TODO2 根据配置实例化Store并加入到Pool中
+	shardStorePool := pool.New()
+
 	distlock, err := distlock.NewService(&config.Cfg().DistLock)
 	if err != nil {
-		log.Fatalf("new ipfs failed, err: %s", err.Error())
+		logger.Fatalf("new ipfs failed, err: %s", err.Error())
 	}
 
-	// 初始化数据切换开关
-	sw := ioswitch.NewSwitch()
-
-	dlder := downloader.NewDownloader(config.Cfg().Downloader)
+	dlder := downloader.NewDownloader(config.Cfg().Downloader, &conCol)
 
-	//处置协调端、客户端命令（可多建几个）
-	wg := sync.WaitGroup{}
-	wg.Add(4)
-
-	taskMgr := task.NewManager(distlock, &sw, &conCol, &dlder)
+	taskMgr := task.NewManager(distlock, &conCol, &dlder, acStat, shardStorePool)
 
 	// 启动命令服务器
-	agtSvr, err := agtmq.NewServer(cmdsvc.NewService(&taskMgr, &sw), config.Cfg().ID, &config.Cfg().RabbitMQ)
+	// TODO 需要设计AgentID持久化机制
+	agtSvr, err := agtmq.NewServer(cmdsvc.NewService(&taskMgr, shardStorePool), config.Cfg().ID, &config.Cfg().RabbitMQ)
 	if err != nil {
-		log.Fatalf("new agent server failed, err: %s", err.Error())
+		logger.Fatalf("new agent server failed, err: %s", err.Error())
 	}
 	agtSvr.OnError(func(err error) {
-		log.Warnf("agent server err: %s", err.Error())
+		logger.Warnf("agent server err: %s", err.Error())
 	})
+	go serveAgentServer(agtSvr)
 
-	go serveAgentServer(agtSvr, &wg)
-
-	// 启动面向客户端的GRPC服务
+	//面向客户端收发数据
 	listenAddr := config.Cfg().GRPC.MakeListenAddress()
 	lis, err := net.Listen("tcp", listenAddr)
 	if err != nil {
-		log.Fatalf("listen on %s failed, err: %s", listenAddr, err.Error())
+		logger.Fatalf("listen on %s failed, err: %s", listenAddr, err.Error())
 	}
-
 	s := grpc.NewServer()
 	agtrpc.RegisterAgentServer(s, grpcsvc.NewService(&sw))
-	go serveGRPC(s, lis, &wg)
+	go serveGRPC(s, lis)
 
-	// 启动分布式锁服务的处理程序
 	go serveDistLock(distlock)
 
-	// 等待所有服务结束
-	wg.Wait()
+	foever := make(chan struct{})
+	<-foever
 }
 
-// serveAgentServer 启动并服务一个命令服务器
-// server: 指向agtmq.Server的指针，代表要被服务的命令服务器
-// wg: 指向sync.WaitGroup的指针，用于等待服务器停止
-func serveAgentServer(server *agtmq.Server, wg *sync.WaitGroup) {
-	log.Info("start serving command server")
+func serveAgentServer(server *agtmq.Server) {
+	logger.Info("start serving command server")
 
 	err := server.Serve()
 
 	if err != nil {
-		log.Errorf("command server stopped with error: %s", err.Error())
+		logger.Errorf("command server stopped with error: %s", err.Error())
 	}
 
-	log.Info("command server stopped")
+	logger.Info("command server stopped")
 
-	wg.Done() // 表示服务器已经停止
+	// TODO 仅简单结束了程序
+	os.Exit(1)
 }
 
-// serveGRPC 启动并服务一个gRPC服务器
-// s: 指向grpc.Server的指针，代表要被服务的gRPC服务器
-// lis: 网络监听器，用于监听gRPC请求
-// wg: 指向sync.WaitGroup的指针，用于等待服务器停止
-func serveGRPC(s *grpc.Server, lis net.Listener, wg *sync.WaitGroup) {
-	log.Info("start serving grpc")
+func serveGRPC(s *grpc.Server, lis net.Listener) {
+	logger.Info("start serving grpc")
 
 	err := s.Serve(lis)
 
 	if err != nil {
-		log.Errorf("grpc stopped with error: %s", err.Error())
+		logger.Errorf("grpc stopped with error: %s", err.Error())
 	}
 
-	log.Info("grpc stopped")
+	logger.Info("grpc stopped")
 
-	wg.Done() // 表示gRPC服务器已经停止
+	// TODO 仅简单结束了程序
+	os.Exit(1)
 }
 
-// serveDistLock 启动并服务一个分布式锁服务
-// svc: 指向distlock.Service的指针，代表要被服务的分布式锁服务
 func serveDistLock(svc *distlock.Service) {
-	log.Info("start serving distlock")
+	logger.Info("start serving distlock")
 
 	err := svc.Serve()
 
 	if err != nil {
-		log.Errorf("distlock stopped with error: %s", err.Error())
+		logger.Errorf("distlock stopped with error: %s", err.Error())
+	}
+
+	logger.Info("distlock stopped")
+
+	// TODO 仅简单结束了程序
+	os.Exit(1)
+}
+
+func serveAccessStat(svc *accessstat.AccessStat) {
+	logger.Info("start serving access stat")
+
+	ch := svc.Start()
+loop:
+	for {
+		val, err := ch.Receive()
+		if err != nil {
+			logger.Errorf("access stat stopped with error: %v", err)
+			break
+		}
+
+		switch val := val.(type) {
+		case error:
+			logger.Errorf("access stat stopped with error: %v", val)
+			break loop
+		}
 	}
+	logger.Info("access stat stopped")
 
-	log.Info("distlock stopped")
+	// TODO 仅简单结束了程序
+	os.Exit(1)
 }

client/internal/cmdline/bucket.go

@@ -7,22 +7,16 @@ import (
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 )
 
-// BucketListUserBuckets 列出指定用户的存储桶列表。
-// ctx: 命令上下文，提供必要的服务和配置。
-// 返回值: 执行错误时返回error。
 func BucketListUserBuckets(ctx CommandContext) error {
 	userID := cdssdk.UserID(1)
 
-	// 获取指定用户ID的存储桶列表
 	buckets, err := ctx.Cmdline.Svc.BucketSvc().GetUserBuckets(userID)
 	if err != nil {
 		return err
 	}
 
-	// 打印找到的存储桶数量和用户ID
 	fmt.Printf("Find %d buckets for user %d:\n", len(buckets), userID)
 
-	// 构建存储桶列表的表格显示
 	tb := table.NewWriter()
 	tb.AppendHeader(table.Row{"ID", "Name", "CreatorID"})
 
@@ -30,55 +24,38 @@ func BucketListUserBuckets(ctx CommandContext) error {
 		tb.AppendRow(table.Row{bucket.BucketID, bucket.Name, bucket.CreatorID})
 	}
 
-	// 打印存储桶列表表格
 	fmt.Print(tb.Render())
 	return nil
 }
 
-// BucketCreateBucket 为指定用户创建一个新的存储桶。
-// ctx: 命令上下文，提供必要的服务和配置。
-// bucketName: 新存储桶的名称。
-// 返回值: 执行错误时返回error。
 func BucketCreateBucket(ctx CommandContext, bucketName string) error {
 	userID := cdssdk.UserID(1)
 
-	// 创建存储桶并获取新存储桶的ID
 	bucketID, err := ctx.Cmdline.Svc.BucketSvc().CreateBucket(userID, bucketName)
 	if err != nil {
 		return err
 	}
 
-	// 打印创建存储桶成功的消息
-	fmt.Printf("Create bucket %s success, id: %d", bucketName, bucketID)
+	fmt.Printf("Create bucket %v success, id: %v", bucketName, bucketID)
 	return nil
 }
 
-// BucketDeleteBucket 删除指定的存储桶。
-// ctx: 命令上下文，提供必要的服务和配置。
-// bucketID: 要删除的存储桶ID。
-// 返回值: 执行错误时返回error。
 func BucketDeleteBucket(ctx CommandContext, bucketID cdssdk.BucketID) error {
 	userID := cdssdk.UserID(1)
 
-	// 删除指定的存储桶
 	err := ctx.Cmdline.Svc.BucketSvc().DeleteBucket(userID, bucketID)
 	if err != nil {
 		return err
 	}
 
-	// 打印删除成功的消息
 	fmt.Printf("Delete bucket %d success ", bucketID)
 	return nil
 }
 
-// 初始化命令注册
 func init() {
-	// 注册列出用户存储桶的命令
 	commands.MustAdd(BucketListUserBuckets, "bucket", "ls")
 
-	// 注册创建存储桶的命令
 	commands.MustAdd(BucketCreateBucket, "bucket", "new")
 
-	// 注册删除存储桶的命令
 	commands.MustAdd(BucketDeleteBucket, "bucket", "delete")
 }

client/internal/cmdline/cache.go

@@ -7,27 +7,19 @@ import (
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 )
 
-// CacheMovePackage 移动缓存包到指定节点。
-// ctx: 命令上下文环境。
-// packageID: 待移动的包ID。
-// nodeID: 目标节点ID。
-// 返回值: 移动成功返回nil，失败返回error。
-func CacheMovePackage(ctx CommandContext, packageID cdssdk.PackageID, nodeID cdssdk.NodeID) error {
+func CacheMovePackage(ctx CommandContext, packageID cdssdk.PackageID, stgID cdssdk.StorageID) error {
 	startTime := time.Now()
 	defer func() {
-		// 打印函数执行时间
 		fmt.Printf("%v\n", time.Since(startTime).Seconds())
 	}()
 
-	// 开始移动缓存包任务
-	taskID, err := ctx.Cmdline.Svc.CacheSvc().StartCacheMovePackage(1, packageID, nodeID)
+	hubID, taskID, err := ctx.Cmdline.Svc.CacheSvc().StartCacheMovePackage(1, packageID, stgID)
 	if err != nil {
 		return fmt.Errorf("start cache moving package: %w", err)
 	}
 
-	// 循环等待缓存包移动完成
 	for {
-		complete, err := ctx.Cmdline.Svc.CacheSvc().WaitCacheMovePackage(nodeID, taskID, time.Second*10)
+		complete, err := ctx.Cmdline.Svc.CacheSvc().WaitCacheMovePackage(hubID, taskID, time.Second*10)
 		if complete {
 			if err != nil {
 				return fmt.Errorf("moving complete with: %w", err)
@@ -42,20 +34,12 @@ func CacheMovePackage(ctx CommandContext, packageID cdssdk.PackageID, nodeID cds
 	}
 }
 
-// CacheRemovePackage 从缓存中移除指定的包。
-// ctx: 命令上下文环境。
-// packageID: 待移除的包ID。
-// nodeID: 缓存节点ID。
-// 返回值: 移除成功返回nil，失败返回error。
 func CacheRemovePackage(ctx CommandContext, packageID cdssdk.PackageID, nodeID cdssdk.NodeID) error {
 	return ctx.Cmdline.Svc.CacheSvc().CacheRemovePackage(packageID, nodeID)
 }
 
-// 初始化命令列表
 func init() {
-	// 添加移动缓存包命令
 	commands.Add(CacheMovePackage, "cache", "move")
 
-	// 添加移除缓存包命令
 	commands.Add(CacheRemovePackage, "cache", "remove")
 }

client/internal/cmdline/commandline.go

@@ -1,45 +1,50 @@
 package cmdline
 
 import (
+	"context"
 	"fmt"
 	"os"
 
+	"github.com/spf13/cobra"
 	"gitlink.org.cn/cloudream/common/pkgs/cmdtrie"
 	"gitlink.org.cn/cloudream/storage/client/internal/services"
 )
 
-// CommandContext 命令上下文，存储与命令行相关的上下文信息。
 type CommandContext struct {
-	Cmdline *Commandline // 指向当前的Commandline实例。
+	Cmdline *Commandline
 }
 
-// commands 用于存储所有已注册的命令及其相关信息的Trie树。
+// TODO 逐步使用cobra代替cmdtrie
 var commands cmdtrie.CommandTrie[CommandContext, error] = cmdtrie.NewCommandTrie[CommandContext, error]()
 
-// Commandline 命令行对象，封装了与服务交互的能力。
+var rootCmd = cobra.Command{}
+
 type Commandline struct {
-	Svc *services.Service // 指向内部服务接口。
+	Svc *services.Service
 }
 
-// NewCommandline 创建一个新的Commandline实例。
-// svc: 指向内部服务的实例。
-// 返回值: 初始化好的Commandline指针及可能的错误。
 func NewCommandline(svc *services.Service) (*Commandline, error) {
 	return &Commandline{
 		Svc: svc,
 	}, nil
 }
 
-// DispatchCommand 分发并执行命令。
-// allArgs: 命令行中所有的参数。
-// 功能: 根据参数执行相应的命令逻辑，出错时退出程序。
 func (c *Commandline) DispatchCommand(allArgs []string) {
 	cmdCtx := CommandContext{
 		Cmdline: c,
 	}
-	// 执行命令，根据命令执行结果做相应处理。
 	cmdErr, err := commands.Execute(cmdCtx, allArgs, cmdtrie.ExecuteOption{ReplaceEmptyArrayWithNil: true})
 	if err != nil {
+		if err == cmdtrie.ErrCommandNotFound {
+			ctx := context.WithValue(context.Background(), "cmdCtx", &cmdCtx)
+			err = rootCmd.ExecuteContext(ctx)
+			if err != nil {
+				fmt.Println(err)
+				os.Exit(1)
+			}
+			return
+		}
+
 		fmt.Printf("execute command failed, err: %s", err.Error())
 		os.Exit(1)
 	}
@@ -49,12 +54,11 @@ func (c *Commandline) DispatchCommand(allArgs []string) {
 	}
 }
 
-// MustAddCmd 必须添加命令。
-// fn: 命令执行的函数。
-// prefixWords: 命令的前缀词。
-// 返回值: 无。
-// 功能: 向命令树中添加命令，添加失败时会抛出异常。
 func MustAddCmd(fn any, prefixWords ...string) any {
 	commands.MustAdd(fn, prefixWords...)
 	return nil
 }
+
+func GetCmdCtx(cmd *cobra.Command) *CommandContext {
+	return cmd.Context().Value("cmdCtx").(*CommandContext)
+}

client/internal/cmdline/getp.go

@@ -0,0 +1,133 @@
+package cmdline
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/inhies/go-bytesize"
+	"github.com/spf13/cobra"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/iterator"
+)
+
+func init() {
+	var usePkgID bool
+	cmd := &cobra.Command{
+		Use:   "getp",
+		Short: "Download whole package by package id or path",
+		Args:  cobra.ExactArgs(2),
+		Run: func(cmd *cobra.Command, args []string) {
+			cmdCtx := GetCmdCtx(cmd)
+
+			if usePkgID {
+				id, err := strconv.ParseInt(args[0], 10, 64)
+				if err != nil {
+					fmt.Printf("Invalid package id: %s\n", args[0])
+					return
+				}
+
+				getpByID(cmdCtx, cdssdk.PackageID(id), args[1])
+			} else {
+				getpByPath(cmdCtx, args[0], args[1])
+			}
+		},
+	}
+	cmd.Flags().BoolVarP(&usePkgID, "id", "i", false, "Download with package id instead of path")
+
+	rootCmd.AddCommand(cmd)
+}
+
+func getpByPath(cmdCtx *CommandContext, path string, output string) {
+	userID := cdssdk.UserID(1)
+
+	comps := strings.Split(strings.Trim(path, cdssdk.ObjectPathSeparator), cdssdk.ObjectPathSeparator)
+	if len(comps) != 2 {
+		fmt.Printf("Package path must be in format of <bucket>/<package>")
+		return
+	}
+
+	pkg, err := cmdCtx.Cmdline.Svc.PackageSvc().GetByName(userID, comps[0], comps[1])
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	getpByID(cmdCtx, pkg.PackageID, output)
+}
+
+func getpByID(cmdCtx *CommandContext, id cdssdk.PackageID, output string) {
+	userID := cdssdk.UserID(1)
+	startTime := time.Now()
+
+	objIter, err := cmdCtx.Cmdline.Svc.PackageSvc().DownloadPackage(userID, id)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	err = os.MkdirAll(output, os.ModePerm)
+	if err != nil {
+		fmt.Printf("Create output directory %s failed, err: %v", output, err)
+		return
+	}
+	defer objIter.Close()
+
+	madeDirs := make(map[string]bool)
+	fileCount := 0
+	totalSize := int64(0)
+
+	for {
+		objInfo, err := objIter.MoveNext()
+		if err == iterator.ErrNoMoreItem {
+			break
+		}
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+
+		err = func() error {
+			defer objInfo.File.Close()
+			fileCount++
+			totalSize += objInfo.Object.Size
+
+			fullPath := filepath.Join(output, objInfo.Object.Path)
+
+			dirPath := filepath.Dir(fullPath)
+			if !madeDirs[dirPath] {
+				if err := os.MkdirAll(dirPath, 0755); err != nil {
+					return fmt.Errorf("creating object dir: %w", err)
+				}
+				madeDirs[dirPath] = true
+			}
+
+			outputFile, err := os.Create(fullPath)
+			if err != nil {
+				return fmt.Errorf("creating object file: %w", err)
+			}
+			defer outputFile.Close()
+
+			_, err = io.Copy(outputFile, objInfo.File)
+			if err != nil {
+				return fmt.Errorf("copy object data to local file failed, err: %w", err)
+			}
+
+			if stgglb.Local.NodeID != nil {
+				cmdCtx.Cmdline.Svc.AccessStat.AddAccessCounter(objInfo.Object.ObjectID, id, *stgglb.Local.NodeID, 1)
+			}
+			return nil
+		}()
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+	}
+
+	fmt.Printf("Get %v files (%v) to %s in %v.\n", fileCount, bytesize.ByteSize(totalSize), output, time.Since(startTime))
+}

client/internal/cmdline/load.go

@@ -0,0 +1,89 @@
+package cmdline
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+)
+
+func init() {
+	var useID bool
+	cmd := cobra.Command{
+		Use:   "load",
+		Short: "Load data from CDS to a storage service",
+		Args:  cobra.ExactArgs(2),
+		Run: func(cmd *cobra.Command, args []string) {
+			cmdCtx := GetCmdCtx(cmd)
+
+			if useID {
+				pkgID, err := strconv.ParseInt(args[0], 10, 64)
+				if err != nil {
+					fmt.Printf("Invalid package ID: %s\n", args[0])
+				}
+
+				stgID, err := strconv.ParseInt(args[1], 10, 64)
+				if err != nil {
+					fmt.Printf("Invalid storage ID: %s\n", args[1])
+				}
+
+				loadByID(cmdCtx, cdssdk.PackageID(pkgID), cdssdk.StorageID(stgID))
+			} else {
+				loadByPath(cmdCtx, args[0], args[1])
+			}
+		},
+	}
+	cmd.Flags().BoolVarP(&useID, "id", "i", false, "Use ID for both package and storage service instead of their name or path")
+	rootCmd.AddCommand(&cmd)
+}
+
+func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string) {
+	userID := cdssdk.UserID(1)
+
+	comps := strings.Split(strings.Trim(pkgPath, cdssdk.ObjectPathSeparator), cdssdk.ObjectPathSeparator)
+	if len(comps) != 2 {
+		fmt.Printf("Package path must be in format of <bucket>/<package>")
+		return
+	}
+
+	pkg, err := cmdCtx.Cmdline.Svc.PackageSvc().GetByName(userID, comps[0], comps[1])
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	stg, err := cmdCtx.Cmdline.Svc.StorageSvc().GetByName(userID, stgName)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	loadByID(cmdCtx, pkg.PackageID, stg.StorageID)
+}
+
+func loadByID(cmdCtx *CommandContext, pkgID cdssdk.PackageID, stgID cdssdk.StorageID) {
+	userID := cdssdk.UserID(1)
+	startTime := time.Now()
+
+	nodeID, taskID, err := cmdCtx.Cmdline.Svc.StorageSvc().StartStorageLoadPackage(userID, pkgID, stgID)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for {
+		complete, fullPath, err := cmdCtx.Cmdline.Svc.StorageSvc().WaitStorageLoadPackage(nodeID, taskID, time.Second*10)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+
+		if complete {
+			fmt.Printf("Package loaded to: %s in %v\n", fullPath, time.Since(startTime))
+			break
+		}
+	}
+}

client/internal/cmdline/lsp.go

@@ -0,0 +1,74 @@
+package cmdline
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/jedib0t/go-pretty/v6/table"
+	"github.com/spf13/cobra"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+)
+
+func init() {
+	var usePkgID *bool
+	cmd := &cobra.Command{
+		Use:   "lsp",
+		Short: "List package information",
+		Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			cmdCtx := GetCmdCtx(cmd)
+
+			if usePkgID != nil && *usePkgID {
+				id, err := strconv.ParseInt(args[0], 10, 64)
+				if err != nil {
+					fmt.Printf("Invalid package id: %s\n", args[0])
+					return
+				}
+
+				lspOneByID(cmdCtx, cdssdk.PackageID(id))
+			} else {
+				lspByPath(cmdCtx, args[0])
+			}
+		},
+	}
+	usePkgID = cmd.Flags().BoolP("id", "i", false, "List with package id instead of path")
+
+	rootCmd.AddCommand(cmd)
+}
+
+func lspByPath(cmdCtx *CommandContext, path string) {
+	userID := cdssdk.UserID(1)
+
+	comps := strings.Split(strings.Trim(path, cdssdk.ObjectPathSeparator), cdssdk.ObjectPathSeparator)
+	if len(comps) != 2 {
+		fmt.Printf("Package path must be in format of <bucket>/<package>")
+		return
+	}
+
+	pkg, err := cmdCtx.Cmdline.Svc.PackageSvc().GetByName(userID, comps[0], comps[1])
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	wr := table.NewWriter()
+	wr.AppendHeader(table.Row{"ID", "Name", "State"})
+	wr.AppendRow(table.Row{pkg.PackageID, pkg.Name, pkg.State})
+	fmt.Println(wr.Render())
+}
+
+func lspOneByID(cmdCtx *CommandContext, id cdssdk.PackageID) {
+	userID := cdssdk.UserID(1)
+
+	pkg, err := cmdCtx.Cmdline.Svc.PackageSvc().Get(userID, id)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	wr := table.NewWriter()
+	wr.AppendHeader(table.Row{"ID", "Name", "State"})
+	wr.AppendRow(table.Row{pkg.PackageID, pkg.Name, pkg.State})
+	fmt.Println(wr.Render())
+}

client/internal/cmdline/put.go

@@ -0,0 +1,124 @@
+package cmdline
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/inhies/go-bytesize"
+	"github.com/spf13/cobra"
+	"gitlink.org.cn/cloudream/common/consts/errorcode"
+	"gitlink.org.cn/cloudream/common/pkgs/mq"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/iterator"
+)
+
+func init() {
+	var nodeID int64
+	cmd := &cobra.Command{
+		Use:   "put",
+		Short: "Upload files to CDS",
+		Args: func(cmd *cobra.Command, args []string) error {
+			if err := cobra.ExactArgs(2)(cmd, args); err != nil {
+				return err
+			}
+
+			remote := args[1]
+			comps := strings.Split(strings.Trim(remote, cdssdk.ObjectPathSeparator), cdssdk.ObjectPathSeparator)
+			if len(comps) != 2 {
+				return fmt.Errorf("invalid remote path: %s, which must be in format of <bucket>/<package>", remote)
+			}
+
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			userID := cdssdk.UserID(1)
+			cmdCtx := GetCmdCtx(cmd)
+
+			local := args[0]
+			remote := args[1]
+			comps := strings.Split(strings.Trim(remote, cdssdk.ObjectPathSeparator), cdssdk.ObjectPathSeparator)
+
+			startTime := time.Now()
+
+			bkt, err := cmdCtx.Cmdline.Svc.BucketSvc().GetBucketByName(userID, comps[0])
+			if err != nil {
+				fmt.Printf("getting bucket: %v\n", err)
+				return
+			}
+
+			pkg, err := cmdCtx.Cmdline.Svc.PackageSvc().GetByName(userID, comps[0], comps[1])
+			if err != nil {
+				if codeMsg, ok := err.(*mq.CodeMessageError); ok && codeMsg.Code == errorcode.DataNotFound {
+					pkg2, err := cmdCtx.Cmdline.Svc.PackageSvc().Create(userID, bkt.BucketID, comps[1])
+					if err != nil {
+						fmt.Printf("creating package: %v\n", err)
+						return
+					}
+					pkg = &pkg2
+
+				} else {
+					fmt.Printf("getting package: %v\n", err)
+					return
+				}
+			}
+
+			var fileCount int
+			var totalSize int64
+			var uploadFilePathes []string
+			err = filepath.WalkDir(local, func(fname string, fi os.DirEntry, err error) error {
+				if err != nil {
+					return nil
+				}
+
+				if !fi.IsDir() {
+					uploadFilePathes = append(uploadFilePathes, fname)
+					fileCount++
+
+					info, err := fi.Info()
+					if err == nil {
+						totalSize += info.Size()
+					}
+				}
+
+				return nil
+			})
+			if err != nil {
+				fmt.Printf("walking directory: %v\n", err)
+				return
+			}
+
+			var nodeAff *cdssdk.NodeID
+			if nodeID != 0 {
+				id := cdssdk.NodeID(nodeID)
+				nodeAff = &id
+			}
+
+			objIter := iterator.NewUploadingObjectIterator(local, uploadFilePathes)
+			taskID, err := cmdCtx.Cmdline.Svc.ObjectSvc().StartUploading(userID, pkg.PackageID, objIter, nodeAff)
+			if err != nil {
+				fmt.Printf("start uploading objects: %v\n", err)
+				return
+			}
+
+			for {
+				complete, _, err := cmdCtx.Cmdline.Svc.ObjectSvc().WaitUploading(taskID, time.Second*5)
+				if err != nil {
+					fmt.Printf("uploading objects: %v\n", err)
+					return
+				}
+
+				if complete {
+					break
+				}
+			}
+
+			fmt.Printf("Put %v files (%v) to %s in %v.\n", fileCount, bytesize.ByteSize(totalSize), remote, time.Since(startTime))
+		},
+	}
+	cmd.Flags().Int64VarP(&nodeID, "node", "n", 0, "node affinity")
+
+	rootCmd.AddCommand(cmd)
+}

client/internal/cmdline/scanner.go

@@ -4,61 +4,44 @@ import (
 	"fmt"
 
 	"gitlink.org.cn/cloudream/common/pkgs/cmdtrie"
-	myreflect "gitlink.org.cn/cloudream/common/utils/reflect"
+	"gitlink.org.cn/cloudream/common/utils/reflect2"
 	scevt "gitlink.org.cn/cloudream/storage/common/pkgs/mq/scanner/event"
 )
 
-// parseScannerEventCmdTrie 是一个静态命令 trie 树，用于解析扫描器事件命令。
 var parseScannerEventCmdTrie cmdtrie.StaticCommandTrie[any] = cmdtrie.NewStaticCommandTrie[any]()
 
-// ScannerPostEvent 发布扫描器事件。
-// ctx: 命令上下文。
-// args: 命令参数数组。
-// 返回值: 执行错误时返回 error。
 func ScannerPostEvent(ctx CommandContext, args []string) error {
-	// 尝试执行解析扫描器事件命令。
 	ret, err := parseScannerEventCmdTrie.Execute(args, cmdtrie.ExecuteOption{ReplaceEmptyArrayWithNil: true})
 	if err != nil {
-		// 解析失败，返回错误信息。
 		return fmt.Errorf("execute parsing event command failed, err: %w", err)
 	}
 
-	// 发布解析得到的事件。
 	err = ctx.Cmdline.Svc.ScannerSvc().PostEvent(ret.(scevt.Event), false, false)
 	if err != nil {
-		// 发布事件失败，返回错误信息。
 		return fmt.Errorf("post event to scanner failed, err: %w", err)
 	}
 
 	return nil
 }
 
-// 初始化函数，用于向 parseScannerEventCmdTrie 注册扫描器事件命令。
 func init() {
-	// 注册 AgentCacheGC 事件。
-	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentCacheGC, myreflect.TypeNameOf[scevt.AgentCacheGC]())
+	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentCacheGC, reflect2.TypeNameOf[scevt.AgentCacheGC]())
 
-	// 注册 AgentCheckCache 事件。
-	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentCheckCache, myreflect.TypeNameOf[scevt.AgentCheckCache]())
+	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentCheckCache, reflect2.TypeNameOf[scevt.AgentCheckCache]())
 
-	// 注册 AgentCheckState 事件。
-	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentCheckState, myreflect.TypeNameOf[scevt.AgentCheckState]())
+	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentCheckState, reflect2.TypeNameOf[scevt.AgentCheckState]())
 
-	// 注册 AgentStorageGC 事件。
-	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentStorageGC, myreflect.TypeNameOf[scevt.AgentStorageGC]())
+	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentStorageGC, reflect2.TypeNameOf[scevt.AgentStorageGC]())
 
-	// 注册 AgentCheckStorage 事件。
-	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentCheckStorage, myreflect.TypeNameOf[scevt.AgentCheckStorage]())
+	parseScannerEventCmdTrie.MustAdd(scevt.NewAgentCheckStorage, reflect2.TypeNameOf[scevt.AgentCheckStorage]())
 
-	// 注册 CheckPackage 事件。
-	parseScannerEventCmdTrie.MustAdd(scevt.NewCheckPackage, myreflect.TypeNameOf[scevt.CheckPackage]())
+	parseScannerEventCmdTrie.MustAdd(scevt.NewCheckPackage, reflect2.TypeNameOf[scevt.CheckPackage]())
 
-	// 注册 CheckPackageRedundancy 事件。
-	parseScannerEventCmdTrie.MustAdd(scevt.NewCheckPackageRedundancy, myreflect.TypeNameOf[scevt.CheckPackageRedundancy]())
+	parseScannerEventCmdTrie.MustAdd(scevt.NewCheckPackageRedundancy, reflect2.TypeNameOf[scevt.CheckPackageRedundancy]())
 
-	// 注册 CleanPinned 事件。
-	parseScannerEventCmdTrie.MustAdd(scevt.NewCleanPinned, myreflect.TypeNameOf[scevt.CleanPinned]())
+	parseScannerEventCmdTrie.MustAdd(scevt.NewCleanPinned, reflect2.TypeNameOf[scevt.CleanPinned]())
+
+	parseScannerEventCmdTrie.MustAdd(scevt.NewUpdatePackageAccessStatAmount, reflect2.TypeNameOf[scevt.UpdatePackageAccessStatAmount]())
 
-	// 向命令行注册 ScannerPostEvent 命令。
 	commands.MustAdd(ScannerPostEvent, "scanner", "event")
 }

client/internal/cmdline/test.go

@@ -0,0 +1,252 @@
+package cmdline
+
+/*
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/spf13/cobra"
+	"gitlink.org.cn/cloudream/common/pkgs/future"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/parser"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc"
+	lrcparser "gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc/parser"
+	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
+)
+
+func init() {
+	rootCmd.AddCommand(&cobra.Command{
+		Use:   "test",
+		Short: "test",
+		// Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			// cmdCtx := GetCmdCtx(cmd)
+			coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+			if err != nil {
+				panic(err)
+			}
+			defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+			nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.NodeID{1, 2}))
+			if err != nil {
+				panic(err)
+			}
+
+			ft := ioswitch2.NewFromTo()
+
+			ft.AddFrom(ioswitch2.NewFromNode("Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", &nodes.Nodes[0], -1))
+			ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], -1, "asd"))
+			le := int64(3)
+			toExec, hd := ioswitch2.NewToDriverWithRange(-1, exec.Range{Offset: 5, Length: &le})
+			ft.AddTo(toExec)
+			ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 0, "0"))
+			ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 1, "1"))
+			ft.AddTo(ioswitch2.NewToNode(nodes.Nodes[1], 2, "2"))
+
+			// ft.AddFrom(ioswitch2.NewFromNode("QmS2s8GRYHEurXL7V1zUtKvf2H1BGcQc5NN1T1hiSnWvbd", &nodes.Nodes[0], 1))
+			// ft.AddFrom(ioswitch2.NewFromNode("QmUgUEUMzdnjPNx6xu9PDGXpSyXTk8wzPWvyYZ9zasE1WW", &nodes.Nodes[1], 2))
+			// le := int64(5)
+			// toExec, hd := ioswitch2.NewToDriverWithRange(-1, exec.Range{Offset: 3, Length: &le})
+			// toExec, hd := plans.NewToExecutorWithRange(1, plans.Range{Offset: 0, Length: nil})
+			// toExec2, hd2 := plans.NewToExecutorWithRange(2, plans.Range{Offset: 0, Length: nil})
+			// ft.AddTo(toExec)
+			// ft.AddTo(toExec2)
+
+			// fromExec, hd := plans.NewFromExecutor(-1)
+			// ft.AddFrom(fromExec)
+			// ft.AddTo(plans.NewToNode(nodes.Nodes[1], -1, "asd"))
+
+			parser := parser.NewParser(cdssdk.DefaultECRedundancy)
+
+			plans := exec.NewPlanBuilder()
+			err = parser.Parse(ft, plans)
+			if err != nil {
+				panic(err)
+			}
+
+			exec := plans.Execute()
+
+			fut := future.NewSetVoid()
+			go func() {
+				mp, err := exec.Wait(context.Background())
+				if err != nil {
+					panic(err)
+				}
+
+				fmt.Printf("mp: %+v\n", mp)
+				fut.SetVoid()
+			}()
+
+			go func() {
+				// exec.BeginWrite(io.NopCloser(bytes.NewBuffer([]byte("hello world"))), hd)
+				// if err != nil {
+				// 	panic(err)
+				// }
+
+				str, err := exec.BeginRead(hd)
+				if err != nil {
+					panic(err)
+				}
+				defer str.Close()
+				data, err := io.ReadAll(str)
+				if err != nil && err != io.EOF {
+					panic(err)
+				}
+				fmt.Printf("data: %v(%v)\n", string(data), len(data))
+			}()
+
+			fut.Wait(context.TODO())
+		},
+	})
+
+	// rootCmd.AddCommand(&cobra.Command{
+	// 	Use:   "test",
+	// 	Short: "test",
+	// 	// Args:  cobra.ExactArgs(1),
+	// 	Run: func(cmd *cobra.Command, args []string) {
+	// 		cmdCtx := GetCmdCtx(cmd)
+	// 		file, _ := cmdCtx.Cmdline.Svc.ObjectSvc().Download(1, downloader.DownloadReqeust{
+	// 			ObjectID: 27379,
+	// 			Length:   -1,
+	// 		})
+	// 		data, _ := io.ReadAll(file.File)
+	// 		fmt.Printf("data: %v(%v)\n", string(data), len(data))
+	// 	},
+	// })
+
+	rootCmd.AddCommand(&cobra.Command{
+		Use:   "test3",
+		Short: "test3",
+		// Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			// cmdCtx := GetCmdCtx(cmd)
+
+			coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+			if err != nil {
+				panic(err)
+			}
+			defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+			nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.NodeID{1, 2}))
+			if err != nil {
+				panic(err)
+			}
+
+			red := cdssdk.DefaultLRCRedundancy
+
+			var toes []ioswitchlrc.To
+			for i := 0; i < red.N; i++ {
+				toes = append(toes, ioswitchlrc.NewToNode(nodes.Nodes[i%2], i, fmt.Sprintf("%d", i)))
+			}
+
+			plans := exec.NewPlanBuilder()
+			err = lrcparser.Encode(ioswitchlrc.NewFromNode("QmNspjDLxQbAsuh37jRXKvLWHE2f7JpqY4HEJ8x7Jgbzqa", &nodes.Nodes[0], -1), toes, plans)
+			if err != nil {
+				panic(err)
+				// return nil, fmt.Errorf("parsing plan: %w", err)
+			}
+
+			ioRet, err := plans.Execute().Wait(context.TODO())
+			if err != nil {
+				panic(err)
+				// return nil, fmt.Errorf("executing io plan: %w", err)
+			}
+
+			fmt.Printf("ioRet: %v\n", ioRet)
+		},
+	})
+
+	rootCmd.AddCommand(&cobra.Command{
+		Use:   "test4",
+		Short: "test4",
+		// Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			// cmdCtx := GetCmdCtx(cmd)
+
+			coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+			if err != nil {
+				panic(err)
+			}
+			defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+			nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.NodeID{1, 2}))
+			if err != nil {
+				panic(err)
+			}
+
+			// red := cdssdk.DefaultLRCRedundancy
+
+			plans := exec.NewPlanBuilder()
+			err = lrcparser.ReconstructGroup([]ioswitchlrc.From{
+				ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[1], 0),
+				ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[1], 1),
+			}, []ioswitchlrc.To{
+				ioswitchlrc.NewToNode(nodes.Nodes[1], 3, "3"),
+			}, plans)
+			if err != nil {
+				panic(err)
+				// return nil, fmt.Errorf("parsing plan: %w", err)
+			}
+
+			ioRet, err := plans.Execute().Wait(context.TODO())
+			if err != nil {
+				panic(err)
+				// return nil, fmt.Errorf("executing io plan: %w", err)
+			}
+
+			fmt.Printf("ioRet: %v\n", ioRet)
+		},
+	})
+
+	rootCmd.AddCommand(&cobra.Command{
+		Use:   "test3",
+		Short: "test3",
+		// Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			// cmdCtx := GetCmdCtx(cmd)
+
+			coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+			if err != nil {
+				panic(err)
+			}
+			defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+			nodes, err := coorCli.GetNodes(coormq.NewGetNodes([]cdssdk.NodeID{1, 2}))
+			if err != nil {
+				panic(err)
+			}
+
+			// red := cdssdk.DefaultLRCRedundancy
+
+			plans := exec.NewPlanBuilder()
+			le := int64(1293)
+			err = lrcparser.ReconstructAny([]ioswitchlrc.From{
+				ioswitchlrc.NewFromNode("QmVAZzVQEvnvTvzSz2SvpziAcDSQ8aYCoTyGrZNuV8raEQ", &nodes.Nodes[0], 0),
+				ioswitchlrc.NewFromNode("QmQBKncEDqxw3BrGr3th3gS3jUC2fizGz1w29ZxxrrKfNv", &nodes.Nodes[0], 2),
+			}, []ioswitchlrc.To{
+				ioswitchlrc.NewToNodeWithRange(nodes.Nodes[1], -1, "-1", exec.Range{0, &le}),
+				ioswitchlrc.NewToNodeWithRange(nodes.Nodes[1], 0, "0", exec.Range{10, &le}),
+				ioswitchlrc.NewToNode(nodes.Nodes[1], 1, "1"),
+				ioswitchlrc.NewToNode(nodes.Nodes[1], 2, "2"),
+				ioswitchlrc.NewToNode(nodes.Nodes[1], 3, "3"),
+			}, plans)
+			if err != nil {
+				panic(err)
+				// return nil, fmt.Errorf("parsing plan: %w", err)
+			}
+
+			ioRet, err := plans.Execute().Wait(context.TODO())
+			if err != nil {
+				panic(err)
+				// return nil, fmt.Errorf("executing io plan: %w", err)
+			}
+
+			fmt.Printf("ioRet: %v\n", ioRet)
+		},
+	})
+}
+*/

client/internal/config/config.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"gitlink.org.cn/cloudream/common/pkgs/distlock"
-	"gitlink.org.cn/cloudream/common/pkgs/ipfs"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	"gitlink.org.cn/cloudream/common/utils/config"
 	stgmodels "gitlink.org.cn/cloudream/storage/common/models"
@@ -17,7 +16,6 @@ type Config struct {
 	AgentGRPC    agtrpc.PoolConfig          `json:"agentGRPC"`
 	Logger       logger.Config              `json:"logger"`
 	RabbitMQ     stgmq.Config               `json:"rabbitMQ"`
-	IPFS         *ipfs.Config               `json:"ipfs"` // 此字段非空代表客户端上存在ipfs daemon
 	DistLock     distlock.Config            `json:"distlock"`
 	Connectivity connectivity.Config        `json:"connectivity"`
 	Downloader   downloader.Config          `json:"downloader"`

client/internal/http/bucket.go

@@ -6,15 +6,13 @@ import (
 	"github.com/gin-gonic/gin"
 	"gitlink.org.cn/cloudream/common/consts/errorcode"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
 )
 
-// BucketService 用于处理与存储桶相关的HTTP请求
 type BucketService struct {
 	*Server
 }
 
-// Bucket 返回BucketService的实例
 func (s *Server) Bucket() *BucketService {
 	return &BucketService{
 		Server: s,
@@ -24,7 +22,7 @@ func (s *Server) Bucket() *BucketService {
 func (s *BucketService) GetByName(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Bucket.GetByName")
 
-	var req cdssdk.BucketGetByName
+	var req cdsapi.BucketGetByName
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding query: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -38,7 +36,7 @@ func (s *BucketService) GetByName(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.BucketGetByNameResp{
+	ctx.JSON(http.StatusOK, OK(cdsapi.BucketGetByNameResp{
 		Bucket: bucket,
 	}))
 }
@@ -46,10 +44,9 @@ func (s *BucketService) GetByName(ctx *gin.Context) {
 func (s *BucketService) Create(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Bucket.Create")
 
-	var req cdssdk.BucketCreate
+	var req cdsapi.BucketCreate
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
-		// 绑定失败，返回错误信息
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
 		return
 	}
@@ -57,46 +54,38 @@ func (s *BucketService) Create(ctx *gin.Context) {
 	bucket, err := s.svc.BucketSvc().CreateBucket(req.UserID, req.Name)
 	if err != nil {
 		log.Warnf("creating bucket: %s", err.Error())
-		// 创建存储桶失败，返回错误信息
 		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "create bucket failed"))
 		return
 	}
 
-	// 创建存储桶成功，返回成功响应
-	ctx.JSON(http.StatusOK, OK(cdssdk.BucketCreateResp{
+	ctx.JSON(http.StatusOK, OK(cdsapi.BucketCreateResp{
 		Bucket: bucket,
 	}))
 }
 
-// Delete 删除指定的存储桶
-// ctx *gin.Context: Gin框架的上下文对象，用于处理HTTP请求和响应
 func (s *BucketService) Delete(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Bucket.Delete")
 
-	var req cdssdk.BucketDelete
+	var req cdsapi.BucketDelete
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
-		// 绑定失败，返回错误信息
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
 		return
 	}
 
-	// 调用服务层方法，删除存储桶
 	if err := s.svc.BucketSvc().DeleteBucket(req.UserID, req.BucketID); err != nil {
 		log.Warnf("deleting bucket: %s", err.Error())
-		// 删除存储桶失败，返回错误信息
 		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "delete bucket failed"))
 		return
 	}
 
-	// 删除存储桶成功，返回成功响应
 	ctx.JSON(http.StatusOK, OK(nil))
 }
 
 func (s *BucketService) ListUserBuckets(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Bucket.ListUserBuckets")
 
-	var req cdssdk.BucketListUserBucketsReq
+	var req cdsapi.BucketListUserBucketsReq
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding query: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -110,7 +99,7 @@ func (s *BucketService) ListUserBuckets(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.BucketListUserBucketsResp{
+	ctx.JSON(http.StatusOK, OK(cdsapi.BucketListUserBucketsResp{
 		Buckets: buckets,
 	}))
 }

client/internal/http/cache.go

@@ -8,36 +8,29 @@ import (
 	"gitlink.org.cn/cloudream/common/consts/errorcode"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
 )
 
-// CacheService 缓存服务结构体，依赖于Server
 type CacheService struct {
 	*Server
 }
 
-// Cache 返回CacheService的实例
 func (s *Server) Cache() *CacheService {
 	return &CacheService{
 		Server: s,
 	}
 }
 
-// CacheMovePackageReq 移动缓存包的请求参数
 type CacheMovePackageReq struct {
-	UserID    *cdssdk.UserID    `json:"userID" binding:"required"`
-	PackageID *cdssdk.PackageID `json:"packageID" binding:"required"`
-	NodeID    *cdssdk.NodeID    `json:"nodeID" binding:"required"`
+	UserID    cdssdk.UserID    `json:"userID" binding:"required"`
+	PackageID cdssdk.PackageID `json:"packageID" binding:"required"`
+	StorageID cdssdk.StorageID `json:"storageID" binding:"required"`
 }
+type CacheMovePackageResp = cdsapi.CacheMovePackageResp
 
-// CacheMovePackageResp 移动缓存包的响应参数
-type CacheMovePackageResp = cdssdk.CacheMovePackageResp
-
-// MovePackage 处理移动缓存包的请求
 func (s *CacheService) MovePackage(ctx *gin.Context) {
-	// 初始化日志
 	log := logger.WithField("HTTP", "Cache.LoadPackage")
 
-	// 绑定请求JSON
 	var req CacheMovePackageReq
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
@@ -45,20 +38,16 @@ func (s *CacheService) MovePackage(ctx *gin.Context) {
 		return
 	}
 
-	// 开始移动缓存包任务
-	taskID, err := s.svc.CacheSvc().StartCacheMovePackage(*req.UserID, *req.PackageID, *req.NodeID)
+	hubID, taskID, err := s.svc.CacheSvc().StartCacheMovePackage(req.UserID, req.PackageID, req.StorageID)
 	if err != nil {
 		log.Warnf("start cache move package: %s", err.Error())
 		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "cache move package failed"))
 		return
 	}
 
-	// 循环等待缓存包移动完成
 	for {
-		// 检查移动是否完成
-		complete, err := s.svc.CacheSvc().WaitCacheMovePackage(*req.NodeID, taskID, time.Second*10)
+		complete, err := s.svc.CacheSvc().WaitCacheMovePackage(hubID, taskID, time.Second*10)
 		if complete {
-			// 移动完成后的处理
 			if err != nil {
 				log.Warnf("moving complete with: %s", err.Error())
 				ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "cache move package failed"))
@@ -69,7 +58,6 @@ func (s *CacheService) MovePackage(ctx *gin.Context) {
 			return
 		}
 
-		// 等待移动过程中的错误处理
 		if err != nil {
 			log.Warnf("wait moving: %s", err.Error())
 			ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "cache move package failed"))

client/internal/http/node.go

@@ -7,55 +7,40 @@ import (
 	"gitlink.org.cn/cloudream/common/consts/errorcode"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
 )
 
-// NodeService 结构体代表了节点服务，它包含了一个Server实例。
 type NodeService struct {
 	*Server
 }
 
-// NodeSvc 为Server结构体提供一个方法，返回一个NodeService的实例。
-// 这个方法主要用于在Server实例中访问NodeService。
 func (s *Server) NodeSvc() *NodeService {
 	return &NodeService{
 		Server: s,
 	}
 }
 
-// GetNodesReq 结构体定义了获取节点信息请求的参数。
-// 它包含一个NodeIDs字段，该字段是需要查询的节点的ID列表，是必需的。
 type GetNodesReq struct {
 	NodeIDs *[]cdssdk.NodeID `form:"nodeIDs" binding:"required"`
 }
+type GetNodesResp = cdsapi.NodeGetNodesResp
 
-// GetNodesResp 结构体与cdssdk包中的NodeGetNodesResp类型相同，用于定义获取节点信息的响应。
-type GetNodesResp = cdssdk.NodeGetNodesResp
-
-// GetNodes 是一个处理获取节点信息请求的方法。
-// 它使用Gin框架的Context来处理HTTP请求，获取请求参数，并返回节点信息。
-// ctx *gin.Context: 代表当前的HTTP请求上下文。
 func (s *ObjectService) GetNodes(ctx *gin.Context) {
-	// 初始化日志记录器，添加"HTTP"字段标识。
 	log := logger.WithField("HTTP", "Node.GetNodes")
 
 	var req GetNodesReq
-	// 尝试绑定查询参数到请求结构体，如果出错则返回错误信息。
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
-		// 参数绑定失败，返回400状态码和错误信息。
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
 		return
 	}
 
-	// 调用NodeSvc获取节点信息，如果出错则返回操作失败的错误信息。
 	nodes, err := s.svc.NodeSvc().GetNodes(*req.NodeIDs)
 	if err != nil {
 		log.Warnf("getting nodes: %s", err.Error())
-		// 获取节点信息失败，返回操作失败的错误信息。
 		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get nodes failed"))
 		return
 	}
 
-	// 节点信息获取成功，返回200状态码和节点信息。
 	ctx.JSON(http.StatusOK, OK(GetNodesResp{Nodes: nodes}))
 }

client/internal/http/object.go

@@ -1,40 +1,36 @@
 package http
 
 import (
-	"fmt"
 	"io"
 	"mime/multipart"
 	"net/http"
+	"net/url"
 	"path"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"gitlink.org.cn/cloudream/common/consts/errorcode"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
-	myhttp "gitlink.org.cn/cloudream/common/utils/http"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
+	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/downloader"
 )
 
-// ObjectService 服务结构体，处理对象相关的HTTP请求
 type ObjectService struct {
 	*Server
 }
 
-// Object 返回ObjectService的实例
 func (s *Server) Object() *ObjectService {
 	return &ObjectService{
 		Server: s,
 	}
 }
 
-// ObjectUploadReq 定义上传对象请求的结构体
 type ObjectUploadReq struct {
-	Info  cdssdk.ObjectUploadInfo `form:"info" binding:"required"` // 上传信息
-	Files []*multipart.FileHeader `form:"files"`                   // 上传文件列表
+	Info  cdsapi.ObjectUploadInfo `form:"info" binding:"required"`
+	Files []*multipart.FileHeader `form:"files"`
 }
 
-// Upload 处理对象上传请求
 func (s *ObjectService) Upload(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Object.Upload")
 
@@ -45,18 +41,18 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
 		return
 	}
 
-	// 将multipart文件转换为上传对象
+	var err error
+
 	objIter := mapMultiPartFileToUploadingObject(req.Files)
 
-	// 开始上传任务
 	taskID, err := s.svc.ObjectSvc().StartUploading(req.Info.UserID, req.Info.PackageID, objIter, req.Info.NodeAffinity)
+
 	if err != nil {
 		log.Warnf("start uploading object task: %s", err.Error())
 		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "start uploading task failed"))
 		return
 	}
 
-	// 等待上传任务完成
 	for {
 		complete, objs, err := s.svc.ObjectSvc().WaitUploading(taskID, time.Second*5)
 		if complete {
@@ -66,20 +62,20 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
 				return
 			}
 
-			uploadeds := make([]cdssdk.UploadedObject, len(objs.Objects))
+			uploadeds := make([]cdsapi.UploadedObject, len(objs.Objects))
 			for i, obj := range objs.Objects {
 				err := ""
 				if obj.Error != nil {
 					err = obj.Error.Error()
 				}
 				o := obj.Object
-				uploadeds[i] = cdssdk.UploadedObject{
+				uploadeds[i] = cdsapi.UploadedObject{
 					Object: &o,
 					Error:  err,
 				}
 			}
 
-			ctx.JSON(http.StatusOK, OK(cdssdk.ObjectUploadResp{Uploadeds: uploadeds}))
+			ctx.JSON(http.StatusOK, OK(cdsapi.ObjectUploadResp{Uploadeds: uploadeds}))
 			return
 		}
 
@@ -94,7 +90,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
 func (s *ObjectService) Download(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Object.Download")
 
-	var req cdssdk.ObjectDownload
+	var req cdsapi.ObjectDownload
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -117,26 +113,27 @@ func (s *ObjectService) Download(ctx *gin.Context) {
 		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "download object failed"))
 		return
 	}
+	defer file.File.Close()
 
-	mw := multipart.NewWriter(ctx.Writer)
-	defer mw.Close()
-
-	ctx.Writer.Header().Set("Content-Type", fmt.Sprintf("%s;boundary=%s", myhttp.ContentTypeMultiPart, mw.Boundary()))
-	ctx.Writer.WriteHeader(http.StatusOK)
+	ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(path.Base(file.Object.Path)))
+	ctx.Header("Content-Type", "application/octet-stream")
+	ctx.Header("Content-Transfer-Encoding", "binary")
 
-	fw, err := mw.CreateFormFile("file", path.Base(file.Object.Path))
+	n, err := io.Copy(ctx.Writer, file.File)
 	if err != nil {
-		log.Warnf("creating form file: %s", err.Error())
-		return
+		log.Warnf("copying file: %s", err.Error())
 	}
 
-	io.Copy(fw, file.File)
+	// TODO 当client不在某个代理节点上时如何处理？
+	if stgglb.Local.NodeID != nil {
+		s.svc.AccessStat.AddAccessCounter(file.Object.ObjectID, file.Object.PackageID, *stgglb.Local.NodeID, float64(n)/float64(file.Object.Size))
+	}
 }
 
 func (s *ObjectService) UpdateInfo(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Object.UpdateInfo")
 
-	var req cdssdk.ObjectUpdateInfo
+	var req cdsapi.ObjectUpdateInfo
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -150,13 +147,13 @@ func (s *ObjectService) UpdateInfo(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.ObjectUpdateInfoResp{Successes: sucs}))
+	ctx.JSON(http.StatusOK, OK(cdsapi.ObjectUpdateInfoResp{Successes: sucs}))
 }
 
 func (s *ObjectService) Move(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Object.Move")
 
-	var req cdssdk.ObjectMove
+	var req cdsapi.ObjectMove
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -170,13 +167,13 @@ func (s *ObjectService) Move(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.ObjectMoveResp{Successes: sucs}))
+	ctx.JSON(http.StatusOK, OK(cdsapi.ObjectMoveResp{Successes: sucs}))
 }
 
 func (s *ObjectService) Delete(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Object.Delete")
 
-	var req cdssdk.ObjectDelete
+	var req cdsapi.ObjectDelete
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -193,11 +190,10 @@ func (s *ObjectService) Delete(ctx *gin.Context) {
 	ctx.JSON(http.StatusOK, OK(nil))
 }
 
-// GetPackageObjects 处理获取包内对象的请求
 func (s *ObjectService) GetPackageObjects(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Object.GetPackageObjects")
 
-	var req cdssdk.ObjectGetPackageObjects
+	var req cdsapi.ObjectGetPackageObjects
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -211,5 +207,5 @@ func (s *ObjectService) GetPackageObjects(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.ObjectGetPackageObjectsResp{Objects: objs}))
+	ctx.JSON(http.StatusOK, OK(cdsapi.ObjectGetPackageObjectsResp{Objects: objs}))
 }

client/internal/http/package.go

@@ -9,7 +9,7 @@ import (
 	"gitlink.org.cn/cloudream/common/consts/errorcode"
 	"gitlink.org.cn/cloudream/common/pkgs/iterator"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
 
 	stgiter "gitlink.org.cn/cloudream/storage/common/pkgs/iterator"
 )
@@ -29,7 +29,7 @@ func (s *Server) Package() *PackageService {
 func (s *PackageService) Get(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Package.Get")
 
-	var req cdssdk.PackageGetReq
+	var req cdsapi.PackageGetReq
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -43,13 +43,13 @@ func (s *PackageService) Get(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.PackageGetResp{Package: *pkg}))
+	ctx.JSON(http.StatusOK, OK(cdsapi.PackageGetResp{Package: *pkg}))
 }
 
 func (s *PackageService) GetByName(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Package.GetByName")
 
-	var req cdssdk.PackageGetByName
+	var req cdsapi.PackageGetByName
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding query: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -63,13 +63,13 @@ func (s *PackageService) GetByName(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.PackageGetByNameResp{Package: *pkg}))
+	ctx.JSON(http.StatusOK, OK(cdsapi.PackageGetByNameResp{Package: *pkg}))
 }
 
 // Create 处理创建新包的HTTP请求。
 func (s *PackageService) Create(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Package.Create")
-	var req cdssdk.PackageCreate
+	var req cdsapi.PackageCreate
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -83,7 +83,7 @@ func (s *PackageService) Create(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.PackageCreateResp{
+	ctx.JSON(http.StatusOK, OK(cdsapi.PackageCreateResp{
 		Package: pkg,
 	}))
 }
@@ -91,7 +91,7 @@ func (s *PackageService) Create(ctx *gin.Context) {
 func (s *PackageService) Delete(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Package.Delete")
 
-	var req cdssdk.PackageDelete
+	var req cdsapi.PackageDelete
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -111,7 +111,7 @@ func (s *PackageService) Delete(ctx *gin.Context) {
 func (s *PackageService) ListBucketPackages(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Package.ListBucketPackages")
 
-	var req cdssdk.PackageListBucketPackages
+	var req cdsapi.PackageListBucketPackages
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding query: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -125,7 +125,7 @@ func (s *PackageService) ListBucketPackages(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.PackageListBucketPackagesResp{
+	ctx.JSON(http.StatusOK, OK(cdsapi.PackageListBucketPackagesResp{
 		Packages: pkgs,
 	}))
 }
@@ -134,7 +134,7 @@ func (s *PackageService) ListBucketPackages(ctx *gin.Context) {
 func (s *PackageService) GetCachedNodes(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Package.GetCachedNodes")
 
-	var req cdssdk.PackageGetCachedNodesReq
+	var req cdsapi.PackageGetCachedNodesReq
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding query: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -148,14 +148,14 @@ func (s *PackageService) GetCachedNodes(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.PackageGetCachedNodesResp{PackageCachingInfo: resp}))
+	ctx.JSON(http.StatusOK, OK(cdsapi.PackageGetCachedNodesResp{PackageCachingInfo: resp}))
 }
 
 // GetLoadedNodes 处理获取包的加载节点的HTTP请求。
 func (s *PackageService) GetLoadedNodes(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Package.GetLoadedNodes")
 
-	var req cdssdk.PackageGetLoadedNodesReq
+	var req cdsapi.PackageGetLoadedNodesReq
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding query: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -169,7 +169,7 @@ func (s *PackageService) GetLoadedNodes(ctx *gin.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(cdssdk.PackageGetLoadedNodesResp{
+	ctx.JSON(http.StatusOK, OK(cdsapi.PackageGetLoadedNodesResp{
 		NodeIDs: nodeIDs,
 	}))
 }

client/internal/http/server.go

@@ -3,21 +3,16 @@ package http
 import (
 	"github.com/gin-gonic/gin"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
 	"gitlink.org.cn/cloudream/storage/client/internal/services"
 )
 
-// Server 结构体定义了HTTP服务的基本配置和操作
 type Server struct {
-	engine     *gin.Engine       // Gin框架的HTTP引擎
-	listenAddr string            // 服务监听地址
-	svc        *services.Service // 业务逻辑服务实例
+	engine     *gin.Engine
+	listenAddr string
+	svc        *services.Service
 }
 
-// NewServer 创建一个新的Server实例
-// listenAddr: 服务监听的地址
-// svc: 用于处理HTTP请求的业务逻辑服务实例
-// 返回值: 初始化好的Server实例和可能发生的错误
 func NewServer(listenAddr string, svc *services.Service) (*Server, error) {
 	engine := gin.New()
 
@@ -28,10 +23,8 @@ func NewServer(listenAddr string, svc *services.Service) (*Server, error) {
 	}, nil
 }
 
-// Serve 启动HTTP服务并监听请求
-// 返回值: 服务停止时可能发生的错误
 func (s *Server) Serve() error {
-	s.initRouters() // 初始化路由
+	s.initRouters()
 
 	logger.Infof("start serving http at: %s", s.listenAddr)
 	err := s.engine.Run(s.listenAddr)
@@ -45,36 +38,34 @@ func (s *Server) Serve() error {
 	return nil
 }
 
-// initRouters 初始化所有HTTP请求的路由
-//
-// 它主要用于配置和初始化与HTTP请求相关的所有路由，
-// 包括对象存储、包管理、存储管理、缓存管理和存储桶管理等。
 func (s *Server) initRouters() {
-	s.engine.GET(cdssdk.ObjectDownloadPath, s.Object().Download)
-	s.engine.POST(cdssdk.ObjectUploadPath, s.Object().Upload)
-	s.engine.GET(cdssdk.ObjectGetPackageObjectsPath, s.Object().GetPackageObjects)
-	s.engine.POST(cdssdk.ObjectUpdateInfoPath, s.Object().UpdateInfo)
-	s.engine.POST(cdssdk.ObjectMovePath, s.Object().Move)
-	s.engine.POST(cdssdk.ObjectDeletePath, s.Object().Delete)
+	rt := s.engine.Use()
 
-	s.engine.GET(cdssdk.PackageGetPath, s.Package().Get)
-	s.engine.GET(cdssdk.PackageGetByNamePath, s.Package().GetByName)
-	s.engine.POST(cdssdk.PackageCreatePath, s.Package().Create)
-	s.engine.POST(cdssdk.PackageDeletePath, s.Package().Delete)
-	s.engine.GET(cdssdk.PackageListBucketPackagesPath, s.Package().ListBucketPackages)
-	s.engine.GET(cdssdk.PackageGetCachedNodesPath, s.Package().GetCachedNodes)
-	s.engine.GET(cdssdk.PackageGetLoadedNodesPath, s.Package().GetLoadedNodes)
+	initTemp(rt, s)
 
-	// 存储管理相关路由配置
-	s.engine.POST("/storage/loadPackage", s.Storage().LoadPackage)     // 处理加载包请求
-	s.engine.POST("/storage/createPackage", s.Storage().CreatePackage) // 处理创建包请求
-	s.engine.GET("/storage/getInfo", s.Storage().GetInfo)              // 处理获取存储信息请求
+	rt.GET(cdsapi.ObjectDownloadPath, s.Object().Download)
+	rt.POST(cdsapi.ObjectUploadPath, s.Object().Upload)
+	rt.GET(cdsapi.ObjectGetPackageObjectsPath, s.Object().GetPackageObjects)
+	rt.POST(cdsapi.ObjectUpdateInfoPath, s.Object().UpdateInfo)
+	rt.POST(cdsapi.ObjectMovePath, s.Object().Move)
+	rt.POST(cdsapi.ObjectDeletePath, s.Object().Delete)
 
-	// 缓存管理相关路由配置
-	s.engine.POST(cdssdk.CacheMovePackagePath, s.Cache().MovePackage) // 处理移动包到缓存请求
+	rt.GET(cdsapi.PackageGetPath, s.Package().Get)
+	rt.GET(cdsapi.PackageGetByNamePath, s.Package().GetByName)
+	rt.POST(cdsapi.PackageCreatePath, s.Package().Create)
+	rt.POST(cdsapi.PackageDeletePath, s.Package().Delete)
+	rt.GET(cdsapi.PackageListBucketPackagesPath, s.Package().ListBucketPackages)
+	rt.GET(cdsapi.PackageGetCachedNodesPath, s.Package().GetCachedNodes)
+	rt.GET(cdsapi.PackageGetLoadedNodesPath, s.Package().GetLoadedNodes)
 
-	s.engine.GET(cdssdk.BucketGetByNamePath, s.Bucket().GetByName)
-	s.engine.POST(cdssdk.BucketCreatePath, s.Bucket().Create)
-	s.engine.POST(cdssdk.BucketDeletePath, s.Bucket().Delete)
-	s.engine.GET(cdssdk.BucketListUserBucketsPath, s.Bucket().ListUserBuckets)
+	rt.POST(cdsapi.StorageLoadPackagePath, s.Storage().LoadPackage)
+	rt.POST(cdsapi.StorageCreatePackagePath, s.Storage().CreatePackage)
+	rt.GET(cdsapi.StorageGetPath, s.Storage().Get)
+
+	rt.POST(cdsapi.CacheMovePackagePath, s.Cache().MovePackage)
+
+	rt.GET(cdsapi.BucketGetByNamePath, s.Bucket().GetByName)
+	rt.POST(cdsapi.BucketCreatePath, s.Bucket().Create)
+	rt.POST(cdsapi.BucketDeletePath, s.Bucket().Delete)
+	rt.GET(cdsapi.BucketListUserBucketsPath, s.Bucket().ListUserBuckets)
 }

client/internal/http/storage.go

@@ -1,102 +1,74 @@
 package http
 
 import (
+	"fmt"
 	"net/http"
+	"path/filepath"
 	"time"
 
 	"github.com/gin-gonic/gin"
 	"gitlink.org.cn/cloudream/common/consts/errorcode"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
 )
 
-// StorageService 用于提供存储服务的相关操作
 type StorageService struct {
 	*Server
 }
 
-// Storage 返回StorageService的实例
 func (s *Server) Storage() *StorageService {
 	return &StorageService{
 		Server: s,
 	}
 }
 
-// StorageLoadPackageReq 定义加载存储包的请求参数
-type StorageLoadPackageReq struct {
-	UserID    *cdssdk.UserID    `json:"userID" binding:"required"`
-	PackageID *cdssdk.PackageID `json:"packageID" binding:"required"`
-	StorageID *cdssdk.StorageID `json:"storageID" binding:"required"`
-}
-
-// StorageLoadPackageResp 定义加载存储包的响应参数
-type StorageLoadPackageResp struct {
-	cdssdk.StorageLoadPackageResp
-}
-
-// LoadPackage 加载存储包
 func (s *StorageService) LoadPackage(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Storage.LoadPackage")
 
-	var req StorageLoadPackageReq
+	var req cdsapi.StorageLoadPackageReq
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
 		return
 	}
 
-	nodeID, taskID, err := s.svc.StorageSvc().StartStorageLoadPackage(*req.UserID, *req.PackageID, *req.StorageID)
+	nodeID, taskID, err := s.svc.StorageSvc().StartStorageLoadPackage(req.UserID, req.PackageID, req.StorageID)
 	if err != nil {
 		log.Warnf("start storage load package: %s", err.Error())
-		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "storage load package failed"))
+		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("start loading: %v", err)))
 		return
 	}
 
 	for {
-		complete, fullPath, err := s.svc.StorageSvc().WaitStorageLoadPackage(nodeID, taskID, time.Second*10)
+		complete, ret, err := s.svc.StorageSvc().WaitStorageLoadPackage(nodeID, taskID, time.Second*10)
 		if complete {
 			if err != nil {
 				log.Warnf("loading complete with: %s", err.Error())
-				ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "storage load package failed"))
+				ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("loading complete with: %v", err)))
 				return
 			}
 
-			ctx.JSON(http.StatusOK, OK(StorageLoadPackageResp{
-				StorageLoadPackageResp: cdssdk.StorageLoadPackageResp{
-					FullPath: fullPath,
-				},
+			ctx.JSON(http.StatusOK, OK(cdsapi.StorageLoadPackageResp{
+				FullPath:    filepath.Join(ret.RemoteBase, ret.PackagePath),
+				PackagePath: ret.PackagePath,
+				LocalBase:   ret.LocalBase,
+				RemoteBase:  ret.RemoteBase,
 			}))
 			return
 		}
 
 		if err != nil {
 			log.Warnf("wait loadding: %s", err.Error())
-			ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "storage load package failed"))
+			ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, fmt.Sprintf("wait loading: %v", err)))
 			return
 		}
 	}
 }
 
-// StorageCreatePackageReq 定义创建存储包的请求参数
-type StorageCreatePackageReq struct {
-	UserID       *cdssdk.UserID    `json:"userID" binding:"required"`
-	StorageID    *cdssdk.StorageID `json:"storageID" binding:"required"`
-	Path         string            `json:"path" binding:"required"`
-	BucketID     *cdssdk.BucketID  `json:"bucketID" binding:"required"`
-	Name         string            `json:"name" binding:"required"`
-	NodeAffinity *cdssdk.NodeID    `json:"nodeAffinity"`
-}
-
-// StorageCreatePackageResp 定义创建存储包的响应参数
-type StorageCreatePackageResp struct {
-	PackageID cdssdk.PackageID `json:"packageID"`
-}
-
-// CreatePackage 创建存储包
 func (s *StorageService) CreatePackage(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Storage.CreatePackage")
 
-	var req StorageCreatePackageReq
+	var req cdsapi.StorageCreatePackageReq
 	if err := ctx.ShouldBindJSON(&req); err != nil {
 		log.Warnf("binding body: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
@@ -104,7 +76,7 @@ func (s *StorageService) CreatePackage(ctx *gin.Context) {
 	}
 
 	nodeID, taskID, err := s.svc.StorageSvc().StartStorageCreatePackage(
-		*req.UserID, *req.BucketID, req.Name, *req.StorageID, req.Path, req.NodeAffinity)
+		req.UserID, req.BucketID, req.Name, req.StorageID, req.Path, req.NodeAffinity)
 	if err != nil {
 		log.Warnf("start storage create package: %s", err.Error())
 		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "storage create package failed"))
@@ -120,7 +92,7 @@ func (s *StorageService) CreatePackage(ctx *gin.Context) {
 				return
 			}
 
-			ctx.JSON(http.StatusOK, OK(StorageCreatePackageResp{
+			ctx.JSON(http.StatusOK, OK(cdsapi.StorageCreatePackageResp{
 				PackageID: packageID,
 			}))
 			return
@@ -134,40 +106,24 @@ func (s *StorageService) CreatePackage(ctx *gin.Context) {
 	}
 }
 
-// StorageGetInfoReq 定义获取存储信息的请求参数
-type StorageGetInfoReq struct {
-	UserID    *cdssdk.UserID    `form:"userID" binding:"required"`
-	StorageID *cdssdk.StorageID `form:"storageID" binding:"required"`
-}
-
-// StorageGetInfoResp 定义获取存储信息的响应参数
-type StorageGetInfoResp struct {
-	cdssdk.StorageGetInfoResp
-}
-
-// GetInfo 获取存储信息
-func (s *StorageService) GetInfo(ctx *gin.Context) {
-	log := logger.WithField("HTTP", "Storage.GetInfo")
+func (s *StorageService) Get(ctx *gin.Context) {
+	log := logger.WithField("HTTP", "Storage.Get")
 
-	var req StorageGetInfoReq
+	var req cdsapi.StorageGet
 	if err := ctx.ShouldBindQuery(&req); err != nil {
 		log.Warnf("binding query: %s", err.Error())
 		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
 		return
 	}
 
-	info, err := s.svc.StorageSvc().GetInfo(*req.UserID, *req.StorageID)
+	info, err := s.svc.StorageSvc().Get(req.UserID, req.StorageID)
 	if err != nil {
 		log.Warnf("getting info: %s", err.Error())
 		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get storage inf failed"))
 		return
 	}
 
-	ctx.JSON(http.StatusOK, OK(StorageGetInfoResp{
-		StorageGetInfoResp: cdssdk.StorageGetInfoResp{
-			Name:      info.Name,
-			NodeID:    info.NodeID,
-			Directory: info.Directory,
-		},
+	ctx.JSON(http.StatusOK, OK(cdsapi.StorageGetResp{
+		Storage: *info,
 	}))
 }

client/internal/http/temp.go

@@ -0,0 +1,391 @@
+package http
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/samber/lo"
+	"gitlink.org.cn/cloudream/common/consts/errorcode"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+)
+
+type TempService struct {
+	*Server
+}
+
+func (s *Server) Temp() *TempService {
+	return &TempService{
+		Server: s,
+	}
+}
+
+type TempListDetailsResp struct {
+	Buckets []BucketDetail `json:"buckets"`
+}
+type BucketDetail struct {
+	BucketID    cdssdk.BucketID `json:"bucketID"`
+	Name        string          `json:"name"`
+	ObjectCount int             `json:"objectCount"`
+}
+
+func (s *TempService) ListDetails(ctx *gin.Context) {
+	log := logger.WithField("HTTP", "Bucket.ListBucketsDetails")
+
+	bkts, err := s.svc.BucketSvc().GetUserBuckets(1)
+	if err != nil {
+		log.Warnf("getting user buckets: %s", err.Error())
+		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get user buckets failed"))
+		return
+	}
+
+	details := make([]BucketDetail, len(bkts))
+	for i := range bkts {
+		details[i].BucketID = bkts[i].BucketID
+		details[i].Name = bkts[i].Name
+		objs, err := s.getBucketObjects(bkts[i].BucketID)
+		if err != nil {
+			log.Warnf("getting bucket objects: %s", err.Error())
+			ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get bucket objects failed"))
+			return
+		}
+		details[i].ObjectCount = len(objs)
+	}
+
+	ctx.JSON(http.StatusOK, OK(TempListDetailsResp{
+		Buckets: details,
+	}))
+}
+
+type TempGetObjects struct {
+	BucketID cdssdk.BucketID `form:"bucketID"`
+}
+type BucketGetObjectsResp struct {
+	Objects []cdssdk.Object `json:"objects"`
+}
+
+func (s *TempService) GetObjects(ctx *gin.Context) {
+	log := logger.WithField("HTTP", "Bucket.ListBucketsDetails")
+
+	var req TempGetObjects
+	if err := ctx.ShouldBindQuery(&req); err != nil {
+		log.Warnf("binding query: %s", err.Error())
+		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
+		return
+	}
+
+	objs, err := s.getBucketObjects(req.BucketID)
+	if err != nil {
+		log.Warnf("getting bucket objects: %s", err.Error())
+		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get bucket objects failed"))
+		return
+	}
+
+	ctx.JSON(http.StatusOK, OK(BucketGetObjectsResp{
+		Objects: objs,
+	}))
+}
+
+type TempGetObjectDetail struct {
+	ObjectID cdssdk.ObjectID `form:"objectID"`
+}
+type TempGetObjectDetailResp struct {
+	Blocks []ObjectBlockDetail `json:"blocks"`
+}
+type ObjectBlockDetail struct {
+	ObjectID     cdssdk.ObjectID `json:"objectID"`
+	Type         string          `json:"type"`
+	FileHash     string          `json:"fileHash"`
+	LocationType string          `json:"locationType"`
+	LocationName string          `json:"locationName"`
+}
+
+func (s *TempService) GetObjectDetail(ctx *gin.Context) {
+	log := logger.WithField("HTTP", "Object.GetObjectDetail")
+
+	var req TempGetObjectDetail
+	if err := ctx.ShouldBindQuery(&req); err != nil {
+		log.Warnf("binding query: %s", err.Error())
+		ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
+		return
+	}
+
+	details, err := s.svc.ObjectSvc().GetObjectDetail(req.ObjectID)
+	if err != nil {
+		log.Warnf("getting object detail: %s", err.Error())
+		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get object detail failed"))
+		return
+	}
+	if details == nil {
+		log.Warnf("object not found")
+		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "object not found"))
+		return
+	}
+
+	loadedNodeIDs, err := s.svc.PackageSvc().GetLoadedNodes(1, details.Object.PackageID)
+	if err != nil {
+		log.Warnf("getting loaded nodes: %s", err.Error())
+		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get loaded nodes failed"))
+		return
+	}
+
+	var allNodeIDs []cdssdk.NodeID
+	allNodeIDs = append(allNodeIDs, details.PinnedAt...)
+	for _, b := range details.Blocks {
+		allNodeIDs = append(allNodeIDs, b.NodeID)
+	}
+	allNodeIDs = append(allNodeIDs, loadedNodeIDs...)
+
+	allNodeIDs = lo.Uniq(allNodeIDs)
+
+	getNodes, err := s.svc.NodeSvc().GetNodes(allNodeIDs)
+	if err != nil {
+		log.Warnf("getting nodes: %s", err.Error())
+		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get nodes failed"))
+		return
+	}
+
+	allNodes := make(map[cdssdk.NodeID]*cdssdk.Node)
+	for _, n := range getNodes {
+		n2 := n
+		allNodes[n.NodeID] = &n2
+	}
+
+	var blocks []ObjectBlockDetail
+
+	for _, nodeID := range details.PinnedAt {
+		blocks = append(blocks, ObjectBlockDetail{
+			Type:         "Rep",
+			FileHash:     details.Object.FileHash,
+			LocationType: "Agent",
+			LocationName: allNodes[nodeID].Name,
+		})
+	}
+
+	switch details.Object.Redundancy.(type) {
+	case *cdssdk.NoneRedundancy:
+		for _, blk := range details.Blocks {
+			if !lo.Contains(details.PinnedAt, blk.NodeID) {
+				blocks = append(blocks, ObjectBlockDetail{
+					Type:         "Rep",
+					FileHash:     blk.FileHash,
+					LocationType: "Agent",
+					LocationName: allNodes[blk.NodeID].Name,
+				})
+			}
+		}
+	case *cdssdk.RepRedundancy:
+		for _, blk := range details.Blocks {
+			if !lo.Contains(details.PinnedAt, blk.NodeID) {
+				blocks = append(blocks, ObjectBlockDetail{
+					Type:         "Rep",
+					FileHash:     blk.FileHash,
+					LocationType: "Agent",
+					LocationName: allNodes[blk.NodeID].Name,
+				})
+			}
+		}
+
+	case *cdssdk.ECRedundancy:
+		for _, blk := range details.Blocks {
+			blocks = append(blocks, ObjectBlockDetail{
+				Type:         "Block",
+				FileHash:     blk.FileHash,
+				LocationType: "Agent",
+				LocationName: allNodes[blk.NodeID].Name,
+			})
+		}
+	}
+
+	for _, nodeID := range loadedNodeIDs {
+		blocks = append(blocks, ObjectBlockDetail{
+			Type:         "Rep",
+			FileHash:     details.Object.FileHash,
+			LocationType: "Storage",
+			LocationName: allNodes[nodeID].Name,
+		})
+	}
+
+	ctx.JSON(http.StatusOK, OK(TempGetObjectDetailResp{
+		Blocks: blocks,
+	}))
+}
+
+func (s *TempService) getBucketObjects(bktID cdssdk.BucketID) ([]cdssdk.Object, error) {
+	pkgs, err := s.svc.PackageSvc().GetBucketPackages(1, bktID)
+	if err != nil {
+		return nil, err
+	}
+
+	var allObjs []cdssdk.Object
+	for _, pkg := range pkgs {
+		objs, err := s.svc.ObjectSvc().GetPackageObjects(1, pkg.PackageID)
+		if err != nil {
+			return nil, err
+		}
+		allObjs = append(allObjs, objs...)
+	}
+
+	return allObjs, nil
+}
+
+type TempGetDatabaseAll struct {
+}
+type TempGetDatabaseAllResp struct {
+	Buckets []BucketDetail      `json:"buckets"`
+	Objects []BucketObject      `json:"objects"`
+	Blocks  []ObjectBlockDetail `json:"blocks"`
+}
+type BucketObject struct {
+	cdssdk.Object
+	BucketID cdssdk.BucketID `json:"bucketID"`
+}
+
+func (s *TempService) GetDatabaseAll(ctx *gin.Context) {
+	log := logger.WithField("HTTP", "Temp.GetDatabaseAll")
+
+	db, err := s.svc.ObjectSvc().GetDatabaseAll()
+	if err != nil {
+		log.Warnf("getting database all: %s", err.Error())
+		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get database all failed"))
+		return
+	}
+
+	nodes, err := s.svc.NodeSvc().GetNodes(nil)
+	if err != nil {
+		log.Warnf("getting nodes: %s", err.Error())
+		ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get nodes failed"))
+		return
+	}
+	allNodes := make(map[cdssdk.NodeID]cdssdk.Node)
+	for _, n := range nodes {
+		allNodes[n.NodeID] = n
+	}
+
+	bkts := make(map[cdssdk.BucketID]*BucketDetail)
+	for _, bkt := range db.Buckets {
+		bkts[bkt.BucketID] = &BucketDetail{
+			BucketID:    bkt.BucketID,
+			Name:        bkt.Name,
+			ObjectCount: 0,
+		}
+	}
+
+	type PackageDetail struct {
+		Package cdssdk.Package
+		Loaded  []cdssdk.Node
+	}
+	pkgs := make(map[cdssdk.PackageID]*PackageDetail)
+	for _, pkg := range db.Packages {
+		p := PackageDetail{
+			Package: pkg,
+			Loaded:  make([]cdssdk.Node, 0),
+		}
+
+		loaded, err := s.svc.PackageSvc().GetLoadedNodes(1, pkg.PackageID)
+		if err != nil {
+			log.Warnf("getting loaded nodes: %s", err.Error())
+			ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get loaded nodes failed"))
+			return
+		}
+
+		for _, nodeID := range loaded {
+			p.Loaded = append(p.Loaded, allNodes[nodeID])
+		}
+
+		pkgs[pkg.PackageID] = &p
+	}
+
+	var objs []BucketObject
+	for _, obj := range db.Objects {
+		o := BucketObject{
+			Object:   obj.Object,
+			BucketID: pkgs[obj.Object.PackageID].Package.BucketID,
+		}
+		objs = append(objs, o)
+	}
+
+	var blocks []ObjectBlockDetail
+	for _, obj := range db.Objects {
+		bkts[pkgs[obj.Object.PackageID].Package.BucketID].ObjectCount++
+
+		for _, nodeID := range obj.PinnedAt {
+			blocks = append(blocks, ObjectBlockDetail{
+				ObjectID:     obj.Object.ObjectID,
+				Type:         "Rep",
+				FileHash:     obj.Object.FileHash,
+				LocationType: "Agent",
+				LocationName: allNodes[nodeID].Name,
+			})
+		}
+
+		switch obj.Object.Redundancy.(type) {
+		case *cdssdk.NoneRedundancy:
+			for _, blk := range obj.Blocks {
+				if !lo.Contains(obj.PinnedAt, blk.NodeID) {
+					blocks = append(blocks, ObjectBlockDetail{
+						ObjectID:     obj.Object.ObjectID,
+						Type:         "Rep",
+						FileHash:     blk.FileHash,
+						LocationType: "Agent",
+						LocationName: allNodes[blk.NodeID].Name,
+					})
+				}
+			}
+		case *cdssdk.RepRedundancy:
+			for _, blk := range obj.Blocks {
+				if !lo.Contains(obj.PinnedAt, blk.NodeID) {
+					blocks = append(blocks, ObjectBlockDetail{
+						ObjectID:     obj.Object.ObjectID,
+						Type:         "Rep",
+						FileHash:     blk.FileHash,
+						LocationType: "Agent",
+						LocationName: allNodes[blk.NodeID].Name,
+					})
+				}
+			}
+
+		case *cdssdk.ECRedundancy:
+			for _, blk := range obj.Blocks {
+				blocks = append(blocks, ObjectBlockDetail{
+					ObjectID:     obj.Object.ObjectID,
+					Type:         "Block",
+					FileHash:     blk.FileHash,
+					LocationType: "Agent",
+					LocationName: allNodes[blk.NodeID].Name,
+				})
+			}
+		}
+
+		for _, node := range pkgs[obj.Object.PackageID].Loaded {
+			blocks = append(blocks, ObjectBlockDetail{
+				ObjectID:     obj.Object.ObjectID,
+				Type:         "Rep",
+				FileHash:     obj.Object.FileHash,
+				LocationType: "Storage",
+				LocationName: allNodes[node.NodeID].Name,
+			})
+		}
+
+	}
+
+	ctx.JSON(http.StatusOK, OK(TempGetDatabaseAllResp{
+		Buckets: lo.Map(lo.Values(bkts), func(b *BucketDetail, _ int) BucketDetail { return *b }),
+		Objects: objs,
+		Blocks:  blocks,
+	}))
+}
+
+func initTemp(rt gin.IRoutes, s *Server) {
+	rt.GET("/bucket/listDetails", s.Temp().ListDetails)
+	rt.GET("/bucket/getObjects", s.Temp().GetObjects)
+	rt.GET("/object/getDetail", s.Temp().GetObjectDetail)
+	rt.GET("/temp/getDatabaseAll", s.Temp().GetDatabaseAll)
+}
+
+func auth(ctx *gin.Context) {
+	token := ctx.Request.Header.Get("X-CDS-Auth")
+	if token != "cloudream@123" {
+		ctx.AbortWithStatus(http.StatusUnauthorized)
+	}
+}

client/internal/services/cache.go

@@ -11,52 +11,51 @@ import (
 	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
 )
 
-// CacheService 缓存服务结构体，继承自Service。
 type CacheService struct {
 	*Service
 }
 
-// CacheSvc 创建并返回一个CacheService的实例。
 func (svc *Service) CacheSvc() *CacheService {
 	return &CacheService{Service: svc}
 }
 
-// StartCacheMovePackage 启动缓存移动包的流程。
-// userID: 用户标识符；
-// packageID: 包标识符；
-// nodeID: 节点标识符；
-// 返回任务ID和可能的错误。
-func (svc *CacheService) StartCacheMovePackage(userID cdssdk.UserID, packageID cdssdk.PackageID, nodeID cdssdk.NodeID) (string, error) {
-	// 获取Agent消息队列客户端
-	agentCli, err := stgglb.AgentMQPool.Acquire(nodeID)
+func (svc *CacheService) StartCacheMovePackage(userID cdssdk.UserID, packageID cdssdk.PackageID, stgID cdssdk.StorageID) (cdssdk.NodeID, string, error) {
+	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+	if err != nil {
+		return 0, "", fmt.Errorf("new coordinator client: %w", err)
+	}
+	defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+	getStg, err := coorCli.GetStorageDetail(coormq.ReqGetStorageDetail(stgID))
+	if err != nil {
+		return 0, "", fmt.Errorf("get storage detail: %w", err)
+	}
+
+	if getStg.Storage.Shard == nil {
+		return 0, "", fmt.Errorf("shard storage is not enabled")
+	}
+
+	agentCli, err := stgglb.AgentMQPool.Acquire(getStg.Storage.Shard.MasterHub)
 	if err != nil {
-		return "", fmt.Errorf("new agent client: %w", err)
+		return 0, "", fmt.Errorf("new agent client: %w", err)
 	}
 	defer stgglb.AgentMQPool.Release(agentCli)
 
-	// 向Agent发起启动缓存移动包的请求
-	startResp, err := agentCli.StartCacheMovePackage(agtmq.NewStartCacheMovePackage(userID, packageID))
+	startResp, err := agentCli.StartCacheMovePackage(agtmq.NewStartCacheMovePackage(userID, packageID, stgID))
 	if err != nil {
-		return "", fmt.Errorf("start cache move package: %w", err)
+		return 0, "", fmt.Errorf("start cache move package: %w", err)
 	}
 
-	return startResp.TaskID, nil
+	return getStg.Storage.Shard.MasterHub, startResp.TaskID, nil
 }
 
-// WaitCacheMovePackage 等待缓存移动包完成。
-// nodeID: 节点标识符；
-// taskID: 任务标识符；
-// waitTimeout: 等待超时时间；
-// 返回任务是否完成和可能的错误。
-func (svc *CacheService) WaitCacheMovePackage(nodeID cdssdk.NodeID, taskID string, waitTimeout time.Duration) (bool, error) {
-	// 获取Agent消息队列客户端
-	agentCli, err := stgglb.AgentMQPool.Acquire(nodeID)
+func (svc *CacheService) WaitCacheMovePackage(hubID cdssdk.NodeID, taskID string, waitTimeout time.Duration) (bool, error) {
+	agentCli, err := stgglb.AgentMQPool.Acquire(hubID)
 	if err != nil {
 		return true, fmt.Errorf("new agent client: %w", err)
 	}
 	defer stgglb.AgentMQPool.Release(agentCli)
 
-	// 向Agent查询缓存移动包状态
 	waitResp, err := agentCli.WaitCacheMovePackage(agtmq.NewWaitCacheMovePackage(taskID, waitTimeout.Milliseconds()))
 	if err != nil {
 		return true, fmt.Errorf("wait cache move package: %w", err)
@@ -73,19 +72,13 @@ func (svc *CacheService) WaitCacheMovePackage(nodeID cdssdk.NodeID, taskID strin
 	return true, nil
 }
 
-// CacheRemovePackage 请求移除缓存包。
-// packageID: 包标识符；
-// nodeID: 节点标识符；
-// 返回可能的错误。
 func (svc *CacheService) CacheRemovePackage(packageID cdssdk.PackageID, nodeID cdssdk.NodeID) error {
-	// 获取协调器消息队列客户端
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
 		return fmt.Errorf("new agent client: %w", err)
 	}
 	defer stgglb.CoordinatorMQPool.Release(coorCli)
 
-	// 向协调器发送移除缓存包的请求
 	_, err = coorCli.CacheRemovePackage(coormq.ReqCacheRemoveMovedPackage(packageID, nodeID))
 	if err != nil {
 		return fmt.Errorf("requesting to coordinator: %w", err)

client/internal/services/object.go

@@ -5,8 +5,10 @@ import (
 	"time"
 
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
 	mytask "gitlink.org.cn/cloudream/storage/client/internal/task"
 	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+	stgmod "gitlink.org.cn/cloudream/storage/common/models"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/downloader"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/iterator"
@@ -47,7 +49,7 @@ func (svc *ObjectService) WaitUploading(taskID string, waitTimeout time.Duration
 	return false, nil, nil
 }
 
-func (svc *ObjectService) UpdateInfo(userID cdssdk.UserID, updatings []cdssdk.UpdatingObject) ([]cdssdk.ObjectID, error) {
+func (svc *ObjectService) UpdateInfo(userID cdssdk.UserID, updatings []cdsapi.UpdatingObject) ([]cdssdk.ObjectID, error) {
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
 		return nil, fmt.Errorf("new coordinator client: %w", err)
@@ -62,7 +64,7 @@ func (svc *ObjectService) UpdateInfo(userID cdssdk.UserID, updatings []cdssdk.Up
 	return resp.Successes, nil
 }
 
-func (svc *ObjectService) Move(userID cdssdk.UserID, movings []cdssdk.MovingObject) ([]cdssdk.ObjectID, error) {
+func (svc *ObjectService) Move(userID cdssdk.UserID, movings []cdsapi.MovingObject) ([]cdssdk.ObjectID, error) {
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
 		return nil, fmt.Errorf("new coordinator client: %w", err)
@@ -83,8 +85,8 @@ func (svc *ObjectService) Download(userID cdssdk.UserID, req downloader.Download
 
 	// 初始化下载过程
 	downloading, err := iter.MoveNext()
-	if downloading.Object == nil {
-		return nil, fmt.Errorf("object not found")
+	if downloading == nil {
+		return nil, fmt.Errorf("object %v not found", req.ObjectID)
 	}
 	if err != nil {
 		return nil, err
@@ -126,3 +128,18 @@ func (svc *ObjectService) GetPackageObjects(userID cdssdk.UserID, packageID cdss
 
 	return getResp.Objects, nil
 }
+
+func (svc *ObjectService) GetObjectDetail(objectID cdssdk.ObjectID) (*stgmod.ObjectDetail, error) {
+	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+	if err != nil {
+		return nil, fmt.Errorf("new coordinator client: %w", err)
+	}
+	defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+	getResp, err := coorCli.GetObjectDetails(coormq.ReqGetObjectDetails([]cdssdk.ObjectID{objectID}))
+	if err != nil {
+		return nil, fmt.Errorf("requsting to coodinator: %w", err)
+	}
+
+	return getResp.Objects[0], nil
+}

client/internal/services/package.go

@@ -45,7 +45,8 @@ func (svc *PackageService) GetByName(userID cdssdk.UserID, bucketName string, pa
 
 	getResp, err := coorCli.GetPackageByName(coormq.ReqGetPackageByName(userID, bucketName, packageName))
 	if err != nil {
-		return nil, fmt.Errorf("requsting to coodinator: %w", err)
+		// TODO 要附加日志信息，但不能直接%w，因为外部需要判断错误吗
+		return nil, err
 	}
 
 	return &getResp.Package, nil

client/internal/services/service.go

@@ -5,6 +5,7 @@ package services
 import (
 	"gitlink.org.cn/cloudream/common/pkgs/distlock"
 	"gitlink.org.cn/cloudream/storage/client/internal/task"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/accessstat"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/downloader"
 )
 
@@ -13,12 +14,14 @@ type Service struct {
 	DistLock   *distlock.Service
 	TaskMgr    *task.Manager
 	Downloader *downloader.Downloader
+	AccessStat *accessstat.AccessStat
 }
 
-func NewService(distlock *distlock.Service, taskMgr *task.Manager, downloader *downloader.Downloader) (*Service, error) {
+func NewService(distlock *distlock.Service, taskMgr *task.Manager, downloader *downloader.Downloader, accStat *accessstat.AccessStat) (*Service, error) {
 	return &Service{
 		DistLock:   distlock,
 		TaskMgr:    taskMgr,
 		Downloader: downloader,
+		AccessStat: accStat,
 	}, nil
 }

client/internal/services/storage.go

@@ -2,119 +2,125 @@ package services
 
 import (
 	"fmt"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
 	"time"
 
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 
 	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
 	agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent"
 	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
 )
 
-// StorageService 存储服务结构体，继承自Service结构体
 type StorageService struct {
 	*Service
 }
 
-// StorageSvc 返回StorageService的实例
 func (svc *Service) StorageSvc() *StorageService {
 	return &StorageService{Service: svc}
 }
 
-// StartStorageLoadPackage 开始加载存储包。
-// userID: 用户ID，用于标识请求的用户。
-// packageID: 包ID，用于标识需要加载的数据包。
-// storageID: 存储ID，用于标识数据存储的位置。
-// 返回值1: 节点ID，标识进行存储操作的节点。
-// 返回值2: 任务ID，标识加载数据包的任务。
-// 返回值3: 错误，如果执行过程中出现错误，则返回错误信息。
+func (svc *StorageService) Get(userID cdssdk.UserID, storageID cdssdk.StorageID) (*model.Storage, error) {
+	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+	if err != nil {
+		return nil, fmt.Errorf("new coordinator client: %w", err)
+	}
+	defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+	getResp, err := coorCli.GetStorage(coormq.ReqGetStorage(userID, storageID))
+	if err != nil {
+		return nil, fmt.Errorf("request to coordinator: %w", err)
+	}
+
+	return &getResp.Storage, nil
+}
+
+func (svc *StorageService) GetByName(userID cdssdk.UserID, name string) (*model.Storage, error) {
+	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+	if err != nil {
+		return nil, fmt.Errorf("new coordinator client: %w", err)
+	}
+	defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+	getResp, err := coorCli.GetStorageByName(coormq.ReqGetStorageByName(userID, name))
+	if err != nil {
+		return nil, fmt.Errorf("request to coordinator: %w", err)
+	}
+
+	return &getResp.Storage, nil
+}
+
 func (svc *StorageService) StartStorageLoadPackage(userID cdssdk.UserID, packageID cdssdk.PackageID, storageID cdssdk.StorageID) (cdssdk.NodeID, string, error) {
-	// 获取协调器MQ客户端
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
 		return 0, "", fmt.Errorf("new coordinator client: %w", err)
 	}
 	defer stgglb.CoordinatorMQPool.Release(coorCli)
 
-	// 从协调器获取存储信息
-	stgResp, err := coorCli.GetStorageInfo(coormq.NewGetStorageInfo(userID, storageID))
+	stgResp, err := coorCli.GetStorageDetail(coormq.ReqGetStorageDetail(storageID))
 	if err != nil {
 		return 0, "", fmt.Errorf("getting storage info: %w", err)
 	}
 
-	// 获取代理MQ客户端
-	agentCli, err := stgglb.AgentMQPool.Acquire(stgResp.NodeID)
+	if stgResp.Storage.Shard == nil {
+		return 0, "", fmt.Errorf("shard storage is not enabled")
+	}
+
+	agentCli, err := stgglb.AgentMQPool.Acquire(stgResp.Storage.Shard.MasterHub)
 	if err != nil {
 		return 0, "", fmt.Errorf("new agent client: %w", err)
 	}
 	defer stgglb.AgentMQPool.Release(agentCli)
 
-	// 向代理发送开始加载存储包的请求
 	startResp, err := agentCli.StartStorageLoadPackage(agtmq.NewStartStorageLoadPackage(userID, packageID, storageID))
 	if err != nil {
 		return 0, "", fmt.Errorf("start storage load package: %w", err)
 	}
 
-	return stgResp.NodeID, startResp.TaskID, nil
+	return stgResp.Storage.Shard.MasterHub, startResp.TaskID, nil
 }
 
-/*
-WaitStorageLoadPackage 等待存储包加载完成。
-参数：
-- nodeID：节点ID
-- taskID：任务ID
-- waitTimeout：等待超时时间
-返回值：
-- bool：任务是否完成
-- string：错误信息
-- error：错误信息
-*/
-func (svc *StorageService) WaitStorageLoadPackage(nodeID cdssdk.NodeID, taskID string, waitTimeout time.Duration) (bool, string, error) {
+type StorageLoadPackageResult struct {
+	PackagePath string
+	LocalBase   string
+	RemoteBase  string
+}
+
+func (svc *StorageService) WaitStorageLoadPackage(nodeID cdssdk.NodeID, taskID string, waitTimeout time.Duration) (bool, *StorageLoadPackageResult, error) {
 	agentCli, err := stgglb.AgentMQPool.Acquire(nodeID)
 	if err != nil {
 		// TODO 失败是否要当做任务已经结束？
-		return true, "", fmt.Errorf("new agent client: %w", err)
+		return true, nil, fmt.Errorf("new agent client: %w", err)
 	}
 	defer stgglb.AgentMQPool.Release(agentCli)
 
 	waitResp, err := agentCli.WaitStorageLoadPackage(agtmq.NewWaitStorageLoadPackage(taskID, waitTimeout.Milliseconds()))
 	if err != nil {
 		// TODO 请求失败是否要当做任务已经结束？
-		return true, "", fmt.Errorf("wait storage load package: %w", err)
+		return true, nil, fmt.Errorf("wait storage load package: %w", err)
 	}
 
 	if !waitResp.IsComplete {
-		return false, "", nil
+		return false, nil, nil
 	}
 
 	if waitResp.Error != "" {
-		return true, "", fmt.Errorf("%s", waitResp.Error)
+		return true, nil, fmt.Errorf("%s", waitResp.Error)
 	}
 
-	return true, waitResp.FullPath, nil
+	return true, &StorageLoadPackageResult{
+		PackagePath: waitResp.PackagePath,
+		LocalBase:   waitResp.LocalBase,
+		RemoteBase:  waitResp.RemoteBase,
+	}, nil
 }
 
-// DeleteStoragePackage 删除存储包的函数，当前未实现。
 func (svc *StorageService) DeleteStoragePackage(userID int64, packageID int64, storageID int64) error {
 	// TODO
 	panic("not implement yet")
 }
 
-/*
-StartStorageCreatePackage 请求节点启动从Storage中上传文件的任务。
-参数：
-- userID：用户ID
-- bucketID：存储桶ID
-- name：文件名
-- storageID：存储ID
-- path：文件路径
-- nodeAffinity：节点亲和性（可选）
-返回值：
-- cdssdk.NodeID：节点ID
-- string：任务ID
-- error：错误信息
-*/
+// 请求节点启动从Storage中上传文件的任务。会返回节点ID和任务ID
 func (svc *StorageService) StartStorageCreatePackage(userID cdssdk.UserID, bucketID cdssdk.BucketID, name string, storageID cdssdk.StorageID, path string, nodeAffinity *cdssdk.NodeID) (cdssdk.NodeID, string, error) {
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
@@ -122,12 +128,16 @@ func (svc *StorageService) StartStorageCreatePackage(userID cdssdk.UserID, bucke
 	}
 	defer stgglb.CoordinatorMQPool.Release(coorCli)
 
-	stgResp, err := coorCli.GetStorageInfo(coormq.NewGetStorageInfo(userID, storageID))
+	stgResp, err := coorCli.GetStorageDetail(coormq.ReqGetStorageDetail(storageID))
 	if err != nil {
 		return 0, "", fmt.Errorf("getting storage info: %w", err)
 	}
 
-	agentCli, err := stgglb.AgentMQPool.Acquire(stgResp.NodeID)
+	if stgResp.Storage.Shard == nil {
+		return 0, "", fmt.Errorf("shard storage is not enabled")
+	}
+
+	agentCli, err := stgglb.AgentMQPool.Acquire(stgResp.Storage.Shard.MasterHub)
 	if err != nil {
 		return 0, "", fmt.Errorf("new agent client: %w", err)
 	}
@@ -138,20 +148,9 @@ func (svc *StorageService) StartStorageCreatePackage(userID cdssdk.UserID, bucke
 		return 0, "", fmt.Errorf("start storage upload package: %w", err)
 	}
 
-	return stgResp.NodeID, startResp.TaskID, nil
+	return stgResp.Storage.Shard.MasterHub, startResp.TaskID, nil
 }
 
-/*
-WaitStorageCreatePackage 等待存储包创建完成。
-参数：
-- nodeID：节点ID
-- taskID：任务ID
-- waitTimeout：等待超时时间
-返回值：
-- bool：任务是否完成
-- cdssdk.PackageID：包ID
-- error：错误信息
-*/
 func (svc *StorageService) WaitStorageCreatePackage(nodeID cdssdk.NodeID, taskID string, waitTimeout time.Duration) (bool, cdssdk.PackageID, error) {
 	agentCli, err := stgglb.AgentMQPool.Acquire(nodeID)
 	if err != nil {
@@ -176,26 +175,3 @@ func (svc *StorageService) WaitStorageCreatePackage(nodeID cdssdk.NodeID, taskID
 
 	return true, waitResp.PackageID, nil
 }
-
-/*
-GetInfo 获取存储信息。
-参数：
-- userID：用户ID
-- storageID：存储ID
-返回值：
--
-*/
-func (svc *StorageService) GetInfo(userID cdssdk.UserID, storageID cdssdk.StorageID) (*model.Storage, error) {
-	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
-	if err != nil {
-		return nil, fmt.Errorf("new coordinator client: %w", err)
-	}
-	defer stgglb.CoordinatorMQPool.Release(coorCli)
-
-	getResp, err := coorCli.GetStorageInfo(coormq.NewGetStorageInfo(userID, storageID))
-	if err != nil {
-		return nil, fmt.Errorf("request to coordinator: %w", err)
-	}
-
-	return &getResp.Storage, nil
-}

client/internal/services/temp.go

@@ -0,0 +1,23 @@
+package services
+
+import (
+	"fmt"
+
+	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
+)
+
+func (svc *ObjectService) GetDatabaseAll() (*coormq.GetDatabaseAllResp, error) {
+	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+	if err != nil {
+		return nil, fmt.Errorf("new coordinator client: %w", err)
+	}
+	defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+	getResp, err := coorCli.GetDatabaseAll(coormq.ReqGetDatabaseAll(1))
+	if err != nil {
+		return nil, fmt.Errorf("requsting to coodinator: %w", err)
+	}
+
+	return getResp, nil
+}

client/main.go

@@ -3,107 +3,107 @@ package main
 import (
 	"fmt"
 	"os"
+	"time"
 
 	_ "google.golang.org/grpc/balancer/grpclb"
 
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 	"gitlink.org.cn/cloudream/storage/client/internal/cmdline"
 	"gitlink.org.cn/cloudream/storage/client/internal/config"
 	"gitlink.org.cn/cloudream/storage/client/internal/services"
 	"gitlink.org.cn/cloudream/storage/client/internal/task"
 	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/accessstat"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/connectivity"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/distlock"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/downloader"
+	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
 )
 
-/*
-该Go程序是一个客户端应用程序，主要负责初始化配置、日志、全局变量，并启动网络检测、分布式锁服务、任务管理器和服务处理客户端请求。具体功能如下：
-程序的主入口函数main()：
-初始化配置，如果失败则结束进程。
-初始化日志系统，如果失败则结束进程。
-初始化全局变量，包括本地配置、消息队列池和Agent RPC池。
-根据IPFS配置初始化IPFS客户端。
-启动网络连通性检测。
-启动分布式锁服务，并在独立的goroutine中运行。
-创建任务管理器。
-创建服务实例。
-创建命令行接口。
-分发命令行指令。
-辅助函数serveDistLock()：
-在独立的goroutine中启动分布式锁服务。
-处理服务停止时的错误。
-该程序使用了多个外部包和模块，包括配置管理、日志系统、全局变量初始化、网络检测、分布式锁服务、任务管理和命令行接口等。这些模块共同协作，提供了一个功能丰富的客户端应用程序。
-*/
-
-// @Description: 程序的主入口函数，负责初始化配置、日志、全局变量，并启动网络检测、分布式锁服务、任务管理器和服务处理客户端请求。
 func main() {
-	// 初始化配置，失败则结束进程
 	err := config.Init()
 	if err != nil {
 		fmt.Printf("init config failed, err: %s", err.Error())
 		os.Exit(1)
 	}
 
-	// 初始化日志系统
 	err = logger.Init(&config.Cfg().Logger)
 	if err != nil {
 		fmt.Printf("init logger failed, err: %s", err.Error())
 		os.Exit(1)
 	}
 
-	// 初始化全局变量
 	stgglb.InitLocal(&config.Cfg().Local)
 	stgglb.InitMQPool(&config.Cfg().RabbitMQ)
 	stgglb.InitAgentRPCPool(&config.Cfg().AgentGRPC)
-	// 如果IPFS配置非空，初始化IPFS客户端
-	if config.Cfg().IPFS != nil {
-		logger.Infof("IPFS config is not empty, so create a ipfs client")
-		stgglb.InitIPFSPool(config.Cfg().IPFS)
-	}
 
-	// 启动网络连通性检测
-	conCol := connectivity.NewCollector(&config.Cfg().Connectivity, nil)
-	conCol.CollectInPlace()
+	var conCol connectivity.Collector
+	if config.Cfg().Local.NodeID != nil {
+		//如果client与某个node处于同一台机器，则使用这个node的连通性信息
+		coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+		if err != nil {
+			logger.Warnf("acquire coordinator mq failed, err: %s", err.Error())
+			os.Exit(1)
+		}
+		getCons, err := coorCli.GetNodeConnectivities(coormq.ReqGetNodeConnectivities([]cdssdk.NodeID{*config.Cfg().Local.NodeID}))
+		if err != nil {
+			logger.Warnf("get node connectivities failed, err: %s", err.Error())
+			os.Exit(1)
+		}
+		consMap := make(map[cdssdk.NodeID]connectivity.Connectivity)
+		for _, con := range getCons.Connectivities {
+			var delay *time.Duration
+			if con.Delay != nil {
+				d := time.Duration(*con.Delay * float32(time.Millisecond))
+				delay = &d
+			}
+			consMap[con.FromNodeID] = connectivity.Connectivity{
+				ToNodeID: con.ToNodeID,
+				Delay:    delay,
+			}
+		}
+		conCol = connectivity.NewCollectorWithInitData(&config.Cfg().Connectivity, nil, consMap)
+		logger.Info("use local node connectivities")
+
+	} else {
+		// 否则需要就地收集连通性信息
+		conCol = connectivity.NewCollector(&config.Cfg().Connectivity, nil)
+		conCol.CollectInPlace()
+	}
 
-	// 启动分布式锁服务
 	distlockSvc, err := distlock.NewService(&config.Cfg().DistLock)
 	if err != nil {
 		logger.Warnf("new distlock service failed, err: %s", err.Error())
 		os.Exit(1)
 	}
-	go serveDistLock(distlockSvc) // 在goroutine中运行分布式锁服务
+	go serveDistLock(distlockSvc)
+
+	acStat := accessstat.NewAccessStat(accessstat.Config{
+		// TODO 考虑放到配置里
+		ReportInterval: time.Second * 10,
+	})
+	go serveAccessStat(acStat)
 
-	// 创建任务管理器
 	taskMgr := task.NewManager(distlockSvc, &conCol)
 
-<<<<<<< HEAD
-	// 创建服务实例
-	svc, err := services.NewService(distlockSvc, &taskMgr)
-=======
-	dlder := downloader.NewDownloader(config.Cfg().Downloader)
+	dlder := downloader.NewDownloader(config.Cfg().Downloader, &conCol)
 
-	svc, err := services.NewService(distlockSvc, &taskMgr, &dlder)
->>>>>>> 770feaf2da11a3de00fa3ec57b16dc54ff31b288
+	svc, err := services.NewService(distlockSvc, &taskMgr, &dlder, acStat)
 	if err != nil {
 		logger.Warnf("new services failed, err: %s", err.Error())
 		os.Exit(1)
 	}
 
-	// 创建命令行接口
 	cmds, err := cmdline.NewCommandline(svc)
 	if err != nil {
 		logger.Warnf("new command line failed, err: %s", err.Error())
 		os.Exit(1)
 	}
 
-	// 分发命令行指令
 	cmds.DispatchCommand(os.Args[1:])
 }
 
-// serveDistLock 启动分布式锁服务
-//
-// @Description: 在独立的goroutine中启动分布式锁服务，并处理服务停止时的错误。
 func serveDistLock(svc *distlock.Service) {
 	logger.Info("start serving distlock")
 
@@ -114,4 +114,31 @@ func serveDistLock(svc *distlock.Service) {
 	}
 
 	logger.Info("distlock stopped")
+
+	// TODO 仅简单结束了程序
+	os.Exit(1)
+}
+
+func serveAccessStat(svc *accessstat.AccessStat) {
+	logger.Info("start serving access stat")
+
+	ch := svc.Start()
+loop:
+	for {
+		val, err := ch.Receive()
+		if err != nil {
+			logger.Errorf("access stat stopped with error: %v", err)
+			break
+		}
+
+		switch val := val.(type) {
+		case error:
+			logger.Errorf("access stat stopped with error: %v", val)
+			break loop
+		}
+	}
+	logger.Info("access stat stopped")
+
+	// TODO 仅简单结束了程序
+	os.Exit(1)
 }

common/assets/confs/agent.config.json

@@ -3,13 +3,13 @@
     "local": {
         "nodeID": 1,
         "localIP": "127.0.0.1",
-        "externalIP": "127.0.0.1"
+        "externalIP": "127.0.0.1",
+        "locationID": 1
     },
     "grpc": {
         "ip": "127.0.0.1",
         "port": 5010
     },
-    "tempFileLifetime": 3600,
     "logger": {
         "output": "file",
         "outputFileName": "agent",
@@ -37,6 +37,8 @@
         "testInterval": 300
     },
     "downloader": {
-        "maxStripCacheCount": 100
+        "maxStripCacheCount": 100,
+        "highLatencyNode": 35,
+        "ecStripPrefetchCount": 1
     }
 }

common/assets/confs/client.config.json

@@ -30,6 +30,8 @@
         "testInterval": 300
     },
     "downloader": {
-        "maxStripCacheCount": 100
+        "maxStripCacheCount": 100,
+        "highLatencyNode": 35,
+        "ecStripPrefetchCount": 1
     }
 }

common/assets/confs/scanner.config.json

@@ -1,4 +1,5 @@
 {
+    "accessStatHistoryAmount": 0.8,
     "ecFileSizeThreshold": 104857600,
     "nodeUnavailableSeconds": 300,
     "logger": {
@@ -8,19 +9,19 @@
         "level": "debug"
     },
     "db": {
-        "address": "127.0.0.1:3306",
+        "address": "106.75.6.194:3306",
         "account": "root",
-        "password": "123456",
+        "password": "cloudream123456",
         "databaseName": "cloudream"
     },
     "rabbitMQ": {
-        "address": "127.0.0.1:5672",
+        "address": "106.75.6.194:5672",
         "account": "cloudream",
-        "password": "123456",
+        "password": "cloudream123456",
         "vhost": "/"
     },
     "distlock": {
-        "etcdAddress": "127.0.0.1:2379",
+        "etcdAddress": "106.75.6.194:2379",
         "etcdUsername": "",
         "etcdPassword": "",
         "etcdLockLeaseTimeSec": 5,

common/assets/scripts/create_database.sql

@@ -44,6 +44,7 @@ create table Storage (
   Name varchar(100) not null comment '存储服务名称',
   NodeID int not null comment '存储服务所在节点的ID',
   Directory varchar(4096) not null comment '存储服务所在节点的目录',
+  Remote varchar(4096) not null,
   State varchar(100) comment '状态'
 ) comment = "存储服务表";
 
@@ -113,7 +114,7 @@ create table Package (
   PackageID int not null auto_increment primary key comment '包ID',
   Name varchar(100) not null comment '对象名',
   BucketID int not null comment '桶ID',
-  State varchar(100) not null comment '状态'
+  State varchar(100) not null comment '状态',
 );
 
 create table Object (
@@ -159,11 +160,12 @@ create table StoragePackage (
   primary key(StorageID, PackageID, UserID)
 );
 
-create table StoragePackageLog (
-  StorageID int not null comment '存储服务ID',
+create table PackageAccessStat (
   PackageID int not null comment '包ID',
-  UserID int not null comment '调度了此文件的用户ID',
-  CreateTime timestamp not null comment '加载Package完成的时间'
+  NodeID int not null comment '节点ID',
+  Amount float not null comment '前一日流量的滑动平均值',
+  Counter float not null comment '本日的流量',
+  primary key(PackageID, NodeID)
 );
 
 create table Location (
@@ -174,4 +176,12 @@ create table Location (
 insert into
   Location (LocationID, Name)
 values
-  (1, "Local");
+  (1, "Local");
+
+create table ObjectAccessStat (
+  ObjectID int not null comment '对象ID',
+  NodeID int not null comment '节点ID',
+  Amount float not null comment '前一日流量的滑动平均值',
+  Counter float not null comment '本日的流量',
+  primary key(ObjectID, NodeID)
+);

common/consts/consts.go

@@ -1,9 +1,6 @@
 package consts
 
 const (
-	IPFSStateOK          = "OK"
-	IPFSStateUnavailable = "Unavailable"
-
 	StorageDirectoryStateOK = "OK"
 
 	NodeStateNormal      = "Normal"
@@ -11,7 +8,8 @@ const (
 )
 
 const (
-	NodeDistanceSameNode     = 0.1
-	NodeDistanceSameLocation = 1
-	NodeDistanceOther        = 5
+	NodeDistanceSameNode        = 0.1
+	NodeDistanceSameLocation    = 1
+	NodeDistanceOther           = 5
+	NodeDistanceHighLatencyNode = 10
 )

common/globals/pools.go

@@ -1,7 +1,6 @@
 package stgglb
 
 import (
-	"gitlink.org.cn/cloudream/common/pkgs/ipfs"
 	agtrpc "gitlink.org.cn/cloudream/storage/common/pkgs/grpc/agent"
 	stgmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq"
 	agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent"
@@ -36,9 +35,3 @@ var AgentRPCPool *agtrpc.Pool
 func InitAgentRPCPool(cfg *agtrpc.PoolConfig) {
 	AgentRPCPool = agtrpc.NewPool(cfg)
 }
-
-var IPFSPool *ipfs.Pool
-
-func InitIPFSPool(cfg *ipfs.Config) {
-	IPFSPool = ipfs.NewPool(cfg)
-}

common/globals/utils.go

@@ -0,0 +1,12 @@
+package stgglb
+
+import cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+
+// 根据当前节点与目标地址的距离关系，选择合适的地址
+func SelectGRPCAddress(node *cdssdk.Node) (string, int) {
+	if Local != nil && Local.LocationID == node.LocationID {
+		return node.LocalIP, node.LocalGRPCPort
+	}
+
+	return node.ExternalIP, node.ExternalGRPCPort
+}

common/models/models.go

@@ -3,6 +3,7 @@ package stgmod
 import (
 	"github.com/samber/lo"
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/utils/sort2"
 )
 
 type ObjectBlock struct {
@@ -26,6 +27,48 @@ func NewObjectDetail(object cdssdk.Object, pinnedAt []cdssdk.NodeID, blocks []Ob
 	}
 }
 
+func DetailsFromObjects(objects []cdssdk.Object) []ObjectDetail {
+	details := make([]ObjectDetail, len(objects))
+	for i, object := range objects {
+		details[i] = ObjectDetail{
+			Object: object,
+		}
+	}
+	return details
+}
+
+// 将blocks放到对应的object中。要求objs和blocks都按ObjectID升序
+func DetailsFillObjectBlocks(objs []ObjectDetail, blocks []ObjectBlock) {
+	blksCur := 0
+	for i := range objs {
+		obj := &objs[i]
+		// 1. 查询Object和ObjectBlock时均按照ObjectID升序排序
+		// 2. ObjectBlock结果集中的不同ObjectID数只会比Object结果集的少
+		// 因此在两个结果集上同时从头开始遍历时，如果两边的ObjectID字段不同，那么一定是ObjectBlock这边的ObjectID > Object的ObjectID，
+		// 此时让Object的遍历游标前进，直到两边的ObjectID再次相等
+		for ; blksCur < len(blocks); blksCur++ {
+			if blocks[blksCur].ObjectID != obj.Object.ObjectID {
+				break
+			}
+			obj.Blocks = append(obj.Blocks, blocks[blksCur])
+		}
+	}
+}
+
+// 将pinnedAt放到对应的object中。要求objs和pinnedAt都按ObjectID升序
+func DetailsFillPinnedAt(objs []ObjectDetail, pinnedAt []cdssdk.PinnedObject) {
+	pinnedCur := 0
+	for i := range objs {
+		obj := &objs[i]
+		for ; pinnedCur < len(pinnedAt); pinnedCur++ {
+			if pinnedAt[pinnedCur].ObjectID != obj.Object.ObjectID {
+				break
+			}
+			obj.PinnedAt = append(obj.PinnedAt, pinnedAt[pinnedCur].StorageID)
+		}
+	}
+}
+
 type GrouppedObjectBlock struct {
 	ObjectID cdssdk.ObjectID
 	Index    int
@@ -48,7 +91,7 @@ func (o *ObjectDetail) GroupBlocks() []GrouppedObjectBlock {
 		grps[block.Index] = grp
 	}
 
-	return lo.Values(grps)
+	return sort2.Sort(lo.Values(grps), func(l, r GrouppedObjectBlock) int { return l.Index - r.Index })
 }
 
 type LocalMachineInfo struct {
@@ -57,3 +100,23 @@ type LocalMachineInfo struct {
 	LocalIP    string            `json:"localIP"`
 	LocationID cdssdk.LocationID `json:"locationID"`
 }
+
+type PackageAccessStat struct {
+	PackageID cdssdk.PackageID `db:"PackageID" json:"packageID"`
+	NodeID    cdssdk.NodeID    `db:"NodeID" json:"nodeID"`
+	Amount    float64          `db:"Amount" json:"Amount"`   // 前一日的读取量的滑动平均值
+	Counter   float64          `db:"Counter" json:"counter"` // 当日的读取量
+}
+
+type ObjectAccessStat struct {
+	ObjectID cdssdk.ObjectID `db:"ObjectID" json:"objectID"`
+	NodeID   cdssdk.NodeID   `db:"NodeID" json:"nodeID"`
+	Amount   float64         `db:"Amount" json:"Amount"`   // 前一日的读取量的滑动平均值
+	Counter  float64         `db:"Counter" json:"counter"` // 当日的读取量
+}
+
+type StorageDetail struct {
+	Storage cdssdk.Storage        `json:"storage"`
+	Shard   *cdssdk.ShardStorage  `json:"shard"`
+	Shared  *cdssdk.SharedStorage `json:"shared"`
+}

common/pkgs/accessstat/access_stat.go

@@ -0,0 +1,76 @@
+package accessstat
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/utils/sync2"
+	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
+)
+
+type AccessStatEvent interface{}
+
+type AccessStat struct {
+	cfg   Config
+	stats []coormq.AddAccessStatEntry
+	lock  sync.Mutex
+}
+
+func NewAccessStat(cfg Config) *AccessStat {
+	return &AccessStat{
+		cfg: cfg,
+	}
+}
+
+func (p *AccessStat) AddAccessCounter(objID cdssdk.ObjectID, pkgID cdssdk.PackageID, nodeID cdssdk.NodeID, value float64) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.stats = append(p.stats, coormq.AddAccessStatEntry{
+		ObjectID:  objID,
+		PackageID: pkgID,
+		NodeID:    nodeID,
+		Counter:   value,
+	})
+}
+
+func (p *AccessStat) Start() *sync2.UnboundChannel[AccessStatEvent] {
+	ch := sync2.NewUnboundChannel[AccessStatEvent]()
+
+	go func() {
+		coorCli, err := stgglb.CoordinatorMQPool.Acquire()
+		if err != nil {
+			ch.Send(fmt.Errorf("new coordinator client: %w", err))
+		}
+		defer stgglb.CoordinatorMQPool.Release(coorCli)
+
+		ticker := time.NewTicker(p.cfg.ReportInterval)
+		for {
+			<-ticker.C
+
+			p.lock.Lock()
+			st := p.stats
+			p.stats = nil
+			p.lock.Unlock()
+
+			if len(st) == 0 {
+				continue
+			}
+
+			err := coorCli.AddAccessStat(coormq.ReqAddAccessStat(st))
+			if err != nil {
+				logger.Errorf("add all package access stat counter: %v", err)
+
+				p.lock.Lock()
+				p.stats = append(p.stats, st...)
+				p.lock.Unlock()
+				continue
+			}
+		}
+	}()
+	return ch
+}

common/pkgs/accessstat/config.go

@@ -0,0 +1,7 @@
+package accessstat
+
+import "time"
+
+type Config struct {
+	ReportInterval time.Duration
+}

common/pkgs/cmd/upload_objects.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"math"
@@ -10,19 +11,19 @@ import (
 	"github.com/samber/lo"
 
 	"gitlink.org.cn/cloudream/common/pkgs/distlock"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 	"gitlink.org.cn/cloudream/common/utils/sort2"
 
 	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/connectivity"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/distlock/reqbuilder"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/parser"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/iterator"
-	agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent"
 	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
 )
 
-// UploadObjects 上传对象的结构体，包含上传所需的用户ID、包ID、对象迭代器和节点亲和性信息。
 type UploadObjects struct {
 	userID       cdssdk.UserID
 	packageID    cdssdk.PackageID
@@ -30,32 +31,27 @@ type UploadObjects struct {
 	nodeAffinity *cdssdk.NodeID
 }
 
-// UploadObjectsResult 上传对象结果的结构体，包含上传结果的数组。
 type UploadObjectsResult struct {
 	Objects []ObjectUploadResult
 }
 
-// ObjectUploadResult 单个对象上传结果的结构体，包含上传信息、错误和对象ID。
 type ObjectUploadResult struct {
 	Info   *iterator.IterUploadingObject
 	Error  error
 	Object cdssdk.Object
 }
 
-// UploadNodeInfo 上传节点信息的结构体，包含节点信息、延迟、是否与客户端在同一位置。
 type UploadNodeInfo struct {
 	Node           cdssdk.Node
 	Delay          time.Duration
 	IsSameLocation bool
 }
 
-// UploadObjectsContext 上传对象上下文的结构体，包含分布式锁服务和连通性收集器。
 type UploadObjectsContext struct {
 	Distlock     *distlock.Service
 	Connectivity *connectivity.Collector
 }
 
-// NewUploadObjects 创建一个新的UploadObjects实例。
 func NewUploadObjects(userID cdssdk.UserID, packageID cdssdk.PackageID, objIter iterator.UploadingObjectIterator, nodeAffinity *cdssdk.NodeID) *UploadObjects {
 	return &UploadObjects{
 		userID:       userID,
@@ -65,23 +61,19 @@ func NewUploadObjects(userID cdssdk.UserID, packageID cdssdk.PackageID, objIter
 	}
 }
 
-// Execute 执行上传对象的操作。
 func (t *UploadObjects) Execute(ctx *UploadObjectsContext) (*UploadObjectsResult, error) {
 	defer t.objectIter.Close()
 
-	// 获取协调器客户端
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
 		return nil, fmt.Errorf("new coordinator client: %w", err)
 	}
 
-	// 获取用户节点信息
 	getUserNodesResp, err := coorCli.GetUserNodes(coormq.NewGetUserNodes(t.userID))
 	if err != nil {
 		return nil, fmt.Errorf("getting user nodes: %w", err)
 	}
 
-	// 获取节点连通性信息
 	cons := ctx.Connectivity.GetAll()
 	userNodes := lo.Map(getUserNodesResp.Nodes, func(node cdssdk.Node, index int) UploadNodeInfo {
 		delay := time.Duration(math.MaxInt64)
@@ -101,8 +93,9 @@ func (t *UploadObjects) Execute(ctx *UploadObjectsContext) (*UploadObjectsResult
 		return nil, fmt.Errorf("user no available nodes")
 	}
 
-	// 对上传节点的IPFS加锁
+	// 给上传节点的IPFS加锁
 	ipfsReqBlder := reqbuilder.NewBuilder()
+	// 如果本地的IPFS也是存储系统的一个节点，那么从本地上传时，需要加锁
 	if stgglb.Local.NodeID != nil {
 		ipfsReqBlder.IPFS().Buzy(*stgglb.Local.NodeID)
 	}
@@ -113,15 +106,14 @@ func (t *UploadObjects) Execute(ctx *UploadObjectsContext) (*UploadObjectsResult
 
 		ipfsReqBlder.IPFS().Buzy(node.Node.NodeID)
 	}
-
-	// 获得IPFS锁
+	// TODO 考虑加Object的Create锁
+	// 防止上传的副本被清除
 	ipfsMutex, err := ipfsReqBlder.MutexLock(ctx.Distlock)
 	if err != nil {
 		return nil, fmt.Errorf("acquire locks failed, err: %w", err)
 	}
 	defer ipfsMutex.Unlock()
 
-	// 上传并更新包信息
 	rets, err := uploadAndUpdatePackage(t.packageID, t.objectIter, userNodes, t.nodeAffinity)
 	if err != nil {
 		return nil, err
@@ -132,8 +124,10 @@ func (t *UploadObjects) Execute(ctx *UploadObjectsContext) (*UploadObjectsResult
 	}, nil
 }
 
-// chooseUploadNode 选择一个上传文件的节点。
-// 首先选择设置了亲和性的节点，然后从与当前客户端相同地域的节点中随机选择一个，最后选择延迟最低的节点。
+// chooseUploadNode 选择一个上传文件的节点
+// 1. 选择设置了亲和性的节点
+// 2. 从与当前客户端相同地域的节点中随机选一个
+// 3. 没有的话从所有节点选择延迟最低的节点
 func chooseUploadNode(nodes []UploadNodeInfo, nodeAffinity *cdssdk.NodeID) UploadNodeInfo {
 	if nodeAffinity != nil {
 		aff, ok := lo.Find(nodes, func(node UploadNodeInfo) bool { return node.Node.NodeID == *nodeAffinity })
@@ -153,63 +147,45 @@ func chooseUploadNode(nodes []UploadNodeInfo, nodeAffinity *cdssdk.NodeID) Uploa
 	return nodes[0]
 }
 
-// uploadAndUpdatePackage 上传文件并更新包信息。
-// packageID：标识待更新的包的ID。
-// objectIter：提供上传对象迭代器，用于遍历上传的文件。
-// userNodes：用户可选的上传节点信息列表。
-// nodeAffinity：用户首选的上传节点。
-// 返回值：上传结果列表和错误信息。
 func uploadAndUpdatePackage(packageID cdssdk.PackageID, objectIter iterator.UploadingObjectIterator, userNodes []UploadNodeInfo, nodeAffinity *cdssdk.NodeID) ([]ObjectUploadResult, error) {
-	// 获取协调器客户端
 	coorCli, err := stgglb.CoordinatorMQPool.Acquire()
 	if err != nil {
 		return nil, fmt.Errorf("new coordinator client: %w", err)
 	}
 	defer stgglb.CoordinatorMQPool.Release(coorCli)
 
-	// 选择上传节点
+	// 为所有文件选择相同的上传节点
 	uploadNode := chooseUploadNode(userNodes, nodeAffinity)
 
 	var uploadRets []ObjectUploadResult
-	// 构建添加对象的列表
+	//上传文件夹
 	var adds []coormq.AddObjectEntry
 	for {
-		// 获取下一个对象信息。如果不存在更多对象，则退出循环。
 		objInfo, err := objectIter.MoveNext()
 		if err == iterator.ErrNoMoreItem {
 			break
 		}
 		if err != nil {
-			// 对象获取发生错误，返回错误信息。
 			return nil, fmt.Errorf("reading object: %w", err)
 		}
-
-		// 执行上传逻辑，每个对象依次执行。
 		err = func() error {
-			// 确保对象文件在函数退出时关闭。
 			defer objInfo.File.Close()
 
-			// 记录上传开始时间。
 			uploadTime := time.Now()
-			// 上传文件，并获取文件哈希值。
 			fileHash, err := uploadFile(objInfo.File, uploadNode)
 			if err != nil {
-				// 文件上传失败，记录错误信息并返回。
 				return fmt.Errorf("uploading file: %w", err)
 			}
 
-			// 收集上传结果。
 			uploadRets = append(uploadRets, ObjectUploadResult{
 				Info:  objInfo,
 				Error: err,
 			})
 
-			// 准备添加到队列的条目，以供后续处理。
 			adds = append(adds, coormq.NewAddObjectEntry(objInfo.Path, objInfo.Size, fileHash, uploadTime, uploadNode.Node.NodeID))
 			return nil
 		}()
 		if err != nil {
-			// 上传操作中出现错误，返回错误信息。
 			return nil, err
 		}
 	}
@@ -237,103 +213,27 @@ func uploadAndUpdatePackage(packageID cdssdk.PackageID, objectIter iterator.Uplo
 	return uploadRets, nil
 }
 
-// uploadFile 上传文件。
-// file：待上传的文件流。
-// uploadNode：指定的上传节点信息。
-// 返回值：文件哈希和错误信息。
 func uploadFile(file io.Reader, uploadNode UploadNodeInfo) (string, error) {
-	// 尝试使用本地IPFS上传
-	if stgglb.IPFSPool != nil {
-		logger.Infof("try to use local IPFS to upload file")
-
-		fileHash, err := uploadToLocalIPFS(file, uploadNode.Node.NodeID, stgglb.Local.NodeID == nil)
-		if err == nil {
-			return fileHash, nil
-		} else {
-			logger.Warnf("upload to local IPFS failed, so try to upload to node %d, err: %s", uploadNode.Node.NodeID, err.Error())
-		}
-	}
-
-	// 否则，发送到agent进行上传
-	nodeIP := uploadNode.Node.ExternalIP
-	grpcPort := uploadNode.Node.ExternalGRPCPort
-	if uploadNode.IsSameLocation {
-		nodeIP = uploadNode.Node.LocalIP
-		grpcPort = uploadNode.Node.LocalGRPCPort
-		logger.Infof("client and node %d are at the same location, use local ip", uploadNode.Node.NodeID)
-	}
-
-	fileHash, err := uploadToNode(file, nodeIP, grpcPort)
-	if err != nil {
-		return "", fmt.Errorf("upload to node %s failed, err: %w", nodeIP, err)
-	}
-
-	return fileHash, nil
-}
-
-// uploadToNode 发送文件到指定的节点。
-// file：文件流。
-// nodeIP：节点的IP地址。
-// grpcPort：节点的gRPC端口。
-// 返回值：文件哈希和错误信息。
-func uploadToNode(file io.Reader, nodeIP string, grpcPort int) (string, error) {
-	rpcCli, err := stgglb.AgentRPCPool.Acquire(nodeIP, grpcPort)
-	if err != nil {
-		return "", fmt.Errorf("new agent rpc client: %w", err)
-	}
-	defer rpcCli.Close()
+	ft := ioswitch2.NewFromTo()
+	fromExec, hd := ioswitch2.NewFromDriver(-1)
+	ft.AddFrom(fromExec).AddTo(ioswitch2.NewToNode(uploadNode.Node, -1, "fileHash"))
 
-	return rpcCli.SendIPFSFile(file)
-}
-
-// uploadToLocalIPFS 将文件上传到本地的IPFS节点，并根据需要将文件固定（pin）在节点上。
-// file: 要上传的文件，作为io.Reader提供。
-// nodeID: 指定上传到的IPFS节点的ID。
-// shouldPin: 指示是否在IPFS节点上固定（pin）上传的文件。如果为true，则文件会被固定，否则不会。
-// 返回上传文件的IPFS哈希值和可能出现的错误。
-func uploadToLocalIPFS(file io.Reader, nodeID cdssdk.NodeID, shouldPin bool) (string, error) {
-	// 从IPFS池获取一个IPFS客户端实例
-	ipfsCli, err := stgglb.IPFSPool.Acquire()
+	parser := parser.NewParser(cdssdk.DefaultECRedundancy)
+	plans := exec.NewPlanBuilder()
+	err := parser.Parse(ft, plans)
 	if err != nil {
-		return "", fmt.Errorf("new ipfs client: %w", err)
+		return "", fmt.Errorf("parsing plan: %w", err)
 	}
-	defer ipfsCli.Close() // 确保IPFS客户端在函数返回前被释放
 
-	// 在IPFS上创建文件并获取其哈希值
-	fileHash, err := ipfsCli.CreateFile(file)
-	if err != nil {
-		return "", fmt.Errorf("creating ipfs file: %w", err)
-	}
+	// TODO2 注入依赖
+	exeCtx := exec.NewExecContext()
 
-	// 如果不需要固定文件，则直接返回文件哈希值
-	if !shouldPin {
-		return fileHash, nil
-	}
-
-	// 将文件固定在IPFS节点上
-	err = pinIPFSFile(nodeID, fileHash)
+	exec := plans.Execute(exeCtx)
+	exec.BeginWrite(io.NopCloser(file), hd)
+	ret, err := exec.Wait(context.TODO())
 	if err != nil {
 		return "", err
 	}
 
-	return fileHash, nil
-}
-
-// pinIPFSFile 将文件Pin到IPFS节点。
-// nodeID：节点ID。
-// fileHash：文件哈希。
-// 返回值：错误信息。
-func pinIPFSFile(nodeID cdssdk.NodeID, fileHash string) error {
-	agtCli, err := stgglb.AgentMQPool.Acquire(nodeID)
-	if err != nil {
-		return fmt.Errorf("new agent client: %w", err)
-	}
-	defer stgglb.AgentMQPool.Release(agtCli)
-
-	_, err = agtCli.PinObject(agtmq.ReqPinObject([]string{fileHash}, false))
-	if err != nil {
-		return fmt.Errorf("start pinning object: %w", err)
-	}
-
-	return nil
+	return ret["fileHash"].(string), nil
 }

common/pkgs/connectivity/collector.go

@@ -39,6 +39,19 @@ func NewCollector(cfg *Config, onCollected func(collector *Collector)) Collector
 	return rpt
 }
 
+func NewCollectorWithInitData(cfg *Config, onCollected func(collector *Collector), initData map[cdssdk.NodeID]Connectivity) Collector {
+	rpt := Collector{
+		cfg:            cfg,
+		collectNow:     make(chan any),
+		close:          make(chan any),
+		connectivities: initData,
+		lock:           &sync.RWMutex{},
+		onCollected:    onCollected,
+	}
+	go rpt.serve()
+	return rpt
+}
+
 func (r *Collector) Get(nodeID cdssdk.NodeID) *Connectivity {
 	r.lock.RLock()
 	defer r.lock.RUnlock()
@@ -202,8 +215,7 @@ func (r *Collector) ping(node cdssdk.Node) Connectivity {
 			}
 		}
 
-		// 此时间差为一个来回的时间，因此单程延迟需要除以2
-		delay := time.Since(start) / 2
+		delay := time.Since(start)
 		avgDelay += delay
 
 		// 每次ping之间间隔1秒

common/pkgs/db/model/model.go

@@ -11,14 +11,7 @@ import (
 )
 
 // TODO 可以考虑逐步迁移到cdssdk中。迁移思路：数据对象应该包含的字段都迁移到cdssdk中，内部使用的一些特殊字段则留在这里
-
-type Storage struct {
-	StorageID cdssdk.StorageID `db:"StorageID" json:"storageID"`
-	Name      string           `db:"Name" json:"name"`
-	NodeID    cdssdk.NodeID    `db:"NodeID" json:"nodeID"`
-	Directory string           `db:"Directory" json:"directory"`
-	State     string           `db:"State" json:"state"`
-}
+type Storage = cdssdk.Storage
 
 type User struct {
 	UserID   cdssdk.UserID `db:"UserID" json:"userID"`
@@ -103,13 +96,6 @@ type StoragePackage struct {
 	State     string           `db:"State" json:"state"`
 }
 
-type StoragePackageLog struct {
-	StorageID  cdssdk.StorageID `db:"StorageID" json:"storageID"`
-	PackageID  cdssdk.PackageID `db:"PackageID" json:"packageID"`
-	UserID     cdssdk.UserID    `db:"UserID" json:"userID"`
-	CreateTime time.Time        `db:"CreateTime" json:"createTime"`
-}
-
 type Location struct {
 	LocationID cdssdk.LocationID `db:"LocationID" json:"locationID"`
 	Name       string            `db:"Name" json:"name"`

common/pkgs/db/node_connectivity.go

@@ -14,14 +14,14 @@ func (db *DB) NodeConnectivity() *NodeConnectivityDB {
 	return &NodeConnectivityDB{DB: db}
 }
 
-func (db *NodeConnectivityDB) BatchGetByFromNode(ctx SQLContext, nodeIDs []cdssdk.NodeID) ([]model.NodeConnectivity, error) {
-	if len(nodeIDs) == 0 {
+func (db *NodeConnectivityDB) BatchGetByFromNode(ctx SQLContext, fromNodeIDs []cdssdk.NodeID) ([]model.NodeConnectivity, error) {
+	if len(fromNodeIDs) == 0 {
 		return nil, nil
 	}
 
 	var ret []model.NodeConnectivity
 
-	sql, args, err := sqlx.In("select * from NodeConnectivity where NodeID in (?)", nodeIDs)
+	sql, args, err := sqlx.In("select * from NodeConnectivity where FromNodeID in (?)", fromNodeIDs)
 	if err != nil {
 		return nil, err
 	}

common/pkgs/db/object.go

@@ -2,11 +2,13 @@ package db
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/jmoiron/sqlx"
 	"github.com/samber/lo"
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/utils/sort2"
 	stgmod "gitlink.org.cn/cloudream/storage/common/models"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
 	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
@@ -26,6 +28,30 @@ func (db *ObjectDB) GetByID(ctx SQLContext, objectID cdssdk.ObjectID) (model.Obj
 	return ret.ToObject(), err
 }
 
+func (db *ObjectDB) BatchTestObjectID(ctx SQLContext, objectIDs []cdssdk.ObjectID) (map[cdssdk.ObjectID]bool, error) {
+	if len(objectIDs) == 0 {
+		return make(map[cdssdk.ObjectID]bool), nil
+	}
+
+	stmt, args, err := sqlx.In("select ObjectID from Object where ObjectID in (?)", lo.Uniq(objectIDs))
+	if err != nil {
+		return nil, err
+	}
+
+	var avaiIDs []cdssdk.ObjectID
+	err = sqlx.Select(ctx, &avaiIDs, stmt, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	avaiIDMap := make(map[cdssdk.ObjectID]bool)
+	for _, pkgID := range avaiIDs {
+		avaiIDMap[pkgID] = true
+	}
+
+	return avaiIDMap, nil
+}
+
 func (db *ObjectDB) BatchGet(ctx SQLContext, objectIDs []cdssdk.ObjectID) ([]model.Object, error) {
 	if len(objectIDs) == 0 {
 		return nil, nil
@@ -117,66 +143,50 @@ func (*ObjectDB) GetPackageObjects(ctx SQLContext, packageID cdssdk.PackageID) (
 	return lo.Map(ret, func(o model.TempObject, idx int) model.Object { return o.ToObject() }), err
 }
 
-// GetPackageObjectDetails 获取指定包ID的对象详情列表。
-//
-// ctx: SQL执行上下文。
-// packageID: 指定的包ID。
-//
-// 返回值为Object详情列表和可能出现的错误。
 func (db *ObjectDB) GetPackageObjectDetails(ctx SQLContext, packageID cdssdk.PackageID) ([]stgmod.ObjectDetail, error) {
-	// 从Object表中查询所有属于指定包ID的对象，按ObjectID升序排序
 	var objs []model.TempObject
 	err := sqlx.Select(ctx, &objs, "select * from Object where PackageID = ? order by ObjectID asc", packageID)
 	if err != nil {
 		return nil, fmt.Errorf("getting objects: %w", err)
 	}
 
-	// 初始化返回的Object详情列表
-	rets := make([]stgmod.ObjectDetail, 0, len(objs))
-
-	// 从ObjectBlock表中查询所有属于指定包ID的对象块，按ObjectID和Index升序排序
 	var allBlocks []stgmod.ObjectBlock
 	err = sqlx.Select(ctx, &allBlocks, "select ObjectBlock.* from ObjectBlock, Object where PackageID = ? and ObjectBlock.ObjectID = Object.ObjectID order by ObjectBlock.ObjectID, `Index` asc", packageID)
 	if err != nil {
 		return nil, fmt.Errorf("getting all object blocks: %w", err)
 	}
 
-	// 从PinnedObject表中查询所有属于指定包ID的被固定的对象，按ObjectID排序
 	var allPinnedObjs []cdssdk.PinnedObject
 	err = sqlx.Select(ctx, &allPinnedObjs, "select PinnedObject.* from PinnedObject, Object where PackageID = ? and PinnedObject.ObjectID = Object.ObjectID order by PinnedObject.ObjectID", packageID)
 	if err != nil {
 		return nil, fmt.Errorf("getting all pinned objects: %w", err)
 	}
 
-	// 遍历查询得到的对象，为每个对象构建详细的Object信息
-	blksCur := 0    // 当前遍历到的对象块索引
-	pinnedsCur := 0 // 当前遍历到的被固定对象索引
-	for _, temp := range objs {
-		detail := stgmod.ObjectDetail{
-			Object: temp.ToObject(),
+	details := make([]stgmod.ObjectDetail, len(objs))
+	for i, obj := range objs {
+		details[i] = stgmod.ObjectDetail{
+			Object: obj.ToObject(),
 		}
+	}
 
-		// 同时遍历对象和对象块的结果集，将属于同一对象的对象块附加到Object详情中
-		for ; blksCur < len(allBlocks); blksCur++ {
-			if allBlocks[blksCur].ObjectID != temp.ObjectID {
-				break
-			}
-			detail.Blocks = append(detail.Blocks, allBlocks[blksCur])
-		}
+	stgmod.DetailsFillObjectBlocks(details, allBlocks)
+	stgmod.DetailsFillPinnedAt(details, allPinnedObjs)
+	return details, nil
+}
 
-		// 遍历被固定对象的结果集，将被固定的信息附加到Object详情中
-		for ; pinnedsCur < len(allPinnedObjs); pinnedsCur++ {
-			if allPinnedObjs[pinnedsCur].ObjectID != temp.ObjectID {
-				break
-			}
-			detail.PinnedAt = append(detail.PinnedAt, allPinnedObjs[pinnedsCur].NodeID)
-		}
+func (*ObjectDB) GetObjectsIfAnyBlockOnNode(ctx SQLContext, nodeID cdssdk.NodeID) ([]cdssdk.Object, error) {
+	var temps []model.TempObject
+	err := sqlx.Select(ctx, &temps, "select * from Object where ObjectID in (select ObjectID from ObjectBlock where NodeID = ?) order by ObjectID asc", nodeID)
+	if err != nil {
+		return nil, fmt.Errorf("getting objects: %w", err)
+	}
 
-		// 将构建好的Object详情添加到返回列表中
-		rets = append(rets, detail)
+	objs := make([]cdssdk.Object, len(temps))
+	for i := range temps {
+		objs[i] = temps[i].ToObject()
 	}
 
-	return rets, nil
+	return objs, nil
 }
 
 func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds []coormq.AddObjectEntry) ([]cdssdk.Object, error) {
@@ -211,6 +221,11 @@ func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds []
 	if err != nil {
 		return nil, fmt.Errorf("batch get object ids: %w", err)
 	}
+
+	// 所有需要按索引来一一对应的数据都需要进行排序
+	adds = sort2.Sort(adds, func(l, r coormq.AddObjectEntry) int { return strings.Compare(l.Path, r.Path) })
+	addedObjs = sort2.Sort(addedObjs, func(l, r cdssdk.Object) int { return strings.Compare(l.Path, r.Path) })
+
 	addedObjIDs := make([]cdssdk.ObjectID, len(addedObjs))
 	for i := range addedObjs {
 		addedObjIDs[i] = addedObjs[i].ObjectID
@@ -326,7 +341,7 @@ func (db *ObjectDB) BatchUpdateRedundancy(ctx SQLContext, objs []coormq.Updating
 		for _, p := range obj.PinnedAt {
 			pinneds = append(pinneds, cdssdk.PinnedObject{
 				ObjectID:   obj.ObjectID,
-				NodeID:     p,
+				StorageID:  p,
 				CreateTime: time.Now(),
 			})
 		}

common/pkgs/db/object_access_stat.go

@@ -0,0 +1,121 @@
+package db
+
+import (
+	"github.com/jmoiron/sqlx"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	stgmod "gitlink.org.cn/cloudream/storage/common/models"
+	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
+)
+
+type ObjectAccessStatDB struct {
+	*DB
+}
+
+func (db *DB) ObjectAccessStat() *ObjectAccessStatDB {
+	return &ObjectAccessStatDB{db}
+}
+
+func (*ObjectAccessStatDB) Get(ctx SQLContext, objID cdssdk.ObjectID, nodeID cdssdk.NodeID) (stgmod.ObjectAccessStat, error) {
+	var ret stgmod.ObjectAccessStat
+	err := sqlx.Get(ctx, &ret, "select * from ObjectAccessStat where ObjectID=? and NodeID=?", objID, nodeID)
+	return ret, err
+}
+
+func (*ObjectAccessStatDB) GetByObjectID(ctx SQLContext, objID cdssdk.ObjectID) ([]stgmod.ObjectAccessStat, error) {
+	var ret []stgmod.ObjectAccessStat
+	err := sqlx.Select(ctx, &ret, "select * from ObjectAccessStat where ObjectID=?", objID)
+	return ret, err
+}
+
+func (*ObjectAccessStatDB) BatchGetByObjectID(ctx SQLContext, objIDs []cdssdk.ObjectID) ([]stgmod.ObjectAccessStat, error) {
+	if len(objIDs) == 0 {
+		return nil, nil
+	}
+
+	var ret []stgmod.ObjectAccessStat
+	stmt, args, err := sqlx.In("select * from ObjectAccessStat where ObjectID in (?)", objIDs)
+	if err != nil {
+		return ret, err
+	}
+
+	err = sqlx.Select(ctx, &ret, stmt, args...)
+	return ret, err
+}
+
+func (*ObjectAccessStatDB) BatchGetByObjectIDOnNode(ctx SQLContext, objIDs []cdssdk.ObjectID, nodeID cdssdk.NodeID) ([]stgmod.ObjectAccessStat, error) {
+	if len(objIDs) == 0 {
+		return nil, nil
+	}
+
+	var ret []stgmod.ObjectAccessStat
+	stmt, args, err := sqlx.In("select * from ObjectAccessStat where ObjectID in (?) and NodeID=?", objIDs, nodeID)
+	if err != nil {
+		return ret, err
+	}
+
+	err = sqlx.Select(ctx, &ret, stmt, args...)
+	return ret, err
+}
+
+func (*ObjectAccessStatDB) BatchAddCounter(ctx SQLContext, entries []coormq.AddAccessStatEntry) error {
+	if len(entries) == 0 {
+		return nil
+	}
+
+	sql := "insert into ObjectAccessStat(ObjectID, NodeID, Counter, Amount) " +
+		" values(:ObjectID, :NodeID, :Counter, 0) as new" +
+		" on duplicate key update ObjectAccessStat.Counter=ObjectAccessStat.Counter+new.Counter"
+	err := BatchNamedExec(ctx, sql, 4, entries, nil)
+	return err
+}
+
+func (*ObjectAccessStatDB) BatchUpdateAmountInPackage(ctx SQLContext, pkgIDs []cdssdk.PackageID, historyWeight float64) error {
+	if len(pkgIDs) == 0 {
+		return nil
+	}
+
+	stmt, args, err := sqlx.In("update ObjectAccessStat inner join Object"+
+		" on ObjectAccessStat.ObjectID = Object.ObjectID"+
+		" set Amount=Amount*?+Counter*(1-?), Counter = 0"+
+		" where PackageID in (?)", historyWeight, historyWeight, pkgIDs)
+	if err != nil {
+		return err
+	}
+
+	_, err = ctx.Exec(stmt, args...)
+	return err
+}
+
+func (*ObjectAccessStatDB) UpdateAllAmount(ctx SQLContext, historyWeight float64) error {
+	stmt, args, err := sqlx.In("update ObjectAccessStat set Amount=Amount*?+Counter*(1-?), Counter = 0", historyWeight, historyWeight)
+	if err != nil {
+		return err
+	}
+
+	_, err = ctx.Exec(stmt, args...)
+	return err
+}
+
+func (*ObjectAccessStatDB) DeleteByObjectID(ctx SQLContext, objID cdssdk.ObjectID) error {
+	_, err := ctx.Exec("delete from ObjectAccessStat where ObjectID=?", objID)
+	return err
+}
+
+func (*ObjectAccessStatDB) BatchDeleteByObjectID(ctx SQLContext, objIDs []cdssdk.ObjectID) error {
+	if len(objIDs) == 0 {
+		return nil
+	}
+
+	stmt, args, err := sqlx.In("delete from ObjectAccessStat where ObjectID in (?)", objIDs)
+	if err != nil {
+		return err
+	}
+
+	_, err = ctx.Exec(stmt, args...)
+	return err
+}
+
+func (*ObjectAccessStatDB) DeleteInPackage(ctx SQLContext, packageID cdssdk.PackageID) error {
+	_, err := ctx.Exec("delete ObjectAccessStat from ObjectAccessStat inner join Object on ObjectAccessStat.ObjectID = Object.ObjectID where PackageID = ?", packageID)
+	return err
+}

common/pkgs/db/package.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/samber/lo"
 
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
@@ -31,6 +32,30 @@ func (db *PackageDB) GetByName(ctx SQLContext, bucketID cdssdk.BucketID, name st
 	return ret, err
 }
 
+func (db *PackageDB) BatchTestPackageID(ctx SQLContext, pkgIDs []cdssdk.PackageID) (map[cdssdk.PackageID]bool, error) {
+	if len(pkgIDs) == 0 {
+		return make(map[cdssdk.PackageID]bool), nil
+	}
+
+	stmt, args, err := sqlx.In("select PackageID from Package where PackageID in (?)", lo.Uniq(pkgIDs))
+	if err != nil {
+		return nil, err
+	}
+
+	var avaiIDs []cdssdk.PackageID
+	err = sqlx.Select(ctx, &avaiIDs, stmt, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	avaiIDMap := make(map[cdssdk.PackageID]bool)
+	for _, pkgID := range avaiIDs {
+		avaiIDMap[pkgID] = true
+	}
+
+	return avaiIDMap, nil
+}
+
 func (*PackageDB) BatchGetAllPackageIDs(ctx SQLContext, start int, count int) ([]cdssdk.PackageID, error) {
 	var ret []cdssdk.PackageID
 	err := sqlx.Select(ctx, &ret, "select PackageID from Package limit ?, ?", start, count)
@@ -136,6 +161,11 @@ func (db *PackageDB) SoftDelete(ctx SQLContext, packageID cdssdk.PackageID) erro
 		return fmt.Errorf("change package state failed, err: %w", err)
 	}
 
+	err = db.ObjectAccessStat().DeleteInPackage(ctx, packageID)
+	if err != nil {
+		return fmt.Errorf("delete from object access stat: %w", err)
+	}
+
 	err = db.ObjectBlock().DeleteInPackage(ctx, packageID)
 	if err != nil {
 		return fmt.Errorf("delete from object rep failed, err: %w", err)

common/pkgs/db/package_access_stat.go

@@ -0,0 +1,84 @@
+package db
+
+import (
+	"github.com/jmoiron/sqlx"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	stgmod "gitlink.org.cn/cloudream/storage/common/models"
+	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
+)
+
+type PackageAccessStatDB struct {
+	*DB
+}
+
+func (db *DB) PackageAccessStat() *PackageAccessStatDB {
+	return &PackageAccessStatDB{db}
+}
+
+func (*PackageAccessStatDB) Get(ctx SQLContext, pkgID cdssdk.PackageID, nodeID cdssdk.NodeID) (stgmod.PackageAccessStat, error) {
+	var ret stgmod.PackageAccessStat
+	err := sqlx.Get(ctx, &ret, "select * from PackageAccessStat where PackageID=? and NodeID=?", pkgID, nodeID)
+	return ret, err
+}
+
+func (*PackageAccessStatDB) GetByPackageID(ctx SQLContext, pkgID cdssdk.PackageID) ([]stgmod.PackageAccessStat, error) {
+	var ret []stgmod.PackageAccessStat
+	err := sqlx.Select(ctx, &ret, "select * from PackageAccessStat where PackageID=?", pkgID)
+	return ret, err
+}
+
+func (*PackageAccessStatDB) BatchGetByPackageID(ctx SQLContext, pkgIDs []cdssdk.PackageID) ([]stgmod.PackageAccessStat, error) {
+	if len(pkgIDs) == 0 {
+		return nil, nil
+	}
+
+	var ret []stgmod.PackageAccessStat
+	stmt, args, err := sqlx.In("select * from PackageAccessStat where PackageID in (?)", pkgIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	err = sqlx.Select(ctx, &ret, stmt, args...)
+	return ret, err
+}
+
+func (*PackageAccessStatDB) BatchAddCounter(ctx SQLContext, entries []coormq.AddAccessStatEntry) error {
+	if len(entries) == 0 {
+		return nil
+	}
+
+	sql := "insert into PackageAccessStat(PackageID, NodeID, Counter, Amount)" +
+		" values(:PackageID, :NodeID, :Counter, 0) as new" +
+		" on duplicate key update Counter=Counter+new.Counter"
+	err := BatchNamedExec(ctx, sql, 4, entries, nil)
+	return err
+}
+
+func (*PackageAccessStatDB) BatchUpdateAmount(ctx SQLContext, pkgIDs []cdssdk.PackageID, historyWeight float64) error {
+	if len(pkgIDs) == 0 {
+		return nil
+	}
+
+	stmt, args, err := sqlx.In("update PackageAccessStat set Amount=Amount*?+Counter*(1-?), Counter = 0 where PackageID in (?)", historyWeight, historyWeight, pkgIDs)
+	if err != nil {
+		return err
+	}
+
+	_, err = ctx.Exec(stmt, args...)
+	return err
+}
+
+func (*PackageAccessStatDB) UpdateAllAmount(ctx SQLContext, historyWeight float64) error {
+	stmt, args, err := sqlx.In("update PackageAccessStat set Amount=Amount*?+Counter*(1-?), Counter = 0", historyWeight, historyWeight)
+	if err != nil {
+		return err
+	}
+
+	_, err = ctx.Exec(stmt, args...)
+	return err
+}
+
+func (*PackageAccessStatDB) DeleteByPackageID(ctx SQLContext, pkgID cdssdk.PackageID) error {
+	_, err := ctx.Exec("delete from PackageAccessStat where PackageID=?", pkgID)
+	return err
+}

common/pkgs/db/pinned_object.go

@@ -1,139 +0,0 @@
-package db
-
-import (
-	"time"
-
-	"github.com/jmoiron/sqlx"
-	"github.com/samber/lo"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
-)
-
-type PinnedObjectDB struct {
-	*DB
-}
-
-func (db *DB) PinnedObject() *PinnedObjectDB {
-	return &PinnedObjectDB{DB: db}
-}
-
-func (*PinnedObjectDB) GetByNodeID(ctx SQLContext, nodeID cdssdk.NodeID) ([]cdssdk.PinnedObject, error) {
-	var ret []cdssdk.PinnedObject
-	err := sqlx.Select(ctx, &ret, "select * from PinnedObject where NodeID = ?", nodeID)
-	return ret, err
-}
-
-func (*PinnedObjectDB) GetObjectsByNodeID(ctx SQLContext, nodeID cdssdk.NodeID) ([]cdssdk.Object, error) {
-	var ret []model.TempObject
-	err := sqlx.Select(ctx, &ret, "select Object.* from PinnedObject, Object where PinnedObject.ObjectID = Object.ObjectID and NodeID = ?", nodeID)
-	return lo.Map(ret, func(o model.TempObject, idx int) cdssdk.Object { return o.ToObject() }), err
-}
-
-func (*PinnedObjectDB) Create(ctx SQLContext, nodeID cdssdk.NodeID, objectID cdssdk.ObjectID, createTime time.Time) error {
-	_, err := ctx.Exec("insert into PinnedObject values(?,?,?)", nodeID, objectID, createTime)
-	return err
-}
-
-func (*PinnedObjectDB) BatchGetByObjectID(ctx SQLContext, objectIDs []cdssdk.ObjectID) ([]cdssdk.PinnedObject, error) {
-	if len(objectIDs) == 0 {
-		return nil, nil
-	}
-
-	stmt, args, err := sqlx.In("select * from PinnedObject where ObjectID in (?) order by ObjectID asc", objectIDs)
-	if err != nil {
-		return nil, err
-	}
-	stmt = ctx.Rebind(stmt)
-
-	var pinneds []cdssdk.PinnedObject
-	err = sqlx.Select(ctx, &pinneds, stmt, args...)
-	if err != nil {
-		return nil, err
-	}
-
-	return pinneds, nil
-}
-
-func (*PinnedObjectDB) TryCreate(ctx SQLContext, nodeID cdssdk.NodeID, objectID cdssdk.ObjectID, createTime time.Time) error {
-	_, err := ctx.Exec("insert ignore into PinnedObject values(?,?,?)", nodeID, objectID, createTime)
-	return err
-}
-
-func (*PinnedObjectDB) BatchTryCreate(ctx SQLContext, pinneds []cdssdk.PinnedObject) error {
-	if len(pinneds) == 0 {
-		return nil
-	}
-
-	return BatchNamedExec(ctx, "insert ignore into PinnedObject values(:NodeID,:ObjectID,:CreateTime)", 3, pinneds, nil)
-}
-
-func (*PinnedObjectDB) CreateFromPackage(ctx SQLContext, packageID cdssdk.PackageID, nodeID cdssdk.NodeID) error {
-	_, err := ctx.Exec(
-		"insert ignore into PinnedObject(NodeID, ObjectID, CreateTime) select ? as NodeID, ObjectID, ? as CreateTime from Object where PackageID = ?",
-		nodeID,
-		time.Now(),
-		packageID,
-	)
-	return err
-}
-
-func (db *PinnedObjectDB) ObjectBatchCreate(ctx SQLContext, objectID cdssdk.ObjectID, nodeIDs []cdssdk.NodeID) error {
-	if len(nodeIDs) == 0 {
-		return nil
-	}
-
-	for _, id := range nodeIDs {
-		err := db.TryCreate(ctx, id, objectID, time.Now())
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (*PinnedObjectDB) Delete(ctx SQLContext, nodeID cdssdk.NodeID, objectID cdssdk.ObjectID) error {
-	_, err := ctx.Exec("delete from PinnedObject where NodeID = ? and ObjectID = ?", nodeID, objectID)
-	return err
-}
-
-func (*PinnedObjectDB) DeleteByObjectID(ctx SQLContext, objectID cdssdk.ObjectID) error {
-	_, err := ctx.Exec("delete from PinnedObject where ObjectID = ?", objectID)
-	return err
-}
-
-func (*PinnedObjectDB) BatchDeleteByObjectID(ctx SQLContext, objectIDs []cdssdk.ObjectID) error {
-	if len(objectIDs) == 0 {
-		return nil
-	}
-
-	// TODO in语句有长度限制
-	query, args, err := sqlx.In("delete from PinnedObject where ObjectID in (?)", objectIDs)
-	if err != nil {
-		return err
-	}
-	_, err = ctx.Exec(query, args...)
-	return err
-}
-
-func (*PinnedObjectDB) DeleteInPackage(ctx SQLContext, packageID cdssdk.PackageID) error {
-	_, err := ctx.Exec("delete PinnedObject from PinnedObject inner join Object on PinnedObject.ObjectID = Object.ObjectID where PackageID = ?", packageID)
-	return err
-}
-
-func (*PinnedObjectDB) DeleteInPackageAtNode(ctx SQLContext, packageID cdssdk.PackageID, nodeID cdssdk.NodeID) error {
-	_, err := ctx.Exec("delete PinnedObject from PinnedObject inner join Object on PinnedObject.ObjectID = Object.ObjectID where PackageID = ? and NodeID = ?", packageID, nodeID)
-	return err
-}
-
-func (*PinnedObjectDB) NodeBatchDelete(ctx SQLContext, nodeID cdssdk.NodeID, objectIDs []cdssdk.ObjectID) error {
-	if len(objectIDs) == 0 {
-		return nil
-	}
-
-	query, args, err := sqlx.In("delete from PinnedObject where NodeID = ? and ObjectID in (?)", nodeID, objectIDs)
-	if err != nil {
-		return err
-	}
-	_, err = ctx.Exec(query, args...)
-	return err
-}

common/pkgs/db/storage.go

@@ -1,65 +0,0 @@
-package db
-
-import (
-	"database/sql"
-	"fmt"
-
-	"github.com/jmoiron/sqlx"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
-)
-
-type StorageDB struct {
-	*DB
-}
-
-func (db *DB) Storage() *StorageDB {
-	return &StorageDB{DB: db}
-}
-
-func (db *StorageDB) GetByID(ctx SQLContext, stgID cdssdk.StorageID) (model.Storage, error) {
-	var stg model.Storage
-	err := sqlx.Get(ctx, &stg, "select * from Storage where StorageID = ?", stgID)
-	return stg, err
-}
-
-func (db *StorageDB) BatchGetAllStorageIDs(ctx SQLContext, start int, count int) ([]cdssdk.StorageID, error) {
-	var ret []cdssdk.StorageID
-	err := sqlx.Select(ctx, &ret, "select StorageID from Storage limit ?, ?", start, count)
-	return ret, err
-}
-
-func (db *StorageDB) IsAvailable(ctx SQLContext, userID cdssdk.UserID, storageID cdssdk.StorageID) (bool, error) {
-	var stgID int64
-	err := sqlx.Get(ctx, &stgID,
-		"select Storage.StorageID from Storage, UserStorage where"+
-			" Storage.StorageID = ? and"+
-			" Storage.StorageID = UserStorage.StorageID and"+
-			" UserStorage.UserID = ?",
-		storageID, userID)
-
-	if err == sql.ErrNoRows {
-		return false, nil
-	}
-
-	if err != nil {
-		return false, fmt.Errorf("find storage failed, err: %w", err)
-	}
-
-	return true, nil
-}
-
-func (db *StorageDB) GetUserStorage(ctx SQLContext, userID cdssdk.UserID, storageID cdssdk.StorageID) (model.Storage, error) {
-	var stg model.Storage
-	err := sqlx.Get(ctx, &stg,
-		"select Storage.* from UserStorage, Storage where UserID = ? and UserStorage.StorageID = ? and UserStorage.StorageID = Storage.StorageID",
-		userID,
-		storageID)
-
-	return stg, err
-}
-
-func (db *StorageDB) ChangeState(ctx SQLContext, storageID cdssdk.StorageID, state string) error {
-	_, err := ctx.Exec("update Storage set State = ? where StorageID = ?", state, storageID)
-	return err
-}

common/pkgs/db/storage_package_log.go

@@ -1,39 +0,0 @@
-package db
-
-import (
-	"time"
-
-	"github.com/jmoiron/sqlx"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
-)
-
-type StoragePackageLogDB struct {
-	*DB
-}
-
-func (db *DB) StoragePackageLog() *StoragePackageLogDB {
-	return &StoragePackageLogDB{DB: db}
-}
-
-func (*StoragePackageLogDB) Get(ctx SQLContext, storageID cdssdk.StorageID, packageID cdssdk.PackageID, userID cdssdk.UserID) (model.StoragePackageLog, error) {
-	var ret model.StoragePackageLog
-	err := sqlx.Get(ctx, &ret, "select * from StoragePackageLog where StorageID = ? and PackageID = ? and UserID = ?", storageID, packageID, userID)
-	return ret, err
-}
-
-func (*StoragePackageLogDB) GetByPackageID(ctx SQLContext, packageID cdssdk.PackageID) ([]model.StoragePackageLog, error) {
-	var ret []model.StoragePackageLog
-	err := sqlx.Select(ctx, &ret, "select * from StoragePackageLog where PackageID = ?", packageID)
-	return ret, err
-}
-
-func (*StoragePackageLogDB) Create(ctx SQLContext, storageID cdssdk.StorageID, packageID cdssdk.PackageID, userID cdssdk.UserID, createTime time.Time) error {
-	_, err := ctx.Exec("insert into StoragePackageLog values(?,?,?,?)", storageID, packageID, userID, createTime)
-	return err
-}
-
-func (*StoragePackageLogDB) Delete(ctx SQLContext, storageID cdssdk.StorageID, packageID cdssdk.PackageID, userID cdssdk.UserID) error {
-	_, err := ctx.Exec("delete from StoragePackageLog where StorageID = ? and PackageID = ? and UserID = ?", storageID, packageID, userID)
-	return err
-}

common/pkgs/db2/db2.go

@@ -0,0 +1,38 @@
+package db2
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/sirupsen/logrus"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/db/config"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+)
+
+type DB struct {
+	db *gorm.DB
+}
+
+func NewDB(cfg *config.Config) (*DB, error) {
+	mydb, err := gorm.Open(mysql.Open(cfg.MakeSourceString()), &gorm.Config{})
+	if err != nil {
+		logrus.Fatalf("failed to connect to database: %v", err)
+	}
+
+	return &DB{
+		db: mydb,
+	}, nil
+}
+
+func (s *DB) DoTx(do func(tx SQLContext) error) error {
+	return s.db.Transaction(func(tx *gorm.DB) error {
+		return do(SQLContext{tx})
+	})
+}
+
+type SQLContext struct {
+	*gorm.DB
+}
+
+func (d *DB) DefCtx() SQLContext {
+	return SQLContext{d.db}
+}

common/pkgs/db2/node.go

@@ -0,0 +1,53 @@
+package db2
+
+import (
+	"time"
+
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+)
+
+type NodeDB struct {
+	*DB
+}
+
+func (nodeDB *DB) Node() *NodeDB {
+	return &NodeDB{DB: nodeDB}
+}
+
+func (*NodeDB) GetAllNodes(ctx SQLContext) ([]cdssdk.Node, error) {
+	var ret []cdssdk.Node
+
+	err := ctx.Table("node").Find(&ret).Error
+	return ret, err
+}
+
+func (*NodeDB) GetByID(ctx SQLContext, nodeID cdssdk.NodeID) (cdssdk.Node, error) {
+	var ret cdssdk.Node
+	err := ctx.Table("node").Where("NodeID = ?", nodeID).Find(&ret).Error
+
+	return ret, err
+}
+
+// GetUserNodes 根据用户id查询可用node
+func (*NodeDB) GetUserNodes(ctx SQLContext, userID cdssdk.UserID) ([]cdssdk.Node, error) {
+	var nodes []cdssdk.Node
+	err := ctx.
+		Table("Node").
+		Select("Node.*").
+		Joins("JOIN UserNode ON UserNode.NodeID = Node.NodeID").
+		Where("UserNode.UserID = ?", userID).
+		Find(&nodes).Error
+	return nodes, err
+}
+
+// UpdateState 更新状态，并且设置上次上报时间为现在
+func (*NodeDB) UpdateState(ctx SQLContext, nodeID cdssdk.NodeID, state string) error {
+	err := ctx.
+		Model(&cdssdk.Node{}).
+		Where("NodeID = ?", nodeID).
+		Updates(map[string]interface{}{
+			"State":          state,
+			"LastReportTime": time.Now(),
+		}).Error
+	return err
+}

common/pkgs/db2/pinned_object.go

@@ -0,0 +1,118 @@
+package db2
+
+import (
+	"time"
+
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gorm.io/gorm/clause"
+)
+
+type PinnedObjectDB struct {
+	*DB
+}
+
+func (db *DB) PinnedObject() *PinnedObjectDB {
+	return &PinnedObjectDB{DB: db}
+}
+
+func (*PinnedObjectDB) GetByStorageID(ctx SQLContext, stgID cdssdk.StorageID) ([]cdssdk.PinnedObject, error) {
+	var ret []cdssdk.PinnedObject
+	err := ctx.Table("PinnedObject").Find(&ret, "StorageID = ?", stgID).Error
+	return ret, err
+}
+
+func (*PinnedObjectDB) GetObjectsByStorageID(ctx SQLContext, stgID cdssdk.StorageID) ([]cdssdk.Object, error) {
+	var ret []cdssdk.Object
+	err := ctx.Table("Object").Joins("inner join PinnedObject on Object.ObjectID = PinnedObject.ObjectID").Where("StorageID = ?", stgID).Find(&ret).Error
+	return ret, err
+}
+
+func (*PinnedObjectDB) Create(ctx SQLContext, stgID cdssdk.StorageID, objectID cdssdk.ObjectID, createTime time.Time) error {
+	return ctx.Table("PinnedObject").Create(&cdssdk.PinnedObject{StorageID: stgID, ObjectID: objectID, CreateTime: createTime}).Error
+}
+
+func (*PinnedObjectDB) BatchGetByObjectID(ctx SQLContext, objectIDs []cdssdk.ObjectID) ([]cdssdk.PinnedObject, error) {
+	if len(objectIDs) == 0 {
+		return nil, nil
+	}
+
+	var pinneds []cdssdk.PinnedObject
+	err := ctx.Table("PinnedObject").Where("ObjectID in (?)", objectIDs).Order("ObjectID asc").Find(&pinneds).Error
+	return pinneds, err
+}
+
+func (*PinnedObjectDB) TryCreate(ctx SQLContext, stgID cdssdk.StorageID, objectID cdssdk.ObjectID, createTime time.Time) error {
+	err := ctx.Clauses(clause.Insert{Modifier: "ignore"}).Table("PinnedObject").Create(&cdssdk.PinnedObject{StorageID: stgID, ObjectID: objectID, CreateTime: createTime}).Error
+	return err
+}
+
+func (*PinnedObjectDB) BatchTryCreate(ctx SQLContext, pinneds []cdssdk.PinnedObject) error {
+	if len(pinneds) == 0 {
+		return nil
+	}
+
+	err := ctx.Clauses(clause.Insert{Modifier: "ignore"}).Table("PinnedObject").Create(pinneds).Error
+	return err
+}
+
+func (*PinnedObjectDB) CreateFromPackage(ctx SQLContext, packageID cdssdk.PackageID, stgID cdssdk.StorageID) error {
+	err := ctx.Exec(
+		"insert ignore into PinnedObject(StorageID, ObjectID, CreateTime) select ? as StorageID, ObjectID, ? as CreateTime from Object where PackageID = ?",
+		stgID,
+		time.Now(),
+		packageID,
+	).Error
+	return err
+}
+
+func (db *PinnedObjectDB) ObjectBatchCreate(ctx SQLContext, objectID cdssdk.ObjectID, stgIDs []cdssdk.StorageID) error {
+	if len(stgIDs) == 0 {
+		return nil
+	}
+
+	for _, id := range stgIDs {
+		err := db.TryCreate(ctx, id, objectID, time.Now())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (*PinnedObjectDB) Delete(ctx SQLContext, stgID cdssdk.StorageID, objectID cdssdk.ObjectID) error {
+	err := ctx.Exec("delete from PinnedObject where StorageID = ? and ObjectID = ?", stgID, objectID).Error
+	return err
+}
+
+func (*PinnedObjectDB) DeleteByObjectID(ctx SQLContext, objectID cdssdk.ObjectID) error {
+	err := ctx.Exec("delete from PinnedObject where ObjectID = ?", objectID).Error
+	return err
+}
+
+func (*PinnedObjectDB) BatchDeleteByObjectID(ctx SQLContext, objectIDs []cdssdk.ObjectID) error {
+	if len(objectIDs) == 0 {
+		return nil
+	}
+
+	err := ctx.Table("PinnedObject").Where("ObjectID in (?)", objectIDs).Delete(&cdssdk.PinnedObject{}).Error
+	return err
+}
+
+func (*PinnedObjectDB) DeleteInPackage(ctx SQLContext, packageID cdssdk.PackageID) error {
+	err := ctx.Table("PinnedObject").Where("ObjectID in (select ObjectID from Object where PackageID = ?)", packageID).Delete(&cdssdk.PinnedObject{}).Error
+	return err
+}
+
+func (*PinnedObjectDB) DeleteInPackageAtStorage(ctx SQLContext, packageID cdssdk.PackageID, stgID cdssdk.StorageID) error {
+	err := ctx.Exec("delete PinnedObject from PinnedObject inner join Object on PinnedObject.ObjectID = Object.ObjectID where PackageID = ? and StorageID = ?", packageID, stgID).Error
+	return err
+}
+
+func (*PinnedObjectDB) StorageBatchDelete(ctx SQLContext, stgID cdssdk.StorageID, objectIDs []cdssdk.ObjectID) error {
+	if len(objectIDs) == 0 {
+		return nil
+	}
+
+	err := ctx.Table("PinnedObject").Where("StorageID = ? and ObjectID in (?)", stgID, objectIDs).Delete(&cdssdk.PinnedObject{}).Error
+	return err
+}

common/pkgs/db2/shard_storage.go

@@ -0,0 +1,19 @@
+package db2
+
+import (
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+)
+
+type ShardStorageDB struct {
+	*DB
+}
+
+func (db *DB) ShardStorage() *ShardStorageDB {
+	return &ShardStorageDB{DB: db}
+}
+
+func (*ShardStorageDB) GetByStorageID(ctx SQLContext, stgID cdssdk.StorageID) (cdssdk.ShardStorage, error) {
+	var ret cdssdk.ShardStorage
+	err := ctx.Table("ShardStorage").First(&ret, stgID).Error
+	return ret, err
+}

common/pkgs/db2/shared_storage.go

@@ -0,0 +1,19 @@
+package db2
+
+import (
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+)
+
+type SharedStorageDB struct {
+	*DB
+}
+
+func (db *DB) SharedStorage() *SharedStorageDB {
+	return &SharedStorageDB{DB: db}
+}
+
+func (*SharedStorageDB) GetByStorageID(ctx SQLContext, stgID cdssdk.StorageID) (cdssdk.SharedStorage, error) {
+	var ret cdssdk.SharedStorage
+	err := ctx.Table("SharedStorage").First(&ret, stgID).Error
+	return ret, err
+}

common/pkgs/db2/storage.go

@@ -0,0 +1,62 @@
+package db2
+
+import (
+	"fmt"
+
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
+)
+
+type StorageDB struct {
+	*DB
+}
+
+func (db *DB) Storage() *StorageDB {
+	return &StorageDB{DB: db}
+}
+
+func (db *StorageDB) GetByID(ctx SQLContext, stgID cdssdk.StorageID) (model.Storage, error) {
+	var stg model.Storage
+	err := ctx.Table("Storage").First(&stg, stgID).Error
+	return stg, err
+}
+
+func (db *StorageDB) BatchGetAllStorageIDs(ctx SQLContext, start int, count int) ([]cdssdk.StorageID, error) {
+	var ret []cdssdk.StorageID
+	err := ctx.Table("Storage").Select("StorageID").Find(ret).Limit(count).Offset(start).Error
+	return ret, err
+}
+
+func (db *StorageDB) IsAvailable(ctx SQLContext, userID cdssdk.UserID, storageID cdssdk.StorageID) (bool, error) {
+	rows, err := ctx.Table("Storage").Select("Storage.StorageID").
+		Joins("inner join UserStorage on Storage.StorageID = UserStorage.StorageID").
+		Where("UserID = ? and StorageID = ?", userID, storageID).Rows()
+	if err != nil {
+		return false, fmt.Errorf("execute sql: %w", err)
+	}
+	defer rows.Close()
+
+	return rows.Next(), nil
+}
+
+func (db *StorageDB) GetUserStorage(ctx SQLContext, userID cdssdk.UserID, storageID cdssdk.StorageID) (model.Storage, error) {
+	var stg model.Storage
+	err := ctx.Table("Storage").Select("Storage.*").
+		Joins("inner join UserStorage on Storage.StorageID = UserStorage.StorageID").
+		Where("UserID = ? and StorageID = ?", userID, storageID).First(&stg).Error
+
+	return stg, err
+}
+
+func (db *StorageDB) GetUserStorageByName(ctx SQLContext, userID cdssdk.UserID, name string) (model.Storage, error) {
+	var stg model.Storage
+	err := ctx.Table("Storage").Select("Storage.*").
+		Joins("inner join UserStorage on Storage.StorageID = UserStorage.StorageID").
+		Where("UserID = ? and Name = ?", userID, name).First(&stg).Error
+
+	return stg, err
+}
+
+// func (db *StorageDB) ChangeState(ctx SQLContext, storageID cdssdk.StorageID, state string) error {
+// 	return ctx.Table("Storage").Where("StorageID = ?", storageID).Update("State", state).Error
+// }

common/pkgs/db2/union_serializer.go

@@ -0,0 +1,44 @@
+package db2
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"gitlink.org.cn/cloudream/common/utils/serder"
+	"gorm.io/gorm/schema"
+)
+
+type UnionSerializer struct {
+}
+
+func (UnionSerializer) Scan(ctx context.Context, field *schema.Field, dst reflect.Value, dbValue interface{}) error {
+	fieldValue := reflect.New(field.FieldType)
+	if dbValue != nil {
+		var data []byte
+		switch v := dbValue.(type) {
+		case []byte:
+			data = v
+		case string:
+			data = []byte(v)
+		default:
+			return fmt.Errorf("failed to unmarshal JSONB value: %#v", dbValue)
+		}
+
+		err := serder.JSONToObjectExRaw(data, fieldValue.Interface())
+		if err != nil {
+			return err
+		}
+	}
+
+	field.ReflectValueOf(ctx, dst).Set(fieldValue.Elem())
+	return nil
+}
+
+func (UnionSerializer) Value(ctx context.Context, field *schema.Field, dst reflect.Value, fieldValue interface{}) (interface{}, error) {
+	return serder.ObjectToJSONEx(fieldValue)
+}
+
+func init() {
+	schema.RegisterSerializer("union", UnionSerializer{})
+}

common/pkgs/distlock/service.go

@@ -8,6 +8,8 @@ import (
 
 type Service = distlock.Service
 
+type Mutex = distlock.Mutex
+
 func NewService(cfg *distlock.Config) (*distlock.Service, error) {
 	srv, err := distlock.NewService(cfg, initProviders())
 	if err != nil {

common/pkgs/downloader/cache.go

@@ -0,0 +1,104 @@
+package downloader
+
+import (
+	"sort"
+
+	lru "github.com/hashicorp/golang-lru/v2"
+	"gitlink.org.cn/cloudream/common/utils/lo2"
+)
+
+type LRUID int64
+
+type cacheFile struct {
+	Segments []*fileSegment
+}
+
+type fileSegment struct {
+	LRUID  LRUID
+	Offset int64
+	Data   []byte
+}
+
+type lruEntry struct {
+	FileHash string
+	Offset   int64
+}
+
+type Cache struct {
+	lru       *lru.Cache[LRUID, lruEntry]
+	nextLRUID LRUID
+	files     map[string]*cacheFile
+}
+
+func NewCache(size int) *Cache {
+	c := &Cache{
+		files: make(map[string]*cacheFile),
+	}
+
+	lru, _ := lru.NewWithEvict(size, c.onEvict)
+	c.lru = lru
+
+	return c
+}
+
+func (c *Cache) Put(fileHash string, offset int64, data []byte) {
+	file, ok := c.files[fileHash]
+	if !ok {
+		file = &cacheFile{}
+		c.files[fileHash] = file
+	}
+
+	idx := sort.Search(len(file.Segments), upperBound(file.Segments, offset))
+
+	// 允许不同Segment之间有重叠，只在Offset相等时替换数据
+	if idx < len(file.Segments) && file.Segments[idx].Offset == offset {
+		file.Segments[idx].Data = data
+		// Get一下更新LRU
+		c.lru.Get(file.Segments[idx].LRUID)
+	} else {
+		file.Segments = lo2.Insert(file.Segments, idx, &fileSegment{
+			LRUID:  c.nextLRUID,
+			Offset: offset,
+			Data:   data,
+		})
+		c.lru.Add(c.nextLRUID, lruEntry{
+			FileHash: fileHash,
+			Offset:   offset,
+		})
+		c.nextLRUID++
+	}
+}
+
+func (c *Cache) Get(fileHash string, offset int64) []byte {
+	file, ok := c.files[fileHash]
+	if !ok {
+		return nil
+	}
+
+	idx := sort.Search(len(file.Segments), upperBound(file.Segments, offset))
+	if idx == 0 {
+		return nil
+	}
+	seg := file.Segments[idx-1]
+	// Get一下更新LRU
+	c.lru.Get(seg.LRUID)
+
+	return seg.Data[offset-seg.Offset:]
+}
+
+func (c *Cache) onEvict(key LRUID, value lruEntry) {
+	// 不应该找不到文件或者分片
+	file := c.files[value.FileHash]
+	idx := sort.Search(len(file.Segments), upperBound(file.Segments, value.Offset))
+	file.Segments = lo2.RemoveAt(file.Segments, idx)
+	if len(file.Segments) == 0 {
+		delete(c.files, value.FileHash)
+	}
+}
+
+// 使用此函数会找到第一个大于等于 target 的索引，如果找不到，则返回 len(seg)
+func upperBound(seg []*fileSegment, target int64) func(int) bool {
+	return func(i int) bool {
+		return seg[i].Offset >= target
+	}
+}

common/pkgs/downloader/config.go

@@ -3,4 +3,8 @@ package downloader
 type Config struct {
 	// EC模式的Object的条带缓存数量
 	MaxStripCacheCount int `json:"maxStripCacheCount"`
+	// 当到下载节点的延迟高于这个值时，该节点在评估时会有更高的分数惩罚，单位：ms
+	HighLatencyNodeMs float64 `json:"highLatencyNodeMs"`
+	// EC模式下，每个Object的条带的预取数量，最少为1
+	ECStripPrefetchCount int `json:"ecStripPrefetchCount"`
 }

common/pkgs/downloader/downloader.go

@@ -9,6 +9,7 @@ import (
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
 	stgmod "gitlink.org.cn/cloudream/storage/common/models"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/connectivity"
 	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
 )
 
@@ -37,9 +38,11 @@ type Downloading struct {
 
 type Downloader struct {
 	strips *StripCache
+	conn   *connectivity.Collector
+	cfg    Config
 }
 
-func NewDownloader(cfg Config) Downloader {
+func NewDownloader(cfg Config, conn *connectivity.Collector) Downloader {
 	if cfg.MaxStripCacheCount == 0 {
 		cfg.MaxStripCacheCount = DefaultMaxStripCacheCount
 	}
@@ -47,6 +50,8 @@ func NewDownloader(cfg Config) Downloader {
 	ch, _ := lru.New[ECStripKey, ObjectECStrip](cfg.MaxStripCacheCount)
 	return Downloader{
 		strips: ch,
+		conn:   conn,
+		cfg:    cfg,
 	}
 }
 
@@ -116,8 +121,8 @@ type ObjectECStrip struct {
 }
 
 type ECStripKey struct {
-	ObjectID      cdssdk.ObjectID
-	StripPosition int64
+	ObjectID   cdssdk.ObjectID
+	StripIndex int64
 }
 
 type StripCache = lru.Cache[ECStripKey, ObjectECStrip]

common/pkgs/downloader/io.go

@@ -1,148 +0,0 @@
-package downloader
-
-import (
-	"fmt"
-	"io"
-
-	"gitlink.org.cn/cloudream/common/pkgs/ipfs"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch/plans"
-)
-
-type IPFSReader struct {
-	node     cdssdk.Node
-	fileHash string
-	stream   io.ReadCloser
-	offset   int64
-}
-
-func NewIPFSReader(node cdssdk.Node, fileHash string) *IPFSReader {
-	return &IPFSReader{
-		node:     node,
-		fileHash: fileHash,
-	}
-}
-
-func NewIPFSReaderWithRange(node cdssdk.Node, fileHash string, rng ipfs.ReadOption) io.ReadCloser {
-	str := &IPFSReader{
-		node:     node,
-		fileHash: fileHash,
-	}
-	str.Seek(rng.Offset, io.SeekStart)
-	if rng.Length > 0 {
-		return io2.Length(str, rng.Length)
-	}
-
-	return str
-}
-
-func (r *IPFSReader) Seek(offset int64, whence int) (int64, error) {
-	if whence == io.SeekEnd {
-		return 0, fmt.Errorf("seek end not supported")
-	}
-
-	if whence == io.SeekCurrent {
-		return 0, fmt.Errorf("seek current not supported")
-	}
-
-	if r.stream == nil {
-		r.offset = offset
-		return r.offset, nil
-	}
-
-	// 如果文件流已经打开，那么如果seek的位置和当前位置不同，那么需要重新打开文件流
-	if offset != r.offset {
-		var err error
-		r.stream.Close()
-		r.offset = offset
-		r.stream, err = r.openStream()
-		if err != nil {
-			return 0, fmt.Errorf("reopen stream: %w", err)
-		}
-	}
-
-	return r.offset, nil
-}
-
-func (r *IPFSReader) Read(buf []byte) (int, error) {
-	if r.stream == nil {
-		var err error
-		r.stream, err = r.openStream()
-		if err != nil {
-			return 0, err
-		}
-	}
-
-	n, err := r.stream.Read(buf)
-	r.offset += int64(n)
-	return n, err
-}
-
-func (r *IPFSReader) Close() error {
-	if r.stream != nil {
-		return r.stream.Close()
-	}
-
-	return nil
-}
-
-func (r *IPFSReader) openStream() (io.ReadCloser, error) {
-	if stgglb.IPFSPool != nil {
-		logger.Infof("try to use local IPFS to download file")
-
-		reader, err := r.fromLocalIPFS()
-		if err == nil {
-			return reader, nil
-		}
-
-		logger.Warnf("download from local IPFS failed, so try to download from node %v, err: %s", r.node.Name, err.Error())
-	}
-
-	return r.fromNode()
-}
-
-func (r *IPFSReader) fromNode() (io.ReadCloser, error) {
-	planBld := plans.NewPlanBuilder()
-	fileStr := planBld.AtAgent(r.node).IPFSRead(r.fileHash, ipfs.ReadOption{
-		Offset: r.offset,
-		Length: -1,
-	}).ToExecutor()
-
-	plan, err := planBld.Build()
-	if err != nil {
-		return nil, fmt.Errorf("building plan: %w", err)
-	}
-
-	waiter, err := plans.Execute(*plan)
-	if err != nil {
-		return nil, fmt.Errorf("execute plan: %w", err)
-	}
-	go func() {
-		waiter.Wait()
-	}()
-
-	return waiter.ReadStream(fileStr)
-}
-
-func (r *IPFSReader) fromLocalIPFS() (io.ReadCloser, error) {
-	ipfsCli, err := stgglb.IPFSPool.Acquire()
-	if err != nil {
-		return nil, fmt.Errorf("new ipfs client: %w", err)
-	}
-
-	reader, err := ipfsCli.OpenRead(r.fileHash, ipfs.ReadOption{
-		Offset: r.offset,
-		Length: -1,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("read ipfs file failed, err: %w", err)
-	}
-
-	reader = io2.AfterReadClosed(reader, func(io.ReadCloser) {
-		ipfsCli.Close()
-	})
-	return reader, nil
-}

common/pkgs/downloader/iterator.go

@@ -1,32 +1,33 @@
 package downloader
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"math"
 	"reflect"
+	"time"
 
 	"github.com/samber/lo"
 
 	"gitlink.org.cn/cloudream/common/pkgs/bitmap"
-	"gitlink.org.cn/cloudream/common/pkgs/ipfs"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
 	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
 
 	"gitlink.org.cn/cloudream/common/utils/io2"
 	"gitlink.org.cn/cloudream/common/utils/math2"
 	"gitlink.org.cn/cloudream/common/utils/sort2"
-	"gitlink.org.cn/cloudream/common/utils/sync2"
 	"gitlink.org.cn/cloudream/storage/common/consts"
 	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
 	stgmod "gitlink.org.cn/cloudream/storage/common/models"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/distlock"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ec"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/parser"
 	"gitlink.org.cn/cloudream/storage/common/pkgs/iterator"
 	coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
 )
 
-var errNoDirectReadBlock = fmt.Errorf("no direct read block")
-
 type DownloadNodeInfo struct {
 	Node         cdssdk.Node
 	ObjectPinned bool
@@ -83,6 +84,10 @@ func (i *DownloadObjectIterator) init() error {
 
 	allNodeIDs := make(map[cdssdk.NodeID]bool)
 	for _, obj := range i.reqs {
+		if obj.Detail == nil {
+			continue
+		}
+
 		for _, p := range obj.Detail.PinnedAt {
 			allNodeIDs[p] = true
 		}
@@ -119,7 +124,7 @@ func (iter *DownloadObjectIterator) doMove() (*Downloading, error) {
 	case *cdssdk.NoneRedundancy:
 		reader, err := iter.downloadNoneOrRepObject(req)
 		if err != nil {
-			return nil, fmt.Errorf("downloading object: %w", err)
+			return nil, fmt.Errorf("downloading object %v: %w", req.Raw.ObjectID, err)
 		}
 
 		return &Downloading{
@@ -131,7 +136,7 @@ func (iter *DownloadObjectIterator) doMove() (*Downloading, error) {
 	case *cdssdk.RepRedundancy:
 		reader, err := iter.downloadNoneOrRepObject(req)
 		if err != nil {
-			return nil, fmt.Errorf("downloading rep object: %w", err)
+			return nil, fmt.Errorf("downloading rep object %v: %w", req.Raw.ObjectID, err)
 		}
 
 		return &Downloading{
@@ -143,7 +148,19 @@ func (iter *DownloadObjectIterator) doMove() (*Downloading, error) {
 	case *cdssdk.ECRedundancy:
 		reader, err := iter.downloadECObject(req, red)
 		if err != nil {
-			return nil, fmt.Errorf("downloading ec object: %w", err)
+			return nil, fmt.Errorf("downloading ec object %v: %w", req.Raw.ObjectID, err)
+		}
+
+		return &Downloading{
+			Object:  &req.Detail.Object,
+			File:    reader,
+			Request: req.Raw,
+		}, nil
+
+	case *cdssdk.LRCRedundancy:
+		reader, err := iter.downloadLRCObject(req, red)
+		if err != nil {
+			return nil, fmt.Errorf("downloading lrc object %v: %w", req.Raw.ObjectID, err)
 		}
 
 		return &Downloading{
@@ -153,7 +170,7 @@ func (iter *DownloadObjectIterator) doMove() (*Downloading, error) {
 		}, nil
 	}
 
-	return nil, fmt.Errorf("unsupported redundancy type: %v", reflect.TypeOf(req.Detail.Object.Redundancy))
+	return nil, fmt.Errorf("unsupported redundancy type: %v of object %v", reflect.TypeOf(req.Detail.Object.Redundancy), req.Raw.ObjectID)
 }
 
 func (i *DownloadObjectIterator) Close() {
@@ -171,22 +188,17 @@ func (iter *DownloadObjectIterator) downloadNoneOrRepObject(obj downloadReqeust2
 	bsc, blocks := iter.getMinReadingBlockSolution(allNodes, 1)
 	osc, node := iter.getMinReadingObjectSolution(allNodes, 1)
 	if bsc < osc {
-
-		return NewIPFSReaderWithRange(blocks[0].Node, blocks[0].Block.FileHash, ipfs.ReadOption{
-			Offset: obj.Raw.Offset,
-			Length: obj.Raw.Length,
-		}), nil
+		logger.Debugf("downloading object %v from node %v(%v)", obj.Raw.ObjectID, blocks[0].Node.Name, blocks[0].Node.NodeID)
+		return iter.downloadFromNode(&blocks[0].Node, obj)
 	}
 
-	// bsc >= osc，如果osc是MaxFloat64，那么bsc也一定是，也就意味着没有足够块来恢复文件
 	if osc == math.MaxFloat64 {
+		// bsc >= osc，如果osc是MaxFloat64，那么bsc也一定是，也就意味着没有足够块来恢复文件
 		return nil, fmt.Errorf("no node has this object")
 	}
 
-	return NewIPFSReaderWithRange(*node, obj.Detail.Object.FileHash, ipfs.ReadOption{
-		Offset: obj.Raw.Offset,
-		Length: obj.Raw.Length,
-	}), nil
+	logger.Debugf("downloading object %v from node %v(%v)", obj.Raw.ObjectID, node.Name, node.NodeID)
+	return iter.downloadFromNode(node, obj)
 }
 
 func (iter *DownloadObjectIterator) downloadECObject(req downloadReqeust2, ecRed *cdssdk.ECRedundancy) (io.ReadCloser, error) {
@@ -199,99 +211,51 @@ func (iter *DownloadObjectIterator) downloadECObject(req downloadReqeust2, ecRed
 	osc, node := iter.getMinReadingObjectSolution(allNodes, ecRed.K)
 
 	if bsc < osc {
-		var fileStrs []*IPFSReader
-		for _, b := range blocks {
-			str := NewIPFSReader(b.Node, b.Block.FileHash)
-
-			fileStrs = append(fileStrs, str)
-		}
-
-		rs, err := ec.NewRs(ecRed.K, ecRed.N)
-		if err != nil {
-			return nil, fmt.Errorf("new rs: %w", err)
+		var logStrs []any = []any{fmt.Sprintf("downloading ec object %v from blocks: ", req.Raw.ObjectID)}
+		for i, b := range blocks {
+			if i > 0 {
+				logStrs = append(logStrs, ", ")
+			}
+			logStrs = append(logStrs, fmt.Sprintf("%v@%v(%v)", b.Block.Index, b.Node.Name, b.Node.NodeID))
 		}
+		logger.Debug(logStrs...)
 
 		pr, pw := io.Pipe()
 		go func() {
-			defer func() {
-				for _, str := range fileStrs {
-					str.Close()
-				}
-			}()
-
 			readPos := req.Raw.Offset
 			totalReadLen := req.Detail.Object.Size - req.Raw.Offset
 			if req.Raw.Length >= 0 {
-				totalReadLen = req.Raw.Length
+				totalReadLen = math2.Min(req.Raw.Length, totalReadLen)
 			}
 
-			for totalReadLen > 0 {
-				curStripPos := readPos / int64(ecRed.K) / int64(ecRed.ChunkSize)
-				curStripPosInBytes := curStripPos * int64(ecRed.K) * int64(ecRed.ChunkSize)
-				nextStripPosInBytes := (curStripPos + 1) * int64(ecRed.K) * int64(ecRed.ChunkSize)
-				curReadLen := math2.Min(totalReadLen, nextStripPosInBytes-readPos)
-				readRelativePos := readPos - curStripPosInBytes
-				cacheKey := ECStripKey{
-					ObjectID:      req.Detail.Object.ObjectID,
-					StripPosition: curStripPos,
-				}
-
-				cache, ok := iter.downloader.strips.Get(cacheKey)
-				if ok {
-					if cache.ObjectFileHash == req.Detail.Object.FileHash {
-						err := io2.WriteAll(pw, cache.Data[readRelativePos:readRelativePos+curReadLen])
-						if err != nil {
-							pw.CloseWithError(err)
-							break
-						}
-						totalReadLen -= curReadLen
-						readPos += curReadLen
-						continue
-					}
-
-					// 如果Object的Hash和Cache的Hash不一致，说明Cache是无效的，需要重新下载
-					iter.downloader.strips.Remove(cacheKey)
-				}
-
-				for _, str := range fileStrs {
-					_, err := str.Seek(curStripPosInBytes, io.SeekStart)
-					if err != nil {
-						pw.CloseWithError(err)
-						break
-					}
-				}
+			firstStripIndex := readPos / ecRed.StripSize()
+			stripIter := NewStripIterator(req.Detail.Object, blocks, ecRed, firstStripIndex, iter.downloader.strips, iter.downloader.cfg.ECStripPrefetchCount)
+			defer stripIter.Close()
 
-				dataBuf := make([]byte, int64(ecRed.K*ecRed.ChunkSize))
-				blockArrs := make([][]byte, ecRed.N)
-				for _, b := range blocks {
-					if b.Block.Index < ecRed.K {
-						blockArrs[b.Block.Index] = dataBuf[b.Block.Index*ecRed.ChunkSize : (b.Block.Index+1)*ecRed.ChunkSize]
-					} else {
-						blockArrs[b.Block.Index] = make([]byte, ecRed.ChunkSize)
-					}
+			for totalReadLen > 0 {
+				strip, err := stripIter.MoveNext()
+				if err == iterator.ErrNoMoreItem {
+					pw.CloseWithError(io.ErrUnexpectedEOF)
+					return
 				}
-
-				err := sync2.ParallelDo(blocks, func(b downloadBlock, idx int) error {
-					_, err := io.ReadFull(fileStrs[idx], blockArrs[b.Block.Index])
-					return err
-				})
 				if err != nil {
 					pw.CloseWithError(err)
-					break
+					return
 				}
 
-				err = rs.ReconstructData(blockArrs)
+				readRelativePos := readPos - strip.Position
+				curReadLen := math2.Min(totalReadLen, ecRed.StripSize()-readRelativePos)
+
+				err = io2.WriteAll(pw, strip.Data[readRelativePos:readRelativePos+curReadLen])
 				if err != nil {
 					pw.CloseWithError(err)
-					break
+					return
 				}
 
-				iter.downloader.strips.Add(cacheKey, ObjectECStrip{
-					Data:           dataBuf,
-					ObjectFileHash: req.Detail.Object.FileHash,
-				})
-				// 下次循环就能从Cache中读取数据
+				totalReadLen -= curReadLen
+				readPos += curReadLen
 			}
+			pw.Close()
 		}()
 
 		return pr, nil
@@ -299,13 +263,11 @@ func (iter *DownloadObjectIterator) downloadECObject(req downloadReqeust2, ecRed
 
 	// bsc >= osc，如果osc是MaxFloat64，那么bsc也一定是，也就意味着没有足够块来恢复文件
 	if osc == math.MaxFloat64 {
-		return nil, fmt.Errorf("no enough blocks to reconstruct the file, want %d, get only %d", ecRed.K, len(blocks))
+		return nil, fmt.Errorf("no enough blocks to reconstruct the object %v , want %d, get only %d", req.Raw.ObjectID, ecRed.K, len(blocks))
 	}
 
-	return NewIPFSReaderWithRange(*node, req.Detail.Object.FileHash, ipfs.ReadOption{
-		Offset: req.Raw.Offset,
-		Length: req.Raw.Length,
-	}), nil
+	logger.Debugf("downloading ec object %v from node %v(%v)", req.Raw.ObjectID, node.Name, node.NodeID)
+	return iter.downloadFromNode(node, req)
 }
 
 func (iter *DownloadObjectIterator) sortDownloadNodes(req downloadReqeust2) ([]*DownloadNodeInfo, error) {
@@ -356,11 +318,6 @@ func (iter *DownloadObjectIterator) sortDownloadNodes(req downloadReqeust2) ([]*
 	}), nil
 }
 
-type downloadBlock struct {
-	Node  cdssdk.Node
-	Block stgmod.ObjectBlock
-}
-
 func (iter *DownloadObjectIterator) getMinReadingBlockSolution(sortedNodes []*DownloadNodeInfo, k int) (float64, []downloadBlock) {
 	gotBlocksMap := bitmap.Bitmap64(0)
 	var gotBlocks []downloadBlock
@@ -391,7 +348,8 @@ func (iter *DownloadObjectIterator) getMinReadingObjectSolution(sortedNodes []*D
 	for _, n := range sortedNodes {
 		if n.ObjectPinned && float64(k)*n.Distance < dist {
 			dist = float64(k) * n.Distance
-			downloadNode = &n.Node
+			node := n.Node
+			downloadNode = &node
 		}
 	}
 
@@ -409,5 +367,40 @@ func (iter *DownloadObjectIterator) getNodeDistance(node cdssdk.Node) float64 {
 		return consts.NodeDistanceSameLocation
 	}
 
+	c := iter.downloader.conn.Get(node.NodeID)
+	if c == nil || c.Delay == nil || *c.Delay > time.Duration(float64(time.Millisecond)*iter.downloader.cfg.HighLatencyNodeMs) {
+		return consts.NodeDistanceHighLatencyNode
+	}
+
 	return consts.NodeDistanceOther
 }
+
+func (iter *DownloadObjectIterator) downloadFromNode(node *cdssdk.Node, req downloadReqeust2) (io.ReadCloser, error) {
+	var strHandle *exec.DriverReadStream
+	ft := ioswitch2.NewFromTo()
+
+	toExec, handle := ioswitch2.NewToDriver(-1)
+	toExec.Range = exec.Range{
+		Offset: req.Raw.Offset,
+	}
+	if req.Raw.Length != -1 {
+		len := req.Raw.Length
+		toExec.Range.Length = &len
+	}
+	ft.AddFrom(ioswitch2.NewFromNode(req.Detail.Object.FileHash, node, -1)).AddTo(toExec)
+	strHandle = handle
+
+	parser := parser.NewParser(cdssdk.DefaultECRedundancy)
+	plans := exec.NewPlanBuilder()
+	if err := parser.Parse(ft, plans); err != nil {
+		return nil, fmt.Errorf("parsing plan: %w", err)
+	}
+
+	// TODO2 注入依赖
+	exeCtx := exec.NewExecContext()
+
+	exec := plans.Execute(exeCtx)
+	go exec.Wait(context.TODO())
+
+	return exec.BeginRead(strHandle)
+}

common/pkgs/downloader/lrc.go

@@ -0,0 +1,87 @@
+package downloader
+
+import (
+	"fmt"
+	"io"
+
+	"gitlink.org.cn/cloudream/common/pkgs/iterator"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/common/utils/io2"
+	"gitlink.org.cn/cloudream/common/utils/math2"
+)
+
+func (iter *DownloadObjectIterator) downloadLRCObject(req downloadReqeust2, red *cdssdk.LRCRedundancy) (io.ReadCloser, error) {
+	allNodes, err := iter.sortDownloadNodes(req)
+	if err != nil {
+		return nil, err
+	}
+
+	var blocks []downloadBlock
+	selectedBlkIdx := make(map[int]bool)
+	for _, node := range allNodes {
+		for _, b := range node.Blocks {
+			if b.Index >= red.M() || selectedBlkIdx[b.Index] {
+				continue
+			}
+			blocks = append(blocks, downloadBlock{
+				Node:  node.Node,
+				Block: b,
+			})
+			selectedBlkIdx[b.Index] = true
+		}
+	}
+	if len(blocks) < red.K {
+		return nil, fmt.Errorf("not enough blocks to download lrc object")
+	}
+
+	var logStrs []any = []any{"downloading lrc object from blocks: "}
+	for i, b := range blocks {
+		if i > 0 {
+			logStrs = append(logStrs, ", ")
+		}
+		logStrs = append(logStrs, fmt.Sprintf("%v@%v(%v)", b.Block.Index, b.Node.Name, b.Node.NodeID))
+	}
+	logger.Debug(logStrs...)
+
+	pr, pw := io.Pipe()
+	go func() {
+		readPos := req.Raw.Offset
+		totalReadLen := req.Detail.Object.Size - req.Raw.Offset
+		if req.Raw.Length >= 0 {
+			totalReadLen = math2.Min(req.Raw.Length, totalReadLen)
+		}
+
+		firstStripIndex := readPos / int64(red.K) / int64(red.ChunkSize)
+		stripIter := NewLRCStripIterator(req.Detail.Object, blocks, red, firstStripIndex, iter.downloader.strips, iter.downloader.cfg.ECStripPrefetchCount)
+		defer stripIter.Close()
+
+		for totalReadLen > 0 {
+			strip, err := stripIter.MoveNext()
+			if err == iterator.ErrNoMoreItem {
+				pw.CloseWithError(io.ErrUnexpectedEOF)
+				return
+			}
+			if err != nil {
+				pw.CloseWithError(err)
+				return
+			}
+
+			readRelativePos := readPos - strip.Position
+			nextStripPos := strip.Position + int64(red.K)*int64(red.ChunkSize)
+			curReadLen := math2.Min(totalReadLen, nextStripPos-readPos)
+
+			err = io2.WriteAll(pw, strip.Data[readRelativePos:readRelativePos+curReadLen])
+			if err != nil {
+				pw.CloseWithError(err)
+				return
+			}
+
+			totalReadLen -= curReadLen
+			readPos += curReadLen
+		}
+		pw.Close()
+	}()
+
+	return pr, nil
+}

common/pkgs/downloader/lrc_strip_iterator.go

@@ -0,0 +1,196 @@
+package downloader
+
+import (
+	"context"
+	"io"
+	"sync"
+
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
+	"gitlink.org.cn/cloudream/common/pkgs/iterator"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitchlrc/parser"
+)
+
+type LRCStripIterator struct {
+	object              cdssdk.Object
+	blocks              []downloadBlock
+	red                 *cdssdk.LRCRedundancy
+	curStripIndex       int64
+	cache               *StripCache
+	dataChan            chan dataChanEntry
+	downloadingDone     chan any
+	downloadingDoneOnce sync.Once
+	inited              bool
+}
+
+func NewLRCStripIterator(object cdssdk.Object, blocks []downloadBlock, red *cdssdk.LRCRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *LRCStripIterator {
+	if maxPrefetch <= 0 {
+		maxPrefetch = 1
+	}
+
+	iter := &LRCStripIterator{
+		object:          object,
+		blocks:          blocks,
+		red:             red,
+		curStripIndex:   beginStripIndex,
+		cache:           cache,
+		dataChan:        make(chan dataChanEntry, maxPrefetch-1),
+		downloadingDone: make(chan any),
+	}
+
+	return iter
+}
+
+func (s *LRCStripIterator) MoveNext() (Strip, error) {
+	if !s.inited {
+		go s.downloading()
+		s.inited = true
+	}
+
+	// 先尝试获取一下，用于判断本次获取是否发生了等待
+	select {
+	case entry, ok := <-s.dataChan:
+		if !ok || entry.Error == io.EOF {
+			return Strip{}, iterator.ErrNoMoreItem
+		}
+
+		if entry.Error != nil {
+			return Strip{}, entry.Error
+		}
+
+		s.curStripIndex++
+		return Strip{Data: entry.Data, Position: entry.Position}, nil
+
+	default:
+		logger.Debugf("waitting for ec strip %v of object %v", s.curStripIndex, s.object.ObjectID)
+	}
+
+	// 发生了等待
+	select {
+	case entry, ok := <-s.dataChan:
+		if !ok || entry.Error == io.EOF {
+			return Strip{}, iterator.ErrNoMoreItem
+		}
+
+		if entry.Error != nil {
+			return Strip{}, entry.Error
+		}
+
+		s.curStripIndex++
+		return Strip{Data: entry.Data, Position: entry.Position}, nil
+
+	case <-s.downloadingDone:
+		return Strip{}, iterator.ErrNoMoreItem
+	}
+}
+
+func (s *LRCStripIterator) Close() {
+	s.downloadingDoneOnce.Do(func() {
+		close(s.downloadingDone)
+	})
+}
+
+func (s *LRCStripIterator) downloading() {
+	var froms []ioswitchlrc.From
+	for _, b := range s.blocks {
+		node := b.Node
+		froms = append(froms, ioswitchlrc.NewFromNode(b.Block.FileHash, &node, b.Block.Index))
+	}
+
+	toExec, hd := ioswitchlrc.NewToDriverWithRange(-1, exec.Range{
+		Offset: s.curStripIndex * int64(s.red.ChunkSize*s.red.K),
+	})
+
+	plans := exec.NewPlanBuilder()
+	err := parser.ReconstructAny(froms, []ioswitchlrc.To{toExec}, plans)
+	if err != nil {
+		s.sendToDataChan(dataChanEntry{Error: err})
+		return
+	}
+
+	// TODO2 注入依赖
+	exeCtx := exec.NewExecContext()
+
+	exec := plans.Execute(exeCtx)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go exec.Wait(ctx)
+	defer cancel()
+
+	str, err := exec.BeginRead(hd)
+	if err != nil {
+		s.sendToDataChan(dataChanEntry{Error: err})
+		return
+	}
+
+	curStripIndex := s.curStripIndex
+loop:
+	for {
+		stripBytesPos := curStripIndex * int64(s.red.K) * int64(s.red.ChunkSize)
+		if stripBytesPos >= s.object.Size {
+			s.sendToDataChan(dataChanEntry{Error: io.EOF})
+			break
+		}
+
+		stripKey := ECStripKey{
+			ObjectID:   s.object.ObjectID,
+			StripIndex: curStripIndex,
+		}
+
+		item, ok := s.cache.Get(stripKey)
+		if ok {
+			if item.ObjectFileHash == s.object.FileHash {
+				if !s.sendToDataChan(dataChanEntry{Data: item.Data, Position: stripBytesPos}) {
+					break loop
+				}
+				curStripIndex++
+				continue
+
+			} else {
+				// 如果Object的Hash和Cache的Hash不一致，说明Cache是无效的，需要重新下载
+				s.cache.Remove(stripKey)
+			}
+		}
+
+		dataBuf := make([]byte, int64(s.red.K*s.red.ChunkSize))
+		n, err := io.ReadFull(str, dataBuf)
+		if err == io.ErrUnexpectedEOF {
+			s.cache.Add(stripKey, ObjectECStrip{
+				Data:           dataBuf,
+				ObjectFileHash: s.object.FileHash,
+			})
+
+			s.sendToDataChan(dataChanEntry{Data: dataBuf[:n], Position: stripBytesPos})
+			s.sendToDataChan(dataChanEntry{Error: io.EOF})
+			break loop
+		}
+		if err != nil {
+			s.sendToDataChan(dataChanEntry{Error: err})
+			break loop
+		}
+
+		s.cache.Add(stripKey, ObjectECStrip{
+			Data:           dataBuf,
+			ObjectFileHash: s.object.FileHash,
+		})
+
+		if !s.sendToDataChan(dataChanEntry{Data: dataBuf, Position: stripBytesPos}) {
+			break loop
+		}
+
+		curStripIndex++
+	}
+
+	close(s.dataChan)
+}
+
+func (s *LRCStripIterator) sendToDataChan(entry dataChanEntry) bool {
+	select {
+	case s.dataChan <- entry:
+		return true
+	case <-s.downloadingDone:
+		return false
+	}
+}

common/pkgs/downloader/strip_iterator.go

@@ -0,0 +1,235 @@
+package downloader
+
+import (
+	"context"
+	"io"
+	"sync"
+
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
+	"gitlink.org.cn/cloudream/common/pkgs/iterator"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
+	stgmod "gitlink.org.cn/cloudream/storage/common/models"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2"
+	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch2/parser"
+)
+
+type downloadBlock struct {
+	Node  cdssdk.Node
+	Block stgmod.ObjectBlock
+}
+
+type Strip struct {
+	Data     []byte
+	Position int64
+}
+
+type StripIterator struct {
+	object                   cdssdk.Object
+	blocks                   []downloadBlock
+	red                      *cdssdk.ECRedundancy
+	curStripIndex            int64
+	cache                    *StripCache
+	dataChan                 chan dataChanEntry
+	downloadingDone          chan any
+	downloadingDoneOnce      sync.Once
+	inited                   bool
+	downloadingStream        io.ReadCloser
+	downloadingStripIndex    int64
+	downloadingPlanCtxCancel func()
+}
+
+type dataChanEntry struct {
+	Data     []byte
+	Position int64 // 条带在文件中的位置。字节为单位
+	Error    error
+}
+
+func NewStripIterator(object cdssdk.Object, blocks []downloadBlock, red *cdssdk.ECRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *StripIterator {
+	if maxPrefetch <= 0 {
+		maxPrefetch = 1
+	}
+
+	iter := &StripIterator{
+		object:          object,
+		blocks:          blocks,
+		red:             red,
+		curStripIndex:   beginStripIndex,
+		cache:           cache,
+		dataChan:        make(chan dataChanEntry, maxPrefetch-1),
+		downloadingDone: make(chan any),
+	}
+
+	return iter
+}
+
+func (s *StripIterator) MoveNext() (Strip, error) {
+	if !s.inited {
+		go s.downloading(s.curStripIndex)
+		s.inited = true
+	}
+
+	// 先尝试获取一下，用于判断本次获取是否发生了等待
+	select {
+	case entry, ok := <-s.dataChan:
+		if !ok || entry.Error == io.EOF {
+			return Strip{}, iterator.ErrNoMoreItem
+		}
+
+		if entry.Error != nil {
+			return Strip{}, entry.Error
+		}
+
+		s.curStripIndex++
+		return Strip{Data: entry.Data, Position: entry.Position}, nil
+
+	default:
+		logger.Debugf("waitting for ec strip %v of object %v", s.curStripIndex, s.object.ObjectID)
+	}
+
+	// 发生了等待
+	select {
+	case entry, ok := <-s.dataChan:
+		if !ok || entry.Error == io.EOF {
+			return Strip{}, iterator.ErrNoMoreItem
+		}
+
+		if entry.Error != nil {
+			return Strip{}, entry.Error
+		}
+
+		s.curStripIndex++
+		return Strip{Data: entry.Data, Position: entry.Position}, nil
+
+	case <-s.downloadingDone:
+		return Strip{}, iterator.ErrNoMoreItem
+	}
+}
+
+func (s *StripIterator) Close() {
+	s.downloadingDoneOnce.Do(func() {
+		close(s.downloadingDone)
+	})
+}
+
+func (s *StripIterator) downloading(startStripIndex int64) {
+	curStripIndex := startStripIndex
+loop:
+	for {
+		stripBytesPos := curStripIndex * int64(s.red.K) * int64(s.red.ChunkSize)
+		if stripBytesPos >= s.object.Size {
+			s.sendToDataChan(dataChanEntry{Error: io.EOF})
+			break
+		}
+
+		stripKey := ECStripKey{
+			ObjectID:   s.object.ObjectID,
+			StripIndex: curStripIndex,
+		}
+
+		item, ok := s.cache.Get(stripKey)
+		if ok {
+			if item.ObjectFileHash == s.object.FileHash {
+				if !s.sendToDataChan(dataChanEntry{Data: item.Data, Position: stripBytesPos}) {
+					break loop
+				}
+				curStripIndex++
+				continue
+
+			} else {
+				// 如果Object的Hash和Cache的Hash不一致，说明Cache是无效的，需要重新下载
+				s.cache.Remove(stripKey)
+			}
+		}
+
+		dataBuf := make([]byte, int64(s.red.K*s.red.ChunkSize))
+		n, err := s.readStrip(curStripIndex, dataBuf)
+		if err == io.ErrUnexpectedEOF {
+			// dataBuf中的内容可能不足一个条带，但仍然将其完整放入cache中，外部应该自行计算该从这个buffer中读多少数据
+			s.cache.Add(stripKey, ObjectECStrip{
+				Data:           dataBuf,
+				ObjectFileHash: s.object.FileHash,
+			})
+
+			s.sendToDataChan(dataChanEntry{Data: dataBuf[:n], Position: stripBytesPos})
+			s.sendToDataChan(dataChanEntry{Error: io.EOF})
+			break loop
+		}
+		if err != nil {
+			s.sendToDataChan(dataChanEntry{Error: err})
+			break loop
+		}
+
+		s.cache.Add(stripKey, ObjectECStrip{
+			Data:           dataBuf,
+			ObjectFileHash: s.object.FileHash,
+		})
+
+		if !s.sendToDataChan(dataChanEntry{Data: dataBuf, Position: stripBytesPos}) {
+			break loop
+		}
+
+		curStripIndex++
+	}
+
+	close(s.dataChan)
+}
+
+func (s *StripIterator) sendToDataChan(entry dataChanEntry) bool {
+	select {
+	case s.dataChan <- entry:
+		return true
+	case <-s.downloadingDone:
+		return false
+	}
+}
+
+func (s *StripIterator) readStrip(stripIndex int64, buf []byte) (int, error) {
+	// 如果需求的条带不当前正在下载的条带的位置不符合，则需要重新打开下载流
+	if s.downloadingStream == nil || s.downloadingStripIndex != stripIndex {
+		if s.downloadingStream != nil {
+			s.downloadingStream.Close()
+			s.downloadingPlanCtxCancel()
+		}
+
+		ft := ioswitch2.NewFromTo()
+		for _, b := range s.blocks {
+			node := b.Node
+			ft.AddFrom(ioswitch2.NewFromNode(b.Block.FileHash, &node, b.Block.Index))
+		}
+
+		toExec, hd := ioswitch2.NewToDriverWithRange(-1, exec.Range{
+			Offset: stripIndex * s.red.StripSize(),
+		})
+		ft.AddTo(toExec)
+
+		parser := parser.NewParser(*s.red)
+		plans := exec.NewPlanBuilder()
+		err := parser.Parse(ft, plans)
+		if err != nil {
+			return 0, err
+		}
+
+		// TODo2 注入依赖
+		exeCtx := exec.NewExecContext()
+
+		exec := plans.Execute(exeCtx)
+
+		ctx, cancel := context.WithCancel(context.Background())
+		go exec.Wait(ctx)
+
+		str, err := exec.BeginRead(hd)
+		if err != nil {
+			cancel()
+			return 0, err
+		}
+
+		s.downloadingStream = str
+		s.downloadingStripIndex = stripIndex
+		s.downloadingPlanCtxCancel = cancel
+	}
+
+	n, err := io.ReadFull(s.downloadingStream, buf)
+	s.downloadingStripIndex += 1
+	return n, err
+}

common/pkgs/ec/block.go

@@ -1,190 +1,190 @@
 package ec
 
-import (
-	"errors"
-	"io"
-	"io/ioutil"
+// import (
+// 	"errors"
+// 	"io"
+// 	"io/ioutil"
 
-	"gitlink.org.cn/cloudream/common/pkgs/ipfs"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
-)
+// 	"gitlink.org.cn/cloudream/common/pkgs/ipfs"
+// 	"gitlink.org.cn/cloudream/common/pkgs/logger"
+// 	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
+// )
 
-type BlockReader struct {
-	ipfsCli *ipfs.PoolClient
-	/*将文件分块相关的属性*/
-	//fileHash
-	fileHash string
-	//fileSize
-	fileSize int64
-	//ecK将文件的分块数
-	ecK int
-	//chunkSize
-	chunkSize int64
+// type BlockReader struct {
+// 	ipfsCli *ipfs.PoolClient
+// 	/*将文件分块相关的属性*/
+// 	//fileHash
+// 	fileHash string
+// 	//fileSize
+// 	fileSize int64
+// 	//ecK将文件的分块数
+// 	ecK int
+// 	//chunkSize
+// 	chunkSize int64
 
-	/*可选项*/
-	//fastRead,true的时候直接通过hash读block
-	jumpReadOpt bool
-}
+// 	/*可选项*/
+// 	//fastRead,true的时候直接通过hash读block
+// 	jumpReadOpt bool
+// }
 
-func NewBlockReader() (*BlockReader, error) {
-	ipfsClient, err := stgglb.IPFSPool.Acquire()
-	if err != nil {
-		return nil, err
-	}
-	//default:fast模式，通过hash直接获取
-	return &BlockReader{ipfsCli: ipfsClient, chunkSize: 256 * 1024, jumpReadOpt: false}, nil
-}
+// func NewBlockReader() (*BlockReader, error) {
+// 	ipfsClient, err := stgglb.IPFSPool.Acquire()
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	//default:fast模式，通过hash直接获取
+// 	return &BlockReader{ipfsCli: ipfsClient, chunkSize: 256 * 1024, jumpReadOpt: false}, nil
+// }
 
-func (r *BlockReader) Close() {
-	r.ipfsCli.Close()
-}
+// func (r *BlockReader) Close() {
+// 	r.ipfsCli.Close()
+// }
 
-func (r *BlockReader) SetJumpRead(fileHash string, fileSize int64, ecK int) {
-	r.fileHash = fileHash
-	r.fileSize = fileSize
-	r.ecK = ecK
-	r.jumpReadOpt = true
-}
+// func (r *BlockReader) SetJumpRead(fileHash string, fileSize int64, ecK int) {
+// 	r.fileHash = fileHash
+// 	r.fileSize = fileSize
+// 	r.ecK = ecK
+// 	r.jumpReadOpt = true
+// }
 
-func (r *BlockReader) SetchunkSize(size int64) {
-	r.chunkSize = size
-}
+// func (r *BlockReader) SetchunkSize(size int64) {
+// 	r.chunkSize = size
+// }
 
-func (r *BlockReader) FetchBLock(blockHash string) (io.ReadCloser, error) {
-	return r.ipfsCli.OpenRead(blockHash)
-}
+// func (r *BlockReader) FetchBLock(blockHash string) (io.ReadCloser, error) {
+// 	return r.ipfsCli.OpenRead(blockHash)
+// }
 
-func (r *BlockReader) FetchBLocks(blockHashs []string) ([]io.ReadCloser, error) {
-	readers := make([]io.ReadCloser, len(blockHashs))
-	for i, hash := range blockHashs {
-		var err error
-		readers[i], err = r.ipfsCli.OpenRead(hash)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return readers, nil
-}
+// func (r *BlockReader) FetchBLocks(blockHashs []string) ([]io.ReadCloser, error) {
+// 	readers := make([]io.ReadCloser, len(blockHashs))
+// 	for i, hash := range blockHashs {
+// 		var err error
+// 		readers[i], err = r.ipfsCli.OpenRead(hash)
+// 		if err != nil {
+// 			return nil, err
+// 		}
+// 	}
+// 	return readers, nil
+// }
 
-func (r *BlockReader) JumpFetchBlock(innerID int) (io.ReadCloser, error) {
-	if !r.jumpReadOpt {
-		return nil, nil
-	}
-	pipeReader, pipeWriter := io.Pipe()
-	go func() {
-		for i := int64(r.chunkSize * int64(innerID)); i < r.fileSize; i += int64(r.ecK) * r.chunkSize {
-			reader, err := r.ipfsCli.OpenRead(r.fileHash, ipfs.ReadOption{Offset: i, Length: r.chunkSize})
-			if err != nil {
-				pipeWriter.CloseWithError(err)
-				return
-			}
-			data, err := ioutil.ReadAll(reader)
-			if err != nil {
-				pipeWriter.CloseWithError(err)
-				return
-			}
-			reader.Close()
-			_, err = pipeWriter.Write(data)
-			if err != nil {
-				pipeWriter.CloseWithError(err)
-				return
-			}
-		}
-		//如果文件大小不是分块的整数倍,可能需要补0
-		if r.fileSize%(r.chunkSize*int64(r.ecK)) != 0 {
-			//pktNum_1:chunkNum-1
-			pktNum_1 := r.fileSize / (r.chunkSize * int64(r.ecK))
-			offset := (r.fileSize - int64(pktNum_1)*int64(r.ecK)*r.chunkSize)
-			count0 := int64(innerID)*int64(r.ecK)*r.chunkSize - offset
-			if count0 > 0 {
-				add0 := make([]byte, count0)
-				pipeWriter.Write(add0)
-			}
-		}
-		pipeWriter.Close()
-	}()
-	return pipeReader, nil
-}
+// func (r *BlockReader) JumpFetchBlock(innerID int) (io.ReadCloser, error) {
+// 	if !r.jumpReadOpt {
+// 		return nil, nil
+// 	}
+// 	pipeReader, pipeWriter := io.Pipe()
+// 	go func() {
+// 		for i := int64(r.chunkSize * int64(innerID)); i < r.fileSize; i += int64(r.ecK) * r.chunkSize {
+// 			reader, err := r.ipfsCli.OpenRead(r.fileHash, ipfs.ReadOption{Offset: i, Length: r.chunkSize})
+// 			if err != nil {
+// 				pipeWriter.CloseWithError(err)
+// 				return
+// 			}
+// 			data, err := ioutil.ReadAll(reader)
+// 			if err != nil {
+// 				pipeWriter.CloseWithError(err)
+// 				return
+// 			}
+// 			reader.Close()
+// 			_, err = pipeWriter.Write(data)
+// 			if err != nil {
+// 				pipeWriter.CloseWithError(err)
+// 				return
+// 			}
+// 		}
+// 		//如果文件大小不是分块的整数倍,可能需要补0
+// 		if r.fileSize%(r.chunkSize*int64(r.ecK)) != 0 {
+// 			//pktNum_1:chunkNum-1
+// 			pktNum_1 := r.fileSize / (r.chunkSize * int64(r.ecK))
+// 			offset := (r.fileSize - int64(pktNum_1)*int64(r.ecK)*r.chunkSize)
+// 			count0 := int64(innerID)*int64(r.ecK)*r.chunkSize - offset
+// 			if count0 > 0 {
+// 				add0 := make([]byte, count0)
+// 				pipeWriter.Write(add0)
+// 			}
+// 		}
+// 		pipeWriter.Close()
+// 	}()
+// 	return pipeReader, nil
+// }
 
-// FetchBlock1这个函数废弃了
-func (r *BlockReader) FetchBlock1(input interface{}, errMsg chan error) (io.ReadCloser, error) {
-	/*两种模式下传入第一个参数，但是input的类型不同：
-	jumpReadOpt-》true：传入blcokHash, string型，通过哈希直接读
-	jumpReadOpt->false: 传入innerID，int型，选择需要获取的数据块的id
-	*/
-	var innerID int
-	var blockHash string
-	switch input.(type) {
-	case int:
-		// 执行针对整数的逻辑分支
-		if r.jumpReadOpt {
-			return nil, errors.New("conflict, wrong input type and jumpReadOpt:true")
-		} else {
-			innerID = input.(int)
-		}
-	case string:
-		if !r.jumpReadOpt {
-			return nil, errors.New("conflict, wrong input type and jumpReadOpt:false")
-		} else {
-			blockHash = input.(string)
-		}
-	default:
-		return nil, errors.New("wrong input type")
-	}
-	//开始执行
-	if r.jumpReadOpt { //快速读
-		ipfsCli, err := stgglb.IPFSPool.Acquire()
-		if err != nil {
-			logger.Warnf("new ipfs client: %s", err.Error())
-			return nil, err
-		}
-		defer ipfsCli.Close()
-		return ipfsCli.OpenRead(blockHash)
-	} else { //跳跃读
-		ipfsCli, err := stgglb.IPFSPool.Acquire()
-		if err != nil {
-			logger.Warnf("new ipfs client: %s", err.Error())
-			return nil, err
-		}
-		defer ipfsCli.Close()
-		pipeReader, pipeWriter := io.Pipe()
-		go func() {
-			for i := int64(r.chunkSize * int64(innerID)); i < r.fileSize; i += int64(r.ecK) * r.chunkSize {
-				reader, err := ipfsCli.OpenRead(r.fileHash, ipfs.ReadOption{i, r.chunkSize})
-				if err != nil {
-					pipeWriter.Close()
-					errMsg <- err
-					return
-				}
-				data, err := ioutil.ReadAll(reader)
-				if err != nil {
-					pipeWriter.Close()
-					errMsg <- err
-					return
-				}
-				reader.Close()
-				_, err = pipeWriter.Write(data)
-				if err != nil {
-					pipeWriter.Close()
-					errMsg <- err
-					return
-				}
-			}
-			//如果文件大小不是分块的整数倍,可能需要补0
-			if r.fileSize%(r.chunkSize*int64(r.ecK)) != 0 {
-				//pktNum_1:chunkNum-1
-				pktNum_1 := r.fileSize / (r.chunkSize * int64(r.ecK))
-				offset := (r.fileSize - int64(pktNum_1)*int64(r.ecK)*r.chunkSize)
-				count0 := int64(innerID)*int64(r.ecK)*r.chunkSize - offset
-				if count0 > 0 {
-					add0 := make([]byte, count0)
-					pipeWriter.Write(add0)
-				}
-			}
-			pipeWriter.Close()
-			errMsg <- nil
-		}()
-		return pipeReader, nil
-	}
-}
+// // FetchBlock1这个函数废弃了
+// func (r *BlockReader) FetchBlock1(input interface{}, errMsg chan error) (io.ReadCloser, error) {
+// 	/*两种模式下传入第一个参数，但是input的类型不同：
+// 	jumpReadOpt-》true：传入blcokHash, string型，通过哈希直接读
+// 	jumpReadOpt->false: 传入innerID，int型，选择需要获取的数据块的id
+// 	*/
+// 	var innerID int
+// 	var blockHash string
+// 	switch input.(type) {
+// 	case int:
+// 		// 执行针对整数的逻辑分支
+// 		if r.jumpReadOpt {
+// 			return nil, errors.New("conflict, wrong input type and jumpReadOpt:true")
+// 		} else {
+// 			innerID = input.(int)
+// 		}
+// 	case string:
+// 		if !r.jumpReadOpt {
+// 			return nil, errors.New("conflict, wrong input type and jumpReadOpt:false")
+// 		} else {
+// 			blockHash = input.(string)
+// 		}
+// 	default:
+// 		return nil, errors.New("wrong input type")
+// 	}
+// 	//开始执行
+// 	if r.jumpReadOpt { //快速读
+// 		ipfsCli, err := stgglb.IPFSPool.Acquire()
+// 		if err != nil {
+// 			logger.Warnf("new ipfs client: %s", err.Error())
+// 			return nil, err
+// 		}
+// 		defer ipfsCli.Close()
+// 		return ipfsCli.OpenRead(blockHash)
+// 	} else { //跳跃读
+// 		ipfsCli, err := stgglb.IPFSPool.Acquire()
+// 		if err != nil {
+// 			logger.Warnf("new ipfs client: %s", err.Error())
+// 			return nil, err
+// 		}
+// 		defer ipfsCli.Close()
+// 		pipeReader, pipeWriter := io.Pipe()
+// 		go func() {
+// 			for i := int64(r.chunkSize * int64(innerID)); i < r.fileSize; i += int64(r.ecK) * r.chunkSize {
+// 				reader, err := ipfsCli.OpenRead(r.fileHash, ipfs.ReadOption{i, r.chunkSize})
+// 				if err != nil {
+// 					pipeWriter.Close()
+// 					errMsg <- err
+// 					return
+// 				}
+// 				data, err := ioutil.ReadAll(reader)
+// 				if err != nil {
+// 					pipeWriter.Close()
+// 					errMsg <- err
+// 					return
+// 				}
+// 				reader.Close()
+// 				_, err = pipeWriter.Write(data)
+// 				if err != nil {
+// 					pipeWriter.Close()
+// 					errMsg <- err
+// 					return
+// 				}
+// 			}
+// 			//如果文件大小不是分块的整数倍,可能需要补0
+// 			if r.fileSize%(r.chunkSize*int64(r.ecK)) != 0 {
+// 				//pktNum_1:chunkNum-1
+// 				pktNum_1 := r.fileSize / (r.chunkSize * int64(r.ecK))
+// 				offset := (r.fileSize - int64(pktNum_1)*int64(r.ecK)*r.chunkSize)
+// 				count0 := int64(innerID)*int64(r.ecK)*r.chunkSize - offset
+// 				if count0 > 0 {
+// 					add0 := make([]byte, count0)
+// 					pipeWriter.Write(add0)
+// 				}
+// 			}
+// 			pipeWriter.Close()
+// 			errMsg <- nil
+// 		}()
+// 		return pipeReader, nil
+// 	}
+// }

common/pkgs/ec/lrc/lrc.go

@@ -0,0 +1,36 @@
+package lrc
+
+import "github.com/klauspost/reedsolomon"
+
+type LRC struct {
+	n      int   // 总块数，包括局部块
+	k      int   // 数据块数量
+	groups []int // 分组校验块生成时使用的数据块
+	l      *reedsolomon.LRC
+}
+
+func New(n int, k int, groups []int) (*LRC, error) {
+	lrc := &LRC{
+		n:      n,
+		k:      k,
+		groups: groups,
+	}
+
+	l, err := reedsolomon.NewLRC(k, n-len(groups)-k, groups)
+	if err != nil {
+		return nil, err
+	}
+
+	lrc.l = l
+	return lrc, nil
+}
+
+// 根据全局修复的原理，生成根据输入修复指定块的矩阵。要求input内元素的值<n-len(r)，且至少包含k个。
+func (l *LRC) GenerateMatrix(inputIdxs []int, outputIdxs []int) ([][]byte, error) {
+	return l.l.GenerateMatrix(inputIdxs, outputIdxs)
+}
+
+// 生成修复组内某个块的矩阵。只支持组内缺少一个块的情况，且默认组内的其他块都存在。
+func (l *LRC) GenerateGroupMatrix(outputIdx int) ([][]byte, error) {
+	return l.l.GenerateGroupMatrix(outputIdx)
+}

common/pkgs/ec/multiply.go

@@ -0,0 +1,7 @@
+package ec
+
+import "github.com/klauspost/reedsolomon"
+
+func GaloisMultiplier() *reedsolomon.MultipilerBuilder {
+	return reedsolomon.DefaultMulOpt()
+}

common/pkgs/ec/rs.go

@@ -42,3 +42,7 @@ func (r *Rs) ReconstructAny(blocks [][]byte, outBlockIdxes []int) error {
 	}
 	return r.encoder.ReconstructAny(blocks, required)
 }
+
+func (r *Rs) GenerateMatrix(inputIdxs []int, outputIdxs []int) ([][]byte, error) {
+	return r.encoder.(reedsolomon.Extensions).GenerateMatrix(inputIdxs, outputIdxs)
+}

common/pkgs/grpc/agent/agent.pb.go

@@ -2,7 +2,7 @@
 
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.30.0
+// 	protoc-gen-go v1.34.2
 // 	protoc        v4.22.3
 // source: pkgs/grpc/agent/agent.proto
 
@@ -71,18 +71,16 @@ func (StreamDataPacketType) EnumDescriptor() ([]byte, []int) {
 	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{0}
 }
 
-// 文件数据。注意：只在Type为Data的时候，Data字段才能有数据
-type FileDataPacket struct {
+type ExecuteIOPlanReq struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Type StreamDataPacketType `protobuf:"varint,1,opt,name=Type,proto3,enum=StreamDataPacketType" json:"Type,omitempty"`
-	Data []byte               `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"`
+	Plan string `protobuf:"bytes,1,opt,name=Plan,proto3" json:"Plan,omitempty"`
 }
 
-func (x *FileDataPacket) Reset() {
-	*x = FileDataPacket{}
+func (x *ExecuteIOPlanReq) Reset() {
+	*x = ExecuteIOPlanReq{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[0]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -90,13 +88,13 @@ func (x *FileDataPacket) Reset() {
 	}
 }
 
-func (x *FileDataPacket) String() string {
+func (x *ExecuteIOPlanReq) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*FileDataPacket) ProtoMessage() {}
+func (*ExecuteIOPlanReq) ProtoMessage() {}
 
-func (x *FileDataPacket) ProtoReflect() protoreflect.Message {
+func (x *ExecuteIOPlanReq) ProtoReflect() protoreflect.Message {
 	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[0]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -108,35 +106,26 @@ func (x *FileDataPacket) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use FileDataPacket.ProtoReflect.Descriptor instead.
-func (*FileDataPacket) Descriptor() ([]byte, []int) {
+// Deprecated: Use ExecuteIOPlanReq.ProtoReflect.Descriptor instead.
+func (*ExecuteIOPlanReq) Descriptor() ([]byte, []int) {
 	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{0}
 }
 
-func (x *FileDataPacket) GetType() StreamDataPacketType {
-	if x != nil {
-		return x.Type
-	}
-	return StreamDataPacketType_EOF
-}
-
-func (x *FileDataPacket) GetData() []byte {
+func (x *ExecuteIOPlanReq) GetPlan() string {
 	if x != nil {
-		return x.Data
+		return x.Plan
 	}
-	return nil
+	return ""
 }
 
-type SendIPFSFileResp struct {
+type ExecuteIOPlanResp struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
-
-	FileHash string `protobuf:"bytes,1,opt,name=FileHash,proto3" json:"FileHash,omitempty"`
 }
 
-func (x *SendIPFSFileResp) Reset() {
-	*x = SendIPFSFileResp{}
+func (x *ExecuteIOPlanResp) Reset() {
+	*x = ExecuteIOPlanResp{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[1]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -144,13 +133,13 @@ func (x *SendIPFSFileResp) Reset() {
 	}
 }
 
-func (x *SendIPFSFileResp) String() string {
+func (x *ExecuteIOPlanResp) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*SendIPFSFileResp) ProtoMessage() {}
+func (*ExecuteIOPlanResp) ProtoMessage() {}
 
-func (x *SendIPFSFileResp) ProtoReflect() protoreflect.Message {
+func (x *ExecuteIOPlanResp) ProtoReflect() protoreflect.Message {
 	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[1]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -162,28 +151,23 @@ func (x *SendIPFSFileResp) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use SendIPFSFileResp.ProtoReflect.Descriptor instead.
-func (*SendIPFSFileResp) Descriptor() ([]byte, []int) {
+// Deprecated: Use ExecuteIOPlanResp.ProtoReflect.Descriptor instead.
+func (*ExecuteIOPlanResp) Descriptor() ([]byte, []int) {
 	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{1}
 }
 
-func (x *SendIPFSFileResp) GetFileHash() string {
-	if x != nil {
-		return x.FileHash
-	}
-	return ""
-}
-
-type GetIPFSFileReq struct {
+// 文件数据。注意：只在Type为Data或EOF的时候，Data字段才能有数据
+type FileDataPacket struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	FileHash string `protobuf:"bytes,1,opt,name=FileHash,proto3" json:"FileHash,omitempty"`
+	Type StreamDataPacketType `protobuf:"varint,1,opt,name=Type,proto3,enum=StreamDataPacketType" json:"Type,omitempty"`
+	Data []byte               `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"`
 }
 
-func (x *GetIPFSFileReq) Reset() {
-	*x = GetIPFSFileReq{}
+func (x *FileDataPacket) Reset() {
+	*x = FileDataPacket{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[2]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -191,13 +175,13 @@ func (x *GetIPFSFileReq) Reset() {
 	}
 }
 
-func (x *GetIPFSFileReq) String() string {
+func (x *FileDataPacket) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*GetIPFSFileReq) ProtoMessage() {}
+func (*FileDataPacket) ProtoMessage() {}
 
-func (x *GetIPFSFileReq) ProtoReflect() protoreflect.Message {
+func (x *FileDataPacket) ProtoReflect() protoreflect.Message {
 	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[2]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -209,16 +193,23 @@ func (x *GetIPFSFileReq) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use GetIPFSFileReq.ProtoReflect.Descriptor instead.
-func (*GetIPFSFileReq) Descriptor() ([]byte, []int) {
+// Deprecated: Use FileDataPacket.ProtoReflect.Descriptor instead.
+func (*FileDataPacket) Descriptor() ([]byte, []int) {
 	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{2}
 }
 
-func (x *GetIPFSFileReq) GetFileHash() string {
+func (x *FileDataPacket) GetType() StreamDataPacketType {
 	if x != nil {
-		return x.FileHash
+		return x.Type
 	}
-	return ""
+	return StreamDataPacketType_EOF
+}
+
+func (x *FileDataPacket) GetData() []byte {
+	if x != nil {
+		return x.Data
+	}
+	return nil
 }
 
 // 注：EOF时data也可能有数据
@@ -227,10 +218,10 @@ type StreamDataPacket struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	Type     StreamDataPacketType `protobuf:"varint,1,opt,name=Type,proto3,enum=StreamDataPacketType" json:"Type,omitempty"`
-	PlanID   string               `protobuf:"bytes,2,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
-	StreamID string               `protobuf:"bytes,3,opt,name=StreamID,proto3" json:"StreamID,omitempty"`
-	Data     []byte               `protobuf:"bytes,4,opt,name=Data,proto3" json:"Data,omitempty"`
+	Type   StreamDataPacketType `protobuf:"varint,1,opt,name=Type,proto3,enum=StreamDataPacketType" json:"Type,omitempty"`
+	PlanID string               `protobuf:"bytes,2,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
+	VarID  int32                `protobuf:"varint,3,opt,name=VarID,proto3" json:"VarID,omitempty"`
+	Data   []byte               `protobuf:"bytes,4,opt,name=Data,proto3" json:"Data,omitempty"`
 }
 
 func (x *StreamDataPacket) Reset() {
@@ -279,11 +270,11 @@ func (x *StreamDataPacket) GetPlanID() string {
 	return ""
 }
 
-func (x *StreamDataPacket) GetStreamID() string {
+func (x *StreamDataPacket) GetVarID() int32 {
 	if x != nil {
-		return x.StreamID
+		return x.VarID
 	}
-	return ""
+	return 0
 }
 
 func (x *StreamDataPacket) GetData() []byte {
@@ -331,17 +322,18 @@ func (*SendStreamResp) Descriptor() ([]byte, []int) {
 	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{4}
 }
 
-type FetchStreamReq struct {
+type GetStreamReq struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	PlanID   string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
-	StreamID string `protobuf:"bytes,2,opt,name=StreamID,proto3" json:"StreamID,omitempty"`
+	PlanID string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
+	VarID  int32  `protobuf:"varint,2,opt,name=VarID,proto3" json:"VarID,omitempty"`
+	Signal string `protobuf:"bytes,3,opt,name=Signal,proto3" json:"Signal,omitempty"`
 }
 
-func (x *FetchStreamReq) Reset() {
-	*x = FetchStreamReq{}
+func (x *GetStreamReq) Reset() {
+	*x = GetStreamReq{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[5]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -349,13 +341,13 @@ func (x *FetchStreamReq) Reset() {
 	}
 }
 
-func (x *FetchStreamReq) String() string {
+func (x *GetStreamReq) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*FetchStreamReq) ProtoMessage() {}
+func (*GetStreamReq) ProtoMessage() {}
 
-func (x *FetchStreamReq) ProtoReflect() protoreflect.Message {
+func (x *GetStreamReq) ProtoReflect() protoreflect.Message {
 	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[5]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -367,21 +359,231 @@ func (x *FetchStreamReq) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use FetchStreamReq.ProtoReflect.Descriptor instead.
-func (*FetchStreamReq) Descriptor() ([]byte, []int) {
+// Deprecated: Use GetStreamReq.ProtoReflect.Descriptor instead.
+func (*GetStreamReq) Descriptor() ([]byte, []int) {
 	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{5}
 }
 
-func (x *FetchStreamReq) GetPlanID() string {
+func (x *GetStreamReq) GetPlanID() string {
+	if x != nil {
+		return x.PlanID
+	}
+	return ""
+}
+
+func (x *GetStreamReq) GetVarID() int32 {
+	if x != nil {
+		return x.VarID
+	}
+	return 0
+}
+
+func (x *GetStreamReq) GetSignal() string {
+	if x != nil {
+		return x.Signal
+	}
+	return ""
+}
+
+type SendVarReq struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	PlanID string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
+	Var    string `protobuf:"bytes,2,opt,name=Var,proto3" json:"Var,omitempty"`
+}
+
+func (x *SendVarReq) Reset() {
+	*x = SendVarReq{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SendVarReq) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SendVarReq) ProtoMessage() {}
+
+func (x *SendVarReq) ProtoReflect() protoreflect.Message {
+	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SendVarReq.ProtoReflect.Descriptor instead.
+func (*SendVarReq) Descriptor() ([]byte, []int) {
+	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *SendVarReq) GetPlanID() string {
+	if x != nil {
+		return x.PlanID
+	}
+	return ""
+}
+
+func (x *SendVarReq) GetVar() string {
+	if x != nil {
+		return x.Var
+	}
+	return ""
+}
+
+type SendVarResp struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *SendVarResp) Reset() {
+	*x = SendVarResp{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SendVarResp) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SendVarResp) ProtoMessage() {}
+
+func (x *SendVarResp) ProtoReflect() protoreflect.Message {
+	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SendVarResp.ProtoReflect.Descriptor instead.
+func (*SendVarResp) Descriptor() ([]byte, []int) {
+	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{7}
+}
+
+type GetVarReq struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	PlanID string `protobuf:"bytes,1,opt,name=PlanID,proto3" json:"PlanID,omitempty"`
+	Var    string `protobuf:"bytes,2,opt,name=Var,proto3" json:"Var,omitempty"`
+	Signal string `protobuf:"bytes,3,opt,name=Signal,proto3" json:"Signal,omitempty"`
+}
+
+func (x *GetVarReq) Reset() {
+	*x = GetVarReq{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GetVarReq) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GetVarReq) ProtoMessage() {}
+
+func (x *GetVarReq) ProtoReflect() protoreflect.Message {
+	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GetVarReq.ProtoReflect.Descriptor instead.
+func (*GetVarReq) Descriptor() ([]byte, []int) {
+	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *GetVarReq) GetPlanID() string {
 	if x != nil {
 		return x.PlanID
 	}
 	return ""
 }
 
-func (x *FetchStreamReq) GetStreamID() string {
+func (x *GetVarReq) GetVar() string {
+	if x != nil {
+		return x.Var
+	}
+	return ""
+}
+
+func (x *GetVarReq) GetSignal() string {
 	if x != nil {
-		return x.StreamID
+		return x.Signal
+	}
+	return ""
+}
+
+type GetVarResp struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Var string `protobuf:"bytes,1,opt,name=Var,proto3" json:"Var,omitempty"` // 此处不使用VarID的原因是，Switch的BindVars函数还需要知道Var的类型
+}
+
+func (x *GetVarResp) Reset() {
+	*x = GetVarResp{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *GetVarResp) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*GetVarResp) ProtoMessage() {}
+
+func (x *GetVarResp) ProtoReflect() protoreflect.Message {
+	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use GetVarResp.ProtoReflect.Descriptor instead.
+func (*GetVarResp) Descriptor() ([]byte, []int) {
+	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{9}
+}
+
+func (x *GetVarResp) GetVar() string {
+	if x != nil {
+		return x.Var
 	}
 	return ""
 }
@@ -395,7 +597,7 @@ type PingReq struct {
 func (x *PingReq) Reset() {
 	*x = PingReq{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[6]
+		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[10]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -408,7 +610,7 @@ func (x *PingReq) String() string {
 func (*PingReq) ProtoMessage() {}
 
 func (x *PingReq) ProtoReflect() protoreflect.Message {
-	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[6]
+	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[10]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -421,7 +623,7 @@ func (x *PingReq) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use PingReq.ProtoReflect.Descriptor instead.
 func (*PingReq) Descriptor() ([]byte, []int) {
-	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{6}
+	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{10}
 }
 
 type PingResp struct {
@@ -433,7 +635,7 @@ type PingResp struct {
 func (x *PingResp) Reset() {
 	*x = PingResp{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[7]
+		mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[11]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -446,7 +648,7 @@ func (x *PingResp) String() string {
 func (*PingResp) ProtoMessage() {}
 
 func (x *PingResp) ProtoReflect() protoreflect.Message {
-	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[7]
+	mi := &file_pkgs_grpc_agent_agent_proto_msgTypes[11]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -459,62 +661,73 @@ func (x *PingResp) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use PingResp.ProtoReflect.Descriptor instead.
 func (*PingResp) Descriptor() ([]byte, []int) {
-	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{7}
+	return file_pkgs_grpc_agent_agent_proto_rawDescGZIP(), []int{11}
 }
 
 var File_pkgs_grpc_agent_agent_proto protoreflect.FileDescriptor
 
 var file_pkgs_grpc_agent_agent_proto_rawDesc = []byte{
 	0x0a, 0x1b, 0x70, 0x6b, 0x67, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x61, 0x67, 0x65, 0x6e,
-	0x74, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x4f, 0x0a,
-	0x0e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x12,
+	0x74, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x26, 0x0a,
+	0x10, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x49, 0x4f, 0x50, 0x6c, 0x61, 0x6e, 0x52, 0x65,
+	0x71, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6c, 0x61, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x04, 0x50, 0x6c, 0x61, 0x6e, 0x22, 0x13, 0x0a, 0x11, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65,
+	0x49, 0x4f, 0x50, 0x6c, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x22, 0x4f, 0x0a, 0x0e, 0x46, 0x69,
+	0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x29, 0x0a, 0x04,
+	0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x53, 0x74, 0x72,
+	0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, 0x70,
+	0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x22, 0x7f, 0x0a, 0x10, 0x53,
+	0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x12,
 	0x29, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e,
 	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74,
-	0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x44, 0x61,
-	0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x22, 0x2e,
-	0x0a, 0x10, 0x53, 0x65, 0x6e, 0x64, 0x49, 0x50, 0x46, 0x53, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65,
-	0x73, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x18, 0x01,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x22, 0x2c,
-	0x0a, 0x0e, 0x47, 0x65, 0x74, 0x49, 0x50, 0x46, 0x53, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71,
-	0x12, 0x1a, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x22, 0x85, 0x01, 0x0a,
-	0x10, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65,
-	0x74, 0x12, 0x29, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32,
-	0x15, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b,
-	0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06,
-	0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x6c,
-	0x61, 0x6e, 0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x44,
-	0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x44,
-	0x12, 0x12, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04,
-	0x44, 0x61, 0x74, 0x61, 0x22, 0x10, 0x0a, 0x0e, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65,
-	0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x22, 0x44, 0x0a, 0x0e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x53,
-	0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e,
+	0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6c,
+	0x61, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x6c, 0x61, 0x6e,
+	0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28,
+	0x05, 0x52, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x22, 0x10, 0x0a, 0x0e,
+	0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x22, 0x54,
+	0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x12, 0x16,
+	0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06,
+	0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x18,
+	0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x56, 0x61, 0x72, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06,
+	0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x69,
+	0x67, 0x6e, 0x61, 0x6c, 0x22, 0x36, 0x0a, 0x0a, 0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52,
+	0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44, 0x12, 0x10, 0x0a, 0x03, 0x56, 0x61,
+	0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x56, 0x61, 0x72, 0x22, 0x0d, 0x0a, 0x0b,
+	0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x4d, 0x0a, 0x09, 0x47,
+	0x65, 0x74, 0x56, 0x61, 0x72, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x6c, 0x61, 0x6e,
 	0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x6c, 0x61, 0x6e, 0x49, 0x44,
-	0x12, 0x1a, 0x0a, 0x08, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01,
-	0x28, 0x09, 0x52, 0x08, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x44, 0x22, 0x09, 0x0a, 0x07,
-	0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x22, 0x0a, 0x0a, 0x08, 0x50, 0x69, 0x6e, 0x67, 0x52,
-	0x65, 0x73, 0x70, 0x2a, 0x37, 0x0a, 0x14, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74,
-	0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x45,
-	0x4f, 0x46, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x10, 0x01, 0x12, 0x0c,
-	0x0a, 0x08, 0x53, 0x65, 0x6e, 0x64, 0x41, 0x72, 0x67, 0x73, 0x10, 0x02, 0x32, 0x80, 0x02, 0x0a,
-	0x05, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x36, 0x0a, 0x0c, 0x53, 0x65, 0x6e, 0x64, 0x49, 0x50,
-	0x46, 0x53, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x0f, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x61, 0x74,
-	0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x1a, 0x11, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x49, 0x50,
-	0x46, 0x53, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x28, 0x01, 0x12, 0x33,
-	0x0a, 0x0b, 0x47, 0x65, 0x74, 0x49, 0x50, 0x46, 0x53, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x0f, 0x2e,
-	0x47, 0x65, 0x74, 0x49, 0x50, 0x46, 0x53, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x1a, 0x0f,
-	0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x22,
-	0x00, 0x30, 0x01, 0x12, 0x34, 0x0a, 0x0a, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61,
-	0x6d, 0x12, 0x11, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61,
-	0x63, 0x6b, 0x65, 0x74, 0x1a, 0x0f, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61,
-	0x6d, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x28, 0x01, 0x12, 0x35, 0x0a, 0x0b, 0x46, 0x65, 0x74,
-	0x63, 0x68, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x0f, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68,
-	0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x1a, 0x11, 0x2e, 0x53, 0x74, 0x72, 0x65,
-	0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x22, 0x00, 0x30, 0x01,
-	0x12, 0x1d, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x08, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52,
-	0x65, 0x71, 0x1a, 0x09, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x42,
-	0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x33,
+	0x12, 0x10, 0x0a, 0x03, 0x56, 0x61, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x56,
+	0x61, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x22, 0x1e, 0x0a, 0x0a, 0x47, 0x65,
+	0x74, 0x56, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x12, 0x10, 0x0a, 0x03, 0x56, 0x61, 0x72, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x56, 0x61, 0x72, 0x22, 0x09, 0x0a, 0x07, 0x50, 0x69,
+	0x6e, 0x67, 0x52, 0x65, 0x71, 0x22, 0x0a, 0x0a, 0x08, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73,
+	0x70, 0x2a, 0x37, 0x0a, 0x14, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50,
+	0x61, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x07, 0x0a, 0x03, 0x45, 0x4f, 0x46,
+	0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x61, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08,
+	0x53, 0x65, 0x6e, 0x64, 0x41, 0x72, 0x67, 0x73, 0x10, 0x02, 0x32, 0x96, 0x02, 0x0a, 0x05, 0x41,
+	0x67, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x0d, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x49,
+	0x4f, 0x50, 0x6c, 0x61, 0x6e, 0x12, 0x11, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x49,
+	0x4f, 0x50, 0x6c, 0x61, 0x6e, 0x52, 0x65, 0x71, 0x1a, 0x12, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75,
+	0x74, 0x65, 0x49, 0x4f, 0x50, 0x6c, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12, 0x34,
+	0x0a, 0x0a, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x11, 0x2e, 0x53,
+	0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x65, 0x74, 0x1a,
+	0x0f, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70,
+	0x22, 0x00, 0x28, 0x01, 0x12, 0x31, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61,
+	0x6d, 0x12, 0x0d, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71,
+	0x1a, 0x11, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x50, 0x61, 0x63,
+	0x6b, 0x65, 0x74, 0x22, 0x00, 0x30, 0x01, 0x12, 0x26, 0x0a, 0x07, 0x53, 0x65, 0x6e, 0x64, 0x56,
+	0x61, 0x72, 0x12, 0x0b, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52, 0x65, 0x71, 0x1a,
+	0x0c, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x56, 0x61, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, 0x12,
+	0x23, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x56, 0x61, 0x72, 0x12, 0x0a, 0x2e, 0x47, 0x65, 0x74, 0x56,
+	0x61, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0b, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x61, 0x72, 0x52, 0x65,
+	0x73, 0x70, 0x22, 0x00, 0x12, 0x1d, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x08, 0x2e, 0x50,
+	0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x1a, 0x09, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73,
+	0x70, 0x22, 0x00, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x3b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x06,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -530,36 +743,42 @@ func file_pkgs_grpc_agent_agent_proto_rawDescGZIP() []byte {
 }
 
 var file_pkgs_grpc_agent_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_pkgs_grpc_agent_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
-var file_pkgs_grpc_agent_agent_proto_goTypes = []interface{}{
+var file_pkgs_grpc_agent_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 12)
+var file_pkgs_grpc_agent_agent_proto_goTypes = []any{
 	(StreamDataPacketType)(0), // 0: StreamDataPacketType
-	(*FileDataPacket)(nil),    // 1: FileDataPacket
-	(*SendIPFSFileResp)(nil),  // 2: SendIPFSFileResp
-	(*GetIPFSFileReq)(nil),    // 3: GetIPFSFileReq
+	(*ExecuteIOPlanReq)(nil),  // 1: ExecuteIOPlanReq
+	(*ExecuteIOPlanResp)(nil), // 2: ExecuteIOPlanResp
+	(*FileDataPacket)(nil),    // 3: FileDataPacket
 	(*StreamDataPacket)(nil),  // 4: StreamDataPacket
 	(*SendStreamResp)(nil),    // 5: SendStreamResp
-	(*FetchStreamReq)(nil),    // 6: FetchStreamReq
-	(*PingReq)(nil),           // 7: PingReq
-	(*PingResp)(nil),          // 8: PingResp
+	(*GetStreamReq)(nil),      // 6: GetStreamReq
+	(*SendVarReq)(nil),        // 7: SendVarReq
+	(*SendVarResp)(nil),       // 8: SendVarResp
+	(*GetVarReq)(nil),         // 9: GetVarReq
+	(*GetVarResp)(nil),        // 10: GetVarResp
+	(*PingReq)(nil),           // 11: PingReq
+	(*PingResp)(nil),          // 12: PingResp
 }
 var file_pkgs_grpc_agent_agent_proto_depIdxs = []int32{
-	0, // 0: FileDataPacket.Type:type_name -> StreamDataPacketType
-	0, // 1: StreamDataPacket.Type:type_name -> StreamDataPacketType
-	1, // 2: Agent.SendIPFSFile:input_type -> FileDataPacket
-	3, // 3: Agent.GetIPFSFile:input_type -> GetIPFSFileReq
-	4, // 4: Agent.SendStream:input_type -> StreamDataPacket
-	6, // 5: Agent.FetchStream:input_type -> FetchStreamReq
-	7, // 6: Agent.Ping:input_type -> PingReq
-	2, // 7: Agent.SendIPFSFile:output_type -> SendIPFSFileResp
-	1, // 8: Agent.GetIPFSFile:output_type -> FileDataPacket
-	5, // 9: Agent.SendStream:output_type -> SendStreamResp
-	4, // 10: Agent.FetchStream:output_type -> StreamDataPacket
-	8, // 11: Agent.Ping:output_type -> PingResp
-	7, // [7:12] is the sub-list for method output_type
-	2, // [2:7] is the sub-list for method input_type
-	2, // [2:2] is the sub-list for extension type_name
-	2, // [2:2] is the sub-list for extension extendee
-	0, // [0:2] is the sub-list for field type_name
+	0,  // 0: FileDataPacket.Type:type_name -> StreamDataPacketType
+	0,  // 1: StreamDataPacket.Type:type_name -> StreamDataPacketType
+	1,  // 2: Agent.ExecuteIOPlan:input_type -> ExecuteIOPlanReq
+	4,  // 3: Agent.SendStream:input_type -> StreamDataPacket
+	6,  // 4: Agent.GetStream:input_type -> GetStreamReq
+	7,  // 5: Agent.SendVar:input_type -> SendVarReq
+	9,  // 6: Agent.GetVar:input_type -> GetVarReq
+	11, // 7: Agent.Ping:input_type -> PingReq
+	2,  // 8: Agent.ExecuteIOPlan:output_type -> ExecuteIOPlanResp
+	5,  // 9: Agent.SendStream:output_type -> SendStreamResp
+	4,  // 10: Agent.GetStream:output_type -> StreamDataPacket
+	8,  // 11: Agent.SendVar:output_type -> SendVarResp
+	10, // 12: Agent.GetVar:output_type -> GetVarResp
+	12, // 13: Agent.Ping:output_type -> PingResp
+	8,  // [8:14] is the sub-list for method output_type
+	2,  // [2:8] is the sub-list for method input_type
+	2,  // [2:2] is the sub-list for extension type_name
+	2,  // [2:2] is the sub-list for extension extendee
+	0,  // [0:2] is the sub-list for field type_name
 }
 
 func init() { file_pkgs_grpc_agent_agent_proto_init() }
@@ -568,8 +787,8 @@ func file_pkgs_grpc_agent_agent_proto_init() {
 		return
 	}
 	if !protoimpl.UnsafeEnabled {
-		file_pkgs_grpc_agent_agent_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*FileDataPacket); i {
+		file_pkgs_grpc_agent_agent_proto_msgTypes[0].Exporter = func(v any, i int) any {
+			switch v := v.(*ExecuteIOPlanReq); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -580,8 +799,8 @@ func file_pkgs_grpc_agent_agent_proto_init() {
 				return nil
 			}
 		}
-		file_pkgs_grpc_agent_agent_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*SendIPFSFileResp); i {
+		file_pkgs_grpc_agent_agent_proto_msgTypes[1].Exporter = func(v any, i int) any {
+			switch v := v.(*ExecuteIOPlanResp); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -592,8 +811,8 @@ func file_pkgs_grpc_agent_agent_proto_init() {
 				return nil
 			}
 		}
-		file_pkgs_grpc_agent_agent_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*GetIPFSFileReq); i {
+		file_pkgs_grpc_agent_agent_proto_msgTypes[2].Exporter = func(v any, i int) any {
+			switch v := v.(*FileDataPacket); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -604,7 +823,7 @@ func file_pkgs_grpc_agent_agent_proto_init() {
 				return nil
 			}
 		}
-		file_pkgs_grpc_agent_agent_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+		file_pkgs_grpc_agent_agent_proto_msgTypes[3].Exporter = func(v any, i int) any {
 			switch v := v.(*StreamDataPacket); i {
 			case 0:
 				return &v.state
@@ -616,7 +835,7 @@ func file_pkgs_grpc_agent_agent_proto_init() {
 				return nil
 			}
 		}
-		file_pkgs_grpc_agent_agent_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+		file_pkgs_grpc_agent_agent_proto_msgTypes[4].Exporter = func(v any, i int) any {
 			switch v := v.(*SendStreamResp); i {
 			case 0:
 				return &v.state
@@ -628,8 +847,56 @@ func file_pkgs_grpc_agent_agent_proto_init() {
 				return nil
 			}
 		}
-		file_pkgs_grpc_agent_agent_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*FetchStreamReq); i {
+		file_pkgs_grpc_agent_agent_proto_msgTypes[5].Exporter = func(v any, i int) any {
+			switch v := v.(*GetStreamReq); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkgs_grpc_agent_agent_proto_msgTypes[6].Exporter = func(v any, i int) any {
+			switch v := v.(*SendVarReq); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkgs_grpc_agent_agent_proto_msgTypes[7].Exporter = func(v any, i int) any {
+			switch v := v.(*SendVarResp); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkgs_grpc_agent_agent_proto_msgTypes[8].Exporter = func(v any, i int) any {
+			switch v := v.(*GetVarReq); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_pkgs_grpc_agent_agent_proto_msgTypes[9].Exporter = func(v any, i int) any {
+			switch v := v.(*GetVarResp); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -640,7 +907,7 @@ func file_pkgs_grpc_agent_agent_proto_init() {
 				return nil
 			}
 		}
-		file_pkgs_grpc_agent_agent_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+		file_pkgs_grpc_agent_agent_proto_msgTypes[10].Exporter = func(v any, i int) any {
 			switch v := v.(*PingReq); i {
 			case 0:
 				return &v.state
@@ -652,7 +919,7 @@ func file_pkgs_grpc_agent_agent_proto_init() {
 				return nil
 			}
 		}
-		file_pkgs_grpc_agent_agent_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+		file_pkgs_grpc_agent_agent_proto_msgTypes[11].Exporter = func(v any, i int) any {
 			switch v := v.(*PingResp); i {
 			case 0:
 				return &v.state
@@ -671,7 +938,7 @@ func file_pkgs_grpc_agent_agent_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_pkgs_grpc_agent_agent_proto_rawDesc,
 			NumEnums:      1,
-			NumMessages:   8,
+			NumMessages:   12,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

common/pkgs/grpc/agent/agent.proto

@@ -5,50 +5,67 @@ syntax = "proto3";
 option go_package = ".;agent";//grpc这里生效了
 
 
+
+message ExecuteIOPlanReq {
+    string Plan = 1;
+}
+
+message ExecuteIOPlanResp {
+}
+
 enum StreamDataPacketType {
     EOF = 0;
     Data = 1;
     SendArgs = 2;
 }
-// 文件数据。注意：只在Type为Data的时候，Data字段才能有数据
+// 文件数据。注意：只在Type为Data或EOF的时候，Data字段才能有数据
 message FileDataPacket {
     StreamDataPacketType Type = 1;
     bytes Data = 2;
 }
 
-message SendIPFSFileResp {
-    string FileHash = 1;
-}
-
-message GetIPFSFileReq {
-    string FileHash = 1;
-}
-
 // 注：EOF时data也可能有数据
 message StreamDataPacket {
     StreamDataPacketType Type = 1;
     string PlanID = 2;
-    string StreamID = 3;
+    int32 VarID = 3;
     bytes Data = 4;
 }
 
-message SendStreamResp {
+message SendStreamResp {}
+
+message GetStreamReq {
+    string PlanID = 1;
+    int32 VarID = 2;
+    string Signal = 3;
+}
+
+message SendVarReq {
+    string PlanID = 1;
+    string Var = 2;
 }
+message SendVarResp {}
 
-message FetchStreamReq {
+message GetVarReq {
     string PlanID = 1;
-    string StreamID = 2;
+    string Var = 2;
+    string Signal = 3;
+}
+message GetVarResp {
+    string Var = 1; // 此处不使用VarID的原因是，Switch的BindVars函数还需要知道Var的类型
 }
 
 message PingReq {}
 message PingResp {}
 
 service Agent {
-    rpc SendIPFSFile(stream FileDataPacket)returns(SendIPFSFileResp){}
-    rpc GetIPFSFile(GetIPFSFileReq)returns(stream FileDataPacket){}
+    rpc ExecuteIOPlan(ExecuteIOPlanReq) returns(ExecuteIOPlanResp){}
 
     rpc SendStream(stream StreamDataPacket)returns(SendStreamResp){}
-    rpc FetchStream(FetchStreamReq)returns(stream StreamDataPacket){}
+    rpc GetStream(GetStreamReq)returns(stream StreamDataPacket){}
+
+    rpc SendVar(SendVarReq)returns(SendVarResp){}
+    rpc GetVar(GetVarReq)returns(GetVarResp){}
 
     rpc Ping(PingReq) returns(PingResp){}
 }

common/pkgs/grpc/agent/agent_grpc.pb.go

@@ -21,21 +21,23 @@ import (
 const _ = grpc.SupportPackageIsVersion7
 
 const (
-	Agent_SendIPFSFile_FullMethodName = "/Agent/SendIPFSFile"
-	Agent_GetIPFSFile_FullMethodName  = "/Agent/GetIPFSFile"
-	Agent_SendStream_FullMethodName   = "/Agent/SendStream"
-	Agent_FetchStream_FullMethodName  = "/Agent/FetchStream"
-	Agent_Ping_FullMethodName         = "/Agent/Ping"
+	Agent_ExecuteIOPlan_FullMethodName = "/Agent/ExecuteIOPlan"
+	Agent_SendStream_FullMethodName    = "/Agent/SendStream"
+	Agent_GetStream_FullMethodName     = "/Agent/GetStream"
+	Agent_SendVar_FullMethodName       = "/Agent/SendVar"
+	Agent_GetVar_FullMethodName        = "/Agent/GetVar"
+	Agent_Ping_FullMethodName          = "/Agent/Ping"
 )
 
 // AgentClient is the client API for Agent service.
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
 type AgentClient interface {
-	SendIPFSFile(ctx context.Context, opts ...grpc.CallOption) (Agent_SendIPFSFileClient, error)
-	GetIPFSFile(ctx context.Context, in *GetIPFSFileReq, opts ...grpc.CallOption) (Agent_GetIPFSFileClient, error)
+	ExecuteIOPlan(ctx context.Context, in *ExecuteIOPlanReq, opts ...grpc.CallOption) (*ExecuteIOPlanResp, error)
 	SendStream(ctx context.Context, opts ...grpc.CallOption) (Agent_SendStreamClient, error)
-	FetchStream(ctx context.Context, in *FetchStreamReq, opts ...grpc.CallOption) (Agent_FetchStreamClient, error)
+	GetStream(ctx context.Context, in *GetStreamReq, opts ...grpc.CallOption) (Agent_GetStreamClient, error)
+	SendVar(ctx context.Context, in *SendVarReq, opts ...grpc.CallOption) (*SendVarResp, error)
+	GetVar(ctx context.Context, in *GetVarReq, opts ...grpc.CallOption) (*GetVarResp, error)
 	Ping(ctx context.Context, in *PingReq, opts ...grpc.CallOption) (*PingResp, error)
 }
 
@@ -47,74 +49,17 @@ func NewAgentClient(cc grpc.ClientConnInterface) AgentClient {
 	return &agentClient{cc}
 }
 
-func (c *agentClient) SendIPFSFile(ctx context.Context, opts ...grpc.CallOption) (Agent_SendIPFSFileClient, error) {
-	stream, err := c.cc.NewStream(ctx, &Agent_ServiceDesc.Streams[0], Agent_SendIPFSFile_FullMethodName, opts...)
+func (c *agentClient) ExecuteIOPlan(ctx context.Context, in *ExecuteIOPlanReq, opts ...grpc.CallOption) (*ExecuteIOPlanResp, error) {
+	out := new(ExecuteIOPlanResp)
+	err := c.cc.Invoke(ctx, Agent_ExecuteIOPlan_FullMethodName, in, out, opts...)
 	if err != nil {
 		return nil, err
 	}
-	x := &agentSendIPFSFileClient{stream}
-	return x, nil
-}
-
-type Agent_SendIPFSFileClient interface {
-	Send(*FileDataPacket) error
-	CloseAndRecv() (*SendIPFSFileResp, error)
-	grpc.ClientStream
-}
-
-type agentSendIPFSFileClient struct {
-	grpc.ClientStream
-}
-
-func (x *agentSendIPFSFileClient) Send(m *FileDataPacket) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *agentSendIPFSFileClient) CloseAndRecv() (*SendIPFSFileResp, error) {
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	m := new(SendIPFSFileResp)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *agentClient) GetIPFSFile(ctx context.Context, in *GetIPFSFileReq, opts ...grpc.CallOption) (Agent_GetIPFSFileClient, error) {
-	stream, err := c.cc.NewStream(ctx, &Agent_ServiceDesc.Streams[1], Agent_GetIPFSFile_FullMethodName, opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &agentGetIPFSFileClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type Agent_GetIPFSFileClient interface {
-	Recv() (*FileDataPacket, error)
-	grpc.ClientStream
-}
-
-type agentGetIPFSFileClient struct {
-	grpc.ClientStream
-}
-
-func (x *agentGetIPFSFileClient) Recv() (*FileDataPacket, error) {
-	m := new(FileDataPacket)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
+	return out, nil
 }
 
 func (c *agentClient) SendStream(ctx context.Context, opts ...grpc.CallOption) (Agent_SendStreamClient, error) {
-	stream, err := c.cc.NewStream(ctx, &Agent_ServiceDesc.Streams[2], Agent_SendStream_FullMethodName, opts...)
+	stream, err := c.cc.NewStream(ctx, &Agent_ServiceDesc.Streams[0], Agent_SendStream_FullMethodName, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -147,12 +92,12 @@ func (x *agentSendStreamClient) CloseAndRecv() (*SendStreamResp, error) {
 	return m, nil
 }
 
-func (c *agentClient) FetchStream(ctx context.Context, in *FetchStreamReq, opts ...grpc.CallOption) (Agent_FetchStreamClient, error) {
-	stream, err := c.cc.NewStream(ctx, &Agent_ServiceDesc.Streams[3], Agent_FetchStream_FullMethodName, opts...)
+func (c *agentClient) GetStream(ctx context.Context, in *GetStreamReq, opts ...grpc.CallOption) (Agent_GetStreamClient, error) {
+	stream, err := c.cc.NewStream(ctx, &Agent_ServiceDesc.Streams[1], Agent_GetStream_FullMethodName, opts...)
 	if err != nil {
 		return nil, err
 	}
-	x := &agentFetchStreamClient{stream}
+	x := &agentGetStreamClient{stream}
 	if err := x.ClientStream.SendMsg(in); err != nil {
 		return nil, err
 	}
@@ -162,16 +107,16 @@ func (c *agentClient) FetchStream(ctx context.Context, in *FetchStreamReq, opts
 	return x, nil
 }
 
-type Agent_FetchStreamClient interface {
+type Agent_GetStreamClient interface {
 	Recv() (*StreamDataPacket, error)
 	grpc.ClientStream
 }
 
-type agentFetchStreamClient struct {
+type agentGetStreamClient struct {
 	grpc.ClientStream
 }
 
-func (x *agentFetchStreamClient) Recv() (*StreamDataPacket, error) {
+func (x *agentGetStreamClient) Recv() (*StreamDataPacket, error) {
 	m := new(StreamDataPacket)
 	if err := x.ClientStream.RecvMsg(m); err != nil {
 		return nil, err
@@ -179,6 +124,24 @@ func (x *agentFetchStreamClient) Recv() (*StreamDataPacket, error) {
 	return m, nil
 }
 
+func (c *agentClient) SendVar(ctx context.Context, in *SendVarReq, opts ...grpc.CallOption) (*SendVarResp, error) {
+	out := new(SendVarResp)
+	err := c.cc.Invoke(ctx, Agent_SendVar_FullMethodName, in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *agentClient) GetVar(ctx context.Context, in *GetVarReq, opts ...grpc.CallOption) (*GetVarResp, error) {
+	out := new(GetVarResp)
+	err := c.cc.Invoke(ctx, Agent_GetVar_FullMethodName, in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *agentClient) Ping(ctx context.Context, in *PingReq, opts ...grpc.CallOption) (*PingResp, error) {
 	out := new(PingResp)
 	err := c.cc.Invoke(ctx, Agent_Ping_FullMethodName, in, out, opts...)
@@ -192,10 +155,11 @@ func (c *agentClient) Ping(ctx context.Context, in *PingReq, opts ...grpc.CallOp
 // All implementations must embed UnimplementedAgentServer
 // for forward compatibility
 type AgentServer interface {
-	SendIPFSFile(Agent_SendIPFSFileServer) error
-	GetIPFSFile(*GetIPFSFileReq, Agent_GetIPFSFileServer) error
+	ExecuteIOPlan(context.Context, *ExecuteIOPlanReq) (*ExecuteIOPlanResp, error)
 	SendStream(Agent_SendStreamServer) error
-	FetchStream(*FetchStreamReq, Agent_FetchStreamServer) error
+	GetStream(*GetStreamReq, Agent_GetStreamServer) error
+	SendVar(context.Context, *SendVarReq) (*SendVarResp, error)
+	GetVar(context.Context, *GetVarReq) (*GetVarResp, error)
 	Ping(context.Context, *PingReq) (*PingResp, error)
 	mustEmbedUnimplementedAgentServer()
 }
@@ -204,17 +168,20 @@ type AgentServer interface {
 type UnimplementedAgentServer struct {
 }
 
-func (UnimplementedAgentServer) SendIPFSFile(Agent_SendIPFSFileServer) error {
-	return status.Errorf(codes.Unimplemented, "method SendIPFSFile not implemented")
-}
-func (UnimplementedAgentServer) GetIPFSFile(*GetIPFSFileReq, Agent_GetIPFSFileServer) error {
-	return status.Errorf(codes.Unimplemented, "method GetIPFSFile not implemented")
+func (UnimplementedAgentServer) ExecuteIOPlan(context.Context, *ExecuteIOPlanReq) (*ExecuteIOPlanResp, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ExecuteIOPlan not implemented")
 }
 func (UnimplementedAgentServer) SendStream(Agent_SendStreamServer) error {
 	return status.Errorf(codes.Unimplemented, "method SendStream not implemented")
 }
-func (UnimplementedAgentServer) FetchStream(*FetchStreamReq, Agent_FetchStreamServer) error {
-	return status.Errorf(codes.Unimplemented, "method FetchStream not implemented")
+func (UnimplementedAgentServer) GetStream(*GetStreamReq, Agent_GetStreamServer) error {
+	return status.Errorf(codes.Unimplemented, "method GetStream not implemented")
+}
+func (UnimplementedAgentServer) SendVar(context.Context, *SendVarReq) (*SendVarResp, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SendVar not implemented")
+}
+func (UnimplementedAgentServer) GetVar(context.Context, *GetVarReq) (*GetVarResp, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetVar not implemented")
 }
 func (UnimplementedAgentServer) Ping(context.Context, *PingReq) (*PingResp, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
@@ -232,51 +199,22 @@ func RegisterAgentServer(s grpc.ServiceRegistrar, srv AgentServer) {
 	s.RegisterService(&Agent_ServiceDesc, srv)
 }
 
-func _Agent_SendIPFSFile_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(AgentServer).SendIPFSFile(&agentSendIPFSFileServer{stream})
-}
-
-type Agent_SendIPFSFileServer interface {
-	SendAndClose(*SendIPFSFileResp) error
-	Recv() (*FileDataPacket, error)
-	grpc.ServerStream
-}
-
-type agentSendIPFSFileServer struct {
-	grpc.ServerStream
-}
-
-func (x *agentSendIPFSFileServer) SendAndClose(m *SendIPFSFileResp) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *agentSendIPFSFileServer) Recv() (*FileDataPacket, error) {
-	m := new(FileDataPacket)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
+func _Agent_ExecuteIOPlan_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ExecuteIOPlanReq)
+	if err := dec(in); err != nil {
 		return nil, err
 	}
-	return m, nil
-}
-
-func _Agent_GetIPFSFile_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(GetIPFSFileReq)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
+	if interceptor == nil {
+		return srv.(AgentServer).ExecuteIOPlan(ctx, in)
 	}
-	return srv.(AgentServer).GetIPFSFile(m, &agentGetIPFSFileServer{stream})
-}
-
-type Agent_GetIPFSFileServer interface {
-	Send(*FileDataPacket) error
-	grpc.ServerStream
-}
-
-type agentGetIPFSFileServer struct {
-	grpc.ServerStream
-}
-
-func (x *agentGetIPFSFileServer) Send(m *FileDataPacket) error {
-	return x.ServerStream.SendMsg(m)
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: Agent_ExecuteIOPlan_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(AgentServer).ExecuteIOPlan(ctx, req.(*ExecuteIOPlanReq))
+	}
+	return interceptor(ctx, in, info, handler)
 }
 
 func _Agent_SendStream_Handler(srv interface{}, stream grpc.ServerStream) error {
@@ -305,27 +243,63 @@ func (x *agentSendStreamServer) Recv() (*StreamDataPacket, error) {
 	return m, nil
 }
 
-func _Agent_FetchStream_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(FetchStreamReq)
+func _Agent_GetStream_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(GetStreamReq)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return srv.(AgentServer).FetchStream(m, &agentFetchStreamServer{stream})
+	return srv.(AgentServer).GetStream(m, &agentGetStreamServer{stream})
 }
 
-type Agent_FetchStreamServer interface {
+type Agent_GetStreamServer interface {
 	Send(*StreamDataPacket) error
 	grpc.ServerStream
 }
 
-type agentFetchStreamServer struct {
+type agentGetStreamServer struct {
 	grpc.ServerStream
 }
 
-func (x *agentFetchStreamServer) Send(m *StreamDataPacket) error {
+func (x *agentGetStreamServer) Send(m *StreamDataPacket) error {
 	return x.ServerStream.SendMsg(m)
 }
 
+func _Agent_SendVar_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SendVarReq)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(AgentServer).SendVar(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: Agent_SendVar_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(AgentServer).SendVar(ctx, req.(*SendVarReq))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Agent_GetVar_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetVarReq)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(AgentServer).GetVar(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: Agent_GetVar_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(AgentServer).GetVar(ctx, req.(*GetVarReq))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _Agent_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(PingReq)
 	if err := dec(in); err != nil {
@@ -352,29 +326,31 @@ var Agent_ServiceDesc = grpc.ServiceDesc{
 	HandlerType: (*AgentServer)(nil),
 	Methods: []grpc.MethodDesc{
 		{
-			MethodName: "Ping",
-			Handler:    _Agent_Ping_Handler,
+			MethodName: "ExecuteIOPlan",
+			Handler:    _Agent_ExecuteIOPlan_Handler,
 		},
-	},
-	Streams: []grpc.StreamDesc{
 		{
-			StreamName:    "SendIPFSFile",
-			Handler:       _Agent_SendIPFSFile_Handler,
-			ClientStreams: true,
+			MethodName: "SendVar",
+			Handler:    _Agent_SendVar_Handler,
 		},
 		{
-			StreamName:    "GetIPFSFile",
-			Handler:       _Agent_GetIPFSFile_Handler,
-			ServerStreams: true,
+			MethodName: "GetVar",
+			Handler:    _Agent_GetVar_Handler,
+		},
+		{
+			MethodName: "Ping",
+			Handler:    _Agent_Ping_Handler,
 		},
+	},
+	Streams: []grpc.StreamDesc{
 		{
 			StreamName:    "SendStream",
 			Handler:       _Agent_SendStream_Handler,
 			ClientStreams: true,
 		},
 		{
-			StreamName:    "FetchStream",
-			Handler:       _Agent_FetchStream_Handler,
+			StreamName:    "GetStream",
+			Handler:       _Agent_GetStream_Handler,
 			ServerStreams: true,
 		},
 	},

common/pkgs/grpc/agent/client.go

@@ -5,7 +5,8 @@ import (
 	"fmt"
 	"io"
 
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
+	"gitlink.org.cn/cloudream/common/utils/serder"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 )
@@ -27,59 +28,29 @@ func NewClient(addr string) (*Client, error) {
 	}, nil
 }
 
-func (c *Client) SendIPFSFile(file io.Reader) (string, error) {
-	sendCli, err := c.cli.SendIPFSFile(context.Background())
+func (c *Client) ExecuteIOPlan(ctx context.Context, plan exec.Plan) error {
+	data, err := serder.ObjectToJSONEx(plan)
 	if err != nil {
-		return "", err
+		return err
 	}
 
-	buf := make([]byte, 4096)
-	for {
-		rd, err := file.Read(buf)
-		if err == io.EOF {
-			err := sendCli.Send(&FileDataPacket{
-				Type: StreamDataPacketType_EOF,
-				Data: buf[:rd],
-			})
-			if err != nil {
-				return "", fmt.Errorf("sending EOF packet: %w", err)
-			}
-
-			resp, err := sendCli.CloseAndRecv()
-			if err != nil {
-				return "", fmt.Errorf("receiving response: %w", err)
-			}
-
-			return resp.FileHash, nil
-		}
-
-		if err != nil {
-			return "", fmt.Errorf("reading file data: %w", err)
-		}
-
-		err = sendCli.Send(&FileDataPacket{
-			Type: StreamDataPacketType_Data,
-			Data: buf[:rd],
-		})
-		if err != nil {
-			return "", fmt.Errorf("sending data packet: %w", err)
-		}
-	}
+	_, err = c.cli.ExecuteIOPlan(ctx, &ExecuteIOPlanReq{
+		Plan: string(data),
+	})
+	return err
 }
 
-type fileReadCloser struct {
+type grpcStreamReadCloser struct {
 	io.ReadCloser
-	// stream      Agent_GetIPFSFileClient
-	// TODO 临时使用
-	recvFn      func() (*StreamDataPacket, error)
+	stream      Agent_GetStreamClient
 	cancelFn    context.CancelFunc
 	readingData []byte
 	recvEOF     bool
 }
 
-func (s *fileReadCloser) Read(p []byte) (int, error) {
+func (s *grpcStreamReadCloser) Read(p []byte) (int, error) {
 	if len(s.readingData) == 0 && !s.recvEOF {
-		resp, err := s.recvFn()
+		resp, err := s.stream.Recv()
 		if err != nil {
 			return 0, err
 		}
@@ -106,63 +77,34 @@ func (s *fileReadCloser) Read(p []byte) (int, error) {
 	return cnt, nil
 }
 
-func (s *fileReadCloser) Close() error {
+func (s *grpcStreamReadCloser) Close() error {
 	s.cancelFn()
 
 	return nil
 }
 
-func (c *Client) GetIPFSFile(fileHash string) (io.ReadCloser, error) {
-	ctx, cancel := context.WithCancel(context.Background())
-
-	stream, err := c.cli.GetIPFSFile(ctx, &GetIPFSFileReq{
-		FileHash: fileHash,
-	})
-	if err != nil {
-		cancel()
-		return nil, fmt.Errorf("request grpc failed, err: %w", err)
-	}
-
-	return &fileReadCloser{
-		// TODO 临时处理方案
-		recvFn: func() (*StreamDataPacket, error) {
-			pkt, err := stream.Recv()
-			if err != nil {
-				return nil, err
-			}
-
-			return &StreamDataPacket{
-				Type: pkt.Type,
-				Data: pkt.Data,
-			}, nil
-		},
-		cancelFn: cancel,
-	}, nil
-}
-
-func (c *Client) SendStream(planID ioswitch.PlanID, streamID ioswitch.StreamID, file io.Reader) error {
-	sendCli, err := c.cli.SendStream(context.Background())
+func (c *Client) SendStream(ctx context.Context, planID exec.PlanID, varID exec.VarID, str io.Reader) error {
+	sendCli, err := c.cli.SendStream(ctx)
 	if err != nil {
 		return err
 	}
 
 	err = sendCli.Send(&StreamDataPacket{
-		Type:     StreamDataPacketType_SendArgs,
-		PlanID:   string(planID),
-		StreamID: string(streamID),
+		Type:   StreamDataPacketType_SendArgs,
+		PlanID: string(planID),
+		VarID:  int32(varID),
 	})
 	if err != nil {
-		return fmt.Errorf("sending stream id packet: %w", err)
+		return fmt.Errorf("sending first stream packet: %w", err)
 	}
 
-	buf := make([]byte, 4096)
+	buf := make([]byte, 1024*64)
 	for {
-		rd, err := file.Read(buf)
+		rd, err := str.Read(buf)
 		if err == io.EOF {
 			err := sendCli.Send(&StreamDataPacket{
-				Type:     StreamDataPacketType_EOF,
-				StreamID: string(streamID),
-				Data:     buf[:rd],
+				Type: StreamDataPacketType_EOF,
+				Data: buf[:rd],
 			})
 			if err != nil {
 				return fmt.Errorf("sending EOF packet: %w", err)
@@ -177,13 +119,12 @@ func (c *Client) SendStream(planID ioswitch.PlanID, streamID ioswitch.StreamID,
 		}
 
 		if err != nil {
-			return fmt.Errorf("reading file data: %w", err)
+			return fmt.Errorf("reading stream data: %w", err)
 		}
 
 		err = sendCli.Send(&StreamDataPacket{
-			Type:     StreamDataPacketType_Data,
-			StreamID: string(streamID),
-			Data:     buf[:rd],
+			Type: StreamDataPacketType_Data,
+			Data: buf[:rd],
 		})
 		if err != nil {
 			return fmt.Errorf("sending data packet: %w", err)
@@ -191,24 +132,77 @@ func (c *Client) SendStream(planID ioswitch.PlanID, streamID ioswitch.StreamID,
 	}
 }
 
-func (c *Client) FetchStream(planID ioswitch.PlanID, streamID ioswitch.StreamID) (io.ReadCloser, error) {
-	ctx, cancel := context.WithCancel(context.Background())
+func (c *Client) GetStream(ctx context.Context, planID exec.PlanID, varID exec.VarID, signal *exec.SignalVar) (io.ReadCloser, error) {
+	ctx, cancel := context.WithCancel(ctx)
+
+	sdata, err := serder.ObjectToJSONEx(signal)
+	if err != nil {
+		cancel()
+		return nil, err
+	}
 
-	stream, err := c.cli.FetchStream(ctx, &FetchStreamReq{
-		PlanID:   string(planID),
-		StreamID: string(streamID),
+	stream, err := c.cli.GetStream(ctx, &GetStreamReq{
+		PlanID: string(planID),
+		VarID:  int32(varID),
+		Signal: string(sdata),
 	})
 	if err != nil {
 		cancel()
 		return nil, fmt.Errorf("request grpc failed, err: %w", err)
 	}
 
-	return &fileReadCloser{
-		recvFn:   stream.Recv,
+	return &grpcStreamReadCloser{
+		stream:   stream,
 		cancelFn: cancel,
 	}, nil
 }
 
+func (c *Client) SendVar(ctx context.Context, planID exec.PlanID, v exec.Var) error {
+	data, err := serder.ObjectToJSONEx(v)
+	if err != nil {
+		return err
+	}
+
+	_, err = c.cli.SendVar(ctx, &SendVarReq{
+		PlanID: string(planID),
+		Var:    string(data),
+	})
+	return err
+}
+
+func (c *Client) GetVar(ctx context.Context, planID exec.PlanID, v exec.Var, signal *exec.SignalVar) error {
+	vdata, err := serder.ObjectToJSONEx(v)
+	if err != nil {
+		return err
+	}
+
+	sdata, err := serder.ObjectToJSONEx(signal)
+	if err != nil {
+		return err
+	}
+
+	resp, err := c.cli.GetVar(ctx, &GetVarReq{
+		PlanID: string(planID),
+		Var:    string(vdata),
+		Signal: string(sdata),
+	})
+	if err != nil {
+		return err
+	}
+
+	v2, err := serder.JSONToObjectEx[exec.Var]([]byte(resp.Var))
+	if err != nil {
+		return err
+	}
+
+	err = exec.AssignVar(v2, v)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (c *Client) Ping() error {
 	_, err := c.cli.Ping(context.Background(), &PingReq{})
 	return err

common/pkgs/ioswitch/ioswitch.go

@@ -1,35 +0,0 @@
-package ioswitch
-
-import (
-	"io"
-)
-
-type PlanID string
-
-type StreamID string
-
-type Plan struct {
-	ID  PlanID
-	Ops []Op
-}
-
-type Stream struct {
-	ID     StreamID
-	Stream io.ReadCloser
-}
-
-func NewStream(id StreamID, stream io.ReadCloser) Stream {
-	return Stream{
-		ID:     id,
-		Stream: stream,
-	}
-}
-
-type Op interface {
-	Execute(sw *Switch, planID PlanID) error
-}
-
-type ResultKV struct {
-	Key   string
-	Value any
-}

common/pkgs/ioswitch/ops/chunked_join.go

@@ -1,49 +0,0 @@
-package ops
-
-import (
-	"context"
-	"io"
-
-	"gitlink.org.cn/cloudream/common/pkgs/future"
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-)
-
-type ChunkedJoin struct {
-	InputIDs  []ioswitch.StreamID `json:"inputIDs"`
-	OutputID  ioswitch.StreamID   `json:"outputID"`
-	ChunkSize int                 `json:"chunkSize"`
-}
-
-func (o *ChunkedJoin) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	strs, err := sw.WaitStreams(planID, o.InputIDs...)
-	if err != nil {
-		return err
-	}
-
-	var strReaders []io.Reader
-	for _, s := range strs {
-		strReaders = append(strReaders, s.Stream)
-	}
-	defer func() {
-		for _, str := range strs {
-			str.Stream.Close()
-		}
-	}()
-
-	fut := future.NewSetVoid()
-	sw.StreamReady(planID,
-		ioswitch.NewStream(o.OutputID,
-			io2.AfterReadClosedOnce(io2.ChunkedJoin(strReaders, o.ChunkSize), func(closer io.ReadCloser) {
-				fut.SetVoid()
-			}),
-		),
-	)
-
-	fut.Wait(context.TODO())
-	return nil
-}
-
-func init() {
-	OpUnion.AddT((*ChunkedJoin)(nil))
-}

common/pkgs/ioswitch/ops/chunked_split.go

@@ -1,49 +0,0 @@
-package ops
-
-import (
-	"io"
-	"sync"
-
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-)
-
-type ChunkedSplit struct {
-	InputID      ioswitch.StreamID   `json:"inputID"`
-	OutputIDs    []ioswitch.StreamID `json:"outputIDs"`
-	ChunkSize    int                 `json:"chunkSize"`
-	StreamCount  int                 `json:"streamCount"`
-	PaddingZeros bool                `json:"paddingZeros"`
-}
-
-func (o *ChunkedSplit) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	str, err := sw.WaitStreams(planID, o.InputID)
-	if err != nil {
-		return err
-	}
-	defer str[0].Stream.Close()
-
-	wg := sync.WaitGroup{}
-	outputs := io2.ChunkedSplit(str[0].Stream, o.ChunkSize, o.StreamCount, io2.ChunkedSplitOption{
-		PaddingZeros: o.PaddingZeros,
-	})
-
-	for i := range outputs {
-		wg.Add(1)
-
-		sw.StreamReady(planID, ioswitch.NewStream(
-			o.OutputIDs[i],
-			io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
-				wg.Done()
-			}),
-		))
-	}
-
-	wg.Wait()
-
-	return nil
-}
-
-func init() {
-	OpUnion.AddT((*ChunkedSplit)(nil))
-}

common/pkgs/ioswitch/ops/clone.go

@@ -1,43 +0,0 @@
-package ops
-
-import (
-	"io"
-	"sync"
-
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-)
-
-type Clone struct {
-	InputID   ioswitch.StreamID   `json:"inputID"`
-	OutputIDs []ioswitch.StreamID `json:"outputIDs"`
-}
-
-func (o *Clone) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	strs, err := sw.WaitStreams(planID, o.InputID)
-	if err != nil {
-		return err
-	}
-	defer strs[0].Stream.Close()
-
-	wg := sync.WaitGroup{}
-	cloned := io2.Clone(strs[0].Stream, len(o.OutputIDs))
-	for i, s := range cloned {
-		wg.Add(1)
-
-		sw.StreamReady(planID,
-			ioswitch.NewStream(o.OutputIDs[i],
-				io2.AfterReadClosedOnce(s, func(closer io.ReadCloser) {
-					wg.Done()
-				}),
-			),
-		)
-	}
-
-	wg.Wait()
-	return nil
-}
-
-func init() {
-	OpUnion.AddT((*Clone)(nil))
-}

common/pkgs/ioswitch/ops/ec.go

@@ -1,102 +0,0 @@
-package ops
-
-import (
-	"fmt"
-	"io"
-	"sync"
-
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ec"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-)
-
-type ECReconstructAny struct {
-	EC                 cdssdk.ECRedundancy `json:"ec"`
-	InputIDs           []ioswitch.StreamID `json:"inputIDs"`
-	OutputIDs          []ioswitch.StreamID `json:"outputIDs"`
-	InputBlockIndexes  []int               `json:"inputBlockIndexes"`
-	OutputBlockIndexes []int               `json:"outputBlockIndexes"`
-}
-
-func (o *ECReconstructAny) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	rs, err := ec.NewStreamRs(o.EC.K, o.EC.N, o.EC.ChunkSize)
-	if err != nil {
-		return fmt.Errorf("new ec: %w", err)
-	}
-
-	strs, err := sw.WaitStreams(planID, o.InputIDs...)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		for _, s := range strs {
-			s.Stream.Close()
-		}
-	}()
-
-	var inputs []io.Reader
-	for _, s := range strs {
-		inputs = append(inputs, s.Stream)
-	}
-
-	outputs := rs.ReconstructAny(inputs, o.InputBlockIndexes, o.OutputBlockIndexes)
-
-	wg := sync.WaitGroup{}
-	for i, id := range o.OutputIDs {
-		wg.Add(1)
-		sw.StreamReady(planID, ioswitch.NewStream(id, io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
-			wg.Done()
-		})))
-	}
-	wg.Wait()
-
-	return nil
-}
-
-type ECReconstruct struct {
-	EC                cdssdk.ECRedundancy `json:"ec"`
-	InputIDs          []ioswitch.StreamID `json:"inputIDs"`
-	OutputIDs         []ioswitch.StreamID `json:"outputIDs"`
-	InputBlockIndexes []int               `json:"inputBlockIndexes"`
-}
-
-func (o *ECReconstruct) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	rs, err := ec.NewStreamRs(o.EC.K, o.EC.N, o.EC.ChunkSize)
-	if err != nil {
-		return fmt.Errorf("new ec: %w", err)
-	}
-
-	strs, err := sw.WaitStreams(planID, o.InputIDs...)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		for _, s := range strs {
-			s.Stream.Close()
-		}
-	}()
-
-	var inputs []io.Reader
-	for _, s := range strs {
-		inputs = append(inputs, s.Stream)
-	}
-
-	outputs := rs.ReconstructData(inputs, o.InputBlockIndexes)
-
-	wg := sync.WaitGroup{}
-	for i, id := range o.OutputIDs {
-		wg.Add(1)
-		sw.StreamReady(planID, ioswitch.NewStream(id, io2.AfterReadClosedOnce(outputs[i], func(closer io.ReadCloser) {
-			wg.Done()
-		})))
-	}
-	wg.Wait()
-
-	return nil
-}
-
-func init() {
-	OpUnion.AddT((*ECReconstructAny)(nil))
-	OpUnion.AddT((*ECReconstruct)(nil))
-}

common/pkgs/ioswitch/ops/file.go

@@ -1,71 +0,0 @@
-package ops
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"os"
-	"path"
-
-	"gitlink.org.cn/cloudream/common/pkgs/future"
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-)
-
-type FileWrite struct {
-	InputID  ioswitch.StreamID `json:"inputID"`
-	FilePath string            `json:"filePath"`
-}
-
-func (o *FileWrite) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	str, err := sw.WaitStreams(planID, o.InputID)
-	if err != nil {
-		return err
-	}
-	defer str[0].Stream.Close()
-
-	dir := path.Dir(o.FilePath)
-	err = os.MkdirAll(dir, 0777)
-	if err != nil {
-		return fmt.Errorf("mkdir: %w", err)
-	}
-
-	file, err := os.Create(o.FilePath)
-	if err != nil {
-		return fmt.Errorf("opening file: %w", err)
-	}
-	defer file.Close()
-
-	_, err = io.Copy(file, str[0].Stream)
-	if err != nil {
-		return fmt.Errorf("copying data to file: %w", err)
-	}
-
-	return nil
-}
-
-type FileRead struct {
-	OutputID ioswitch.StreamID `json:"outputID"`
-	FilePath string            `json:"filePath"`
-}
-
-func (o *FileRead) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	file, err := os.Open(o.FilePath)
-	if err != nil {
-		return fmt.Errorf("opening file: %w", err)
-	}
-
-	fut := future.NewSetVoid()
-	sw.StreamReady(planID, ioswitch.NewStream(o.OutputID, io2.AfterReadClosed(file, func(closer io.ReadCloser) {
-		fut.SetVoid()
-	})))
-
-	fut.Wait(context.TODO())
-
-	return nil
-}
-
-func init() {
-	OpUnion.AddT((*FileRead)(nil))
-	OpUnion.AddT((*FileWrite)(nil))
-}

common/pkgs/ioswitch/ops/grpc.go

@@ -1,84 +0,0 @@
-package ops
-
-import (
-	"context"
-	"fmt"
-	"io"
-
-	"gitlink.org.cn/cloudream/common/pkgs/future"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-)
-
-type GRPCSend struct {
-	LocalID  ioswitch.StreamID `json:"localID"`
-	RemoteID ioswitch.StreamID `json:"remoteID"`
-	Node     cdssdk.Node       `json:"node"`
-}
-
-func (o *GRPCSend) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	logger.
-		WithField("LocalID", o.LocalID).
-		WithField("RemoteID", o.RemoteID).
-		Debugf("grpc send")
-
-	strs, err := sw.WaitStreams(planID, o.LocalID)
-	if err != nil {
-		return err
-	}
-	defer strs[0].Stream.Close()
-
-	// TODO 根据客户端地址选择IP和端口
-	agtCli, err := stgglb.AgentRPCPool.Acquire(o.Node.ExternalIP, o.Node.ExternalGRPCPort)
-	if err != nil {
-		return fmt.Errorf("new agent rpc client: %w", err)
-	}
-	defer stgglb.AgentRPCPool.Release(agtCli)
-
-	err = agtCli.SendStream(planID, o.RemoteID, strs[0].Stream)
-	if err != nil {
-		return fmt.Errorf("sending stream: %w", err)
-	}
-
-	return nil
-}
-
-type GRPCFetch struct {
-	RemoteID ioswitch.StreamID `json:"remoteID"`
-	LocalID  ioswitch.StreamID `json:"localID"`
-	Node     cdssdk.Node       `json:"node"`
-}
-
-func (o *GRPCFetch) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	// TODO 根据客户端地址选择IP和端口
-	agtCli, err := stgglb.AgentRPCPool.Acquire(o.Node.ExternalIP, o.Node.ExternalGRPCPort)
-	if err != nil {
-		return fmt.Errorf("new agent rpc client: %w", err)
-	}
-	defer stgglb.AgentRPCPool.Release(agtCli)
-
-	str, err := agtCli.FetchStream(planID, o.RemoteID)
-	if err != nil {
-		return fmt.Errorf("fetching stream: %w", err)
-	}
-
-	fut := future.NewSetVoid()
-	str = io2.AfterReadClosedOnce(str, func(closer io.ReadCloser) {
-		fut.SetVoid()
-	})
-
-	sw.StreamReady(planID, ioswitch.NewStream(o.LocalID, str))
-
-	// TODO
-	fut.Wait(context.TODO())
-
-	return err
-}
-
-func init() {
-	OpUnion.AddT((*GRPCSend)(nil))
-	OpUnion.AddT((*GRPCFetch)(nil))
-}

common/pkgs/ioswitch/ops/ipfs.go

@@ -1,93 +0,0 @@
-package ops
-
-import (
-	"context"
-	"fmt"
-	"io"
-
-	"gitlink.org.cn/cloudream/common/pkgs/future"
-	"gitlink.org.cn/cloudream/common/pkgs/ipfs"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	stgglb "gitlink.org.cn/cloudream/storage/common/globals"
-	"gitlink.org.cn/cloudream/storage/common/pkgs/ioswitch"
-)
-
-type IPFSRead struct {
-	Output   ioswitch.StreamID `json:"output"`
-	FileHash string            `json:"fileHash"`
-	Option   ipfs.ReadOption   `json:"option"`
-}
-
-func (o *IPFSRead) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	logger.
-		WithField("FileHash", o.FileHash).
-		WithField("Output", o.Output).
-		Debugf("ipfs read op")
-	defer logger.Debugf("ipfs read op finished")
-
-	ipfsCli, err := stgglb.IPFSPool.Acquire()
-	if err != nil {
-		return fmt.Errorf("new ipfs client: %w", err)
-	}
-	defer stgglb.IPFSPool.Release(ipfsCli)
-
-	file, err := ipfsCli.OpenRead(o.FileHash, o.Option)
-	if err != nil {
-		return fmt.Errorf("reading ipfs: %w", err)
-	}
-
-	fut := future.NewSetVoid()
-	file = io2.AfterReadClosedOnce(file, func(closer io.ReadCloser) {
-		fut.SetVoid()
-	})
-
-	sw.StreamReady(planID, ioswitch.NewStream(o.Output, file))
-
-	// TODO context
-	fut.Wait(context.TODO())
-	return nil
-}
-
-type IPFSWrite struct {
-	Input     ioswitch.StreamID `json:"input"`
-	ResultKey string            `json:"resultKey"`
-}
-
-func (o *IPFSWrite) Execute(sw *ioswitch.Switch, planID ioswitch.PlanID) error {
-	logger.
-		WithField("ResultKey", o.ResultKey).
-		WithField("Input", o.Input).
-		Debugf("ipfs write op")
-
-	ipfsCli, err := stgglb.IPFSPool.Acquire()
-	if err != nil {
-		return fmt.Errorf("new ipfs client: %w", err)
-	}
-	defer stgglb.IPFSPool.Release(ipfsCli)
-
-	strs, err := sw.WaitStreams(planID, o.Input)
-	if err != nil {
-		return err
-	}
-	defer strs[0].Stream.Close()
-
-	fileHash, err := ipfsCli.CreateFile(strs[0].Stream)
-	if err != nil {
-		return fmt.Errorf("creating ipfs file: %w", err)
-	}
-
-	if o.ResultKey != "" {
-		sw.AddResultValue(planID, ioswitch.ResultKV{
-			Key:   o.ResultKey,
-			Value: fileHash,
-		})
-	}
-
-	return nil
-}
-
-func init() {
-	OpUnion.AddT((*IPFSRead)(nil))
-	OpUnion.AddT((*IPFSWrite)(nil))
-}