Merge branch 'master' of https://gitlink.org.cn/JointCloud/JCS-pub

client/internal/http/v1/object.go

@@ -3,6 +3,7 @@ package http
 import (
 	"fmt"
 	"io"
+	"mime"
 	"mime/multipart"
 	"net/http"
 	"net/url"
@@ -11,7 +12,9 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	"gitlink.org.cn/cloudream/common/utils/http2"
 	"gitlink.org.cn/cloudream/common/utils/math2"
+	"gitlink.org.cn/cloudream/common/utils/serder"
 	"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
 	"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
 	cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
@@ -69,27 +72,53 @@ func (s *ObjectService) ListByIDs(ctx *gin.Context) {
 	ctx.JSON(http.StatusOK, types.OK(cliapi.ObjectListByIDsResp{Objects: objs}))
 }
 
-type ObjectUpload struct {
-	Info  cliapi.ObjectUploadInfo `form:"info" binding:"required"`
-	Files []*multipart.FileHeader `form:"files"`
-}
-
 func (s *ObjectService) Upload(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Object.Upload")
 
-	var req ObjectUpload
-	if err := ctx.ShouldBind(&req); err != nil {
-		log.Warnf("binding body: %s", err.Error())
-		ctx.JSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "missing argument or invalid argument"))
+	contType := ctx.GetHeader("Content-Type")
+
+	mtype, params, err := mime.ParseMediaType(contType)
+	if err != nil {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "parse content-type: %v", err))
+		return
+	}
+	if mtype != http2.ContentTypeMultiPart {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "content-type %v not supported", mtype))
+		return
+	}
+
+	boundary := params["boundary"]
+	if boundary == "" {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "missing boundary in content-type"))
+		return
+	}
+
+	mr := multipart.NewReader(ctx.Request.Body, boundary)
+
+	p, err := mr.NextPart()
+	if err != nil {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "read next part: %v", err))
 		return
 	}
 
-	copyToPath := make([]clitypes.JPath, 0, len(req.Info.CopyToPath))
-	for _, p := range req.Info.CopyToPath {
+	var info cliapi.ObjectUploadInfo
+	err = serder.JSONToObjectStream(p, &info)
+	if err != nil {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "parse upload info: %v", err))
+		return
+	}
+
+	if info.PackageID == 0 {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "missing packageID in upload info"))
+		return
+	}
+
+	copyToPath := make([]clitypes.JPath, 0, len(info.CopyToPath))
+	for _, p := range info.CopyToPath {
 		copyToPath = append(copyToPath, clitypes.PathFromJcsPathString(p))
 	}
 
-	up, err := s.svc.Uploader.BeginUpdate(req.Info.PackageID, req.Info.Affinity, req.Info.CopyTo, copyToPath)
+	up, err := s.svc.Uploader.BeginUpdate(info.PackageID, info.Affinity, info.CopyTo, copyToPath)
 	if err != nil {
 		log.Warnf("begin update: %s", err.Error())
 		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("begin update: %v", err)))
@@ -98,26 +127,30 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
 	defer up.Abort()
 
 	var pathes []string
-	for _, file := range req.Files {
-		f, err := file.Open()
+
+	for {
+		file, err := mr.NextPart()
+		if err == io.EOF {
+			break
+		}
 		if err != nil {
-			log.Warnf("open file: %s", err.Error())
-			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("open file %v: %v", file.Filename, err)))
+			log.Warnf("read next part: %s", err.Error())
+			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("read next part: %v", err)))
 			return
 		}
 
-		path, err := url.PathUnescape(file.Filename)
+		path, err := url.PathUnescape(file.FileName())
 		if err != nil {
 			log.Warnf("unescape filename: %s", err.Error())
-			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("unescape filename %v: %v", file.Filename, err)))
+			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("unescape filename %v: %v", file.FileName(), err)))
 			return
 		}
 		path = filepath.ToSlash(path)
 
-		err = up.Upload(clitypes.PathFromJcsPathString(path), f)
+		err = up.Upload(clitypes.PathFromJcsPathString(path), file)
 		if err != nil {
 			log.Warnf("uploading file: %s", err.Error())
-			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.Filename, err)))
+			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
 			return
 		}
 		pathes = append(pathes, path)

client/internal/http/v1/package.go

@@ -5,6 +5,7 @@ import (
 	"archive/zip"
 	"fmt"
 	"io"
+	"mime"
 	"mime/multipart"
 	"net/http"
 	"net/url"
@@ -13,6 +14,8 @@ import (
 	"github.com/gin-gonic/gin"
 	"gitlink.org.cn/cloudream/common/consts/errorcode"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	"gitlink.org.cn/cloudream/common/utils/http2"
+	"gitlink.org.cn/cloudream/common/utils/serder"
 	"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
 	"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
 	cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
@@ -106,33 +109,54 @@ func (s *PackageService) Create(ctx *gin.Context) {
 	}))
 }
 
-type PackageCreateUpload struct {
-	Info  cliapi.PackageCreateUploadInfo `form:"info" binding:"required"`
-	Files []*multipart.FileHeader        `form:"files"`
-}
-
 func (s *PackageService) CreateLoad(ctx *gin.Context) {
 	log := logger.WithField("HTTP", "Package.CreateUpload")
 
-	var req PackageCreateUpload
-	if err := ctx.ShouldBind(&req); err != nil {
-		log.Warnf("binding body: %s", err.Error())
-		ctx.JSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "missing argument or invalid argument"))
+	contType := ctx.GetHeader("Content-Type")
+
+	mtype, params, err := mime.ParseMediaType(contType)
+	if err != nil {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "parse content-type: %v", err))
+		return
+	}
+	if mtype != http2.ContentTypeMultiPart {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "content-type %v not supported", mtype))
+		return
+	}
+
+	boundary := params["boundary"]
+	if boundary == "" {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "missing boundary in content-type"))
 		return
 	}
 
-	if len(req.Info.CopyTo) != len(req.Info.CopyToPath) {
+	mr := multipart.NewReader(ctx.Request.Body, boundary)
+
+	p, err := mr.NextPart()
+	if err != nil {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "read next part: %v", err))
+		return
+	}
+
+	var info cliapi.PackageCreateUploadInfo
+	err = serder.JSONToObjectStream(p, &info)
+	if err != nil {
+		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "parse upload info: %v", err))
+		return
+	}
+
+	if len(info.CopyTo) != len(info.CopyToPath) {
 		log.Warnf("CopyTo and CopyToPath count not match")
 		ctx.JSON(http.StatusOK, types.Failed(ecode.BadArgument, "CopyTo and CopyToPath count not match"))
 		return
 	}
 
-	copyToPath := make([]clitypes.JPath, 0, len(req.Info.CopyToPath))
-	for _, p := range req.Info.CopyToPath {
+	copyToPath := make([]clitypes.JPath, 0, len(info.CopyToPath))
+	for _, p := range info.CopyToPath {
 		copyToPath = append(copyToPath, clitypes.PathFromJcsPathString(p))
 	}
 
-	up, err := s.svc.Uploader.BeginCreateUpload(req.Info.BucketID, req.Info.Name, req.Info.CopyTo, copyToPath)
+	up, err := s.svc.Uploader.BeginCreateUpload(info.BucketID, info.Name, info.CopyTo, copyToPath)
 	if err != nil {
 		log.Warnf("begin package create upload: %s", err.Error())
 		ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "%v", err))
@@ -141,26 +165,29 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
 	defer up.Abort()
 
 	var pathes []string
-	for _, file := range req.Files {
-		f, err := file.Open()
+	for {
+		file, err := mr.NextPart()
+		if err == io.EOF {
+			break
+		}
 		if err != nil {
-			log.Warnf("open file: %s", err.Error())
-			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("open file %v: %v", file.Filename, err)))
+			log.Warnf("read next part: %s", err.Error())
+			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("read next part: %v", err)))
 			return
 		}
 
-		path, err := url.PathUnescape(file.Filename)
+		path, err := url.PathUnescape(file.FileName())
 		if err != nil {
 			log.Warnf("unescape filename: %s", err.Error())
-			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("unescape filename %v: %v", file.Filename, err)))
+			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("unescape filename %v: %v", file.FileName(), err)))
 			return
 		}
 		path = filepath.ToSlash(path)
 
-		err = up.Upload(clitypes.PathFromJcsPathString(path), f)
+		err = up.Upload(clitypes.PathFromJcsPathString(path), file)
 		if err != nil {
 			log.Warnf("uploading file: %s", err.Error())
-			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.Filename, err)))
+			ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
 			return
 		}
 		pathes = append(pathes, path)

client/sdk/api/v1/object.go

@@ -82,7 +82,7 @@ func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, er
 const ObjectUploadPath = "/object/upload"
 
 type ObjectUpload struct {
-	ObjectUploadInfo
+	Info  ObjectUploadInfo
 	Files UploadObjectIterator `json:"-"`
 }
 
@@ -114,7 +114,7 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) {
 		return nil, err
 	}
 
-	infoJSON, err := serder.ObjectToJSON(req)
+	infoJSON, err := serder.ObjectToJSON(req.Info)
 	if err != nil {
 		return nil, fmt.Errorf("upload info to json: %w", err)
 	}

client/sdk/api/v1/storage_test.go

@@ -32,7 +32,7 @@ func Test_PackageGet(t *testing.T) {
 		So(err, ShouldBeNil)
 
 		_, err = cli.Object().Upload(ObjectUpload{
-			ObjectUploadInfo: ObjectUploadInfo{
+			Info: ObjectUploadInfo{
 				PackageID: createResp.Package.PackageID,
 			},
 			Files: iterator.Array(
@@ -84,7 +84,7 @@ func Test_Object(t *testing.T) {
 		So(err, ShouldBeNil)
 
 		_, err = cli.Object().Upload(ObjectUpload{
-			ObjectUploadInfo: ObjectUploadInfo{
+			Info: ObjectUploadInfo{
 				PackageID: createResp.Package.PackageID,
 				Affinity:  stgAff,
 			},
@@ -153,7 +153,7 @@ func Test_Storage(t *testing.T) {
 		So(err, ShouldBeNil)
 
 		_, err = cli.Object().Upload(ObjectUpload{
-			ObjectUploadInfo: ObjectUploadInfo{
+			Info: ObjectUploadInfo{
 				PackageID: createResp.Package.PackageID,
 			},
 			Files: iterator.Array(
@@ -250,7 +250,7 @@ func Test_Sign(t *testing.T) {
 		So(err, ShouldBeNil)
 
 		_, err = cli.Object().Upload(ObjectUpload{
-			ObjectUploadInfo: ObjectUploadInfo{
+			Info: ObjectUploadInfo{
 				PackageID: createResp.Package.PackageID,
 			},
 			Files: iterator.Array(

jcsctl/cmd/puto/puto.go

@@ -91,7 +91,7 @@ func puto(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string)
 	startTime := time.Now()
 
 	_, err = ctx.Client.Object().Upload(cliapi.ObjectUpload{
-		ObjectUploadInfo: cliapi.ObjectUploadInfo{
+		Info: cliapi.ObjectUploadInfo{
 			PackageID: pkgID,
 		},
 		Files: iterator.Array(&cliapi.UploadingObject{

jcsctl/cmd/putp/putp.go

@@ -130,7 +130,7 @@ func putp(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string)
 		startTime := time.Now()
 
 		_, err = ctx.Client.Object().Upload(cliapi.ObjectUpload{
-			ObjectUploadInfo: cliapi.ObjectUploadInfo{
+			Info: cliapi.ObjectUploadInfo{
 				PackageID: pkgID,
 			},
 			Files: iterator.Array(&cliapi.UploadingObject{
@@ -154,7 +154,7 @@ func putp(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string)
 
 	startTime := time.Now()
 	_, err = ctx.Client.Object().Upload(cliapi.ObjectUpload{
-		ObjectUploadInfo: cliapi.ObjectUploadInfo{
+		Info: cliapi.ObjectUploadInfo{
 			PackageID: pkgID,
 		},
 		Files: iter,