| @@ -44,6 +44,17 @@ func DoTx01[R any](db *DB, do func(tx SQLContext) (R, error)) (R, error) { | |||
| return ret, err | |||
| } | |||
| func DoTx02[R1, R2 any](db *DB, do func(tx SQLContext) (R1, R2, error)) (R1, R2, error) { | |||
| var ret1 R1 | |||
| var ret2 R2 | |||
| err := db.db.Transaction(func(tx *gorm.DB) error { | |||
| var err error | |||
| ret1, ret2, err = do(SQLContext{tx}) | |||
| return err | |||
| }) | |||
| return ret1, ret2, err | |||
| } | |||
| func DoTx11[T any, R any](db *DB, do func(tx SQLContext, t T) (R, error), t T) (R, error) { | |||
| var ret R | |||
| err := db.db.Transaction(func(tx *gorm.DB) error { | |||
| @@ -8,7 +8,7 @@ import ( | |||
| "gitlink.org.cn/cloudream/common/pkgs/iterator" | |||
| "gitlink.org.cn/cloudream/jcs-pub/client/internal/db" | |||
| "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader/strategy" | |||
| "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity" | |||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" | |||
| ) | |||
| @@ -20,18 +20,18 @@ const ( | |||
| type DownloadIterator = iterator.Iterator[*Downloading] | |||
| type DownloadReqeust struct { | |||
| ObjectID types.ObjectID | |||
| ObjectID clitypes.ObjectID | |||
| Offset int64 | |||
| Length int64 | |||
| } | |||
| type downloadReqeust2 struct { | |||
| Detail *types.ObjectDetail | |||
| Detail *clitypes.ObjectDetail | |||
| Raw DownloadReqeust | |||
| } | |||
| type Downloading struct { | |||
| Object *types.Object | |||
| Object *clitypes.Object | |||
| File io.ReadCloser // 文件流,如果文件不存在,那么为nil | |||
| Request DownloadReqeust | |||
| } | |||
| @@ -62,7 +62,7 @@ func NewDownloader(cfg Config, conn *connectivity.Collector, stgPool *pool.Pool, | |||
| } | |||
| func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator { | |||
| objIDs := make([]types.ObjectID, len(reqs)) | |||
| objIDs := make([]clitypes.ObjectID, len(reqs)) | |||
| for i, req := range reqs { | |||
| objIDs[i] = req.ObjectID | |||
| } | |||
| @@ -76,7 +76,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator { | |||
| return iterator.FuseError[*Downloading](fmt.Errorf("request to db: %w", err)) | |||
| } | |||
| detailsMap := make(map[types.ObjectID]*types.ObjectDetail) | |||
| detailsMap := make(map[clitypes.ObjectID]*clitypes.ObjectDetail) | |||
| for _, detail := range objDetails { | |||
| d := detail | |||
| detailsMap[detail.Object.ObjectID] = &d | |||
| @@ -93,7 +93,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator { | |||
| return NewDownloadObjectIterator(d, req2s) | |||
| } | |||
| func (d *Downloader) DownloadObjectByDetail(detail types.ObjectDetail, off int64, length int64) (*Downloading, error) { | |||
| func (d *Downloader) DownloadObjectByDetail(detail clitypes.ObjectDetail, off int64, length int64) (*Downloading, error) { | |||
| req2s := []downloadReqeust2{{ | |||
| Detail: &detail, | |||
| Raw: DownloadReqeust{ | |||
| @@ -107,10 +107,56 @@ func (d *Downloader) DownloadObjectByDetail(detail types.ObjectDetail, off int64 | |||
| return iter.MoveNext() | |||
| } | |||
| func (d *Downloader) DownloadPackage(pkgID types.PackageID) DownloadIterator { | |||
| details, err := db.DoTx11(d.db, d.db.Object().GetPackageObjectDetails, pkgID) | |||
| func (d *Downloader) DownloadPackage(pkgID clitypes.PackageID, prefix string) (clitypes.Package, DownloadIterator, error) { | |||
| pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (clitypes.Package, []clitypes.ObjectDetail, error) { | |||
| pkg, err := d.db.Package().GetByID(tx, pkgID) | |||
| if err != nil { | |||
| return clitypes.Package{}, nil, err | |||
| } | |||
| var details []clitypes.ObjectDetail | |||
| if prefix != "" { | |||
| objs, err := d.db.Object().GetWithPathPrefix(tx, pkgID, prefix) | |||
| if err != nil { | |||
| return clitypes.Package{}, nil, err | |||
| } | |||
| objIDs := make([]clitypes.ObjectID, len(objs)) | |||
| for i, obj := range objs { | |||
| objIDs[i] = obj.ObjectID | |||
| } | |||
| allBlocks, err := d.db.ObjectBlock().BatchGetByObjectID(tx, objIDs) | |||
| if err != nil { | |||
| return clitypes.Package{}, nil, err | |||
| } | |||
| allPinnedObjs, err := d.db.PinnedObject().BatchGetByObjectID(tx, objIDs) | |||
| if err != nil { | |||
| return clitypes.Package{}, nil, err | |||
| } | |||
| details = make([]clitypes.ObjectDetail, 0, len(objs)) | |||
| for _, obj := range objs { | |||
| detail := clitypes.ObjectDetail{ | |||
| Object: obj, | |||
| } | |||
| details = append(details, detail) | |||
| } | |||
| clitypes.DetailsFillObjectBlocks(details, allBlocks) | |||
| clitypes.DetailsFillPinnedAt(details, allPinnedObjs) | |||
| } else { | |||
| details, err = d.db.Object().GetPackageObjectDetails(tx, pkgID) | |||
| if err != nil { | |||
| return clitypes.Package{}, nil, err | |||
| } | |||
| } | |||
| return pkg, details, nil | |||
| }) | |||
| if err != nil { | |||
| return iterator.FuseError[*Downloading](fmt.Errorf("get package object details: %w", err)) | |||
| return clitypes.Package{}, nil, err | |||
| } | |||
| req2s := make([]downloadReqeust2, len(details)) | |||
| @@ -126,16 +172,16 @@ func (d *Downloader) DownloadPackage(pkgID types.PackageID) DownloadIterator { | |||
| } | |||
| } | |||
| return NewDownloadObjectIterator(d, req2s) | |||
| return pkg, NewDownloadObjectIterator(d, req2s), nil | |||
| } | |||
| type ObjectECStrip struct { | |||
| Data []byte | |||
| ObjectFileHash types.FileHash // 添加这条缓存时,Object的FileHash | |||
| ObjectFileHash clitypes.FileHash // 添加这条缓存时,Object的FileHash | |||
| } | |||
| type ECStripKey struct { | |||
| ObjectID types.ObjectID | |||
| ObjectID clitypes.ObjectID | |||
| StripIndex int64 | |||
| } | |||
| @@ -22,6 +22,28 @@ func (s *Server) Bucket() *BucketService { | |||
| } | |||
| } | |||
| func (s *BucketService) Get(ctx *gin.Context) { | |||
| log := logger.WithField("HTTP", "Bucket.Get") | |||
| var req cliapi.BucketGet | |||
| if err := ctx.ShouldBindQuery(&req); err != nil { | |||
| log.Warnf("binding query: %s", err.Error()) | |||
| ctx.JSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "missing argument or invalid argument")) | |||
| return | |||
| } | |||
| bucket, err := s.svc.DB.Bucket().GetByID(s.svc.DB.DefCtx(), req.BucketID) | |||
| if err != nil { | |||
| log.Warnf("getting bucket by name: %s", err.Error()) | |||
| ctx.JSON(http.StatusOK, types.FailedError(err)) | |||
| return | |||
| } | |||
| ctx.JSON(http.StatusOK, types.OK(cliapi.BucketGetResp{ | |||
| Bucket: bucket, | |||
| })) | |||
| } | |||
| func (s *BucketService) GetByName(ctx *gin.Context) { | |||
| log := logger.WithField("HTTP", "Bucket.GetByName") | |||
| @@ -69,7 +69,7 @@ func (s *ObjectService) ListByIDs(ctx *gin.Context) { | |||
| ctx.JSON(http.StatusOK, types.OK(cliapi.ObjectListByIDsResp{Objects: objs})) | |||
| } | |||
| type ObjectUploadReq struct { | |||
| type ObjectUpload struct { | |||
| Info cliapi.ObjectUploadInfo `form:"info" binding:"required"` | |||
| Files []*multipart.FileHeader `form:"files"` | |||
| } | |||
| @@ -77,7 +77,7 @@ type ObjectUploadReq struct { | |||
| func (s *ObjectService) Upload(ctx *gin.Context) { | |||
| log := logger.WithField("HTTP", "Object.Upload") | |||
| var req ObjectUploadReq | |||
| var req ObjectUpload | |||
| if err := ctx.ShouldBind(&req); err != nil { | |||
| log.Warnf("binding body: %s", err.Error()) | |||
| ctx.JSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "missing argument or invalid argument")) | |||
| @@ -1,18 +1,24 @@ | |||
| package http | |||
| import ( | |||
| "archive/tar" | |||
| "archive/zip" | |||
| "fmt" | |||
| "io" | |||
| "mime/multipart" | |||
| "net/http" | |||
| "net/url" | |||
| "path/filepath" | |||
| "github.com/gin-gonic/gin" | |||
| "gitlink.org.cn/cloudream/common/consts/errorcode" | |||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | |||
| "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader" | |||
| "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/common/ecode" | |||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator" | |||
| ) | |||
| // PackageService 包服务,负责处理包相关的HTTP请求。 | |||
| @@ -30,7 +36,7 @@ func (s *Server) Package() *PackageService { | |||
| func (s *PackageService) Get(ctx *gin.Context) { | |||
| log := logger.WithField("HTTP", "Package.Get") | |||
| var req cliapi.PackageGetReq | |||
| var req cliapi.PackageGet | |||
| if err := ctx.ShouldBindQuery(&req); err != nil { | |||
| log.Warnf("binding body: %s", err.Error()) | |||
| ctx.JSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "missing argument or invalid argument")) | |||
| @@ -164,6 +170,122 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) { | |||
| ctx.JSON(http.StatusOK, types.OK(cliapi.PackageCreateUploadResp{Package: ret.Package, Objects: objs})) | |||
| } | |||
| func (s *PackageService) Download(ctx *gin.Context) { | |||
| log := logger.WithField("HTTP", "Package.Download") | |||
| var req cliapi.PackageDownload | |||
| if err := ctx.ShouldBindQuery(&req); err != nil { | |||
| log.Warnf("binding query: %s", err.Error()) | |||
| ctx.JSON(http.StatusBadRequest, types.Failed(errorcode.BadArgument, "missing argument or invalid argument")) | |||
| return | |||
| } | |||
| pkg, iter, err := s.svc.Downloader.DownloadPackage(req.PackageID, req.Prefix) | |||
| if err != nil { | |||
| log.Warnf("downloading package: %s", err.Error()) | |||
| ctx.JSON(http.StatusOK, types.Failed(errorcode.OperationFailed, err.Error())) | |||
| return | |||
| } | |||
| defer iter.Close() | |||
| if req.Zip { | |||
| s.downloadZip(ctx, req, pkg, iter) | |||
| } else { | |||
| s.downloadTar(ctx, req, pkg, iter) | |||
| } | |||
| } | |||
| func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg clitypes.Package, iter downloader.DownloadIterator) { | |||
| log := logger.WithField("HTTP", "Package.Download") | |||
| ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".zip") | |||
| ctx.Header("Content-Type", "application/zip") | |||
| ctx.Header("Content-Transfer-Encoding", "binary") | |||
| zipFile := zip.NewWriter(ctx.Writer) | |||
| defer zipFile.Close() | |||
| for { | |||
| item, err := iter.MoveNext() | |||
| if err == iterator.ErrNoMoreItem { | |||
| return | |||
| } | |||
| if err != nil { | |||
| log.Warnf("iterating next object: %v", err) | |||
| return | |||
| } | |||
| filePath := item.Object.Path | |||
| if req.Prefix != "" && req.NewPrefix != nil { | |||
| filePath = *req.NewPrefix + filePath[len(req.Prefix):] | |||
| } | |||
| zf, err := zipFile.Create(filePath) | |||
| if err != nil { | |||
| log.Warnf("creating zip file: %v", err) | |||
| item.File.Close() | |||
| return | |||
| } | |||
| _, err = io.Copy(zf, item.File) | |||
| if err != nil { | |||
| log.Warnf("copying file to zip: %v", err) | |||
| item.File.Close() | |||
| return | |||
| } | |||
| item.File.Close() | |||
| } | |||
| } | |||
| func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg clitypes.Package, iter downloader.DownloadIterator) { | |||
| log := logger.WithField("HTTP", "Package.Download") | |||
| ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".tar") | |||
| ctx.Header("Content-Type", "application/x-tar") | |||
| ctx.Header("Content-Transfer-Encoding", "binary") | |||
| tarFile := tar.NewWriter(ctx.Writer) | |||
| defer tarFile.Close() | |||
| for { | |||
| item, err := iter.MoveNext() | |||
| if err == iterator.ErrNoMoreItem { | |||
| return | |||
| } | |||
| if err != nil { | |||
| log.Warnf("iterating next object: %v", err) | |||
| return | |||
| } | |||
| filePath := item.Object.Path | |||
| if req.Prefix != "" && req.NewPrefix != nil { | |||
| filePath = *req.NewPrefix + filePath[len(req.Prefix):] | |||
| } | |||
| err = tarFile.WriteHeader(&tar.Header{ | |||
| Typeflag: tar.TypeReg, | |||
| Name: filePath, | |||
| Size: item.Object.Size, | |||
| ModTime: item.Object.CreateTime, | |||
| }) | |||
| if err != nil { | |||
| log.Warnf("creating tar header: %v", err) | |||
| item.File.Close() | |||
| return | |||
| } | |||
| _, err = io.Copy(tarFile, item.File) | |||
| if err != nil { | |||
| log.Warnf("copying file to tar: %v", err) | |||
| item.File.Close() | |||
| return | |||
| } | |||
| item.File.Close() | |||
| } | |||
| } | |||
| func (s *PackageService) Delete(ctx *gin.Context) { | |||
| log := logger.WithField("HTTP", "Package.Delete") | |||
| @@ -43,6 +43,7 @@ func (s *Server) InitRouters(rt gin.IRoutes, ah *auth.Auth) { | |||
| rt.POST(cliapi.PackageCreateUploadPath, certAuth, s.Package().CreateLoad) | |||
| rt.POST(cliapi.PackageDeletePath, certAuth, s.Package().Delete) | |||
| rt.POST(cliapi.PackageClonePath, certAuth, s.Package().Clone) | |||
| rt.GET(cliapi.PackageDownloadPath, certAuth, s.Package().Download) | |||
| rt.GET(cliapi.PackageListBucketPackagesPath, certAuth, s.Package().ListBucketPackages) | |||
| rt.POST(cliapi.UserSpaceDownloadPackagePath, certAuth, s.UserSpace().DownloadPackage) | |||
| @@ -55,6 +56,7 @@ func (s *Server) InitRouters(rt gin.IRoutes, ah *auth.Auth) { | |||
| rt.POST(cliapi.UserSpaceDeletePath, certAuth, s.UserSpace().Delete) | |||
| rt.POST(cliapi.UserSpaceTestPath, certAuth, s.UserSpace().Test) | |||
| rt.GET(cliapi.BucketGetPath, certAuth, s.Bucket().Get) | |||
| rt.GET(cliapi.BucketGetByNamePath, certAuth, s.Bucket().GetByName) | |||
| rt.POST(cliapi.BucketCreatePath, certAuth, s.Bucket().Create) | |||
| rt.POST(cliapi.BucketDeletePath, certAuth, s.Bucket().Delete) | |||
| @@ -7,7 +7,6 @@ import ( | |||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | |||
| "gitlink.org.cn/cloudream/jcs-pub/client/internal/db" | |||
| "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader" | |||
| "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/common/models/datamap" | |||
| ) | |||
| @@ -47,10 +46,6 @@ func (svc *PackageService) Create(bucketID types.BucketID, name string) (types.P | |||
| return pkg, nil | |||
| } | |||
| func (svc *PackageService) DownloadPackage(packageID types.PackageID) (downloader.DownloadIterator, error) { | |||
| return svc.Downloader.DownloadPackage(packageID), nil | |||
| } | |||
| // DeletePackage 删除指定的包 | |||
| func (svc *PackageService) DeletePackage(packageID types.PackageID) error { | |||
| err := svc.DB.Package().DeleteComplete(svc.DB.DefCtx(), packageID) | |||
| @@ -37,14 +37,14 @@ func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj cl | |||
| return &clitypes.DefaultECRedundancy, newStgs | |||
| } | |||
| return clitypes.DefaultRepRedundancy, t.rechooseUserSpacesForRep(ctx, &clitypes.DefaultRepRedundancy) | |||
| return &clitypes.DefaultRepRedundancy, t.rechooseUserSpacesForRep(ctx, &clitypes.DefaultRepRedundancy) | |||
| case *clitypes.ECRedundancy: | |||
| if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold { | |||
| return &clitypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &clitypes.DefaultRepRedundancy) | |||
| } | |||
| return clitypes.DefaultECRedundancy, t.rechooseUserSpacesForEC(ctx, obj, &clitypes.DefaultECRedundancy) | |||
| return &clitypes.DefaultECRedundancy, t.rechooseUserSpacesForEC(ctx, obj, &clitypes.DefaultECRedundancy) | |||
| case *clitypes.LRCRedundancy: | |||
| newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &clitypes.DefaultLRCRedundancy) | |||
| @@ -15,6 +15,28 @@ func (c *Client) Bucket() *BucketService { | |||
| return &BucketService{c} | |||
| } | |||
| const BucketGetPath = "/bucket/get" | |||
| type BucketGet struct { | |||
| BucketID clitypes.BucketID `json:"bucketID" binding:"required"` | |||
| } | |||
| func (r *BucketGet) MakeParam() *sdks.RequestParam { | |||
| return sdks.MakeJSONParam(http.MethodGet, BucketGetPath, r) | |||
| } | |||
| type BucketGetResp struct { | |||
| Bucket clitypes.Bucket `json:"bucket"` | |||
| } | |||
| func (r *BucketGetResp) ParseResponse(resp *http.Response) error { | |||
| return sdks.ParseCodeDataJSONResponse(resp, r) | |||
| } | |||
| func (c *BucketService) Get(req BucketGet) (*BucketGetResp, error) { | |||
| return JSONAPI(&c.cfg, c.httpCli, &req, &BucketGetResp{}) | |||
| } | |||
| const BucketGetByNamePath = "/bucket/getByName" | |||
| type BucketGetByName struct { | |||
| @@ -119,7 +119,7 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { | |||
| return nil, fmt.Errorf("upload info to json: %w", err) | |||
| } | |||
| resp, err := PostMultiPart(&c.cfg, url, | |||
| resp, err := PostMultiPart(&c.cfg, c.httpCli, url, | |||
| uploadInfo{Info: string(infoJSON)}, | |||
| iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { | |||
| return &http2.IterMultiPartFile{ | |||
| @@ -169,7 +169,11 @@ type DownloadingObject struct { | |||
| } | |||
| func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) { | |||
| httpReq, err := req.MakeParam().MakeRequest(c.cfg.EndPoint) | |||
| u, err := url.JoinPath(c.cfg.EndPoint, "v1") | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| httpReq, err := req.MakeParam().MakeRequest(u) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| @@ -215,7 +219,11 @@ func (r *ObjectDownloadByPath) MakeParam() *sdks.RequestParam { | |||
| } | |||
| func (c *ObjectService) DownloadByPath(req ObjectDownloadByPath) (*DownloadingObject, error) { | |||
| httpReq, err := req.MakeParam().MakeRequest(c.cfg.EndPoint) | |||
| u, err := url.JoinPath(c.cfg.EndPoint, "v1") | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| httpReq, err := req.MakeParam().MakeRequest(u) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| @@ -2,8 +2,11 @@ package api | |||
| import ( | |||
| "fmt" | |||
| "io" | |||
| "mime" | |||
| "net/http" | |||
| "net/url" | |||
| "strings" | |||
| "gitlink.org.cn/cloudream/common/consts/errorcode" | |||
| "gitlink.org.cn/cloudream/common/pkgs/iterator" | |||
| @@ -23,23 +26,23 @@ func (c *Client) Package() *PackageService { | |||
| const PackageGetPath = "/package/get" | |||
| type PackageGetReq struct { | |||
| type PackageGet struct { | |||
| PackageID clitypes.PackageID `form:"packageID" url:"packageID" binding:"required"` | |||
| } | |||
| func (r *PackageGetReq) MakeParam() *sdks.RequestParam { | |||
| func (r *PackageGet) MakeParam() *sdks.RequestParam { | |||
| return sdks.MakeQueryParam(http.MethodGet, PackageGetPath, r) | |||
| } | |||
| type PackageGetResp struct { | |||
| clitypes.Package | |||
| Package clitypes.Package `json:"package"` | |||
| } | |||
| func (r *PackageGetResp) ParseResponse(resp *http.Response) error { | |||
| return sdks.ParseCodeDataJSONResponse(resp, r) | |||
| } | |||
| func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { | |||
| func (c *PackageService) Get(req PackageGet) (*PackageGetResp, error) { | |||
| return JSONAPI(&c.cfg, c.httpCli, &req, &PackageGetResp{}) | |||
| } | |||
| @@ -62,7 +65,7 @@ func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error { | |||
| return sdks.ParseCodeDataJSONResponse(resp, r) | |||
| } | |||
| func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { | |||
| func (c *PackageService) GetByFullName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { | |||
| return JSONAPI(&c.cfg, c.httpCli, &req, &PackageGetByFullNameResp{}) | |||
| } | |||
| @@ -117,7 +120,7 @@ func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUp | |||
| return nil, fmt.Errorf("upload info to json: %w", err) | |||
| } | |||
| resp, err := PostMultiPart(&c.cfg, url, | |||
| resp, err := PostMultiPart(&c.cfg, c.httpCli, url, | |||
| map[string]string{"info": string(infoJSON)}, | |||
| iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { | |||
| return &http2.IterMultiPartFile{ | |||
| @@ -142,6 +145,66 @@ func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUp | |||
| return nil, codeResp.ToError() | |||
| } | |||
| const PackageDownloadPath = "/package/download" | |||
| type PackageDownload struct { | |||
| PackageID clitypes.PackageID `url:"packageID" form:"packageID" binding:"required"` | |||
| Prefix string `url:"prefix" form:"prefix"` | |||
| NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"` | |||
| Zip bool `url:"zip,omitempty" form:"zip"` | |||
| } | |||
| func (r *PackageDownload) MakeParam() *sdks.RequestParam { | |||
| return sdks.MakeQueryParam(http.MethodGet, PackageDownloadPath, r) | |||
| } | |||
| type DownloadingPackage struct { | |||
| Name string | |||
| File io.ReadCloser | |||
| } | |||
| func (c *PackageService) Download(req PackageDownload) (*DownloadingPackage, error) { | |||
| url, err := url.JoinPath(c.cfg.EndPoint, "v1") | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| httpReq, err := req.MakeParam().MakeRequest(url) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| resp, err := c.Client.httpCli.Do(httpReq) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| contType := resp.Header.Get("Content-Type") | |||
| if resp.StatusCode != http.StatusOK { | |||
| return nil, fmt.Errorf("response status code: %d", resp.StatusCode) | |||
| } | |||
| if strings.Contains(contType, http2.ContentTypeJSON) { | |||
| var codeResp response[any] | |||
| if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { | |||
| return nil, fmt.Errorf("parsing response: %w", err) | |||
| } | |||
| return nil, codeResp.ToError() | |||
| } | |||
| _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) | |||
| if err != nil { | |||
| return nil, fmt.Errorf("parsing content disposition: %w", err) | |||
| } | |||
| return &DownloadingPackage{ | |||
| Name: params["filename"], | |||
| File: resp.Body, | |||
| }, nil | |||
| } | |||
| const PackageDeletePath = "/package/delete" | |||
| type PackageDelete struct { | |||
| @@ -48,12 +48,12 @@ func Test_PackageGet(t *testing.T) { | |||
| }) | |||
| So(err, ShouldBeNil) | |||
| getResp, err := cli.Package().Get(PackageGetReq{ | |||
| getResp, err := cli.Package().Get(PackageGet{ | |||
| PackageID: createResp.Package.PackageID, | |||
| }) | |||
| So(err, ShouldBeNil) | |||
| So(getResp.PackageID, ShouldEqual, createResp.Package.PackageID) | |||
| So(getResp.Package.PackageID, ShouldEqual, createResp.Package.PackageID) | |||
| So(getResp.Package.Name, ShouldEqual, pkgName) | |||
| err = cli.Package().Delete(PackageDelete{ | |||
| @@ -266,12 +266,12 @@ func Test_Sign(t *testing.T) { | |||
| }) | |||
| So(err, ShouldBeNil) | |||
| getResp, err := cli.Package().Get(PackageGetReq{ | |||
| getResp, err := cli.Package().Get(PackageGet{ | |||
| PackageID: createResp.Package.PackageID, | |||
| }) | |||
| So(err, ShouldBeNil) | |||
| So(getResp.PackageID, ShouldEqual, createResp.Package.PackageID) | |||
| So(getResp.Package.PackageID, ShouldEqual, createResp.Package.PackageID) | |||
| So(getResp.Package.Name, ShouldEqual, pkgName) | |||
| err = cli.Package().Delete(PackageDelete{ | |||
| @@ -105,7 +105,7 @@ func calcSha256(body sdks.RequestBody) string { | |||
| } | |||
| } | |||
| func PostMultiPart(cfg *api.Config, url string, info any, files http2.MultiPartFileIterator) (*http.Response, error) { | |||
| func PostMultiPart(cfg *api.Config, cli *http.Client, url string, info any, files http2.MultiPartFileIterator) (*http.Response, error) { | |||
| req, err := http.NewRequest(http.MethodPost, url, nil) | |||
| if err != nil { | |||
| return nil, err | |||
| @@ -158,7 +158,6 @@ func PostMultiPart(cfg *api.Config, url string, info any, files http2.MultiPartF | |||
| req.Body = pr | |||
| cli := http.Client{} | |||
| resp, err := cli.Do(req) | |||
| if err != nil { | |||
| return nil, err | |||
| @@ -21,11 +21,14 @@ var RedundancyUnion = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTyp | |||
| )), "type") | |||
| type NoneRedundancy struct { | |||
| Redundancy `json:"-"` | |||
| serder.Metadata `union:"none"` | |||
| Type string `json:"type"` | |||
| } | |||
| func (r *NoneRedundancy) GetRedundancyType() string { | |||
| return "none" | |||
| } | |||
| func NewNoneRedundancy() *NoneRedundancy { | |||
| return &NoneRedundancy{ | |||
| Type: "none", | |||
| @@ -35,12 +38,15 @@ func NewNoneRedundancy() *NoneRedundancy { | |||
| var DefaultRepRedundancy = *NewRepRedundancy(2) | |||
| type RepRedundancy struct { | |||
| Redundancy `json:"-"` | |||
| serder.Metadata `union:"rep"` | |||
| Type string `json:"type"` | |||
| RepCount int `json:"repCount"` | |||
| } | |||
| func (r *RepRedundancy) GetRedundancyType() string { | |||
| return "rep" | |||
| } | |||
| func NewRepRedundancy(repCount int) *RepRedundancy { | |||
| return &RepRedundancy{ | |||
| Type: "rep", | |||
| @@ -51,7 +57,6 @@ func NewRepRedundancy(repCount int) *RepRedundancy { | |||
| var DefaultECRedundancy = *NewECRedundancy(2, 3, 1024*1024*5) | |||
| type ECRedundancy struct { | |||
| Redundancy `json:"-"` | |||
| serder.Metadata `union:"ec"` | |||
| Type string `json:"type"` | |||
| K int `json:"k"` | |||
| @@ -59,6 +64,10 @@ type ECRedundancy struct { | |||
| ChunkSize int `json:"chunkSize"` | |||
| } | |||
| func (b *ECRedundancy) GetRedundancyType() string { | |||
| return "ec" | |||
| } | |||
| func NewECRedundancy(k int, n int, chunkSize int) *ECRedundancy { | |||
| return &ECRedundancy{ | |||
| Type: "ec", | |||
| @@ -75,7 +84,6 @@ func (b *ECRedundancy) StripSize() int64 { | |||
| var DefaultLRCRedundancy = *NewLRCRedundancy(2, 4, []int{2}, 1024*1024*5) | |||
| type LRCRedundancy struct { | |||
| Redundancy `json:"-"` | |||
| serder.Metadata `union:"lrc"` | |||
| Type string `json:"type"` | |||
| K int `json:"k"` | |||
| @@ -84,6 +92,10 @@ type LRCRedundancy struct { | |||
| ChunkSize int `json:"chunkSize"` | |||
| } | |||
| func (b *LRCRedundancy) GetRedundancyType() string { | |||
| return "lrc" | |||
| } | |||
| func NewLRCRedundancy(k int, n int, groups []int, chunkSize int) *LRCRedundancy { | |||
| return &LRCRedundancy{ | |||
| Type: "lrc", | |||
| @@ -132,12 +144,15 @@ func (b *LRCRedundancy) GetGroupElements(grp int) []int { | |||
| } | |||
| type SegmentRedundancy struct { | |||
| Redundancy `json:"-"` | |||
| serder.Metadata `union:"segment"` | |||
| Type string `json:"type"` | |||
| Segments []int64 `json:"segments"` // 每一段的大小 | |||
| } | |||
| func (r *SegmentRedundancy) GetRedundancyType() string { | |||
| return "segment" | |||
| } | |||
| func NewSegmentRedundancy(totalSize int64, segmentCount int) *SegmentRedundancy { | |||
| return &SegmentRedundancy{ | |||
| Type: "segment", | |||
| @@ -201,11 +216,14 @@ func (b *SegmentRedundancy) CalcSegmentRange(start int64, end *int64) (segIdxSta | |||
| } | |||
| type MultipartUploadRedundancy struct { | |||
| Redundancy `json:"-"` | |||
| serder.Metadata `union:"multipartUpload"` | |||
| Type string `json:"type"` | |||
| } | |||
| func (r *MultipartUploadRedundancy) GetRedundancyType() string { | |||
| return "multipartUpload" | |||
| } | |||
| func NewMultipartUploadRedundancy() *MultipartUploadRedundancy { | |||
| return &MultipartUploadRedundancy{ | |||
| Type: "multipartUpload", | |||
| @@ -59,7 +59,7 @@ func (r *DirReader) Next() (types.DirEntry, error) { | |||
| if entry.entry.IsDir() { | |||
| es, err := os.ReadDir(filepath.Join(r.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name())) | |||
| if err != nil { | |||
| return types.DirEntry{}, nil | |||
| return types.DirEntry{}, err | |||
| } | |||
| // 多个entry对象共享同一个JPath对象,但因为不会修改JPath,所以没问题 | |||
| @@ -54,6 +54,7 @@ require ( | |||
| github.com/cloudwego/base64x v0.1.4 // indirect | |||
| github.com/cloudwego/iasm v0.2.0 // indirect | |||
| github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect | |||
| github.com/fatih/color v1.18.0 // indirect | |||
| github.com/gabriel-vasile/mimetype v1.4.6 // indirect | |||
| github.com/goccy/go-json v0.10.3 // indirect | |||
| github.com/jinzhu/inflection v1.0.0 // indirect | |||
| @@ -56,6 +56,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc | |||
| github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= | |||
| github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= | |||
| github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= | |||
| github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= | |||
| github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= | |||
| github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= | |||
| github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= | |||
| github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= | |||
| @@ -2,5 +2,11 @@ package all | |||
| import ( | |||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/bucket" | |||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/geto" | |||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/getp" | |||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/ls" | |||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/package" | |||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/puto" | |||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/putp" | |||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/userspace" | |||
| ) | |||
| @@ -6,7 +6,8 @@ import ( | |||
| ) | |||
| var BucketCmd = &cobra.Command{ | |||
| Use: "bucket", | |||
| Use: "bucket", | |||
| Aliases: []string{"bkt"}, | |||
| } | |||
| func init() { | |||
| @@ -1,49 +0,0 @@ | |||
| package bucket | |||
| import ( | |||
| "fmt" | |||
| "github.com/jedib0t/go-pretty/v6/table" | |||
| "github.com/spf13/cobra" | |||
| "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func init() { | |||
| var opt lsOpt | |||
| cmd := cobra.Command{ | |||
| Use: "ls", | |||
| Run: func(c *cobra.Command, args []string) { | |||
| ctx := cmd.GetCmdCtx(c) | |||
| ls(c, ctx, opt) | |||
| }, | |||
| } | |||
| cmd.Flags().BoolVarP(&opt.Long, "", "l", false, "listing in long format") | |||
| BucketCmd.AddCommand(&cmd) | |||
| } | |||
| type lsOpt struct { | |||
| Long bool | |||
| } | |||
| func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt) { | |||
| resp, err := ctx.Client.Bucket().ListAll(api.BucketListAll{}) | |||
| if err != nil { | |||
| cmd.ErrorExitln(err.Error()) | |||
| } | |||
| if opt.Long { | |||
| fmt.Printf("total: %d\n", len(resp.Buckets)) | |||
| tb := table.NewWriter() | |||
| tb.AppendHeader(table.Row{"Bucket ID", "Name", "Create Time"}) | |||
| for _, b := range resp.Buckets { | |||
| tb.AppendRow(table.Row{b.BucketID, b.Name, b.CreateTime}) | |||
| } | |||
| fmt.Println(tb.Render()) | |||
| } else { | |||
| for _, b := range resp.Buckets { | |||
| fmt.Println(b.Name) | |||
| } | |||
| } | |||
| } | |||
| @@ -74,7 +74,7 @@ func RootExecute() { | |||
| } | |||
| if endpoint == "" { | |||
| endpoint = "https://127.0.0.1:8890" | |||
| endpoint = "https://127.0.0.1:7890" | |||
| } | |||
| cli := cliapi.NewClient(api.Config{ | |||
| @@ -112,7 +112,3 @@ func searchCertDir() string { | |||
| return "" | |||
| } | |||
| func loadCert() { | |||
| } | |||
| @@ -0,0 +1,143 @@ | |||
| package geto | |||
| import ( | |||
| "fmt" | |||
| "io" | |||
| "os" | |||
| "path/filepath" | |||
| "strconv" | |||
| "time" | |||
| "github.com/inhies/go-bytesize" | |||
| "github.com/spf13/cobra" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func init() { | |||
| var opt option | |||
| c := &cobra.Command{ | |||
| Use: "geto <bucket_name>/<package_name>:<object_path> <local_dir>", | |||
| Short: "download object to local disk", | |||
| Args: cobra.ExactArgs(2), | |||
| RunE: func(c *cobra.Command, args []string) error { | |||
| ctx := cmd.GetCmdCtx(c) | |||
| return geto(c, ctx, opt, args) | |||
| }, | |||
| } | |||
| c.Flags().BoolVar(&opt.UseID, "id", false, "treat first argument as object id") | |||
| c.Flags().Int64Var(&opt.Offset, "offset", 0, "offset of object to download") | |||
| c.Flags().Int64Var(&opt.Length, "length", 0, "length of object to download") | |||
| c.Flags().Int64Var(&opt.Seek, "seek", 0, "seek position when save to local file, if set, will not truncate local file") | |||
| c.Flags().StringVarP(&opt.Output, "output", "o", "", "output file name") | |||
| cmd.RootCmd.AddCommand(c) | |||
| } | |||
| type option struct { | |||
| UseID bool | |||
| Offset int64 | |||
| Length int64 | |||
| Seek int64 | |||
| Output string | |||
| } | |||
| func geto(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error { | |||
| var obj clitypes.Object | |||
| if opt.UseID { | |||
| id, err := strconv.ParseInt(args[0], 10, 64) | |||
| if err != nil { | |||
| return fmt.Errorf("invalid object id: %v", err) | |||
| } | |||
| resp, err := ctx.Client.Object().ListByIDs(cliapi.ObjectListByIDs{ | |||
| ObjectIDs: []clitypes.ObjectID{clitypes.ObjectID(id)}, | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("list objects by ids: %v", err) | |||
| } | |||
| if resp.Objects[0] == nil { | |||
| return fmt.Errorf("object not found") | |||
| } | |||
| obj = *resp.Objects[0] | |||
| } else { | |||
| bkt, pkg, objPath, ok := cmd.SplitObjectPath(args[0]) | |||
| if !ok { | |||
| return fmt.Errorf("invalid object path") | |||
| } | |||
| pkgResp, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{ | |||
| BucketName: bkt, | |||
| PackageName: pkg, | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("get package by name: %v", err) | |||
| } | |||
| objResp, err := ctx.Client.Object().ListByPath(cliapi.ObjectListByPath{ | |||
| PackageID: pkgResp.Package.PackageID, | |||
| Path: objPath, | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("list objects by path: %v", err) | |||
| } | |||
| if len(objResp.Objects) != 1 { | |||
| return fmt.Errorf("object not found") | |||
| } | |||
| obj = objResp.Objects[0] | |||
| } | |||
| filePath := args[1] | |||
| if opt.Output != "" { | |||
| filePath = filepath.Join(filePath, opt.Output) | |||
| } else { | |||
| filePath = filepath.Join(filePath, clitypes.BaseName(obj.Path)) | |||
| } | |||
| flag := os.O_CREATE | os.O_WRONLY | |||
| if !c.Flags().Changed("seek") { | |||
| flag |= os.O_TRUNC | |||
| } | |||
| file, err := os.OpenFile(filePath, flag, 0644) | |||
| if err != nil { | |||
| return fmt.Errorf("open file: %v", err) | |||
| } | |||
| defer file.Close() | |||
| if opt.Seek != 0 { | |||
| if _, err := file.Seek(opt.Seek, 0); err != nil { | |||
| return fmt.Errorf("seek file: %v", err) | |||
| } | |||
| } | |||
| fmt.Printf("%v\n", filePath) | |||
| var len *int64 | |||
| if c.Flags().Changed("length") { | |||
| len = &opt.Length | |||
| } | |||
| resp, err := ctx.Client.Object().Download(cliapi.ObjectDownload{ | |||
| ObjectID: obj.ObjectID, | |||
| Offset: opt.Offset, | |||
| Length: len, | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("download object: %v", err) | |||
| } | |||
| startTime := time.Now() | |||
| n, err := io.Copy(file, resp.File) | |||
| if err != nil { | |||
| return fmt.Errorf("copy object to file: %v", err) | |||
| } | |||
| dt := time.Since(startTime) | |||
| fmt.Printf("size: %v, time: %v, speed: %v/s\n", bytesize.ByteSize(n), dt, bytesize.ByteSize(int64(float64(n)/dt.Seconds()))) | |||
| return nil | |||
| } | |||
| @@ -0,0 +1,172 @@ | |||
| package getp | |||
| import ( | |||
| "archive/tar" | |||
| "fmt" | |||
| "io" | |||
| "os" | |||
| "path/filepath" | |||
| "strconv" | |||
| "strings" | |||
| "time" | |||
| "github.com/inhies/go-bytesize" | |||
| "github.com/spf13/cobra" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func init() { | |||
| var opt option | |||
| c := &cobra.Command{ | |||
| Use: "getp <bucket_name>/<package_name> <local_path>", | |||
| Short: "download package all files to local disk", | |||
| Args: cobra.ExactArgs(2), | |||
| RunE: func(c *cobra.Command, args []string) error { | |||
| ctx := cmd.GetCmdCtx(c) | |||
| return getp(c, ctx, opt, args) | |||
| }, | |||
| } | |||
| c.Flags().BoolVar(&opt.UseID, "id", false, "treat first argument as package id") | |||
| c.Flags().StringVar(&opt.Prefix, "prefix", "", "download objects with this prefix") | |||
| c.Flags().StringVar(&opt.NewPrefix, "new", "", "replace prefix specified by --prefix with this prefix") | |||
| c.Flags().BoolVar(&opt.Zip, "zip", false, "download as zip file") | |||
| c.Flags().StringVarP(&opt.Output, "output", "o", "", "output zip file name") | |||
| cmd.RootCmd.AddCommand(c) | |||
| } | |||
| type option struct { | |||
| UseID bool | |||
| Prefix string | |||
| NewPrefix string | |||
| Zip bool | |||
| Output string | |||
| } | |||
| func getp(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error { | |||
| var pkgID clitypes.PackageID | |||
| if opt.UseID { | |||
| id, err := strconv.ParseInt(args[0], 10, 64) | |||
| if err != nil { | |||
| return fmt.Errorf("invalid package id") | |||
| } | |||
| pkgID = clitypes.PackageID(id) | |||
| } else { | |||
| comps := strings.Split(args[0], "/") | |||
| if len(comps) != 2 { | |||
| return fmt.Errorf("invalid package name") | |||
| } | |||
| resp, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{ | |||
| BucketName: comps[0], | |||
| PackageName: comps[1], | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("get package by name: %w", err) | |||
| } | |||
| pkgID = resp.Package.PackageID | |||
| } | |||
| info, err := os.Stat(args[1]) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| if !info.IsDir() { | |||
| return fmt.Errorf("local path should be a directory") | |||
| } | |||
| req := cliapi.PackageDownload{ | |||
| PackageID: pkgID, | |||
| Prefix: opt.Prefix, | |||
| } | |||
| if c.Flags().Changed("new") { | |||
| req.NewPrefix = &opt.NewPrefix | |||
| } | |||
| if opt.Zip { | |||
| req.Zip = true | |||
| } | |||
| downResp, err := ctx.Client.Package().Download(req) | |||
| if err != nil { | |||
| return fmt.Errorf("download package: %w", err) | |||
| } | |||
| defer downResp.File.Close() | |||
| if opt.Zip { | |||
| fileName := downResp.Name | |||
| if opt.Output != "" { | |||
| fileName = opt.Output | |||
| } | |||
| localFilePath := filepath.Join(args[1], fileName) | |||
| file, err := os.OpenFile(localFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| fmt.Printf("%v\n", localFilePath) | |||
| startTime := time.Now() | |||
| n, err := io.Copy(file, downResp.File) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| dt := time.Since(startTime) | |||
| fmt.Printf("size: %v, time: %v, speed: %v/s\n", bytesize.ByteSize(n), dt, bytesize.ByteSize(int64(float64(n)/dt.Seconds()))) | |||
| return nil | |||
| } | |||
| startTime := time.Now() | |||
| totalSize := int64(0) | |||
| fileCnt := 0 | |||
| tr := tar.NewReader(downResp.File) | |||
| for { | |||
| header, err := tr.Next() | |||
| if err == io.EOF { | |||
| break | |||
| } | |||
| if err != nil { | |||
| return err | |||
| } | |||
| localPath := filepath.Join(args[1], header.Name) | |||
| fmt.Printf("%v", localPath) | |||
| dir := filepath.Dir(localPath) | |||
| err = os.MkdirAll(dir, 0755) | |||
| if err != nil { | |||
| fmt.Printf("\tx") | |||
| return err | |||
| } | |||
| fileStartTime := time.Now() | |||
| file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) | |||
| if err != nil { | |||
| fmt.Printf("\tx") | |||
| return err | |||
| } | |||
| _, err = io.Copy(file, tr) | |||
| if err != nil { | |||
| fmt.Printf("\tx") | |||
| return err | |||
| } | |||
| dt := time.Since(fileStartTime) | |||
| fmt.Printf("\t%v\t%v\n", bytesize.ByteSize(header.Size), dt) | |||
| fileCnt++ | |||
| totalSize += header.Size | |||
| } | |||
| dt := time.Since(startTime) | |||
| fmt.Printf("%v files, total size: %v, time: %v, speed: %v/s\n", fileCnt, bytesize.ByteSize(totalSize), dt, bytesize.ByteSize(int64(float64(totalSize)/dt.Seconds()))) | |||
| return nil | |||
| } | |||
| @@ -0,0 +1,76 @@ | |||
| package ls | |||
| import ( | |||
| "fmt" | |||
| "strings" | |||
| "github.com/spf13/cobra" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func init() { | |||
| var opt option | |||
| c := &cobra.Command{ | |||
| Use: "ls [bucket_name]/[package_name]:[object_path]", | |||
| Short: "download package all files to local disk", | |||
| Args: cobra.MaximumNArgs(1), | |||
| RunE: func(c *cobra.Command, args []string) error { | |||
| ctx := cmd.GetCmdCtx(c) | |||
| return ls(c, ctx, opt, args) | |||
| }, | |||
| } | |||
| c.Flags().Int64Var(&opt.BucketID, "bid", 0, "bucket id, if set, you should not set any path") | |||
| c.Flags().Int64Var(&opt.PackageID, "pid", 0, "package id, if set, you should not set <bucket_name>/<package_name>") | |||
| c.Flags().BoolVarP(&opt.Recursive, "recursive", "r", false, "list all files in package recursively, only valid when list in a package") | |||
| c.Flags().BoolVarP(&opt.Long, "long", "l", false, "show more details") | |||
| cmd.RootCmd.AddCommand(c) | |||
| } | |||
| type option struct { | |||
| Long bool | |||
| BucketID int64 | |||
| PackageID int64 | |||
| Recursive bool | |||
| } | |||
| func ls(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error { | |||
| if opt.PackageID != 0 { | |||
| objPath := "" | |||
| if len(args) > 0 { | |||
| objPath = args[0] | |||
| } | |||
| return lsObject(ctx, opt, "", "", objPath) | |||
| } | |||
| if opt.BucketID != 0 { | |||
| if len(args) > 0 { | |||
| return fmt.Errorf("list package objects is not supported when use bucket id") | |||
| } | |||
| return lsPackage(ctx, opt, "") | |||
| } | |||
| if len(args) == 0 { | |||
| return lsBucket(ctx, opt) | |||
| } | |||
| comps := strings.SplitN(args[0], ":", 2) | |||
| if len(comps) > 1 { | |||
| objPath := comps[1] | |||
| comps = strings.SplitN(comps[0], "/", 2) | |||
| if len(comps) != 2 { | |||
| return fmt.Errorf("invalid path format, should be <bucket_name>/<package_name>:<object_path>") | |||
| } | |||
| return lsObject(ctx, opt, comps[0], comps[1], objPath) | |||
| } | |||
| comps = strings.SplitN(args[0], "/", 2) | |||
| if len(comps) == 1 { | |||
| return lsPackage(ctx, opt, comps[0]) | |||
| } | |||
| return lsObject(ctx, opt, comps[0], comps[1], "") | |||
| } | |||
| @@ -0,0 +1,33 @@ | |||
| package ls | |||
| import ( | |||
| "fmt" | |||
| "github.com/jedib0t/go-pretty/v6/table" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func lsBucket(ctx *cmd.CommandContext, opt option) error { | |||
| resp, err := ctx.Client.Bucket().ListAll(cliapi.BucketListAll{}) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| if opt.Long { | |||
| fmt.Printf("total: %d buckets\n", len(resp.Buckets)) | |||
| tb := table.NewWriter() | |||
| tb.AppendHeader(table.Row{"Bucket ID", "Name", "Create Time"}) | |||
| for _, b := range resp.Buckets { | |||
| tb.AppendRow(table.Row{b.BucketID, b.Name, b.CreateTime}) | |||
| } | |||
| fmt.Println(tb.Render()) | |||
| } else { | |||
| for _, b := range resp.Buckets { | |||
| fmt.Println(b.Name) | |||
| } | |||
| } | |||
| return nil | |||
| } | |||
| @@ -0,0 +1,92 @@ | |||
| package ls | |||
| import ( | |||
| "fmt" | |||
| "github.com/jedib0t/go-pretty/v6/table" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func lsObject(ctx *cmd.CommandContext, opt option, bktName string, pkgName string, objPath string) error { | |||
| var pkgID clitypes.PackageID | |||
| if opt.PackageID != 0 { | |||
| pkgID = clitypes.PackageID(opt.PackageID) | |||
| } else { | |||
| resp, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{ | |||
| BucketName: bktName, | |||
| PackageName: pkgName, | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("get package %v: %w", pkgName, err) | |||
| } | |||
| pkgID = resp.Package.PackageID | |||
| } | |||
| var objs []clitypes.Object | |||
| var commonPrefixes []string | |||
| req := cliapi.ObjectListByPath{ | |||
| PackageID: pkgID, | |||
| Path: objPath, | |||
| IsPrefix: true, | |||
| } | |||
| if !opt.Recursive { | |||
| req.NoRecursive = true | |||
| } | |||
| var nextConToken string | |||
| for { | |||
| req.ContinuationToken = nextConToken | |||
| resp, err := ctx.Client.Object().ListByPath(req) | |||
| if err != nil { | |||
| return fmt.Errorf("list objects: %w", err) | |||
| } | |||
| objs = append(objs, resp.Objects...) | |||
| commonPrefixes = append(commonPrefixes, resp.CommonPrefixes...) | |||
| if !resp.IsTruncated { | |||
| break | |||
| } | |||
| nextConToken = resp.NextContinuationToken | |||
| } | |||
| if opt.Long { | |||
| fmt.Printf("total %d objects, %d common prefixes in package %v\n", len(objs), len(commonPrefixes), pkgName) | |||
| } | |||
| if len(commonPrefixes) > 0 { | |||
| for _, prefix := range commonPrefixes { | |||
| fmt.Printf("%s\n", prefix) | |||
| } | |||
| fmt.Printf("\n") | |||
| } | |||
| if len(objs) > 0 { | |||
| if opt.Long { | |||
| tb := table.NewWriter() | |||
| tb.AppendHeader(table.Row{"ID", "Path", "Size", "Hash", "Redundancy", "Create Time", "Update Time"}) | |||
| for _, obj := range objs { | |||
| tb.AppendRow(table.Row{ | |||
| obj.ObjectID, | |||
| obj.Path, | |||
| obj.Size, | |||
| obj.FileHash, | |||
| obj.Redundancy.GetRedundancyType(), | |||
| obj.CreateTime, | |||
| obj.UpdateTime, | |||
| }) | |||
| } | |||
| fmt.Println(tb.Render()) | |||
| } else { | |||
| for _, obj := range objs { | |||
| fmt.Printf("%s\n", obj.Path) | |||
| } | |||
| } | |||
| } | |||
| return nil | |||
| } | |||
| @@ -0,0 +1,48 @@ | |||
| package ls | |||
| import ( | |||
| "fmt" | |||
| "github.com/jedib0t/go-pretty/v6/table" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func lsPackage(ctx *cmd.CommandContext, opt option, bktName string) error { | |||
| var bktID clitypes.BucketID | |||
| if opt.BucketID == 0 { | |||
| bktResp, err := ctx.Client.Bucket().GetByName(cliapi.BucketGetByName{ | |||
| Name: bktName, | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("find bucket %v: %w", bktName, err) | |||
| } | |||
| bktID = bktResp.Bucket.BucketID | |||
| } else { | |||
| bktID = clitypes.BucketID(opt.BucketID) | |||
| } | |||
| pkgResp, err := ctx.Client.Package().ListBucketPackages(cliapi.PackageListBucketPackages{ | |||
| BucketID: bktID, | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("list packages in bucket %v: %w", bktName, err) | |||
| } | |||
| if opt.Long { | |||
| fmt.Printf("total %v:\n", len(pkgResp.Packages)) | |||
| tb := table.NewWriter() | |||
| tb.AppendHeader(table.Row{"Package ID", "Bucket ID", "Name", "CreateTime"}) | |||
| for _, p := range pkgResp.Packages { | |||
| tb.AppendRow(table.Row{p.PackageID, p.BucketID, p.Name, p.CreateTime}) | |||
| } | |||
| fmt.Println(tb.Render()) | |||
| } else { | |||
| for _, p := range pkgResp.Packages { | |||
| fmt.Println(p.Name) | |||
| } | |||
| } | |||
| return nil | |||
| } | |||
| @@ -0,0 +1,15 @@ | |||
| package object | |||
| import ( | |||
| "github.com/spf13/cobra" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| var ObjectCmd = &cobra.Command{ | |||
| Use: "object", | |||
| Aliases: []string{"obj"}, | |||
| } | |||
| func init() { | |||
| cmd.RootCmd.AddCommand(ObjectCmd) | |||
| } | |||
| @@ -0,0 +1,66 @@ | |||
| package pkg | |||
| import ( | |||
| "fmt" | |||
| "strings" | |||
| "github.com/spf13/cobra" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func init() { | |||
| var opt newOpt | |||
| cmd := cobra.Command{ | |||
| Use: "new <bucket_name>/<package_name>", | |||
| Args: cobra.ExactArgs(1), | |||
| RunE: func(c *cobra.Command, args []string) error { | |||
| ctx := cmd.GetCmdCtx(c) | |||
| return new(c, ctx, opt, args) | |||
| }, | |||
| } | |||
| cmd.Flags().Int64Var(&opt.BucketID, "bid", 0, "set bucket id, if set, you should not set bucket name") | |||
| PackageCmd.AddCommand(&cmd) | |||
| } | |||
| type newOpt struct { | |||
| BucketID int64 | |||
| } | |||
| func new(c *cobra.Command, ctx *cmd.CommandContext, opt newOpt, args []string) error { | |||
| if opt.BucketID != 0 { | |||
| resp, err := ctx.Client.Package().Create(cliapi.PackageCreate{ | |||
| BucketID: clitypes.BucketID(opt.BucketID), | |||
| Name: args[0], | |||
| }) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| printOnePackage(resp.Package) | |||
| return nil | |||
| } | |||
| comps := strings.Split(args[0], "/") | |||
| if len(comps) != 2 { | |||
| return fmt.Errorf("invalid package name") | |||
| } | |||
| bktResp, err := ctx.Client.Bucket().GetByName(cliapi.BucketGetByName{ | |||
| Name: comps[0], | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("get bucket by name %v: %v", comps[0], err) | |||
| } | |||
| resp, err := ctx.Client.Package().Create(cliapi.PackageCreate{ | |||
| BucketID: bktResp.Bucket.BucketID, | |||
| Name: comps[1], | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("create package %v: %v", args[0], err) | |||
| } | |||
| printOnePackage(resp.Package) | |||
| return nil | |||
| } | |||
| @@ -0,0 +1,15 @@ | |||
| package pkg | |||
| import ( | |||
| "github.com/spf13/cobra" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| var PackageCmd = &cobra.Command{ | |||
| Use: "package", | |||
| Aliases: []string{"pkg"}, | |||
| } | |||
| func init() { | |||
| cmd.RootCmd.AddCommand(PackageCmd) | |||
| } | |||
| @@ -0,0 +1,15 @@ | |||
| package pkg | |||
| import ( | |||
| "fmt" | |||
| "github.com/jedib0t/go-pretty/v6/table" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| ) | |||
| func printOnePackage(pkg clitypes.Package) { | |||
| tb := table.NewWriter() | |||
| tb.AppendHeader(table.Row{"Package ID", "Bucket ID", "Name", "CreateTime"}) | |||
| tb.AppendRow(table.Row{pkg.PackageID, pkg.BucketID, pkg.Name, pkg.CreateTime}) | |||
| fmt.Println(tb.Render()) | |||
| } | |||
| @@ -0,0 +1,109 @@ | |||
| package puto | |||
| import ( | |||
| "fmt" | |||
| "os" | |||
| "strconv" | |||
| "time" | |||
| "github.com/inhies/go-bytesize" | |||
| "github.com/spf13/cobra" | |||
| "gitlink.org.cn/cloudream/common/pkgs/iterator" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func init() { | |||
| var opt option | |||
| c := &cobra.Command{ | |||
| Use: "puto <local_path> <bucket_name>/<package_name>:<object_path>", | |||
| Short: "upload local file as a object", | |||
| Args: cobra.ExactArgs(2), | |||
| RunE: func(c *cobra.Command, args []string) error { | |||
| ctx := cmd.GetCmdCtx(c) | |||
| return puto(c, ctx, opt, args) | |||
| }, | |||
| } | |||
| c.Flags().BoolVar(&opt.UseID, "id", false, "treat the second argument as object id") | |||
| cmd.RootCmd.AddCommand(c) | |||
| } | |||
| type option struct { | |||
| UseID bool | |||
| } | |||
| func puto(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error { | |||
| var pkgID clitypes.PackageID | |||
| var objPath string | |||
| if opt.UseID { | |||
| id, err := strconv.ParseInt(args[1], 10, 64) | |||
| if err != nil { | |||
| return fmt.Errorf("invalid object id: %v", err) | |||
| } | |||
| resp, err := ctx.Client.Object().ListByIDs(cliapi.ObjectListByIDs{ | |||
| ObjectIDs: []clitypes.ObjectID{clitypes.ObjectID(id)}, | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("list objects by ids: %v", err) | |||
| } | |||
| if resp.Objects[0] == nil { | |||
| return fmt.Errorf("object not found") | |||
| } | |||
| pkgID = resp.Objects[0].PackageID | |||
| objPath = resp.Objects[0].Path | |||
| } else { | |||
| bkt, pkg, objPath2, ok := cmd.SplitObjectPath(args[1]) | |||
| if !ok { | |||
| return fmt.Errorf("invalid object path") | |||
| } | |||
| pkgResp, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{ | |||
| BucketName: bkt, | |||
| PackageName: pkg, | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("get package by name: %v", err) | |||
| } | |||
| pkgID = pkgResp.Package.PackageID | |||
| objPath = objPath2 | |||
| } | |||
| file, err := os.Open(args[0]) | |||
| if err != nil { | |||
| return fmt.Errorf("open file: %v", err) | |||
| } | |||
| defer file.Close() | |||
| info, err := file.Stat() | |||
| if err != nil { | |||
| return err | |||
| } | |||
| fmt.Printf("%v\n", objPath) | |||
| startTime := time.Now() | |||
| _, err = ctx.Client.Object().Upload(cliapi.ObjectUpload{ | |||
| ObjectUploadInfo: cliapi.ObjectUploadInfo{ | |||
| PackageID: pkgID, | |||
| }, | |||
| Files: iterator.Array(&cliapi.UploadingObject{ | |||
| Path: objPath, | |||
| File: file, | |||
| }), | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("upload file %v: %w", objPath, err) | |||
| } | |||
| dt := time.Since(startTime) | |||
| fmt.Printf("size: %v, time: %v, speed: %v/s\n", bytesize.ByteSize(info.Size()), dt, bytesize.ByteSize(int64(float64(info.Size())/dt.Seconds()))) | |||
| return nil | |||
| } | |||
| @@ -0,0 +1,106 @@ | |||
| package putp | |||
| import ( | |||
| "fmt" | |||
| "os" | |||
| "path/filepath" | |||
| "time" | |||
| "github.com/inhies/go-bytesize" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator" | |||
| ) | |||
| type FileIterator struct { | |||
| absRootPath string | |||
| jpathRoot clitypes.JPath | |||
| init bool | |||
| curEntries []dirEntry | |||
| lastStartTime time.Time | |||
| totalSize int64 | |||
| fileCount int | |||
| } | |||
| func (i *FileIterator) MoveNext() (*cliapi.UploadingObject, error) { | |||
| if !i.init { | |||
| es, err := os.ReadDir(i.absRootPath) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| for _, e := range es { | |||
| i.curEntries = append(i.curEntries, dirEntry{ | |||
| dir: clitypes.JPath{}, | |||
| entry: e, | |||
| }) | |||
| } | |||
| i.init = true | |||
| } | |||
| for { | |||
| if len(i.curEntries) == 0 { | |||
| return nil, iterator.ErrNoMoreItem | |||
| } | |||
| entry := i.curEntries[0] | |||
| i.curEntries = i.curEntries[1:] | |||
| if entry.entry.IsDir() { | |||
| es, err := os.ReadDir(filepath.Join(i.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name())) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| // 多个entry对象共享同一个JPath对象,但因为不会修改JPath,所以没问题 | |||
| dir := entry.dir.Clone() | |||
| dir.Push(entry.entry.Name()) | |||
| for _, e := range es { | |||
| i.curEntries = append(i.curEntries, dirEntry{ | |||
| dir: dir, | |||
| entry: e, | |||
| }) | |||
| } | |||
| continue | |||
| } | |||
| file, err := os.Open(filepath.Join(i.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name())) | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| info, err := file.Stat() | |||
| if err != nil { | |||
| return nil, err | |||
| } | |||
| i.totalSize += info.Size() | |||
| i.fileCount++ | |||
| jpath := i.jpathRoot.ConcatNew(entry.dir) | |||
| path := jpath.ConcatCompsNew(entry.entry.Name()).String() | |||
| now := time.Now() | |||
| if !i.lastStartTime.IsZero() { | |||
| dt := now.Sub(i.lastStartTime) | |||
| fmt.Printf("\t%v\n", dt) | |||
| } | |||
| i.lastStartTime = now | |||
| fmt.Printf("%v\t%v", path, bytesize.ByteSize(info.Size())) | |||
| return &cliapi.UploadingObject{ | |||
| Path: path, | |||
| File: file, | |||
| }, nil | |||
| } | |||
| } | |||
| func (i *FileIterator) Close() { | |||
| } | |||
| type dirEntry struct { | |||
| dir clitypes.JPath | |||
| entry os.DirEntry | |||
| } | |||
| @@ -0,0 +1,175 @@ | |||
| package putp | |||
| import ( | |||
| "fmt" | |||
| "os" | |||
| "path" | |||
| "path/filepath" | |||
| "strconv" | |||
| "strings" | |||
| "time" | |||
| "github.com/inhies/go-bytesize" | |||
| "github.com/spf13/cobra" | |||
| "gitlink.org.cn/cloudream/common/pkgs/iterator" | |||
| "gitlink.org.cn/cloudream/common/sdks" | |||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||
| clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" | |||
| "gitlink.org.cn/cloudream/jcs-pub/common/ecode" | |||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||
| ) | |||
| func init() { | |||
| var opt option | |||
| c := &cobra.Command{ | |||
| Use: "putp <local_path> <bucket_name>/<package_name>", | |||
| Short: "upload local files to a package", | |||
| Args: cobra.ExactArgs(2), | |||
| RunE: func(c *cobra.Command, args []string) error { | |||
| ctx := cmd.GetCmdCtx(c) | |||
| return putp(c, ctx, opt, args) | |||
| }, | |||
| } | |||
| c.Flags().BoolVar(&opt.UseID, "id", false, "treat the second argument as package id") | |||
| c.Flags().StringVar(&opt.Prefix, "prefix", "", "add prefix to every uploaded file") | |||
| c.Flags().BoolVar(&opt.Create, "create", false, "create package if not exists") | |||
| cmd.RootCmd.AddCommand(c) | |||
| } | |||
| type option struct { | |||
| UseID bool | |||
| Prefix string | |||
| Create bool | |||
| } | |||
| func putp(c *cobra.Command, ctx *cmd.CommandContext, opt option, args []string) error { | |||
| absLocal, err := filepath.Abs(args[0]) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| local, err := os.Stat(absLocal) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| var pkgID clitypes.PackageID | |||
| if opt.UseID { | |||
| id, err := strconv.ParseInt(args[1], 10, 64) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| pkgID = clitypes.PackageID(id) | |||
| _, err = ctx.Client.Package().Get(cliapi.PackageGet{ | |||
| PackageID: pkgID, | |||
| }) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| } else { | |||
| comps := strings.Split(args[1], "/") | |||
| if len(comps) != 2 { | |||
| return fmt.Errorf("invalid package name") | |||
| } | |||
| pkg, err := ctx.Client.Package().GetByFullName(cliapi.PackageGetByFullName{ | |||
| BucketName: comps[0], | |||
| PackageName: comps[1], | |||
| }) | |||
| if err != nil { | |||
| if !sdks.IsErrorCode(err, string(ecode.DataNotFound)) { | |||
| return err | |||
| } | |||
| if !opt.Create { | |||
| return fmt.Errorf("package not found") | |||
| } | |||
| bkt, err := ctx.Client.Bucket().GetByName(cliapi.BucketGetByName{ | |||
| Name: comps[0], | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("get bucket %v: %w", comps[0], err) | |||
| } | |||
| cpkg, err := ctx.Client.Package().Create(cliapi.PackageCreate{ | |||
| BucketID: bkt.Bucket.BucketID, | |||
| Name: comps[1], | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("create package %v: %w", args[1], err) | |||
| } | |||
| pkgID = cpkg.Package.PackageID | |||
| } else { | |||
| pkgID = pkg.Package.PackageID | |||
| } | |||
| } | |||
| if !local.IsDir() { | |||
| file, err := os.Open(absLocal) | |||
| if err != nil { | |||
| return err | |||
| } | |||
| defer file.Close() | |||
| info, err := file.Stat() | |||
| if err != nil { | |||
| return err | |||
| } | |||
| pat := filepath.Base(absLocal) | |||
| if opt.Prefix != "" { | |||
| pat = path.Join(opt.Prefix, pat) | |||
| } | |||
| fmt.Printf("%v\n", pat) | |||
| startTime := time.Now() | |||
| _, err = ctx.Client.Object().Upload(cliapi.ObjectUpload{ | |||
| ObjectUploadInfo: cliapi.ObjectUploadInfo{ | |||
| PackageID: pkgID, | |||
| }, | |||
| Files: iterator.Array(&cliapi.UploadingObject{ | |||
| Path: pat, | |||
| File: file, | |||
| }), | |||
| }) | |||
| if err != nil { | |||
| return fmt.Errorf("upload file %v: %w", pat, err) | |||
| } | |||
| dt := time.Since(startTime) | |||
| fmt.Printf("size: %v, time: %v, speed: %v/s\n", bytesize.ByteSize(info.Size()), dt, bytesize.ByteSize(int64(float64(info.Size())/dt.Seconds()))) | |||
| return nil | |||
| } | |||
| iter := &FileIterator{ | |||
| absRootPath: absLocal, | |||
| jpathRoot: clitypes.PathFromJcsPathString(opt.Prefix), | |||
| } | |||
| startTime := time.Now() | |||
| _, err = ctx.Client.Object().Upload(cliapi.ObjectUpload{ | |||
| ObjectUploadInfo: cliapi.ObjectUploadInfo{ | |||
| PackageID: pkgID, | |||
| }, | |||
| Files: iter, | |||
| }) | |||
| if err != nil { | |||
| if !iter.lastStartTime.IsZero() { | |||
| fmt.Printf("\tx\n") | |||
| } | |||
| return fmt.Errorf("upload files: %w", err) | |||
| } | |||
| dt := time.Since(startTime) | |||
| if !iter.lastStartTime.IsZero() { | |||
| fileDt := time.Since(iter.lastStartTime) | |||
| fmt.Printf("\t%v\n", fileDt) | |||
| } | |||
| fmt.Printf("%v files, total size: %v, time: %v, speed: %v/s\n", iter.fileCount, bytesize.ByteSize(iter.totalSize), dt, bytesize.ByteSize(int64(float64(iter.totalSize)/dt.Seconds()))) | |||
| return nil | |||
| } | |||
| @@ -16,9 +16,9 @@ func init() { | |||
| cmd := cobra.Command{ | |||
| Use: "ls", | |||
| Args: cobra.MaximumNArgs(1), | |||
| Run: func(c *cobra.Command, args []string) { | |||
| RunE: func(c *cobra.Command, args []string) error { | |||
| ctx := cmd.GetCmdCtx(c) | |||
| ls(c, ctx, opt, args) | |||
| return ls(c, ctx, opt, args) | |||
| }, | |||
| } | |||
| @@ -32,13 +32,12 @@ type lsOpt struct { | |||
| ShowPassword bool | |||
| } | |||
| func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) { | |||
| func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) error { | |||
| // 仅ls无参数 | |||
| if len(args) == 0 { | |||
| resp, err := ctx.Client.UserSpaceGetAll() | |||
| if err != nil { | |||
| cmd.ErrorExitln(err.Error()) | |||
| return | |||
| return err | |||
| } | |||
| fmt.Printf("total: %d\n", len(resp.UserSpaces)) | |||
| @@ -48,7 +47,7 @@ func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) { | |||
| tb.AppendRow(table.Row{userSpace.UserSpaceID, userSpace.Name, userSpace.Storage.GetStorageType()}) | |||
| } | |||
| fmt.Println(tb.Render()) | |||
| return | |||
| return nil | |||
| } | |||
| searchKey := args[0] | |||
| @@ -56,16 +55,14 @@ func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) { | |||
| if opt.ByID { | |||
| id, err := strconv.Atoi(searchKey) | |||
| if err != nil { | |||
| cmd.ErrorExitln("ID必须是数字") | |||
| return | |||
| return fmt.Errorf("ID必须是数字") | |||
| } | |||
| result, err := ctx.Client.UserSpaceGet(cliapi.UserSpaceGet{ | |||
| UserSpaceID: clitypes.UserSpaceID(id), | |||
| }) | |||
| if err != nil { | |||
| cmd.ErrorExitln(err.Error()) | |||
| return | |||
| return err | |||
| } | |||
| userSpace = &result.UserSpace | |||
| @@ -74,15 +71,13 @@ func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) { | |||
| Name: searchKey, | |||
| }) | |||
| if err != nil { | |||
| cmd.ErrorExitln(err.Error()) | |||
| return | |||
| return err | |||
| } | |||
| userSpace = &result.UserSpace | |||
| } | |||
| if userSpace == nil { | |||
| cmd.ErrorExitln(fmt.Sprintf("未找到匹配的云存储: %s", searchKey)) | |||
| return | |||
| return fmt.Errorf("未找到匹配的云存储: %s", searchKey) | |||
| } | |||
| fmt.Println("\n\033[1;36m云存储详情\033[0m") | |||
| @@ -103,4 +98,5 @@ func ls(c *cobra.Command, ctx *cmd.CommandContext, opt lsOpt, args []string) { | |||
| fmt.Printf("\033[1m%-8s\033[0m %s\n", "WorkingDir:", userSpace.WorkingDir) | |||
| fmt.Println("----------------------------------") | |||
| return nil | |||
| } | |||
| @@ -6,7 +6,8 @@ import ( | |||
| ) | |||
| var UserSpaceCmd = &cobra.Command{ | |||
| Use: "userspace", | |||
| Use: "userspace", | |||
| Aliases: []string{"us"}, | |||
| } | |||
| func init() { | |||
| @@ -1,16 +1,18 @@ | |||
| package cmd | |||
| import ( | |||
| "fmt" | |||
| "os" | |||
| ) | |||
| import "strings" | |||
| func ErrorExitf(format string, args ...interface{}) { | |||
| fmt.Printf(format, args...) | |||
| os.Exit(1) | |||
| } | |||
| func SplitObjectPath(str string) (bkt string, pkg string, obj string, ok bool) { | |||
| comps := strings.Split(str, ":") | |||
| if len(comps) != 2 { | |||
| return "", "", "", false | |||
| } | |||
| pat := comps[1] | |||
| comps = strings.Split(comps[0], "/") | |||
| if len(comps) != 2 { | |||
| return "", "", "", false | |||
| } | |||
| func ErrorExitln(msg string) { | |||
| fmt.Println(msg) | |||
| os.Exit(1) | |||
| return comps[0], comps[1], pat, true | |||
| } | |||