Browse Source

coor的types移动到common

master
Sydonian 4 months ago
parent
commit
dd23f68926
100 changed files with 871 additions and 891 deletions
  1. +5
    -5
      client/internal/accessstat/access_stat.go
  2. +3
    -3
      client/internal/accesstoken/accesstoken.go
  3. +10
    -10
      client/internal/cmdline/migrate.go
  4. +2
    -2
      client/internal/cmdline/test.go
  5. +23
    -23
      client/internal/downloader/downloader.go
  6. +4
    -4
      client/internal/downloader/lrc_strip_iterator.go
  7. +3
    -3
      client/internal/downloader/strategy/selector.go
  8. +6
    -6
      client/internal/http/v1/object.go
  9. +7
    -7
      client/internal/http/v1/package.go
  10. +4
    -4
      client/internal/http/v1/presigned.go
  11. +7
    -7
      client/internal/http/v1/space_syncer.go
  12. +2
    -2
      client/internal/http/v1/user_space.go
  13. +9
    -9
      client/internal/metacache/connectivity.go
  14. +8
    -8
      client/internal/metacache/hubmeta.go
  15. +2
    -2
      client/internal/metacache/user_space_meta.go
  16. +4
    -4
      client/internal/mount/mount_linux.go
  17. +4
    -4
      client/internal/mount/mount_win.go
  18. +7
    -7
      client/internal/mount/vfs/cache/cache.go
  19. +3
    -3
      client/internal/mount/vfs/cache/file.go
  20. +17
    -17
      client/internal/mount/vfs/fuse.go
  21. +2
    -2
      client/internal/mount/vfs/fuse_bucket.go
  22. +2
    -2
      client/internal/mount/vfs/fuse_dir.go
  23. +3
    -3
      client/internal/mount/vfs/fuse_file.go
  24. +2
    -2
      client/internal/mount/vfs/fuse_root.go
  25. +4
    -4
      client/internal/repl/load.go
  26. +4
    -4
      client/internal/repl/lsp.go
  27. +26
    -26
      client/internal/services/user_space.go
  28. +6
    -6
      client/internal/services/utils.go
  29. +7
    -7
      client/internal/spacesyncer/execute_diff.go
  30. +3
    -3
      client/internal/spacesyncer/filter.go
  31. +16
    -16
      client/internal/spacesyncer/space_syncer.go
  32. +3
    -3
      client/internal/spacesyncer/trigger.go
  33. +6
    -6
      client/internal/speedstats/speedstats.go
  34. +9
    -9
      client/internal/ticktock/change_redundancy.go
  35. +9
    -9
      client/internal/ticktock/check_shardstore.go
  36. +88
    -89
      client/internal/ticktock/redundancy_recover.go
  37. +52
    -52
      client/internal/ticktock/redundancy_shrink.go
  38. +13
    -13
      client/internal/uploader/uploader.go
  39. +9
    -10
      client/internal/uploader/user_space_upload.go
  40. +7
    -7
      client/sdk/api/v1/bucket.go
  41. +24
    -24
      client/sdk/api/v1/package.go
  42. +22
    -22
      client/sdk/api/v1/presigned.go
  43. +13
    -13
      client/sdk/api/v1/space_syncer.go
  44. +2
    -2
      client/sdk/api/v1/storage_test.go
  45. +2
    -2
      client/sdk/api/v1/system.go
  46. +26
    -27
      client/sdk/api/v1/user_space.go
  47. +1
    -1
      common/globals/globals.go
  48. +2
    -2
      common/globals/utils.go
  49. +6
    -6
      common/pkgs/accesstoken/accesstoken.go
  50. +10
    -10
      common/pkgs/connectivity/collector.go
  51. +15
    -15
      common/pkgs/ioswitch2/fromto.go
  52. +4
    -4
      common/pkgs/ioswitch2/http_hub_worker.go
  53. +4
    -4
      common/pkgs/ioswitch2/hub_worker.go
  54. +14
    -14
      common/pkgs/ioswitch2/ops2/base_store.go
  55. +6
    -6
      common/pkgs/ioswitch2/ops2/bypass.go
  56. +7
    -7
      common/pkgs/ioswitch2/ops2/ec.go
  57. +7
    -7
      common/pkgs/ioswitch2/ops2/multipart.go
  58. +17
    -17
      common/pkgs/ioswitch2/ops2/s2s.go
  59. +9
    -9
      common/pkgs/ioswitch2/ops2/shard_store.go
  60. +4
    -5
      common/pkgs/ioswitch2/parser/gen/generator.go
  61. +4
    -5
      common/pkgs/ioswitch2/parser/opt/utils.go
  62. +2
    -2
      common/pkgs/ioswitch2/plans/complete_multipart.go
  63. +4
    -5
      common/pkgs/ioswitch2/plans/utils.go
  64. +7
    -7
      common/pkgs/ioswitchlrc/fromto.go
  65. +3
    -3
      common/pkgs/ioswitchlrc/hub_worker.go
  66. +14
    -14
      common/pkgs/ioswitchlrc/ops2/base_store.go
  67. +5
    -5
      common/pkgs/ioswitchlrc/ops2/ec.go
  68. +9
    -9
      common/pkgs/ioswitchlrc/ops2/shard_store.go
  69. +5
    -5
      common/pkgs/ioswitchlrc/parser/generator.go
  70. +4
    -5
      common/pkgs/ioswitchlrc/parser/utils.go
  71. +4
    -4
      common/pkgs/publock/reqbuilder/user_space.go
  72. +7
    -7
      common/pkgs/rpc/auth.go
  73. +15
    -15
      common/pkgs/rpc/coordinator/hub.go
  74. +3
    -3
      common/pkgs/rpc/coordinator/storage.go
  75. +7
    -7
      common/pkgs/rpc/coordinator/user.go
  76. +5
    -5
      common/pkgs/rpc/hub/cache.go
  77. +3
    -3
      common/pkgs/rpc/hub/user.go
  78. +3
    -3
      common/pkgs/rpc/hub/user_space.go
  79. +8
    -8
      common/pkgs/servicestats/hub_strorage_transfer.go
  80. +8
    -8
      common/pkgs/servicestats/hub_transfter.go
  81. +5
    -5
      common/pkgs/servicestats/service_stats.go
  82. +4
    -5
      common/pkgs/storage/efile/ec_multiplier.go
  83. +7
    -8
      common/pkgs/storage/efile/efile.go
  84. +2
    -2
      common/pkgs/storage/factory/factory.go
  85. +4
    -5
      common/pkgs/storage/factory/reg/reg.go
  86. +8
    -8
      common/pkgs/storage/local/base_store.go
  87. +4
    -4
      common/pkgs/storage/local/dir_reader.go
  88. +9
    -10
      common/pkgs/storage/local/local.go
  89. +6
    -7
      common/pkgs/storage/local/multipart_upload.go
  90. +9
    -10
      common/pkgs/storage/local/s2s.go
  91. +12
    -12
      common/pkgs/storage/local/shard_store.go
  92. +17
    -18
      common/pkgs/storage/obs/obs.go
  93. +9
    -10
      common/pkgs/storage/obs/obs_test.go
  94. +12
    -13
      common/pkgs/storage/obs/s2s.go
  95. +5
    -6
      common/pkgs/storage/obs/shard_store.go
  96. +10
    -11
      common/pkgs/storage/pool/pool.go
  97. +9
    -9
      common/pkgs/storage/s3/base_store.go
  98. +3
    -3
      common/pkgs/storage/s3/dir_reader.go
  99. +7
    -8
      common/pkgs/storage/s3/multipart_upload.go
  100. +13
    -14
      common/pkgs/storage/s3/s3.go

+ 5
- 5
client/internal/accessstat/access_stat.go View File

@@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type AccessStatEventChan = async.UnboundChannel[AccessStatEvent]
@@ -30,9 +30,9 @@ type AccessStat struct {
}

type entryKey struct {
objID jcsypes.ObjectID
pkgID jcsypes.PackageID
spaceID jcsypes.UserSpaceID
objID jcstypes.ObjectID
pkgID jcstypes.PackageID
spaceID jcstypes.UserSpaceID
}

func NewAccessStat(cfg Config, db *db.DB) *AccessStat {
@@ -44,7 +44,7 @@ func NewAccessStat(cfg Config, db *db.DB) *AccessStat {
}
}

func (p *AccessStat) AddAccessCounter(objID jcsypes.ObjectID, pkgID jcsypes.PackageID, spaceID jcsypes.UserSpaceID, value float64) {
func (p *AccessStat) AddAccessCounter(objID jcstypes.ObjectID, pkgID jcstypes.PackageID, spaceID jcstypes.UserSpaceID, value float64) {
p.lock.Lock()
defer p.lock.Unlock()



+ 3
- 3
client/internal/accesstoken/accesstoken.go View File

@@ -15,7 +15,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/accesstoken"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type KeeperEvent interface {
@@ -30,7 +30,7 @@ type ExitEvent struct {
type Keeper struct {
cfg Config
enabled bool
token cortypes.UserAccessToken
token jcstypes.UserAccessToken
priKey ed25519.PrivateKey
lock sync.RWMutex
done chan any
@@ -144,7 +144,7 @@ func (k *Keeper) Stop() {
}
}

func (k *Keeper) GetToken() cortypes.UserAccessToken {
func (k *Keeper) GetToken() jcstypes.UserAccessToken {
k.lock.RLock()
defer k.lock.RUnlock()



+ 10
- 10
client/internal/cmdline/migrate.go View File

@@ -6,7 +6,7 @@ import (

"github.com/spf13/cobra"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/config"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/driver/mysql"
"gorm.io/gorm"
)
@@ -38,15 +38,15 @@ func migrate(configPath string) {
}
db = db.Set("gorm:table_options", "CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci")

migrateOne(db, jcsypes.Bucket{})
migrateOne(db, jcsypes.ObjectAccessStat{})
migrateOne(db, jcsypes.ObjectBlock{})
migrateOne(db, jcsypes.Object{})
migrateOne(db, jcsypes.PackageAccessStat{})
migrateOne(db, jcsypes.Package{})
migrateOne(db, jcsypes.PinnedObject{})
migrateOne(db, jcsypes.UserSpace{})
migrateOne(db, jcsypes.SpaceSyncTask{})
migrateOne(db, jcstypes.Bucket{})
migrateOne(db, jcstypes.ObjectAccessStat{})
migrateOne(db, jcstypes.ObjectBlock{})
migrateOne(db, jcstypes.Object{})
migrateOne(db, jcstypes.PackageAccessStat{})
migrateOne(db, jcstypes.Package{})
migrateOne(db, jcstypes.PinnedObject{})
migrateOne(db, jcstypes.UserSpace{})
migrateOne(db, jcstypes.SpaceSyncTask{})

fmt.Println("migrate success")
}


+ 2
- 2
client/internal/cmdline/test.go View File

@@ -27,7 +27,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
)

@@ -61,7 +61,7 @@ func doTest(svc *services.Service) {

ft = ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore("Full1AE5436AF72D8EF93923486E0E167315CEF0C91898064DADFAC22216FFBC5E3D", *space1, ioswitch2.RawStream()))
ft.AddTo(ioswitch2.NewToBaseStore(*space2, jcsypes.PathFromComps("test3.txt")))
ft.AddTo(ioswitch2.NewToBaseStore(*space2, jcstypes.PathFromComps("test3.txt")))
plans := exec.NewPlanBuilder()
parser.Parse(ft, plans)
fmt.Println(plans)


+ 23
- 23
client/internal/downloader/downloader.go View File

@@ -11,7 +11,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

const (
@@ -21,18 +21,18 @@ const (
type DownloadIterator = iterator.Iterator[*Downloading]

type DownloadReqeust struct {
ObjectID jcsypes.ObjectID
ObjectID jcstypes.ObjectID
Offset int64
Length int64
}

type downloadReqeust2 struct {
Detail *jcsypes.ObjectDetail
Detail *jcstypes.ObjectDetail
Raw DownloadReqeust
}

type Downloading struct {
Object *jcsypes.Object
Object *jcstypes.Object
File io.ReadCloser // 文件流,如果文件不存在,那么为nil
Request DownloadReqeust
}
@@ -65,7 +65,7 @@ func NewDownloader(cfg Config, conn *connectivity.Collector, stgPool *pool.Pool,
}

func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
objIDs := make([]jcsypes.ObjectID, len(reqs))
objIDs := make([]jcstypes.ObjectID, len(reqs))
for i, req := range reqs {
objIDs[i] = req.ObjectID
}
@@ -79,7 +79,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return iterator.FuseError[*Downloading](fmt.Errorf("request to db: %w", err))
}

detailsMap := make(map[jcsypes.ObjectID]*jcsypes.ObjectDetail)
detailsMap := make(map[jcstypes.ObjectID]*jcstypes.ObjectDetail)
for _, detail := range objDetails {
d := detail
detailsMap[detail.Object.ObjectID] = &d
@@ -96,7 +96,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return NewDownloadObjectIterator(d, req2s)
}

func (d *Downloader) DownloadObjectByDetail(detail jcsypes.ObjectDetail, off int64, length int64) (*Downloading, error) {
func (d *Downloader) DownloadObjectByDetail(detail jcstypes.ObjectDetail, off int64, length int64) (*Downloading, error) {
req2s := []downloadReqeust2{{
Detail: &detail,
Raw: DownloadReqeust{
@@ -110,56 +110,56 @@ func (d *Downloader) DownloadObjectByDetail(detail jcsypes.ObjectDetail, off int
return iter.MoveNext()
}

func (d *Downloader) DownloadPackage(pkgID jcsypes.PackageID, prefix string) (jcsypes.Package, DownloadIterator, error) {
pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (jcsypes.Package, []jcsypes.ObjectDetail, error) {
func (d *Downloader) DownloadPackage(pkgID jcstypes.PackageID, prefix string) (jcstypes.Package, DownloadIterator, error) {
pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (jcstypes.Package, []jcstypes.ObjectDetail, error) {
pkg, err := d.db.Package().GetByID(tx, pkgID)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}

var details []jcsypes.ObjectDetail
var details []jcstypes.ObjectDetail
if prefix != "" {
objs, err := d.db.Object().GetWithPathPrefix(tx, pkgID, prefix)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}

objIDs := make([]jcsypes.ObjectID, len(objs))
objIDs := make([]jcstypes.ObjectID, len(objs))
for i, obj := range objs {
objIDs[i] = obj.ObjectID
}

allBlocks, err := d.db.ObjectBlock().BatchGetByObjectID(tx, objIDs)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}

allPinnedObjs, err := d.db.PinnedObject().BatchGetByObjectID(tx, objIDs)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err

}
details = make([]jcsypes.ObjectDetail, 0, len(objs))
details = make([]jcstypes.ObjectDetail, 0, len(objs))
for _, obj := range objs {
detail := jcsypes.ObjectDetail{
detail := jcstypes.ObjectDetail{
Object: obj,
}
details = append(details, detail)
}

jcsypes.DetailsFillObjectBlocks(details, allBlocks)
jcsypes.DetailsFillPinnedAt(details, allPinnedObjs)
jcstypes.DetailsFillObjectBlocks(details, allBlocks)
jcstypes.DetailsFillPinnedAt(details, allPinnedObjs)
} else {
details, err = d.db.Object().GetPackageObjectDetails(tx, pkgID)
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}
}

return pkg, details, nil
})
if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
}

req2s := make([]downloadReqeust2, len(details))
@@ -180,11 +180,11 @@ func (d *Downloader) DownloadPackage(pkgID jcsypes.PackageID, prefix string) (jc

type ObjectECStrip struct {
Data []byte
ObjectFileHash jcsypes.FileHash // 添加这条缓存时,Object的FileHash
ObjectFileHash jcstypes.FileHash // 添加这条缓存时,Object的FileHash
}

type ECStripKey struct {
ObjectID jcsypes.ObjectID
ObjectID jcstypes.ObjectID
StripIndex int64
}



+ 4
- 4
client/internal/downloader/lrc_strip_iterator.go View File

@@ -12,14 +12,14 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type LRCStripIterator struct {
downloader *Downloader
object jcsypes.Object
object jcstypes.Object
blocks []downloadBlock
red jcsypes.LRCRedundancy
red jcstypes.LRCRedundancy
curStripIndex int64
cache *StripCache
dataChan chan dataChanEntry
@@ -28,7 +28,7 @@ type LRCStripIterator struct {
inited bool
}

func NewLRCStripIterator(downloder *Downloader, object jcsypes.Object, blocks []downloadBlock, red jcsypes.LRCRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *LRCStripIterator {
func NewLRCStripIterator(downloder *Downloader, object jcstypes.Object, blocks []downloadBlock, red jcstypes.LRCRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *LRCStripIterator {
if maxPrefetch <= 0 {
maxPrefetch = 1
}


+ 3
- 3
client/internal/downloader/strategy/selector.go View File

@@ -12,13 +12,13 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache"
"gitlink.org.cn/cloudream/jcs-pub/common/consts"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type Request struct {
Detail types.ObjectDetail
Range math2.Range
DestLocation cortypes.Location
DestLocation jcstypes.Location
}

type Strategy interface {
@@ -113,7 +113,7 @@ type downloadBlock struct {
type request2 struct {
Detail types.ObjectDetail
Range math2.Range
DestLocation cortypes.Location
DestLocation jcstypes.Location
}

func (s *Selector) selectForNoneOrRep(req request2) (Strategy, error) {


+ 6
- 6
client/internal/http/v1/object.go View File

@@ -19,7 +19,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type ObjectService struct {
@@ -113,9 +113,9 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
return
}

copyToPath := make([]jcsypes.JPath, 0, len(info.CopyToPath))
copyToPath := make([]jcstypes.JPath, 0, len(info.CopyToPath))
for _, p := range info.CopyToPath {
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcstypes.PathFromJcsPathString(p))
}

up, err := s.svc.Uploader.BeginUpdate(info.PackageID, info.Affinity, info.CopyTo, copyToPath)
@@ -147,7 +147,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
}
path = filepath.ToSlash(path)

err = up.Upload(jcsypes.PathFromJcsPathString(path), file)
err = up.Upload(jcstypes.PathFromJcsPathString(path), file)
if err != nil {
log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
@@ -163,7 +163,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
return
}

uploadeds := make([]jcsypes.Object, len(pathes))
uploadeds := make([]jcstypes.Object, len(pathes))
for i := range pathes {
uploadeds[i] = ret.Objects[pathes[i]]
}
@@ -398,7 +398,7 @@ func (s *ObjectService) DeleteByPath(ctx *gin.Context) {
return
}

err = s.svc.ObjectSvc().Delete([]jcsypes.ObjectID{resp.Objects[0].ObjectID})
err = s.svc.ObjectSvc().Delete([]jcstypes.ObjectID{resp.Objects[0].ObjectID})
if err != nil {
log.Warnf("deleting objects: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "delete objects failed"))


+ 7
- 7
client/internal/http/v1/package.go View File

@@ -21,7 +21,7 @@ import (
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)

@@ -151,9 +151,9 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
return
}

copyToPath := make([]jcsypes.JPath, 0, len(info.CopyToPath))
copyToPath := make([]jcstypes.JPath, 0, len(info.CopyToPath))
for _, p := range info.CopyToPath {
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcstypes.PathFromJcsPathString(p))
}

up, err := s.svc.Uploader.BeginCreateUpload(info.BucketID, info.Name, info.CopyTo, copyToPath)
@@ -184,7 +184,7 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
}
path = filepath.ToSlash(path)

err = up.Upload(jcsypes.PathFromJcsPathString(path), file)
err = up.Upload(jcstypes.PathFromJcsPathString(path), file)
if err != nil {
log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
@@ -200,7 +200,7 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
return
}

objs := make([]jcsypes.Object, len(pathes))
objs := make([]jcstypes.Object, len(pathes))
for i := range pathes {
objs[i] = ret.Objects[pathes[i]]
}
@@ -233,7 +233,7 @@ func (s *PackageService) Download(ctx *gin.Context) {
}
}

func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg jcsypes.Package, iter downloader.DownloadIterator) {
func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg jcstypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download")

ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".zip")
@@ -276,7 +276,7 @@ func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownloa
}
}

func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg jcsypes.Package, iter downloader.DownloadIterator) {
func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg jcstypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download")

ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".tar")


+ 4
- 4
client/internal/http/v1/presigned.go View File

@@ -15,7 +15,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type PresignedService struct {
@@ -156,9 +156,9 @@ func (s *PresignedService) ObjectUpload(ctx *gin.Context) {
return
}

copyToPath := make([]jcsypes.JPath, 0, len(req.CopyToPath))
copyToPath := make([]jcstypes.JPath, 0, len(req.CopyToPath))
for _, p := range req.CopyToPath {
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcstypes.PathFromJcsPathString(p))
}

up, err := s.svc.Uploader.BeginUpdate(req.PackageID, req.Affinity, req.CopyTo, copyToPath)
@@ -171,7 +171,7 @@ func (s *PresignedService) ObjectUpload(ctx *gin.Context) {

path := filepath.ToSlash(req.Path)

err = up.Upload(jcsypes.PathFromJcsPathString(path), ctx.Request.Body)
err = up.Upload(jcstypes.PathFromJcsPathString(path), ctx.Request.Body)
if err != nil {
log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", req.Path, err)))


+ 7
- 7
client/internal/http/v1/space_syncer.go View File

@@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type SpaceSyncerService struct {
@@ -41,21 +41,21 @@ func (s *SpaceSyncerService) CreateTask(ctx *gin.Context) {
return
}

dests := make([]jcsypes.SpaceSyncDest, 0, len(req.DestUserSpaceIDs))
dests := make([]jcstypes.SpaceSyncDest, 0, len(req.DestUserSpaceIDs))
for _, id := range req.DestUserSpaceIDs {
dests = append(dests, jcsypes.SpaceSyncDest{
DestUserSpaceID: jcsypes.UserSpaceID(id),
DestPath: jcsypes.PathFromJcsPathString(req.DestPathes[0]),
dests = append(dests, jcstypes.SpaceSyncDest{
DestUserSpaceID: jcstypes.UserSpaceID(id),
DestPath: jcstypes.PathFromJcsPathString(req.DestPathes[0]),
})
}

info, err := s.svc.SpaceSyncer.CreateTask(jcsypes.SpaceSyncTask{
info, err := s.svc.SpaceSyncer.CreateTask(jcstypes.SpaceSyncTask{
Trigger: req.Trigger,
Mode: req.Mode,
Filters: req.Filters,
Options: req.Options,
SrcUserSpaceID: req.SrcUserSpaceID,
SrcPath: jcsypes.PathFromJcsPathString(req.SrcPath),
SrcPath: jcstypes.PathFromJcsPathString(req.SrcPath),
Dests: dests,
})
if err != nil {


+ 2
- 2
client/internal/http/v1/user_space.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type UserSpaceService struct {
@@ -52,7 +52,7 @@ func (s *UserSpaceService) CreatePackage(ctx *gin.Context) {
return
}

pkg, err := s.svc.Uploader.UserSpaceUpload(req.UserSpaceID, jcsypes.PathFromJcsPathString(req.Path), req.BucketID, req.Name, req.SpaceAffinity)
pkg, err := s.svc.Uploader.UserSpaceUpload(req.UserSpaceID, jcstypes.PathFromJcsPathString(req.Path), req.BucketID, req.Name, req.SpaceAffinity)
if err != nil {
log.Warnf("userspace create package: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("userspace create package: %v", err)))


+ 9
- 9
client/internal/metacache/connectivity.go View File

@@ -8,12 +8,12 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func (m *MetaCacheHost) AddConnectivity() *Connectivity {
cache := &Connectivity{
entries: make(map[cortypes.HubID]*ConnectivityEntry),
entries: make(map[jcstypes.HubID]*ConnectivityEntry),
}

m.caches = append(m.caches, cache)
@@ -22,10 +22,10 @@ func (m *MetaCacheHost) AddConnectivity() *Connectivity {

type Connectivity struct {
lock sync.RWMutex
entries map[cortypes.HubID]*ConnectivityEntry
entries map[jcstypes.HubID]*ConnectivityEntry
}

func (c *Connectivity) Get(from cortypes.HubID, to cortypes.HubID) *time.Duration {
func (c *Connectivity) Get(from jcstypes.HubID, to jcstypes.HubID) *time.Duration {
for i := 0; i < 2; i++ {
c.lock.RLock()
entry, ok := c.entries[from]
@@ -60,12 +60,12 @@ func (c *Connectivity) ClearOutdated() {
}
}

func (c *Connectivity) load(hubID cortypes.HubID) {
func (c *Connectivity) load(hubID jcstypes.HubID) {
coorCli := stgglb.CoordinatorRPCPool.Get()

defer coorCli.Release()

get, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]cortypes.HubID{hubID}))
get, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]jcstypes.HubID{hubID}))
if cerr != nil {
logger.Warnf("get hub connectivities: %v", cerr)
return
@@ -76,7 +76,7 @@ func (c *Connectivity) load(hubID cortypes.HubID) {

ce := &ConnectivityEntry{
From: hubID,
To: make(map[cortypes.HubID]cortypes.HubConnectivity),
To: make(map[jcstypes.HubID]jcstypes.HubConnectivity),
UpdateTime: time.Now(),
}

@@ -88,7 +88,7 @@ func (c *Connectivity) load(hubID cortypes.HubID) {
}

type ConnectivityEntry struct {
From cortypes.HubID
To map[cortypes.HubID]cortypes.HubConnectivity
From jcstypes.HubID
To map[jcstypes.HubID]jcstypes.HubConnectivity
UpdateTime time.Time
}

+ 8
- 8
client/internal/metacache/hubmeta.go View File

@@ -7,12 +7,12 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func (m *MetaCacheHost) AddHubMeta() *HubMeta {
meta := &HubMeta{}
meta.cache = NewSimpleMetaCache(SimpleMetaCacheConfig[cortypes.HubID, cortypes.Hub]{
meta.cache = NewSimpleMetaCache(SimpleMetaCacheConfig[jcstypes.HubID, jcstypes.Hub]{
Getter: meta.load,
Expire: time.Minute * 5,
})
@@ -22,10 +22,10 @@ func (m *MetaCacheHost) AddHubMeta() *HubMeta {
}

type HubMeta struct {
cache *SimpleMetaCache[cortypes.HubID, cortypes.Hub]
cache *SimpleMetaCache[jcstypes.HubID, jcstypes.Hub]
}

func (h *HubMeta) Get(hubID cortypes.HubID) *cortypes.Hub {
func (h *HubMeta) Get(hubID jcstypes.HubID) *jcstypes.Hub {
v, ok := h.cache.Get(hubID)
if ok {
return &v
@@ -33,9 +33,9 @@ func (h *HubMeta) Get(hubID cortypes.HubID) *cortypes.Hub {
return nil
}

func (h *HubMeta) GetMany(hubIDs []cortypes.HubID) []*cortypes.Hub {
func (h *HubMeta) GetMany(hubIDs []jcstypes.HubID) []*jcstypes.Hub {
vs, oks := h.cache.GetMany(hubIDs)
ret := make([]*cortypes.Hub, len(vs))
ret := make([]*jcstypes.Hub, len(vs))
for i := range vs {
if oks[i] {
ret[i] = &vs[i]
@@ -48,8 +48,8 @@ func (h *HubMeta) ClearOutdated() {
h.cache.ClearOutdated()
}

func (h *HubMeta) load(keys []cortypes.HubID) ([]cortypes.Hub, []bool) {
vs := make([]cortypes.Hub, len(keys))
func (h *HubMeta) load(keys []jcstypes.HubID) ([]jcstypes.Hub, []bool) {
vs := make([]jcstypes.Hub, len(keys))
oks := make([]bool, len(keys))

coorCli := stgglb.CoordinatorRPCPool.Get()


+ 2
- 2
client/internal/metacache/user_space_meta.go View File

@@ -8,7 +8,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func (m *MetaCacheHost) AddStorageMeta() *UserSpaceMeta {
@@ -78,7 +78,7 @@ func (s *UserSpaceMeta) load(keys []types.UserSpaceID) ([]types.UserSpaceDetail,
coorCli := stgglb.CoordinatorRPCPool.Get()
defer coorCli.Release()

stgs := make([]cortypes.StorageType, len(spaces))
stgs := make([]jcstypes.StorageType, len(spaces))
for i := range spaces {
stgs[i] = spaces[i].Storage
}


+ 4
- 4
client/internal/mount/mount_linux.go View File

@@ -12,7 +12,7 @@ import (
fuse2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type Mount struct {
@@ -107,14 +107,14 @@ func (m *Mount) StartReclaimSpace() {
m.vfs.ReclaimSpace()
}

func (m *Mount) NotifyObjectInvalid(obj jcsypes.Object) {
func (m *Mount) NotifyObjectInvalid(obj jcstypes.Object) {

}

func (m *Mount) NotifyPackageInvalid(pkg jcsypes.Package) {
func (m *Mount) NotifyPackageInvalid(pkg jcstypes.Package) {

}

func (m *Mount) NotifyBucketInvalid(bkt jcsypes.Bucket) {
func (m *Mount) NotifyBucketInvalid(bkt jcstypes.Bucket) {

}

+ 4
- 4
client/internal/mount/mount_win.go View File

@@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type Mount struct {
@@ -43,14 +43,14 @@ func (m *Mount) StartReclaimSpace() {

}

func (m *Mount) NotifyObjectInvalid(obj jcsypes.Object) {
func (m *Mount) NotifyObjectInvalid(obj jcstypes.Object) {

}

func (m *Mount) NotifyPackageInvalid(pkg jcsypes.Package) {
func (m *Mount) NotifyPackageInvalid(pkg jcstypes.Package) {

}

func (m *Mount) NotifyBucketInvalid(bkt jcsypes.Bucket) {
func (m *Mount) NotifyBucketInvalid(bkt jcstypes.Bucket) {

}

+ 7
- 7
client/internal/mount/vfs/cache/cache.go View File

@@ -21,7 +21,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type CacheEntry interface {
@@ -196,7 +196,7 @@ func (c *Cache) CreateFile(pathComps []string) *CacheFile {
// 尝试加载缓存文件,如果文件不存在,则使用obj的信息创建一个新缓存文件,而如果obj为nil,那么会返回nil。
//
// 记得使用Release减少引用计数
func (c *Cache) LoadFile(pathComps []string, obj *jcsypes.Object) *CacheFile {
func (c *Cache) LoadFile(pathComps []string, obj *jcstypes.Object) *CacheFile {
c.lock.Lock()
defer c.lock.Unlock()

@@ -490,7 +490,7 @@ func (c *Cache) Move(pathComps []string, newPathComps []string) error {
type syncPackage struct {
bktName string
pkgName string
pkg jcsypes.Package
pkg jcstypes.Package
upObjs []*uploadingObject
}

@@ -917,7 +917,7 @@ func (c *Cache) doUpdatingOnly(pkgs []*syncPackage) {
pathes := make([]string, 0, len(p.upObjs))
modTimes := make([]time.Time, 0, len(p.upObjs))
for _, obj := range p.upObjs {
pathes = append(pathes, jcsypes.JoinObjectPath(obj.pathComps[2:]...))
pathes = append(pathes, jcstypes.JoinObjectPath(obj.pathComps[2:]...))
modTimes = append(modTimes, obj.modTime)
}

@@ -1008,7 +1008,7 @@ func (c *Cache) doUploading(pkgs []*syncPackage) {

counter := io2.Counter(&rd)

err = upder.Upload(jcsypes.PathFromComps(o.pathComps[2:]...), counter, uploader.UploadOption{
err = upder.Upload(jcstypes.PathFromComps(o.pathComps[2:]...), counter, uploader.UploadOption{
CreateTime: o.modTime,
})
if err != nil {
@@ -1036,8 +1036,8 @@ func (c *Cache) doUploading(pkgs []*syncPackage) {
continue
}

oldPath := jcsypes.JoinObjectPath(o.pathComps[2:]...)
newPath := jcsypes.JoinObjectPath(o.cache.pathComps[2:]...)
oldPath := jcstypes.JoinObjectPath(o.pathComps[2:]...)
newPath := jcstypes.JoinObjectPath(o.cache.pathComps[2:]...)

if o.isDeleted {
upder.CancelObject(oldPath)


+ 3
- 3
client/internal/mount/vfs/cache/file.go View File

@@ -14,7 +14,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type CacheLevel int
@@ -102,7 +102,7 @@ type CacheFile struct {
cache *Cache
pathComps []string
info FileInfo
remoteObj *jcsypes.Object
remoteObj *jcstypes.Object
rwLock *sync.RWMutex
readers []*CacheFileHandle
writers []*CacheFileHandle
@@ -262,7 +262,7 @@ func loadCacheFile(cache *Cache, pathComps []string) (*CacheFile, error) {
return ch, nil
}

func newCacheFileFromObject(cache *Cache, pathComps []string, obj *jcsypes.Object) (*CacheFile, error) {
func newCacheFileFromObject(cache *Cache, pathComps []string, obj *jcstypes.Object) (*CacheFile, error) {
metaPath := cache.GetCacheMetaPath(pathComps...)
dataPath := cache.GetCacheDataPath(pathComps...)



+ 17
- 17
client/internal/mount/vfs/fuse.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)

@@ -35,7 +35,7 @@ func child(vfs *Vfs, ctx context.Context, parent FuseNode, name string) (fuse.Fs
return nil
}

objPath := jcsypes.JoinObjectPath(childPathComps[2:]...)
objPath := jcstypes.JoinObjectPath(childPathComps[2:]...)
obj, err := d.Object().GetByPath(tx, pkg.PackageID, objPath)
if err == nil {
ret = newFileFromObject(vfs, childPathComps, obj)
@@ -45,7 +45,7 @@ func child(vfs *Vfs, ctx context.Context, parent FuseNode, name string) (fuse.Fs
return err
}

err = d.Object().HasObjectWithPrefix(tx, pkg.PackageID, objPath+jcsypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, pkg.PackageID, objPath+jcstypes.ObjectPathSeparator)
if err == nil {
dir := vfs.cache.LoadDir(childPathComps, &cache.CreateDirOption{
ModTime: time.Now(),
@@ -98,10 +98,10 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
return err
}

objPath := jcsypes.JoinObjectPath(myPathComps[2:]...)
objPath := jcstypes.JoinObjectPath(myPathComps[2:]...)
objPrefix := objPath
if objPath != "" {
objPrefix += jcsypes.ObjectPathSeparator
objPrefix += jcstypes.ObjectPathSeparator
}

objs, coms, err := d.Object().GetByPrefixGrouped(tx, pkg.PackageID, objPrefix)
@@ -110,8 +110,8 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
}

for _, dir := range coms {
dir = strings.TrimSuffix(dir, jcsypes.ObjectPathSeparator)
pathComps := lo2.AppendNew(myPathComps, jcsypes.BaseName(dir))
dir = strings.TrimSuffix(dir, jcstypes.ObjectPathSeparator)
pathComps := lo2.AppendNew(myPathComps, jcstypes.BaseName(dir))

cd := vfs.cache.LoadDir(pathComps, &cache.CreateDirOption{
ModTime: time.Now(),
@@ -124,7 +124,7 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
}

for _, obj := range objs {
pathComps := lo2.AppendNew(myPathComps, jcsypes.BaseName(obj.Path))
pathComps := lo2.AppendNew(myPathComps, jcstypes.BaseName(obj.Path))
file := newFileFromObject(vfs, pathComps, obj)
dbEntries[file.Name()] = file
}
@@ -179,14 +179,14 @@ func newFile(vfs *Vfs, ctx context.Context, name string, parent FuseNode, flags

func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) error {
pathComps := lo2.AppendNew(parent.PathComps(), name)
joinedPath := jcsypes.JoinObjectPath(pathComps[2:]...)
joinedPath := jcstypes.JoinObjectPath(pathComps[2:]...)
d := vfs.db

// TODO 生成系统事件
return vfs.db.DoTx(func(tx db.SQLContext) error {
pkg, err := d.Package().GetByFullName(tx, pathComps[0], pathComps[1])
if err == nil {
err := d.Object().HasObjectWithPrefix(tx, pkg.PackageID, joinedPath+jcsypes.ObjectPathSeparator)
err := d.Object().HasObjectWithPrefix(tx, pkg.PackageID, joinedPath+jcstypes.ObjectPathSeparator)
if err == nil {
return fuse.ErrNotEmpty
}
@@ -211,7 +211,7 @@ func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) er
func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode, newName string, newParent FuseNode) error {
newParentPath := newParent.PathComps()
newChildPath := lo2.AppendNew(newParentPath, newName)
newChildPathJoined := jcsypes.JoinObjectPath(newChildPath[2:]...)
newChildPathJoined := jcstypes.JoinObjectPath(newChildPath[2:]...)

// 不允许移动任何内容到Package层级以上
if len(newParentPath) < 2 {
@@ -219,7 +219,7 @@ func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode
}

oldChildPath := lo2.AppendNew(oldParent.PathComps(), oldName)
oldChildPathJoined := jcsypes.JoinObjectPath(oldChildPath[2:]...)
oldChildPathJoined := jcstypes.JoinObjectPath(oldChildPath[2:]...)

// 先更新远程,再更新本地,因为远程使用事务更新,可以回滚,而本地不行
return vfs.db.DoTx(func(tx db.SQLContext) error {
@@ -259,7 +259,7 @@ func moveRemote(vfs *Vfs, tx db.SQLContext, oldChildPath []string, newParentPath
return fuse.ErrExists
}

err = d.Object().HasObjectWithPrefix(tx, newPkg.PackageID, newChildPathJoined+jcsypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, newPkg.PackageID, newChildPathJoined+jcstypes.ObjectPathSeparator)
if err == nil {
return fuse.ErrExists
}
@@ -283,17 +283,17 @@ func moveRemote(vfs *Vfs, tx db.SQLContext, oldChildPath []string, newParentPath
oldObj.PackageID = newPkg.PackageID
oldObj.Path = newChildPathJoined

return d.Object().BatchUpdate(tx, []jcsypes.Object{oldObj})
return d.Object().BatchUpdate(tx, []jcstypes.Object{oldObj})
}
if err != gorm.ErrRecordNotFound {
return err
}

err = d.Object().HasObjectWithPrefix(tx, oldPkg.PackageID, oldChildPathJoined+jcsypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, oldPkg.PackageID, oldChildPathJoined+jcstypes.ObjectPathSeparator)
if err == nil {
return d.Object().MoveByPrefix(tx,
oldPkg.PackageID, oldChildPathJoined+jcsypes.ObjectPathSeparator,
newPkg.PackageID, newChildPathJoined+jcsypes.ObjectPathSeparator,
oldPkg.PackageID, oldChildPathJoined+jcstypes.ObjectPathSeparator,
newPkg.PackageID, newChildPathJoined+jcstypes.ObjectPathSeparator,
)
}
if err == gorm.ErrRecordNotFound {


+ 2
- 2
client/internal/mount/vfs/fuse_bucket.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)

@@ -117,7 +117,7 @@ func (r *FuseBucket) listChildren() ([]fuse.FsEntry, error) {
return nil, err
}

pkgMap := make(map[string]*jcsypes.Package)
pkgMap := make(map[string]*jcstypes.Package)
for _, pkg := range pkgs {
p := pkg
pkgMap[pkg.Name] = &p


+ 2
- 2
client/internal/mount/vfs/fuse_dir.go View File

@@ -8,7 +8,7 @@ import (
db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)

@@ -108,7 +108,7 @@ func (r *FuseDir) loadCacheDir() *cache.CacheDir {
return err
}

err = r.vfs.db.Object().HasObjectWithPrefix(tx, pkg.PackageID, jcsypes.JoinObjectPath(r.pathComps[2:]...))
err = r.vfs.db.Object().HasObjectWithPrefix(tx, pkg.PackageID, jcstypes.JoinObjectPath(r.pathComps[2:]...))
if err == nil {
createOpt = &cache.CreateDirOption{
ModTime: time.Now(),


+ 3
- 3
client/internal/mount/vfs/fuse_file.go View File

@@ -6,7 +6,7 @@ import (

"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)

@@ -28,7 +28,7 @@ func newFileFromCache(info cache.CacheEntryInfo, vfs *Vfs) *FuseFileNode {
}
}

func newFileFromObject(vfs *Vfs, pathComps []string, obj jcsypes.Object) *FuseFileNode {
func newFileFromObject(vfs *Vfs, pathComps []string, obj jcstypes.Object) *FuseFileNode {
return &FuseFileNode{
vfs: vfs,
pathComps: pathComps,
@@ -117,7 +117,7 @@ func (n *FuseFileNode) loadCacheFile() *cache.CacheFile {
return n.vfs.cache.LoadFile(n.pathComps, nil)
}

cdsObj, err := n.vfs.db.Object().GetByFullPath(n.vfs.db.DefCtx(), n.pathComps[0], n.pathComps[1], jcsypes.JoinObjectPath(n.pathComps[2:]...))
cdsObj, err := n.vfs.db.Object().GetByFullPath(n.vfs.db.DefCtx(), n.pathComps[0], n.pathComps[1], jcstypes.JoinObjectPath(n.pathComps[2:]...))
if err == nil {
file := n.vfs.cache.LoadFile(n.pathComps, &cdsObj)
if file == nil {


+ 2
- 2
client/internal/mount/vfs/fuse_root.go View File

@@ -8,7 +8,7 @@ import (
db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"
)

@@ -104,7 +104,7 @@ func (r *FuseRoot) listChildren() ([]fuse.FsEntry, error) {
return nil, err
}

bktMap := make(map[string]*jcsypes.Bucket)
bktMap := make(map[string]*jcstypes.Bucket)
for _, bkt := range bkts {
b := bkt
bktMap[bkt.Name] = &b


+ 4
- 4
client/internal/repl/load.go View File

@@ -8,7 +8,7 @@ import (
"time"

"github.com/spf13/cobra"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -31,7 +31,7 @@ func init() {
fmt.Printf("Invalid user space ID: %s\n", args[1])
}

loadByID(cmdCtx, jcsypes.PackageID(pkgID), jcsypes.UserSpaceID(userSpaceID), args[2])
loadByID(cmdCtx, jcstypes.PackageID(pkgID), jcstypes.UserSpaceID(userSpaceID), args[2])
} else {
loadByPath(cmdCtx, args[0], args[1], args[2])
}
@@ -42,7 +42,7 @@ func init() {
}

func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) {
comps := strings.Split(strings.Trim(pkgPath, jcsypes.ObjectPathSeparator), jcsypes.ObjectPathSeparator)
comps := strings.Split(strings.Trim(pkgPath, jcstypes.ObjectPathSeparator), jcstypes.ObjectPathSeparator)
if len(comps) != 2 {
fmt.Printf("Package path must be in format of <bucket>/<package>")
return
@@ -63,7 +63,7 @@ func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath
loadByID(cmdCtx, pkg.PackageID, stg.StorageID, rootPath)
}

func loadByID(cmdCtx *CommandContext, pkgID jcsypes.PackageID, stgID jcsypes.StorageID, rootPath string) {
func loadByID(cmdCtx *CommandContext, pkgID jcstypes.PackageID, stgID jcstypes.StorageID, rootPath string) {
startTime := time.Now()

err := cmdCtx.Cmdline.Svc.StorageSvc().LoadPackage(pkgID, stgID, rootPath)


+ 4
- 4
client/internal/repl/lsp.go View File

@@ -7,7 +7,7 @@ import (

"github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -26,7 +26,7 @@ func init() {
return
}

lspOneByID(cmdCtx, jcsypes.PackageID(id))
lspOneByID(cmdCtx, jcstypes.PackageID(id))
} else {
lspByPath(cmdCtx, args[0])
}
@@ -40,7 +40,7 @@ func init() {
func lspByPath(cmdCtx *CommandContext, path string) {
db2 := cmdCtx.repl.db

comps := strings.Split(strings.Trim(path, jcsypes.ObjectPathSeparator), jcsypes.ObjectPathSeparator)
comps := strings.Split(strings.Trim(path, jcstypes.ObjectPathSeparator), jcstypes.ObjectPathSeparator)
if len(comps) != 2 {
fmt.Printf("Package path must be in format of <bucket>/<package>")
return
@@ -58,7 +58,7 @@ func lspByPath(cmdCtx *CommandContext, path string) {
fmt.Println(wr.Render())
}

func lspOneByID(cmdCtx *CommandContext, id jcsypes.PackageID) {
func lspOneByID(cmdCtx *CommandContext, id jcstypes.PackageID) {
db2 := cmdCtx.repl.db

pkg, err := db2.Package().GetByID(db2.DefCtx(), id)


+ 26
- 26
client/internal/services/user_space.go View File

@@ -8,7 +8,7 @@ import (
"github.com/samber/lo"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm"

"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
@@ -31,41 +31,41 @@ func (svc *Service) UserSpaceSvc() *UserSpaceService {
return &UserSpaceService{Service: svc}
}

func (svc *UserSpaceService) Get(userspaceID jcsypes.UserSpaceID) (jcsypes.UserSpace, error) {
func (svc *UserSpaceService) Get(userspaceID jcstypes.UserSpaceID) (jcstypes.UserSpace, error) {
return svc.DB.UserSpace().GetByID(svc.DB.DefCtx(), userspaceID)
}

func (svc *UserSpaceService) GetByName(name string) (jcsypes.UserSpace, error) {
func (svc *UserSpaceService) GetByName(name string) (jcstypes.UserSpace, error) {
return svc.DB.UserSpace().GetByName(svc.DB.DefCtx(), name)
}

func (svc *UserSpaceService) GetAll() ([]jcsypes.UserSpace, error) {
func (svc *UserSpaceService) GetAll() ([]jcstypes.UserSpace, error) {
return svc.DB.UserSpace().GetAll(svc.DB.DefCtx())
}

func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpaceCreateResp, *ecode.CodeError) {
db2 := svc.DB
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcsypes.UserSpace, error) {
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcstypes.UserSpace, error) {
space, err := db2.UserSpace().GetByName(tx, req.Name)
if err == nil {
return jcsypes.UserSpace{}, gorm.ErrDuplicatedKey
return jcstypes.UserSpace{}, gorm.ErrDuplicatedKey
}
if err != gorm.ErrRecordNotFound {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
}

space = jcsypes.UserSpace{
space = jcstypes.UserSpace{
Name: req.Name,
Storage: req.Storage,
Credential: req.Credential,
ShardStore: req.ShardStore,
Features: req.Features,
WorkingDir: jcsypes.PathFromJcsPathString(req.WorkingDir),
WorkingDir: jcstypes.PathFromJcsPathString(req.WorkingDir),
Revision: 0,
}
err = db2.UserSpace().Create(tx, &space)
if err != nil {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
}
return space, nil
})
@@ -80,19 +80,19 @@ func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpa

func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpaceUpdateResp, *ecode.CodeError) {
db2 := svc.DB
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcsypes.UserSpace, error) {
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcstypes.UserSpace, error) {
space, err := db2.UserSpace().GetByID(tx, req.UserSpaceID)
if err != nil {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
}

if space.Name != req.Name {
_, err = db2.UserSpace().GetByName(tx, req.Name)
if err == nil {
return jcsypes.UserSpace{}, gorm.ErrDuplicatedKey
return jcstypes.UserSpace{}, gorm.ErrDuplicatedKey
}
if err != gorm.ErrRecordNotFound {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
}
}

@@ -110,7 +110,7 @@ func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpa
}

// 通知元数据缓存无效
svc.UserSpaceMeta.Drop([]jcsypes.UserSpaceID{req.UserSpaceID})
svc.UserSpaceMeta.Drop([]jcstypes.UserSpaceID{req.UserSpaceID})

// 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理
svc.StgPool.Drop(stgglb.UserID, space.UserSpaceID)
@@ -155,7 +155,7 @@ func (svc *UserSpaceService) Delete(req cliapi.UserSpaceDelete) (*cliapi.UserSpa
}

// 通知元数据缓存无效
svc.UserSpaceMeta.Drop([]jcsypes.UserSpaceID{req.UserSpaceID})
svc.UserSpaceMeta.Drop([]jcstypes.UserSpaceID{req.UserSpaceID})

// 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理
svc.StgPool.Drop(stgglb.UserID, req.UserSpaceID)
@@ -166,13 +166,13 @@ func (svc *UserSpaceService) Delete(req cliapi.UserSpaceDelete) (*cliapi.UserSpa
}

func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTestResp, *ecode.CodeError) {
detail := jcsypes.UserSpaceDetail{
detail := jcstypes.UserSpaceDetail{
UserID: stgglb.UserID,
UserSpace: jcsypes.UserSpace{
UserSpace: jcstypes.UserSpace{
Name: "test",
Storage: req.Storage,
Credential: req.Credential,
WorkingDir: jcsypes.PathFromJcsPathString(req.WorikingDir),
WorkingDir: jcstypes.PathFromJcsPathString(req.WorikingDir),
},
}
blder := factory.GetBuilder(&detail)
@@ -189,7 +189,7 @@ func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTe
return &cliapi.UserSpaceTestResp{}, nil
}

func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, userspaceID jcsypes.UserSpaceID, rootPath string) error {
func (svc *UserSpaceService) DownloadPackage(packageID jcstypes.PackageID, userspaceID jcstypes.UserSpaceID, rootPath string) error {
destSpace := svc.UserSpaceMeta.Get(userspaceID)
if destSpace == nil {
return fmt.Errorf("userspace not found: %d", userspaceID)
@@ -208,10 +208,10 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
}
defer mutex.Unlock()

rootJPath := jcsypes.PathFromJcsPathString(rootPath)
rootJPath := jcstypes.PathFromJcsPathString(rootPath)

dIndex := 0
var pinned []jcsypes.PinnedObject
var pinned []jcstypes.PinnedObject
for dIndex < len(details) {
plans := exec.NewPlanBuilder()
for i := 0; i < 10 && dIndex < len(details); i++ {
@@ -252,7 +252,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
return fmt.Errorf("unsupported download strategy: %T", strg)
}

objPath := jcsypes.PathFromJcsPathString(details[dIndex].Object.Path)
objPath := jcstypes.PathFromJcsPathString(details[dIndex].Object.Path)
dstPath := rootJPath.ConcatNew(objPath)

newDstSpace := *destSpace
@@ -265,7 +265,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
// 顺便保存到同存储服务的分片存储中
if destSpace.UserSpace.ShardStore != nil {
ft.AddTo(ioswitch2.NewToShardStore(newDstSpace, ioswitch2.RawStream(), ""))
pinned = append(pinned, jcsypes.PinnedObject{
pinned = append(pinned, jcstypes.PinnedObject{
ObjectID: details[dIndex].Object.ObjectID,
UserSpaceID: destSpace.UserSpace.UserSpaceID,
CreateTime: time.Now(),
@@ -297,7 +297,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
}

err = svc.DB.DoTx(func(tx db.SQLContext) error {
objIDs := make([]jcsypes.ObjectID, len(pinned))
objIDs := make([]jcstypes.ObjectID, len(pinned))
for i, obj := range pinned {
objIDs[i] = obj.ObjectID
}
@@ -307,7 +307,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
return err
}

pinned = lo.Filter(pinned, func(p jcsypes.PinnedObject, idx int) bool { return avaiIDs[p.ObjectID] })
pinned = lo.Filter(pinned, func(p jcstypes.PinnedObject, idx int) bool { return avaiIDs[p.ObjectID] })
return svc.DB.PinnedObject().BatchTryCreate(svc.DB.DefCtx(), pinned)
})
if err != nil {


+ 6
- 6
client/internal/services/utils.go View File

@@ -1,22 +1,22 @@
package services

import (
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
)

func getBlockTypeFromRed(red jcsypes.Redundancy) string {
func getBlockTypeFromRed(red jcstypes.Redundancy) string {
switch red.(type) {
case *jcsypes.NoneRedundancy:
case *jcstypes.NoneRedundancy:
return datamap.BlockTypeRaw

case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
return datamap.BlockTypeEC

case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
return datamap.BlockTypeEC

case *jcsypes.SegmentRedundancy:
case *jcstypes.SegmentRedundancy:
return datamap.BlockTypeSegment
}
return ""


+ 7
- 7
client/internal/spacesyncer/execute_diff.go View File

@@ -12,10 +12,10 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDiff) {
func executeDiff(syncer *SpaceSyncer, task *task, mode *jcstypes.SpaceSyncModeDiff) {
log := logger.WithField("Mod", logMod).WithField("TaskID", task.Task.TaskID)

startTime := time.Now()
@@ -112,7 +112,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDif
}

var willSync []stgtypes.DirEntry
var willMkdirs []jcsypes.JPath
var willMkdirs []jcstypes.JPath

dirTree.Iterate(func(path []string, node *trie.Node[srcDstDirEntry], isWordNode bool) trie.VisitCtrl {
if node.Value.src == nil {
@@ -123,7 +123,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDif
if node.Value.src.IsDir {
if node.Value.dst == nil {
if node.IsEmpty() {
willMkdirs = append(willMkdirs, jcsypes.PathFromComps(path...))
willMkdirs = append(willMkdirs, jcstypes.PathFromComps(path...))
}
}
} else {
@@ -193,7 +193,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDif
}
}

func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *stgtypes.DirEntry) {
func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcstypes.JPath, e *stgtypes.DirEntry) {
var ptr = &tree.Root
for _, c := range path.Comps() {
if ptr.Value.src != nil && ptr.Value.src.IsDir {
@@ -205,7 +205,7 @@ func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *s
ptr.Value.src = e
}

func diffCreateDstNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *stgtypes.DirEntry) {
func diffCreateDstNode(tree *trie.Trie[srcDstDirEntry], path jcstypes.JPath, e *stgtypes.DirEntry) {
var ptr = &tree.Root
for _, c := range path.Comps() {
if ptr.Value.src != nil && ptr.Value.src.IsDir {
@@ -227,7 +227,7 @@ type srcDstDirEntry struct {
dst *stgtypes.DirEntry
}

func cmpFile(diff *jcsypes.SpaceSyncModeDiff, src, dst *stgtypes.DirEntry) bool {
func cmpFile(diff *jcstypes.SpaceSyncModeDiff, src, dst *stgtypes.DirEntry) bool {
if diff.IncludeSize && src.Size != dst.Size {
return false
}


+ 3
- 3
client/internal/spacesyncer/filter.go View File

@@ -2,7 +2,7 @@ package spacesyncer

import (
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type FilterFn func(info stgtypes.DirEntry) bool
@@ -11,7 +11,7 @@ func buildFilter(task *task) FilterFn {
var fns []FilterFn
for _, f := range task.Task.Filters {
switch f := f.(type) {
case *jcsypes.SpaceSyncFilterSize:
case *jcstypes.SpaceSyncFilterSize:
fns = append(fns, filterSize(f))
}
}
@@ -26,7 +26,7 @@ func buildFilter(task *task) FilterFn {
}
}

func filterSize(filter *jcsypes.SpaceSyncFilterSize) FilterFn {
func filterSize(filter *jcstypes.SpaceSyncFilterSize) FilterFn {
return func(info stgtypes.DirEntry) bool {
if filter.MinSize > 0 && info.Size < filter.MinSize {
return false


+ 16
- 16
client/internal/spacesyncer/space_syncer.go View File

@@ -10,7 +10,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache"
stgpool "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

const (
@@ -26,7 +26,7 @@ type SpaceSyncer struct {
stgPool *stgpool.Pool
spaceMeta *metacache.UserSpaceMeta
lock sync.Mutex
tasks map[jcsypes.SpaceSyncTaskID]*task
tasks map[jcstypes.SpaceSyncTaskID]*task
}

func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *SpaceSyncer {
@@ -34,7 +34,7 @@ func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *
db: db,
stgPool: stgPool,
spaceMeta: spaceMeta,
tasks: make(map[jcsypes.SpaceSyncTaskID]*task),
tasks: make(map[jcstypes.SpaceSyncTaskID]*task),
}
}

@@ -50,7 +50,7 @@ func (s *SpaceSyncer) Start() *async.UnboundChannel[SpaceSyncerEvent] {
if err != nil {
log.Warnf("load task from db: %v", err)
} else {
var rms []jcsypes.SpaceSyncTaskID
var rms []jcstypes.SpaceSyncTaskID
for _, t := range allTask {
ctx, cancel := context.WithCancel(context.Background())
tsk := task{
@@ -60,14 +60,14 @@ func (s *SpaceSyncer) Start() *async.UnboundChannel[SpaceSyncerEvent] {
}

switch tr := t.Trigger.(type) {
case *jcsypes.SpaceSyncTriggerOnce:
case *jcstypes.SpaceSyncTriggerOnce:
// Once类型的任务没有执行完也不执行了
rms = append(rms, t.TaskID)

case *jcsypes.SpaceSyncTriggerInterval:
case *jcstypes.SpaceSyncTriggerInterval:
triggerInterval(s, &tsk, tr)

case *jcsypes.SpaceSyncTriggerAt:
case *jcstypes.SpaceSyncTriggerAt:
triggerAt(s, &tsk, tr)
}

@@ -95,10 +95,10 @@ func (s *SpaceSyncer) Stop() {
t.CancelFn()
}

s.tasks = make(map[jcsypes.SpaceSyncTaskID]*task)
s.tasks = make(map[jcstypes.SpaceSyncTaskID]*task)
}

func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
func (s *SpaceSyncer) CreateTask(t jcstypes.SpaceSyncTask) (*TaskInfo, error) {
log := logger.WithField("Mod", logMod)

d := s.db
@@ -126,13 +126,13 @@ func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
s.lock.Unlock()

switch tr := t.Trigger.(type) {
case *jcsypes.SpaceSyncTriggerOnce:
case *jcstypes.SpaceSyncTriggerOnce:
triggerOnce(s, &tsk)

case *jcsypes.SpaceSyncTriggerInterval:
case *jcstypes.SpaceSyncTriggerInterval:
triggerInterval(s, &tsk, tr)

case *jcsypes.SpaceSyncTriggerAt:
case *jcstypes.SpaceSyncTriggerAt:
triggerAt(s, &tsk, tr)
}

@@ -143,7 +143,7 @@ func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
}, nil
}

func (s *SpaceSyncer) CancelTask(taskID jcsypes.SpaceSyncTaskID) {
func (s *SpaceSyncer) CancelTask(taskID jcstypes.SpaceSyncTaskID) {
log := logger.WithField("Mod", logMod)

s.lock.Lock()
@@ -166,7 +166,7 @@ func (s *SpaceSyncer) CancelTask(taskID jcsypes.SpaceSyncTaskID) {
log.Infof("task %v canceled", taskID)
}

func (s *SpaceSyncer) GetTask(taskID jcsypes.SpaceSyncTaskID) *jcsypes.SpaceSyncTask {
func (s *SpaceSyncer) GetTask(taskID jcstypes.SpaceSyncTaskID) *jcstypes.SpaceSyncTask {
s.lock.Lock()
defer s.lock.Unlock()

@@ -181,11 +181,11 @@ func (s *SpaceSyncer) GetTask(taskID jcsypes.SpaceSyncTaskID) *jcsypes.SpaceSync
}

type TaskInfo struct {
Task jcsypes.SpaceSyncTask
Task jcstypes.SpaceSyncTask
}

type task struct {
Task jcsypes.SpaceSyncTask
Task jcstypes.SpaceSyncTask
Context context.Context
CancelFn func()
}

+ 3
- 3
client/internal/spacesyncer/trigger.go View File

@@ -5,7 +5,7 @@ import (

"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/sort2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func triggerOnce(syncer *SpaceSyncer, task *task) {
@@ -31,7 +31,7 @@ func triggerOnce(syncer *SpaceSyncer, task *task) {
}()
}

func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSyncTriggerInterval) {
func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcstypes.SpaceSyncTriggerInterval) {
go func() {
log := logger.WithField("Mod", logMod)

@@ -66,7 +66,7 @@ func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSync
}()
}

func triggerAt(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSyncTriggerAt) {
func triggerAt(syncer *SpaceSyncer, task *task, trigger *jcstypes.SpaceSyncTriggerAt) {
go func() {
log := logger.WithField("Mod", logMod)



+ 6
- 6
client/internal/speedstats/speedstats.go View File

@@ -5,7 +5,7 @@ import (
"sync"
"time"

jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

const (
@@ -121,9 +121,9 @@ func (p *SpeedStats) ShouldAtClient(size int64) bool {
return v < prob
}

func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
return jcsypes.SpeedStatsStatus{
Below100M: []jcsypes.SpeedStatsStatusEntry{
func (p *SpeedStats) DumpStatus() jcstypes.SpeedStatsStatus {
return jcstypes.SpeedStatsStatus{
Below100M: []jcstypes.SpeedStatsStatusEntry{
{
TotalSize: p.stats100M[0].TotalSize,
TotalTime: p.stats100M[0].TotalTime,
@@ -137,7 +137,7 @@ func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
LastSpeed: p.stats100M[1].LastSpeed,
},
},
Below1G: []jcsypes.SpeedStatsStatusEntry{
Below1G: []jcstypes.SpeedStatsStatusEntry{
{
TotalSize: p.stats1G[0].TotalSize,
TotalTime: p.stats1G[0].TotalTime,
@@ -151,7 +151,7 @@ func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
LastSpeed: p.stats1G[1].LastSpeed,
},
},
Above1G: []jcsypes.SpeedStatsStatusEntry{
Above1G: []jcstypes.SpeedStatsStatusEntry{
{
TotalSize: p.statsAbove1G[0].TotalSize,
TotalTime: p.statsAbove1G[0].TotalTime,


+ 9
- 9
client/internal/ticktock/change_redundancy.go View File

@@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/reflect2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
)

@@ -34,7 +34,7 @@ func (j *ChangeRedundancy) Execute(t *TickTock) {

ctx := &changeRedundancyContext{
ticktock: t,
allUserSpaces: make(map[jcsypes.UserSpaceID]*userSpaceUsageInfo),
allUserSpaces: make(map[jcstypes.UserSpaceID]*userSpaceUsageInfo),
}

spaceIDs, err := t.db.UserSpace().GetAllIDs(t.db.DefCtx())
@@ -58,7 +58,7 @@ func (j *ChangeRedundancy) Execute(t *TickTock) {
return
}

lastPkgID := jcsypes.PackageID(0)
lastPkgID := jcstypes.PackageID(0)

loop:
for {
@@ -90,16 +90,16 @@ loop:

type changeRedundancyContext struct {
ticktock *TickTock
allUserSpaces map[jcsypes.UserSpaceID]*userSpaceUsageInfo
mostBlockStgIDs []jcsypes.UserSpaceID
allUserSpaces map[jcstypes.UserSpaceID]*userSpaceUsageInfo
mostBlockStgIDs []jcstypes.UserSpaceID
}

type userSpaceUsageInfo struct {
UserSpace *jcsypes.UserSpaceDetail
UserSpace *jcstypes.UserSpaceDetail
AccessAmount float64
}

func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.PackageDetail) error {
func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcstypes.PackageDetail) error {
log := logger.WithType[ChangeRedundancy]("TickTock")
db2 := ctx.ticktock.db

@@ -121,7 +121,7 @@ func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.P
info.AccessAmount = stat.Amount
}

lastObjID := jcsypes.ObjectID(0)
lastObjID := jcstypes.ObjectID(0)
for {
objs, err := db.DoTx31(db2, db2.Object().BatchGetDetailsPaged, pkg.Package.PackageID, lastObjID, BatchGetObjectDetailCount)
if err != nil {
@@ -139,7 +139,7 @@ func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.P

ctx.mostBlockStgIDs = j.summaryRepObjectBlockUserSpaces(ctx, objs, 2)

var willShrinks []jcsypes.ObjectDetail
var willShrinks []jcstypes.ObjectDetail

for _, obj := range objs {
newRed, selectedSpaces := j.chooseRedundancy(ctx, obj)


+ 9
- 9
client/internal/ticktock/check_shardstore.go View File

@@ -10,7 +10,7 @@ import (

"gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

// CheckShardStore 代表一个用于处理代理缓存检查事件的结构体
@@ -52,8 +52,8 @@ func (j *CheckShardStore) Execute(t *TickTock) {
}
}

func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail) error {
// addr, ok := space.RecommendHub.Address.(*cortypes.GRPCAddressInfo)
func (j *CheckShardStore) checkOne(t *TickTock, space *jcstypes.UserSpaceDetail) error {
// addr, ok := space.RecommendHub.Address.(*jcstypes.GRPCAddressInfo)
// if !ok {
// return fmt.Errorf("master of user space %v has no grpc address", space.UserSpace)
// }
@@ -80,9 +80,9 @@ func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail)
return fmt.Errorf("listing all files: %w", err)
}

fileHashes := lo.Map(infos, func(info stgtypes.FileInfo, _ int) jcsypes.FileHash { return info.Hash })
fileHashes := lo.Map(infos, func(info stgtypes.FileInfo, _ int) jcstypes.FileHash { return info.Hash })

realFileHashes := lo.SliceToMap(fileHashes, func(hash jcsypes.FileHash) (jcsypes.FileHash, bool) { return hash, true })
realFileHashes := lo.SliceToMap(fileHashes, func(hash jcstypes.FileHash) (jcstypes.FileHash, bool) { return hash, true })

// 在事务中执行缓存更新操作
t.db.DoTx(func(tx db.SQLContext) error {
@@ -95,7 +95,7 @@ func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail)
}

// checkPinnedObject 对比PinnedObject表,若实际文件不存在,则进行删除操作
func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *jcsypes.UserSpaceDetail, realFileHashes map[jcsypes.FileHash]bool) {
func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *jcstypes.UserSpaceDetail, realFileHashes map[jcstypes.FileHash]bool) {
log := logger.WithType[CheckShardStore]("TickTock")

objs, err := t.db.PinnedObject().GetObjectsByUserSpaceID(tx, space.UserSpace.UserSpaceID)
@@ -104,7 +104,7 @@ func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *
return
}

var rms []jcsypes.ObjectID
var rms []jcstypes.ObjectID
for _, c := range objs {
if realFileHashes[c.FileHash] {
continue
@@ -121,7 +121,7 @@ func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *
}

// checkObjectBlock 对比ObjectBlock表,若实际文件不存在,则进行删除操作
func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *jcsypes.UserSpaceDetail, realFileHashes map[jcsypes.FileHash]bool) {
func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *jcstypes.UserSpaceDetail, realFileHashes map[jcstypes.FileHash]bool) {
log := logger.WithType[CheckShardStore]("TickTock")

blocks, err := t.db.ObjectBlock().GetByUserSpaceID(tx, space.UserSpace.UserSpaceID)
@@ -130,7 +130,7 @@ func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *j
return
}

var rms []jcsypes.FileHash
var rms []jcstypes.FileHash
for _, b := range blocks {
if realFileHashes[b.FileHash] {
continue


+ 88
- 89
client/internal/ticktock/redundancy_recover.go View File

@@ -16,56 +16,55 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
lrcparser "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
)

func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail) (jcsypes.Redundancy, []*userSpaceUsageInfo) {
func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail) (jcstypes.Redundancy, []*userSpaceUsageInfo) {
switch obj.Object.Redundancy.(type) {
case *jcsypes.NoneRedundancy:
case *jcstypes.NoneRedundancy:
if obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold {
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcsypes.DefaultECRedundancy)
return &jcsypes.DefaultECRedundancy, newStgs
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
return &jcstypes.DefaultECRedundancy, newStgs
}

return &jcsypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)

case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
if obj.Object.Size >= ctx.ticktock.cfg.ECFileSizeThreshold {
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcsypes.DefaultECRedundancy)
return &jcsypes.DefaultECRedundancy, newStgs
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
return &jcstypes.DefaultECRedundancy, newStgs
}

newSpaces := t.rechooseUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
newSpaces := t.rechooseUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
for _, s := range newSpaces {
if !obj.ContainsBlock(0, s.UserSpace.UserSpace.UserSpaceID) && !obj.ContainsPinned(s.UserSpace.UserSpace.UserSpaceID) {
return &jcsypes.DefaultRepRedundancy, newSpaces
return &jcstypes.DefaultRepRedundancy, newSpaces
}
}

return nil, nil

case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold {
return &jcsypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
}

newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &jcsypes.DefaultECRedundancy)
newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &jcstypes.DefaultECRedundancy)
for i, s := range newSpaces {
if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
return &jcsypes.DefaultECRedundancy, newSpaces
return &jcstypes.DefaultECRedundancy, newSpaces
}
}

return nil, nil

case *jcsypes.LRCRedundancy:
newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &jcsypes.DefaultLRCRedundancy)
case *jcstypes.LRCRedundancy:
newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &jcstypes.DefaultLRCRedundancy)

for i, s := range newLRCStgs {
if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
return &jcsypes.DefaultLRCRedundancy, newLRCStgs
return &jcstypes.DefaultLRCRedundancy, newLRCStgs
}
}

@@ -74,7 +73,7 @@ func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jc
return nil, nil
}

func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, newRed jcsypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, newRed jcstypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
log := logger.WithType[ChangeRedundancy]("TickTock")

var updating *db.UpdatingObjectRedundancy
@@ -82,48 +81,48 @@ func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj
var err error

switch srcRed := obj.Object.Redundancy.(type) {
case *jcsypes.NoneRedundancy:
case *jcstypes.NoneRedundancy:
switch newRed := newRed.(type) {
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep")
updating, evt, err = t.noneToRep(ctx, obj, newRed, selectedUserSpaces)

case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec")
updating, evt, err = t.noneToEC(ctx, obj, newRed, selectedUserSpaces)

case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc")
updating, evt, err = t.noneToLRC(ctx, obj, newRed, selectedUserSpaces)

case *jcsypes.SegmentRedundancy:
case *jcstypes.SegmentRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment")
updating, evt, err = t.noneToSeg(ctx, obj, newRed, selectedUserSpaces)
}

case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
switch newRed := newRed.(type) {
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
updating, evt, err = t.repToRep(ctx, obj, srcRed, selectedUserSpaces)

case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec")
updating, evt, err = t.repToEC(ctx, obj, newRed, selectedUserSpaces)
}

case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
switch newRed := newRed.(type) {
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep")
updating, evt, err = t.ecToRep(ctx, obj, srcRed, newRed, selectedUserSpaces)

case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
updating, evt, err = t.ecToEC(ctx, obj, srcRed, newRed, selectedUserSpaces)
}

case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
switch newRed := newRed.(type) {
case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
updating, evt, err = t.lrcToLRC(ctx, obj, srcRed, newRed, selectedUserSpaces)
}
}
@@ -132,16 +131,16 @@ func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj
}

// 统计每个对象块所在的节点,选出块最多的不超过userspaceCnt个节点
func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []jcsypes.ObjectDetail, userspaceCnt int) []jcsypes.UserSpaceID {
func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []jcstypes.ObjectDetail, userspaceCnt int) []jcstypes.UserSpaceID {
type stgBlocks struct {
UserSpaceID jcsypes.UserSpaceID
UserSpaceID jcstypes.UserSpaceID
Count int
}

stgBlocksMap := make(map[jcsypes.UserSpaceID]*stgBlocks)
stgBlocksMap := make(map[jcstypes.UserSpaceID]*stgBlocks)
for _, obj := range objs {
shouldUseEC := obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold
if _, ok := obj.Object.Redundancy.(*jcsypes.RepRedundancy); ok && !shouldUseEC {
if _, ok := obj.Object.Redundancy.(*jcstypes.RepRedundancy); ok && !shouldUseEC {
for _, block := range obj.Blocks {
if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
stgBlocksMap[block.UserSpaceID] = &stgBlocks{
@@ -159,14 +158,14 @@ func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancy
return right.Count - left.Count
})

ids := lo.Map(userspaces, func(item *stgBlocks, idx int) jcsypes.UserSpaceID { return item.UserSpaceID })
ids := lo.Map(userspaces, func(item *stgBlocks, idx int) jcstypes.UserSpaceID { return item.UserSpaceID })
if len(ids) > userspaceCnt {
ids = ids[:userspaceCnt]
}
return ids
}

func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *jcsypes.RepRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount)
})
@@ -174,7 +173,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContex
return t.chooseSoManyUserSpaces(red.RepCount, sortedUserSpaces)
}

func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *jcsypes.ECRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount)
})
@@ -182,7 +181,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext
return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces)
}

func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *jcsypes.LRCRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount)
})
@@ -198,7 +197,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForSeg(ctx *changeRedundancyContex
return t.chooseSoManyUserSpaces(segCount, sortedUserSpaces)
}

func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *jcsypes.RepRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
type rechooseUserSpace struct {
*userSpaceUsageInfo
HasBlock bool
@@ -233,7 +232,7 @@ func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext
return t.chooseSoManyUserSpaces(red.RepCount, lo.Map(sortedStgs, func(userspace *rechooseUserSpace, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
}

func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
type rechooseStg struct {
*userSpaceUsageInfo
CachedBlockIndex int
@@ -269,7 +268,7 @@ func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext,
return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
}

func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.LRCRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
type rechooseStg struct {
*userSpaceUsageInfo
CachedBlockIndex int
@@ -322,7 +321,7 @@ func (t *ChangeRedundancy) chooseSoManyUserSpaces(count int, stgs []*userSpaceUs
var chosen []*userSpaceUsageInfo
for len(chosen) < count {
// 在每一轮内都选不同地区的节点,如果节点数不够,那么就再来一轮
chosenLocations := make(map[cortypes.Location]bool)
chosenLocations := make(map[jcstypes.Location]bool)
for i, stg := range extendStgs {
if stg == nil {
continue
@@ -341,7 +340,7 @@ func (t *ChangeRedundancy) chooseSoManyUserSpaces(count int, stgs []*userSpaceUs
return chosen
}

func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
}
@@ -352,7 +351,7 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.O
}

// 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })

ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
@@ -374,11 +373,11 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}

var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var blockChgs []datamap.BlockChange
for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: 0,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@@ -412,7 +411,7 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.O
}, nil
}

func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
}
@@ -441,12 +440,12 @@ func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.Ob
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}

var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer
for i := 0; i < red.N; i++ {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@@ -494,7 +493,7 @@ func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.Ob
}, nil
}

func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
}
@@ -522,12 +521,12 @@ func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}

var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer
for i := 0; i < red.N; i++ {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@@ -576,7 +575,7 @@ func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.O
nil
}

func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
}
@@ -587,7 +586,7 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.O
}

// 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })

ft := ioswitch2.NewFromTo()
ft.SegmentParam = red
@@ -610,12 +609,12 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}

var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer
for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@@ -664,7 +663,7 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.O
nil
}

func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
}
@@ -675,7 +674,7 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.Ob
}

// 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })

ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
@@ -697,11 +696,11 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.Ob
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}

var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var blockChgs []datamap.BlockChange
for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: 0,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@@ -737,14 +736,14 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.Ob
nil
}

func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
return t.noneToEC(ctx, obj, red, uploadUserSpaces)
}

func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.ECRedundancy, tarRed *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcsypes.GrouppedObjectBlock
func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcstypes.GrouppedObjectBlock
var chosenBlockIndexes []int
var chosenBlockStg []jcsypes.UserSpaceDetail
var chosenBlockStg []jcstypes.UserSpaceDetail
for _, block := range obj.GroupBlocks() {
if len(block.UserSpaceIDs) > 0 {
// TODO 考虑选择最优的节点
@@ -768,7 +767,7 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.Obj
}

// 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })

planBlder := exec.NewPlanBuilder()
ft := ioswitch2.NewFromTo()
@@ -795,11 +794,11 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.Obj
return nil, nil, fmt.Errorf("executing io plan: %w", err)
}

var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock

for i := range uploadStgs {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: 0,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@@ -866,11 +865,11 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.Obj
nil
}

func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.ECRedundancy, tarRed *jcsypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
grpBlocks := obj.GroupBlocks()

var chosenBlocks []jcsypes.GrouppedObjectBlock
var chosenBlockStg []jcsypes.UserSpaceDetail
var chosenBlocks []jcstypes.GrouppedObjectBlock
var chosenBlockStg []jcstypes.UserSpaceDetail
for _, block := range grpBlocks {
if len(block.UserSpaceIDs) > 0 {
stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
@@ -909,16 +908,16 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.Obje
})
}

var newBlocks []jcsypes.ObjectBlock
var newBlocks []jcstypes.ObjectBlock
shouldUpdateBlocks := false
for i, stg := range uploadUserSpaces {
newBlock := jcsypes.ObjectBlock{
newBlock := jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
}

grp, ok := lo.Find(grpBlocks, func(grp jcsypes.GrouppedObjectBlock) bool { return grp.Index == i })
grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })

// 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
if ok && lo.Contains(grp.UserSpaceIDs, stg.UserSpace.UserSpace.UserSpaceID) {
@@ -988,7 +987,7 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.Obje

var blockChgs []datamap.BlockChange
for _, block := range obj.Blocks {
keep := lo.ContainsBy(newBlocks, func(newBlock jcsypes.ObjectBlock) bool {
keep := lo.ContainsBy(newBlocks, func(newBlock jcstypes.ObjectBlock) bool {
return newBlock.Index == block.Index && newBlock.UserSpaceID == block.UserSpaceID
})
if !keep {
@@ -1019,7 +1018,7 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.Obje
nil
}

func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.LRCRedundancy, tarRed *jcsypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.LRCRedundancy, tarRed *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {

blocksGrpByIndex := obj.GroupBlocks()

@@ -1062,8 +1061,8 @@ func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcsypes.Ob
/*
TODO2 修复这一块的代码

func (t *ChangeRedundancy) groupReconstructLRC(obj jcsypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []jcsypes.GrouppedObjectBlock, red *jcsypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
grped := make(map[int]jcsypes.GrouppedObjectBlock)
func (t *ChangeRedundancy) groupReconstructLRC(obj jcstypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
grped := make(map[int]jcstypes.GrouppedObjectBlock)
for _, b := range grpedBlocks {
grped[b.Index] = b
}
@@ -1098,9 +1097,9 @@ TODO2 修复这一块的代码
return nil, fmt.Errorf("executing io plan: %w", err)
}

var newBlocks []jcsypes.ObjectBlock
var newBlocks []jcstypes.ObjectBlock
for _, i := range lostBlocks {
newBlocks = append(newBlocks, jcsypes.ObjectBlock{
newBlocks = append(newBlocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: uploadUserSpaces[i].UserSpace.UserSpace.UserSpaceID,
@@ -1109,7 +1108,7 @@ TODO2 修复这一块的代码
}
for _, b := range grpedBlocks {
for _, hubID := range b.UserSpaceIDs {
newBlocks = append(newBlocks, jcsypes.ObjectBlock{
newBlocks = append(newBlocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: b.Index,
UserSpaceID: hubID,
@@ -1125,9 +1124,9 @@ TODO2 修复这一块的代码
}, nil
}
*/
func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, grpBlocks []jcsypes.GrouppedObjectBlock, red *jcsypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcsypes.GrouppedObjectBlock
var chosenBlockStg []jcsypes.UserSpaceDetail
func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, grpBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcstypes.GrouppedObjectBlock
var chosenBlockStg []jcstypes.UserSpaceDetail

for _, block := range grpBlocks {
if len(block.UserSpaceIDs) > 0 && block.Index < red.M() {
@@ -1154,16 +1153,16 @@ func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcsy

var froms []ioswitchlrc.From
var toes []ioswitchlrc.To
var newBlocks []jcsypes.ObjectBlock
var newBlocks []jcstypes.ObjectBlock
shouldUpdateBlocks := false
for i, userspace := range uploadUserSpaces {
newBlock := jcsypes.ObjectBlock{
newBlock := jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: i,
UserSpaceID: userspace.UserSpace.UserSpace.UserSpaceID,
}

grp, ok := lo.Find(grpBlocks, func(grp jcsypes.GrouppedObjectBlock) bool { return grp.Index == i })
grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })

// 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
if ok && lo.Contains(grp.UserSpaceIDs, userspace.UserSpace.UserSpace.UserSpaceID) {


+ 52
- 52
client/internal/ticktock/redundancy_shrink.go View File

@@ -21,14 +21,14 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
)

func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext, pkg jcsypes.PackageDetail, objs []jcsypes.ObjectDetail, reen *publock.Reentrant) ([]db.UpdatingObjectRedundancy, []datamap.SysEventBody, error) {
func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext, pkg jcstypes.PackageDetail, objs []jcstypes.ObjectDetail, reen *publock.Reentrant) ([]db.UpdatingObjectRedundancy, []datamap.SysEventBody, error) {
log := logger.WithType[ChangeRedundancy]("TickTock")

var readerStgIDs []jcsypes.UserSpaceID
var readerStgIDs []jcstypes.UserSpaceID
for _, space := range execCtx.allUserSpaces {
// TODO 可以考虑做成配置
if space.AccessAmount >= float64(pkg.ObjectCount/2) {
@@ -37,17 +37,17 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
}

// 只对ec和rep对象进行处理
var ecObjects []jcsypes.ObjectDetail
var repObjects []jcsypes.ObjectDetail
var ecObjects []jcstypes.ObjectDetail
var repObjects []jcstypes.ObjectDetail
for _, obj := range objs {
if _, ok := obj.Object.Redundancy.(*jcsypes.ECRedundancy); ok {
if _, ok := obj.Object.Redundancy.(*jcstypes.ECRedundancy); ok {
ecObjects = append(ecObjects, obj)
} else if _, ok := obj.Object.Redundancy.(*jcsypes.RepRedundancy); ok {
} else if _, ok := obj.Object.Redundancy.(*jcstypes.RepRedundancy); ok {
repObjects = append(repObjects, obj)
}
}

planningStgIDs := make(map[jcsypes.UserSpaceID]bool)
planningStgIDs := make(map[jcstypes.UserSpaceID]bool)

var sysEvents []datamap.SysEventBody

@@ -80,7 +80,7 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
// 对于ec对象,则每个对象单独进行退火算法
var ecObjectsUpdating []db.UpdatingObjectRedundancy
for i, obj := range ecObjects {
ecRed := obj.Object.Redundancy.(*jcsypes.ECRedundancy)
ecRed := obj.Object.Redundancy.(*jcstypes.ECRedundancy)
solu := t.startAnnealing(execCtx, readerStgIDs, annealingObject{
totalBlockCount: ecRed.N,
minBlockCnt: ecRed.K,
@@ -105,15 +105,15 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
return append(repObjectsUpdating, ecObjectsUpdating...), sysEvents, nil
}

func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcsypes.ObjectDetail) []jcsypes.UserSpaceID {
func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcstypes.ObjectDetail) []jcstypes.UserSpaceID {
type stgBlocks struct {
UserSpaceID jcsypes.UserSpaceID
UserSpaceID jcstypes.UserSpaceID
Count int
}

stgBlocksMap := make(map[jcsypes.UserSpaceID]*stgBlocks)
stgBlocksMap := make(map[jcstypes.UserSpaceID]*stgBlocks)
for _, obj := range objs {
cacheBlockStgs := make(map[jcsypes.UserSpaceID]bool)
cacheBlockStgs := make(map[jcstypes.UserSpaceID]bool)
for _, block := range obj.Blocks {
if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
stgBlocksMap[block.UserSpaceID] = &stgBlocks{
@@ -153,17 +153,17 @@ func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcsypes.ObjectDetai
}
}

return lo.Map(stgs, func(item *stgBlocks, idx int) jcsypes.UserSpaceID { return item.UserSpaceID })
return lo.Map(stgs, func(item *stgBlocks, idx int) jcstypes.UserSpaceID { return item.UserSpaceID })
}

type annealingState struct {
ctx *changeRedundancyContext
readerStgIDs []jcsypes.UserSpaceID // 近期可能访问此对象的节点
stgsSortedByReader map[jcsypes.UserSpaceID][]stgDist // 拥有数据的节点到每个可能访问对象的节点按距离排序
object annealingObject // 进行退火的对象
blockList []objectBlock // 排序后的块分布情况
stgBlockBitmaps map[jcsypes.UserSpaceID]*bitmap.Bitmap64 // 用位图的形式表示每一个节点上有哪些块
stgCombTree combinatorialTree // 节点组合树,用于加速计算容灾度
readerStgIDs []jcstypes.UserSpaceID // 近期可能访问此对象的节点
stgsSortedByReader map[jcstypes.UserSpaceID][]stgDist // 拥有数据的节点到每个可能访问对象的节点按距离排序
object annealingObject // 进行退火的对象
blockList []objectBlock // 排序后的块分布情况
stgBlockBitmaps map[jcstypes.UserSpaceID]*bitmap.Bitmap64 // 用位图的形式表示每一个节点上有哪些块
stgCombTree combinatorialTree // 节点组合树,用于加速计算容灾度

maxScore float64 // 搜索过程中得到过的最大分数
maxScoreRmBlocks []bool // 最大分数对应的删除方案
@@ -178,30 +178,30 @@ type annealingState struct {

type objectBlock struct {
Index int
UserSpaceID jcsypes.UserSpaceID
HasEntity bool // 节点拥有实际的文件数据块
HasShadow bool // 如果节点拥有完整文件数据,那么认为这个节点拥有所有块,这些块被称为影子块
FileHash jcsypes.FileHash // 只有在拥有实际文件数据块时,这个字段才有值
Size int64 // 块大小
UserSpaceID jcstypes.UserSpaceID
HasEntity bool // 节点拥有实际的文件数据块
HasShadow bool // 如果节点拥有完整文件数据,那么认为这个节点拥有所有块,这些块被称为影子块
FileHash jcstypes.FileHash // 只有在拥有实际文件数据块时,这个字段才有值
Size int64 // 块大小
}

type stgDist struct {
UserSpaceID jcsypes.UserSpaceID
UserSpaceID jcstypes.UserSpaceID
Distance float64
}

type combinatorialTree struct {
nodes []combinatorialTreeNode
blocksMaps map[int]bitmap.Bitmap64
stgIDToLocalStgID map[jcsypes.UserSpaceID]int
localStgIDToStgID []jcsypes.UserSpaceID
stgIDToLocalStgID map[jcstypes.UserSpaceID]int
localStgIDToStgID []jcstypes.UserSpaceID
}

type annealingObject struct {
totalBlockCount int
minBlockCnt int
pinnedAt []jcsypes.UserSpaceID
blocks []jcsypes.ObjectBlock
pinnedAt []jcstypes.UserSpaceID
blocks []jcstypes.ObjectBlock
}

const (
@@ -210,10 +210,10 @@ const (
iterActionBreak = 2
)

func newCombinatorialTree(stgBlocksMaps map[jcsypes.UserSpaceID]*bitmap.Bitmap64) combinatorialTree {
func newCombinatorialTree(stgBlocksMaps map[jcstypes.UserSpaceID]*bitmap.Bitmap64) combinatorialTree {
tree := combinatorialTree{
blocksMaps: make(map[int]bitmap.Bitmap64),
stgIDToLocalStgID: make(map[jcsypes.UserSpaceID]int),
stgIDToLocalStgID: make(map[jcstypes.UserSpaceID]int),
}

tree.nodes = make([]combinatorialTreeNode, (1 << len(stgBlocksMaps)))
@@ -271,7 +271,7 @@ func (t *combinatorialTree) GetDepth(index int) int {

// 更新某一个算力中心节点的块分布位图,同时更新它对应组合树节点的所有子节点。
// 如果更新到某个节点时,已有K个块,那么就不会再更新它的子节点
func (t *combinatorialTree) UpdateBitmap(stgID jcsypes.UserSpaceID, mp bitmap.Bitmap64, k int) {
func (t *combinatorialTree) UpdateBitmap(stgID jcstypes.UserSpaceID, mp bitmap.Bitmap64, k int) {
t.blocksMaps[t.stgIDToLocalStgID[stgID]] = mp
// 首先定义两种遍历树节点时的移动方式:
// 1. 竖直移动(深度增加):从一个节点移动到它最左边的子节点。每移动一步,index+1
@@ -416,13 +416,13 @@ type annealingSolution struct {
minAccessCost float64 // 本方案的最小访问费用
}

func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerStgIDs []jcsypes.UserSpaceID, object annealingObject) annealingSolution {
func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerStgIDs []jcstypes.UserSpaceID, object annealingObject) annealingSolution {
state := &annealingState{
ctx: ctx,
readerStgIDs: readerStgIDs,
stgsSortedByReader: make(map[jcsypes.UserSpaceID][]stgDist),
stgsSortedByReader: make(map[jcstypes.UserSpaceID][]stgDist),
object: object,
stgBlockBitmaps: make(map[jcsypes.UserSpaceID]*bitmap.Bitmap64),
stgBlockBitmaps: make(map[jcstypes.UserSpaceID]*bitmap.Bitmap64),
}

t.initBlockList(state)
@@ -486,7 +486,7 @@ func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerSt
}

func (t *ChangeRedundancy) initBlockList(ctx *annealingState) {
blocksMap := make(map[jcsypes.UserSpaceID][]objectBlock)
blocksMap := make(map[jcstypes.UserSpaceID][]objectBlock)

// 先生成所有的影子块
for _, pinned := range ctx.object.pinnedAt {
@@ -680,7 +680,7 @@ func (t *ChangeRedundancy) alwaysAccept(curTemp float64, dScore float64, cooling
return v > rand.Float64()
}

func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcsypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcsypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcstypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcstypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
entry := db.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
FileHash: obj.Object.FileHash,
@@ -694,8 +694,8 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *fromStg, ioswitch2.RawStream()))

for i, f := range solu.rmBlocks {
hasCache := lo.ContainsBy(obj.Blocks, func(b jcsypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcsypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
hasCache := lo.ContainsBy(obj.Blocks, func(b jcstypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcstypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
willRm := f

if !willRm {
@@ -706,7 +706,7 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s

planningHubIDs[solu.blockList[i].UserSpaceID] = true
}
entry.Blocks = append(entry.Blocks, jcsypes.ObjectBlock{
entry.Blocks = append(entry.Blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: solu.blockList[i].Index,
UserSpaceID: solu.blockList[i].UserSpaceID,
@@ -724,12 +724,12 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s
return entry
}

func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution, obj jcsypes.ObjectDetail) []datamap.SysEventBody {
func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution, obj jcstypes.ObjectDetail) []datamap.SysEventBody {
var blockChgs []datamap.BlockChange

for i, f := range solu.rmBlocks {
hasCache := lo.ContainsBy(obj.Blocks, func(b jcsypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcsypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
hasCache := lo.ContainsBy(obj.Blocks, func(b jcstypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcstypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
willRm := f

if !willRm {
@@ -782,7 +782,7 @@ func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution,
return []datamap.SysEventBody{transEvt, distEvt}
}

func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcsypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcsypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcstypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcstypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
entry := db.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
FileHash: obj.Object.FileHash,
@@ -790,11 +790,11 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
Redundancy: obj.Object.Redundancy,
}

reconstrct := make(map[jcsypes.UserSpaceID]*[]int)
reconstrct := make(map[jcstypes.UserSpaceID]*[]int)
for i, f := range solu.rmBlocks {
block := solu.blockList[i]
if !f {
entry.Blocks = append(entry.Blocks, jcsypes.ObjectBlock{
entry.Blocks = append(entry.Blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID,
Index: block.Index,
UserSpaceID: block.UserSpaceID,
@@ -815,7 +815,7 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
}
}

ecRed := obj.Object.Redundancy.(*jcsypes.ECRedundancy)
ecRed := obj.Object.Redundancy.(*jcstypes.ECRedundancy)

for id, idxs := range reconstrct {
// 依次生成每个节点上的执行计划,因为如果放到一个计划里一起生成,不能保证每个节点上的块用的都是本节点上的副本
@@ -838,10 +838,10 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
return entry
}

func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, obj jcsypes.ObjectDetail) []datamap.SysEventBody {
func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, obj jcstypes.ObjectDetail) []datamap.SysEventBody {
var blockChgs []datamap.BlockChange

reconstrct := make(map[jcsypes.UserSpaceID]*[]int)
reconstrct := make(map[jcstypes.UserSpaceID]*[]int)
for i, f := range solu.rmBlocks {
block := solu.blockList[i]
if !f {
@@ -917,7 +917,7 @@ func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, o
return []datamap.SysEventBody{transEvt, distEvt}
}

func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *exec.PlanBuilder, planningSpaceIDs map[jcsypes.UserSpaceID]bool, reen *publock.Reentrant) (exec.PlanResult, error) {
func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *exec.PlanBuilder, planningSpaceIDs map[jcstypes.UserSpaceID]bool, reen *publock.Reentrant) (exec.PlanResult, error) {
reqBlder := reqbuilder.NewBuilder()
for id, _ := range planningSpaceIDs {
reqBlder.UserSpace().Buzy(id)
@@ -955,7 +955,7 @@ func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *e
return ioSwRets, nil
}

func (t *ChangeRedundancy) populateECObjectEntry(entry *db.UpdatingObjectRedundancy, obj jcsypes.ObjectDetail, ioRets exec.PlanResult) {
func (t *ChangeRedundancy) populateECObjectEntry(entry *db.UpdatingObjectRedundancy, obj jcstypes.ObjectDetail, ioRets exec.PlanResult) {
for i := range entry.Blocks {
if entry.Blocks[i].FileHash != "" {
continue


+ 13
- 13
client/internal/uploader/uploader.go View File

@@ -22,7 +22,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type Uploader struct {
@@ -43,7 +43,7 @@ func NewUploader(pubLock *publock.Service, connectivity *connectivity.Collector,
}
}

func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpaceID, copyTo []jcsypes.UserSpaceID, copyToPath []jcsypes.JPath) (*UpdateUploader, error) {
func (u *Uploader) BeginUpdate(pkgID jcstypes.PackageID, affinity jcstypes.UserSpaceID, copyTo []jcstypes.UserSpaceID, copyToPath []jcstypes.JPath) (*UpdateUploader, error) {
spaceIDs, err := u.db.UserSpace().GetAllIDs(u.db.DefCtx())
if err != nil {
return nil, fmt.Errorf("getting user space ids: %w", err)
@@ -84,9 +84,9 @@ func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpa
return nil, fmt.Errorf("user no available userspaces")
}

copyToSpaces := make([]jcsypes.UserSpaceDetail, len(copyTo))
copyToSpaces := make([]jcstypes.UserSpaceDetail, len(copyTo))
for i, spaceID := range copyTo {
space, ok := lo.Find(spaceDetails, func(space *jcsypes.UserSpaceDetail) bool {
space, ok := lo.Find(spaceDetails, func(space *jcstypes.UserSpaceDetail) bool {
return space.UserSpace.UserSpaceID == spaceID
})
if !ok {
@@ -118,7 +118,7 @@ func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpa
// 1. 选择设置了亲和性的节点
// 2. 从与当前客户端相同地域的节点中随机选一个
// 3. 没有的话从所有节点选择延迟最低的节点
func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity jcsypes.UserSpaceID) UploadSpaceInfo {
func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity jcstypes.UserSpaceID) UploadSpaceInfo {
if spaceAffinity > 0 {
aff, ok := lo.Find(spaces, func(space UploadSpaceInfo) bool { return space.Space.UserSpace.UserSpaceID == spaceAffinity })
if ok {
@@ -137,10 +137,10 @@ func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity j
return spaces[0]
}

func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, copyTo []jcsypes.UserSpaceID, copyToPath []jcsypes.JPath) (*CreateUploader, error) {
func (u *Uploader) BeginCreateUpload(bktID jcstypes.BucketID, pkgName string, copyTo []jcstypes.UserSpaceID, copyToPath []jcstypes.JPath) (*CreateUploader, error) {
getSpaces := u.spaceMeta.GetMany(copyTo)

spacesStgs := make([]jcsypes.UserSpaceDetail, len(copyTo))
spacesStgs := make([]jcstypes.UserSpaceDetail, len(copyTo))
for i, stg := range getSpaces {
if stg == nil {
return nil, fmt.Errorf("storage %v not found", copyTo[i])
@@ -148,10 +148,10 @@ func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, cop
spacesStgs[i] = *stg
}

pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcsypes.Package, error) {
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcstypes.Package, error) {
_, err := u.db.Bucket().GetByID(tx, bktID)
if err != nil {
return jcsypes.Package{}, err
return jcstypes.Package{}, err
}

return u.db.Package().Create(u.db.DefCtx(), bktID, pkgName, time.Now())
@@ -178,19 +178,19 @@ func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, cop
}, nil
}

func (u *Uploader) UploadPart(objID jcsypes.ObjectID, index int, stream io.Reader) error {
func (u *Uploader) UploadPart(objID jcstypes.ObjectID, index int, stream io.Reader) error {
detail, err := u.db.Object().GetDetail(u.db.DefCtx(), objID)
if err != nil {
return fmt.Errorf("getting object detail: %w", err)
}

objDe := detail
_, ok := objDe.Object.Redundancy.(*jcsypes.MultipartUploadRedundancy)
_, ok := objDe.Object.Redundancy.(*jcstypes.MultipartUploadRedundancy)
if !ok {
return fmt.Errorf("object %v is not a multipart upload", objID)
}

var space jcsypes.UserSpaceDetail
var space jcstypes.UserSpaceDetail
if len(objDe.Blocks) > 0 {
cstg := u.spaceMeta.Get(objDe.Blocks[0].UserSpaceID)
if cstg == nil {
@@ -272,7 +272,7 @@ func (u *Uploader) UploadPart(objID jcsypes.ObjectID, index int, stream io.Reade

shardInfo := ret.Get("shard").(*ops2.FileInfoValue)
err = u.db.DoTx(func(tx db.SQLContext) error {
return u.db.Object().AppendPart(tx, jcsypes.ObjectBlock{
return u.db.Object().AppendPart(tx, jcstypes.ObjectBlock{
ObjectID: objID,
Index: index,
UserSpaceID: space.UserSpace.UserSpaceID,


+ 9
- 10
client/internal/uploader/user_space_upload.go View File

@@ -17,20 +17,19 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcsypes.JPath, targetBktID jcsypes.BucketID, newPkgName string, uploadAffinity jcsypes.UserSpaceID) (*jcsypes.Package, error) {
func (u *Uploader) UserSpaceUpload(userSpaceID jcstypes.UserSpaceID, rootPath jcstypes.JPath, targetBktID jcstypes.BucketID, newPkgName string, uploadAffinity jcstypes.UserSpaceID) (*jcstypes.Package, error) {
srcSpace := u.spaceMeta.Get(userSpaceID)
if srcSpace == nil {
return nil, fmt.Errorf("user space %d not found", userSpaceID)
}

pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcsypes.Package, error) {
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcstypes.Package, error) {
_, err := u.db.Bucket().GetByID(tx, targetBktID)
if err != nil {
return jcsypes.Package{}, err
return jcstypes.Package{}, err
}

return u.db.Package().Create(tx, targetBktID, newPkgName, time.Now())
@@ -49,7 +48,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcs
}

spaceDetails := u.spaceMeta.GetMany(spaceIDs)
spaceDetails = lo.Filter(spaceDetails, func(e *jcsypes.UserSpaceDetail, i int) bool {
spaceDetails = lo.Filter(spaceDetails, func(e *jcstypes.UserSpaceDetail, i int) bool {
return e != nil && e.UserSpace.ShardStore != nil
})

@@ -59,13 +58,13 @@ func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcs
coorCli := stgglb.CoordinatorRPCPool.Get()
defer coorCli.Release()

resp, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]cortypes.HubID{srcSpace.RecommendHub.HubID}))
resp, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]jcstypes.HubID{srcSpace.RecommendHub.HubID}))
if cerr != nil {
delPkg()
return nil, fmt.Errorf("getting hub connectivities: %w", cerr.ToError())
}

cons := make(map[cortypes.HubID]cortypes.HubConnectivity)
cons := make(map[jcstypes.HubID]jcstypes.HubConnectivity)
for _, c := range resp.Connectivities {
cons[c.ToHubID] = c
}
@@ -153,7 +152,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcs
return &pkg, nil
}

func (u *Uploader) uploadFromBaseStore(srcSpace *jcsypes.UserSpaceDetail, targetSpace *jcsypes.UserSpaceDetail, entries []types.DirEntry, rootPath jcsypes.JPath) ([]db.AddObjectEntry, error) {
func (u *Uploader) uploadFromBaseStore(srcSpace *jcstypes.UserSpaceDetail, targetSpace *jcstypes.UserSpaceDetail, entries []types.DirEntry, rootPath jcstypes.JPath) ([]db.AddObjectEntry, error) {
ft := ioswitch2.FromTo{}

for _, e := range entries {
@@ -198,7 +197,7 @@ func (u *Uploader) uploadFromBaseStore(srcSpace *jcsypes.UserSpaceDetail, target
Size: info.Size,
FileHash: info.Hash,
CreateTime: time.Now(),
UserSpaceIDs: []jcsypes.UserSpaceID{targetSpace.UserSpace.UserSpaceID},
UserSpaceIDs: []jcstypes.UserSpaceID{targetSpace.UserSpace.UserSpaceID},
})
}



+ 7
- 7
client/sdk/api/v1/bucket.go View File

@@ -4,7 +4,7 @@ import (
"net/http"

"gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type BucketService struct {
@@ -18,7 +18,7 @@ func (c *Client) Bucket() *BucketService {
const BucketGetPath = "/bucket/get"

type BucketGet struct {
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
}

func (r *BucketGet) MakeParam() *sdks.RequestParam {
@@ -26,7 +26,7 @@ func (r *BucketGet) MakeParam() *sdks.RequestParam {
}

type BucketGetResp struct {
Bucket jcsypes.Bucket `json:"bucket"`
Bucket jcstypes.Bucket `json:"bucket"`
}

func (r *BucketGetResp) ParseResponse(resp *http.Response) error {
@@ -48,7 +48,7 @@ func (r *BucketGetByName) MakeParam() *sdks.RequestParam {
}

type BucketGetByNameResp struct {
Bucket jcsypes.Bucket `json:"bucket"`
Bucket jcstypes.Bucket `json:"bucket"`
}

func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error {
@@ -70,7 +70,7 @@ func (r *BucketCreate) MakeParam() *sdks.RequestParam {
}

type BucketCreateResp struct {
Bucket jcsypes.Bucket `json:"bucket"`
Bucket jcstypes.Bucket `json:"bucket"`
}

func (r *BucketCreateResp) ParseResponse(resp *http.Response) error {
@@ -84,7 +84,7 @@ func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) {
const BucketDeletePath = "/bucket/delete"

type BucketDelete struct {
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
}

func (r *BucketDelete) MakeParam() *sdks.RequestParam {
@@ -111,7 +111,7 @@ func (r *BucketListAll) MakeParam() *sdks.RequestParam {
}

type BucketListAllResp struct {
Buckets []jcsypes.Bucket `json:"buckets"`
Buckets []jcstypes.Bucket `json:"buckets"`
}

func (r *BucketListAllResp) ParseResponse(resp *http.Response) error {


+ 24
- 24
client/sdk/api/v1/package.go View File

@@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/common/sdks"
"gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type PackageService struct {
@@ -27,7 +27,7 @@ func (c *Client) Package() *PackageService {
const PackageGetPath = "/package/get"

type PackageGet struct {
PackageID jcsypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
PackageID jcstypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
}

func (r *PackageGet) MakeParam() *sdks.RequestParam {
@@ -35,7 +35,7 @@ func (r *PackageGet) MakeParam() *sdks.RequestParam {
}

type PackageGetResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}

func (r *PackageGetResp) ParseResponse(resp *http.Response) error {
@@ -58,7 +58,7 @@ func (r *PackageGetByFullName) MakeParam() *sdks.RequestParam {
}

type PackageGetByFullNameResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}

func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error {
@@ -72,8 +72,8 @@ func (c *PackageService) GetByFullName(req PackageGetByFullName) (*PackageGetByF
const PackageCreatePath = "/package/create"

type PackageCreate struct {
BucketID jcsypes.BucketID `json:"bucketID"`
Name string `json:"name"`
BucketID jcstypes.BucketID `json:"bucketID"`
Name string `json:"name"`
}

func (r *PackageCreate) MakeParam() *sdks.RequestParam {
@@ -81,7 +81,7 @@ func (r *PackageCreate) MakeParam() *sdks.RequestParam {
}

type PackageCreateResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}

func (r *PackageCreateResp) ParseResponse(resp *http.Response) error {
@@ -99,14 +99,14 @@ type PackageCreateUpload struct {
Files UploadObjectIterator `json:"-"`
}
type PackageCreateUploadInfo struct {
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
CopyTo []jcsypes.UserSpaceID `json:"copyTo"`
CopyToPath []string `json:"copyToPath"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
CopyTo []jcstypes.UserSpaceID `json:"copyTo"`
CopyToPath []string `json:"copyToPath"`
}
type PackageCreateUploadResp struct {
Package jcsypes.Package `json:"package"`
Objects []jcsypes.Object `json:"objects"`
Package jcstypes.Package `json:"package"`
Objects []jcstypes.Object `json:"objects"`
}

func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUploadResp, error) {
@@ -148,10 +148,10 @@ func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUp
const PackageDownloadPath = "/package/download"

type PackageDownload struct {
PackageID jcsypes.PackageID `url:"packageID" form:"packageID" binding:"required"`
Prefix string `url:"prefix" form:"prefix"`
NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"`
Zip bool `url:"zip,omitempty" form:"zip"`
PackageID jcstypes.PackageID `url:"packageID" form:"packageID" binding:"required"`
Prefix string `url:"prefix" form:"prefix"`
NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"`
Zip bool `url:"zip,omitempty" form:"zip"`
}

func (r *PackageDownload) MakeParam() *sdks.RequestParam {
@@ -208,7 +208,7 @@ func (c *PackageService) Download(req PackageDownload) (*DownloadingPackage, err
const PackageDeletePath = "/package/delete"

type PackageDelete struct {
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
PackageID jcstypes.PackageID `json:"packageID" binding:"required"`
}

func (r *PackageDelete) MakeParam() *sdks.RequestParam {
@@ -228,9 +228,9 @@ func (c *PackageService) Delete(req PackageDelete) error {
const PackageClonePath = "/package/clone"

type PackageClone struct {
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
PackageID jcstypes.PackageID `json:"packageID" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
}

func (r *PackageClone) MakeParam() *sdks.RequestParam {
@@ -238,7 +238,7 @@ func (r *PackageClone) MakeParam() *sdks.RequestParam {
}

type PackageCloneResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}

func (r *PackageCloneResp) ParseResponse(resp *http.Response) error {
@@ -252,7 +252,7 @@ func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) {
const PackageListBucketPackagesPath = "/package/listBucketPackages"

type PackageListBucketPackages struct {
BucketID jcsypes.BucketID `form:"bucketID" url:"bucketID" binding:"required"`
BucketID jcstypes.BucketID `form:"bucketID" url:"bucketID" binding:"required"`
}

func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam {
@@ -260,7 +260,7 @@ func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam {
}

type PackageListBucketPackagesResp struct {
Packages []jcsypes.Package `json:"packages"`
Packages []jcstypes.Package `json:"packages"`
}

func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error {


+ 22
- 22
client/sdk/api/v1/presigned.go View File

@@ -7,7 +7,7 @@ import (

"github.com/google/go-querystring/query"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/signer"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type PresignedService struct {
@@ -33,10 +33,10 @@ func (c *PresignedService) ObjectListByPath(req PresignedObjectListByPath, expir
const PresignedObjectDownloadByPathPath = "/presigned/object/downloadByPath"

type PresignedObjectDownloadByPath struct {
PackageID jcsypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
Path string `form:"path" url:"path" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
PackageID jcstypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
Path string `form:"path" url:"path" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
}

func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) {
@@ -46,9 +46,9 @@ func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPat
const PresignedObjectDownloadPath = "/presigned/object/download"

type PresignedObjectDownload struct {
ObjectID jcsypes.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
ObjectID jcstypes.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
}

func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) {
@@ -58,15 +58,15 @@ func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn
const PresignedObjectUploadPath = "/presigned/object/upload"

type PresignedObjectUpload struct {
PackageID jcsypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
Affinity jcsypes.UserSpaceID `form:"affinity" url:"affinity,omitempty"`
CopyTo []jcsypes.UserSpaceID `form:"copyTo" url:"copyTo,omitempty"`
CopyToPath []string `form:"copyToPath" url:"copyToPath,omitempty"`
PackageID jcstypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
Affinity jcstypes.UserSpaceID `form:"affinity" url:"affinity,omitempty"`
CopyTo []jcstypes.UserSpaceID `form:"copyTo" url:"copyTo,omitempty"`
CopyToPath []string `form:"copyToPath" url:"copyToPath,omitempty"`
}

type PresignedObjectUploadResp struct {
Object jcsypes.Object `json:"object"`
Object jcstypes.Object `json:"object"`
}

func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int) (string, error) {
@@ -76,12 +76,12 @@ func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int)
const PresignedObjectNewMultipartUploadPath = "/presigned/object/newMultipartUpload"

type PresignedObjectNewMultipartUpload struct {
PackageID jcsypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
PackageID jcstypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
}

type PresignedObjectNewMultipartUploadResp struct {
Object jcsypes.Object `json:"object"`
Object jcstypes.Object `json:"object"`
}

func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultipartUpload, expireIn int) (string, error) {
@@ -91,8 +91,8 @@ func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultip
const PresignedObjectUploadPartPath = "/presigned/object/uploadPart"

type PresignedObjectUploadPart struct {
ObjectID jcsypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Index int `form:"index" binding:"required" url:"index"`
ObjectID jcstypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Index int `form:"index" binding:"required" url:"index"`
}

type PresignedUploadPartResp struct{}
@@ -104,12 +104,12 @@ func (c *PresignedService) ObjectUploadPart(req PresignedObjectUploadPart, expir
const PresignedObjectCompleteMultipartUploadPath = "/presigned/object/completeMultipartUpload"

type PresignedObjectCompleteMultipartUpload struct {
ObjectID jcsypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Indexes []int `form:"indexes" binding:"required" url:"indexes"`
ObjectID jcstypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Indexes []int `form:"indexes" binding:"required" url:"indexes"`
}

type PresignedObjectCompleteMultipartUploadResp struct {
Object jcsypes.Object `json:"object"`
Object jcstypes.Object `json:"object"`
}

func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectCompleteMultipartUpload, expireIn int) (string, error) {


+ 13
- 13
client/sdk/api/v1/space_syncer.go View File

@@ -4,7 +4,7 @@ import (
"net/http"

"gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type SpaceSyncerService struct {
@@ -20,14 +20,14 @@ func (c *Client) SpaceSyncer() *SpaceSyncerService {
const SpaceSyncerCreateTaskPath = "/spaceSyncer/createTask"

type SpaceSyncerCreateTask struct {
Trigger jcsypes.SpaceSyncTrigger `json:"trigger" binding:"required"`
Mode jcsypes.SpaceSyncMode `json:"mode" binding:"required"`
Filters []jcsypes.SpaceSyncFilter `json:"filters"`
Options jcsypes.SpaceSyncOptions `json:"options" binding:"required"`
SrcUserSpaceID jcsypes.UserSpaceID `json:"srcUserSpaceID" binding:"required"`
SrcPath string `json:"srcPath"`
DestUserSpaceIDs []jcsypes.UserSpaceID `json:"destUserSpaceIDs" binding:"required"`
DestPathes []string `json:"destPathes" binding:"required"`
Trigger jcstypes.SpaceSyncTrigger `json:"trigger" binding:"required"`
Mode jcstypes.SpaceSyncMode `json:"mode" binding:"required"`
Filters []jcstypes.SpaceSyncFilter `json:"filters"`
Options jcstypes.SpaceSyncOptions `json:"options" binding:"required"`
SrcUserSpaceID jcstypes.UserSpaceID `json:"srcUserSpaceID" binding:"required"`
SrcPath string `json:"srcPath"`
DestUserSpaceIDs []jcstypes.UserSpaceID `json:"destUserSpaceIDs" binding:"required"`
DestPathes []string `json:"destPathes" binding:"required"`
}

func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam {
@@ -35,7 +35,7 @@ func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam {
}

type SpaceSyncerCreateTaskResp struct {
Task jcsypes.SpaceSyncTask `json:"task"`
Task jcstypes.SpaceSyncTask `json:"task"`
}

func (r *SpaceSyncerCreateTaskResp) ParseResponse(resp *http.Response) error {
@@ -49,7 +49,7 @@ func (c *SpaceSyncerService) CreateTask(req SpaceSyncerCreateTask) (*SpaceSyncer
const SpaceSyncerGetTaskPath = "/spaceSyncer/getTask"

type SpaceSyncerGetTask struct {
TaskID jcsypes.SpaceSyncTaskID `url:"taskID" binding:"required"`
TaskID jcstypes.SpaceSyncTaskID `url:"taskID" binding:"required"`
}

func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam {
@@ -57,7 +57,7 @@ func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam {
}

type SpaceSyncerGetTaskResp struct {
Task jcsypes.SpaceSyncTask `json:"task"`
Task jcstypes.SpaceSyncTask `json:"task"`
}

func (r *SpaceSyncerGetTaskResp) ParseResponse(resp *http.Response) error {
@@ -71,7 +71,7 @@ func (c *SpaceSyncerService) GetTask(req SpaceSyncerGetTask) (*SpaceSyncerGetTas
const SpaceSyncerCancelTaskPath = "/spaceSyncer/cancelTask"

type SpaceSyncerCancelTask struct {
TaskID jcsypes.SpaceSyncTaskID `json:"taskID" binding:"required"`
TaskID jcstypes.SpaceSyncTaskID `json:"taskID" binding:"required"`
}

func (r *SpaceSyncerCancelTask) MakeParam() *sdks.RequestParam {


+ 2
- 2
client/sdk/api/v1/storage_test.go View File

@@ -10,7 +10,7 @@ import (
. "github.com/smartystreets/goconvey/convey"
"gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/api"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func Test_PackageGet(t *testing.T) {
@@ -74,7 +74,7 @@ func Test_Object(t *testing.T) {
fileData[i] = byte(i)
}

stgAff := jcsypes.UserSpaceID(2)
stgAff := jcstypes.UserSpaceID(2)

pkgName := uuid.NewString()
createResp, err := cli.Package().Create(PackageCreate{


+ 2
- 2
client/sdk/api/v1/system.go View File

@@ -4,7 +4,7 @@ import (
"net/http"

"gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type SystemService struct {
@@ -25,7 +25,7 @@ func (r *SystemStatus) MakeParam() *sdks.RequestParam {
}

type SystemStatusResp struct {
SpeedStats jcsypes.SpeedStatsStatus `json:"speedStats"`
SpeedStats jcstypes.SpeedStatsStatus `json:"speedStats"`
}

func (r *SystemStatusResp) ParseResponse(resp *http.Response) error {


+ 26
- 27
client/sdk/api/v1/user_space.go View File

@@ -4,8 +4,7 @@ import (
"net/http"

"gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type UserSpaceService struct {
@@ -19,9 +18,9 @@ func (c *Client) UserSpace() *UserSpaceService {
const UserSpaceDownloadPackagePath = "/userSpace/downloadPackage"

type UserSpaceDownloadPackageReq struct {
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
RootPath string `json:"rootPath"`
PackageID jcstypes.PackageID `json:"packageID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
RootPath string `json:"rootPath"`
}

func (r *UserSpaceDownloadPackageReq) MakeParam() *sdks.RequestParam {
@@ -41,11 +40,11 @@ func (c *UserSpaceService) DownloadPackage(req UserSpaceDownloadPackageReq) (*Us
const UserSpaceCreatePackagePath = "/userSpace/createPackage"

type UserSpaceCreatePackageReq struct {
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Path string `json:"path" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
SpaceAffinity jcsypes.UserSpaceID `json:"spaceAffinity"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Path string `json:"path" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
SpaceAffinity jcstypes.UserSpaceID `json:"spaceAffinity"`
}

func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam {
@@ -53,7 +52,7 @@ func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam {
}

type UserSpaceCreatePackageResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
}

func (r *UserSpaceCreatePackageResp) ParseResponse(resp *http.Response) error {
@@ -67,7 +66,7 @@ func (c *UserSpaceService) CreatePackage(req UserSpaceCreatePackageReq) (*UserSp
const UserSpaceGetPath = "/userSpace/get"

type UserSpaceGet struct {
UserSpaceID jcsypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"`
}

func (r *UserSpaceGet) MakeParam() *sdks.RequestParam {
@@ -75,7 +74,7 @@ func (r *UserSpaceGet) MakeParam() *sdks.RequestParam {
}

type UserSpaceGetResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
}

func (r *UserSpaceGetResp) ParseResponse(resp *http.Response) error {
@@ -97,7 +96,7 @@ func (r *UserSpaceGetByName) MakeParam() *sdks.RequestParam {
}

type UserSpaceGetByNameResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
}

func (r *UserSpaceGetByNameResp) ParseResponse(resp *http.Response) error {
@@ -117,7 +116,7 @@ func (r *UserSpaceGetAll) MakeParam() *sdks.RequestParam {
}

type UserSpaceGetAllResp struct {
UserSpaces []jcsypes.UserSpace `json:"userSpaces"`
UserSpaces []jcstypes.UserSpace `json:"userSpaces"`
}

func (r *UserSpaceGetAllResp) ParseResponse(resp *http.Response) error {
@@ -134,10 +133,10 @@ const UserSpaceCreatePath = "/userSpace/create"

type UserSpaceCreate struct {
Name string `json:"name" binding:"required"`
Storage cortypes.StorageType `json:"storage" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"`
ShardStore *cortypes.ShardStoreUserConfig `json:"shardStore"`
Features []cortypes.StorageFeature `json:"features"`
Storage jcstypes.StorageType `json:"storage" binding:"required"`
Credential jcstypes.StorageCredential `json:"credential" binding:"required"`
ShardStore *jcstypes.ShardStoreUserConfig `json:"shardStore"`
Features []jcstypes.StorageFeature `json:"features"`
WorkingDir string `json:"workingDir"`
}

@@ -146,7 +145,7 @@ func (r *UserSpaceCreate) MakeParam() *sdks.RequestParam {
}

type UserSpaceCreateResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
}

func (r *UserSpaceCreateResp) ParseResponse(resp *http.Response) error {
@@ -161,10 +160,10 @@ func (c *UserSpaceService) Create(req UserSpaceCreate) (*UserSpaceCreateResp, er
const UserSpaceUpdatePath = "/userSpace/update"

type UserSpaceUpdate struct {
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Name string `json:"name" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"`
Features []cortypes.StorageFeature `json:"features"`
Credential jcstypes.StorageCredential `json:"credential" binding:"required"`
Features []jcstypes.StorageFeature `json:"features"`
}

func (r *UserSpaceUpdate) MakeParam() *sdks.RequestParam {
@@ -172,7 +171,7 @@ func (r *UserSpaceUpdate) MakeParam() *sdks.RequestParam {
}

type UserSpaceUpdateResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
}

func (r *UserSpaceUpdateResp) ParseResponse(resp *http.Response) error {
@@ -187,7 +186,7 @@ func (c *UserSpaceService) Update(req UserSpaceUpdate) (*UserSpaceUpdateResp, er
const UserSpaceDeletePath = "/userSpace/delete"

type UserSpaceDelete struct {
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
}

func (r *UserSpaceDelete) MakeParam() *sdks.RequestParam {
@@ -208,8 +207,8 @@ func (c *UserSpaceService) Delete(req UserSpaceDelete) (*UserSpaceDeleteResp, er
const UserSpaceTestPath = "/userSpace/test"

type UserSpaceTest struct {
Storage cortypes.StorageType `json:"storage" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"`
Storage jcstypes.StorageType `json:"storage" binding:"required"`
Credential jcstypes.StorageCredential `json:"credential" binding:"required"`
WorikingDir string `json:"workingDir"`
}



+ 1
- 1
common/globals/globals.go View File

@@ -1,7 +1,7 @@
package stgglb

import (
"gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type LocalMachineInfo struct {


+ 2
- 2
common/globals/utils.go View File

@@ -1,9 +1,9 @@
package stgglb

import cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
import jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"

// 根据当前节点与目标地址的距离关系,选择合适的地址
func SelectGRPCAddress(hub *cortypes.Hub, addr *cortypes.GRPCAddressInfo) (string, int) {
func SelectGRPCAddress(hub *jcstypes.Hub, addr *jcstypes.GRPCAddressInfo) (string, int) {
// TODO 重新设计选择LocalIP的策略
return addr.ExternalIP, addr.ExternalGRPCPort
}

+ 6
- 6
common/pkgs/accesstoken/accesstoken.go View File

@@ -10,7 +10,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type CacheEvent interface {
@@ -23,17 +23,17 @@ type ExitEvent struct {
}

type CacheKey struct {
UserID cortypes.UserID
TokenID cortypes.AccessTokenID
UserID jcstypes.UserID
TokenID jcstypes.AccessTokenID
}

var ErrTokenNotFound = fmt.Errorf("token not found")

type AccessTokenLoader func(key CacheKey) (cortypes.UserAccessToken, error)
type AccessTokenLoader func(key CacheKey) (jcstypes.UserAccessToken, error)

type CacheEntry struct {
IsTokenValid bool
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
PublicKey ed25519.PublicKey
LoadedAt time.Time
LastUsedAt time.Time
@@ -227,6 +227,6 @@ func (mc *Cache) Verify(authInfo rpc.AccessTokenAuthInfo) bool {
return ed25519.Verify(token.PublicKey, []byte(MakeStringToSign(authInfo.UserID, authInfo.AccessTokenID, authInfo.Nonce)), []byte(sig))
}

func MakeStringToSign(userID cortypes.UserID, tokenID cortypes.AccessTokenID, nonce string) string {
func MakeStringToSign(userID jcstypes.UserID, tokenID jcstypes.AccessTokenID, nonce string) string {
return fmt.Sprintf("%v.%v.%v", userID, tokenID, nonce)
}

+ 10
- 10
common/pkgs/connectivity/collector.go View File

@@ -11,7 +11,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type CollectorEvent interface {
@@ -28,7 +28,7 @@ type CollectedEvent struct {
}

type Connectivity struct {
ToHubID cortypes.HubID
ToHubID jcstypes.HubID
Latency *time.Duration
TestTime time.Time
}
@@ -38,7 +38,7 @@ type Collector struct {
enabled bool
collectNow chan any
done chan any
connectivities map[cortypes.HubID]Connectivity
connectivities map[jcstypes.HubID]Connectivity
lock *sync.RWMutex
}

@@ -48,7 +48,7 @@ func NewEnabled(cfg Config) *Collector {
enabled: true,
collectNow: make(chan any, 1),
done: make(chan any, 1),
connectivities: make(map[cortypes.HubID]Connectivity),
connectivities: make(map[jcstypes.HubID]Connectivity),
lock: &sync.RWMutex{},
}
return &rpt
@@ -58,16 +58,16 @@ func NewDisabled() *Collector {
enabled: false,
collectNow: make(chan any, 1),
done: make(chan any, 1),
connectivities: make(map[cortypes.HubID]Connectivity),
connectivities: make(map[jcstypes.HubID]Connectivity),
lock: &sync.RWMutex{},
}
}

func (r *Collector) GetAll() map[cortypes.HubID]Connectivity {
func (r *Collector) GetAll() map[jcstypes.HubID]Connectivity {
r.lock.RLock()
defer r.lock.RUnlock()

ret := make(map[cortypes.HubID]Connectivity)
ret := make(map[jcstypes.HubID]Connectivity)
for k, v := range r.connectivities {
ret[k] = v
}
@@ -170,7 +170,7 @@ func (r *Collector) testing() bool {

r.lock.Lock()
// 删除所有hub的记录,然后重建,避免hub数量变化时导致残余数据
r.connectivities = make(map[cortypes.HubID]Connectivity)
r.connectivities = make(map[jcstypes.HubID]Connectivity)
for _, con := range cons {
r.connectivities[con.ToHubID] = con
}
@@ -179,13 +179,13 @@ func (r *Collector) testing() bool {
return true
}

func (r *Collector) ping(hub cortypes.Hub) Connectivity {
func (r *Collector) ping(hub jcstypes.Hub) Connectivity {
log := logger.WithType[Collector]("").WithField("HubID", hub.HubID)

var ip string
var port int
switch addr := hub.Address.(type) {
case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
// TODO 重新设计选择LocalIP的策略
ip = addr.ExternalIP
port = addr.ExternalGRPCPort


+ 15
- 15
common/pkgs/ioswitch2/fromto.go View File

@@ -4,7 +4,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type From interface {
@@ -69,9 +69,9 @@ type FromTos []FromTo

type FromTo struct {
// 如果输入或者输出用到了EC编码的流,则需要提供EC参数。
ECParam *jcsypes.ECRedundancy
ECParam *jcstypes.ECRedundancy
// 同上
SegmentParam *jcsypes.SegmentRedundancy
SegmentParam *jcstypes.SegmentRedundancy
Froms []From
Toes []To
}
@@ -110,12 +110,12 @@ func (f *FromDriver) GetStreamIndex() StreamIndex {
}

type FromShardStore struct {
FileHash jcsypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcstypes.FileHash
UserSpace jcstypes.UserSpaceDetail
StreamIndex StreamIndex
}

func NewFromShardstore(fileHash jcsypes.FileHash, space jcsypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
func NewFromShardstore(fileHash jcstypes.FileHash, space jcstypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
return &FromShardStore{
FileHash: fileHash,
UserSpace: space,
@@ -128,11 +128,11 @@ func (f *FromShardStore) GetStreamIndex() StreamIndex {
}

type FromBaseStore struct {
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
}

func NewFromBaseStore(space jcsypes.UserSpaceDetail, path jcsypes.JPath) *FromBaseStore {
func NewFromBaseStore(space jcstypes.UserSpaceDetail, path jcstypes.JPath) *FromBaseStore {
return &FromBaseStore{
UserSpace: space,
Path: path,
@@ -177,13 +177,13 @@ func (t *ToDriver) GetRange() math2.Range {
}

type ToShardStore struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
StreamIndex StreamIndex
Range math2.Range
ResultStoreKey string
}

func NewToShardStore(space jcsypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
func NewToShardStore(space jcstypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
return &ToShardStore{
UserSpace: space,
StreamIndex: strIdx,
@@ -191,7 +191,7 @@ func NewToShardStore(space jcsypes.UserSpaceDetail, strIdx StreamIndex, retStore
}
}

func NewToShardStoreWithRange(space jcsypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
func NewToShardStoreWithRange(space jcstypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
return &ToShardStore{
UserSpace: space,
StreamIndex: streamIndex,
@@ -209,12 +209,12 @@ func (t *ToShardStore) GetRange() math2.Range {
}

type ToBaseStore struct {
UserSpace jcsypes.UserSpaceDetail
ObjectPath jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
ObjectPath jcstypes.JPath
Option types.WriteOption
}

func NewToBaseStore(space jcsypes.UserSpaceDetail, objectPath jcsypes.JPath) *ToBaseStore {
func NewToBaseStore(space jcstypes.UserSpaceDetail, objectPath jcstypes.JPath) *ToBaseStore {
return &ToBaseStore{
UserSpace: space,
ObjectPath: objectPath,


+ 4
- 4
common/pkgs/ioswitch2/http_hub_worker.go View File

@@ -9,16 +9,16 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
hubapi "gitlink.org.cn/cloudream/jcs-pub/hub/sdk/api"
)

type HttpHubWorker struct {
Hub cortypes.Hub
Hub jcstypes.Hub
}

func (w *HttpHubWorker) NewClient() (exec.WorkerClient, error) {
addressInfo := w.Hub.Address.(*cortypes.HttpAddressInfo)
addressInfo := w.Hub.Address.(*jcstypes.HttpAddressInfo)
baseUrl := "http://" + addressInfo.ExternalIP + ":" + strconv.Itoa(addressInfo.Port)
config := hubapi.Config{
URL: baseUrl,
@@ -47,7 +47,7 @@ func (w *HttpHubWorker) Equals(worker exec.WorkerInfo) bool {
}

type HttpHubWorkerClient struct {
hubID cortypes.HubID
hubID jcstypes.HubID
cli *hubapi.Client
}



+ 4
- 4
common/pkgs/ioswitch2/hub_worker.go View File

@@ -11,7 +11,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.WorkerInfo](
@@ -20,8 +20,8 @@ var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.Wo
)))

type HubWorker struct {
Hub cortypes.Hub
Address cortypes.GRPCAddressInfo
Hub jcstypes.Hub
Address jcstypes.GRPCAddressInfo
}

func (w *HubWorker) NewClient() (exec.WorkerClient, error) {
@@ -43,7 +43,7 @@ func (w *HubWorker) Equals(worker exec.WorkerInfo) bool {
}

type HubWorkerClient struct {
hubID cortypes.HubID
hubID jcstypes.HubID
cli *hubrpc.Client
}



+ 14
- 14
common/pkgs/ioswitch2/ops2/base_store.go View File

@@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

const (
@@ -43,8 +43,8 @@ func (v *BaseReadStatsValue) Clone() exec.VarValue {

type BaseRead struct {
Output exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
}

@@ -101,7 +101,7 @@ func (o *BaseRead) String() string {
}

type BaseReadDyn struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Output exec.VarID
FileInfo exec.VarID
Option types.OpenOption
@@ -167,8 +167,8 @@ func (o *BaseReadDyn) String() string {

type BaseWrite struct {
Input exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
FileInfo exec.VarID
Option types.WriteOption
}
@@ -213,12 +213,12 @@ func (o *BaseWrite) String() string {
type BaseReadNode struct {
dag.NodeBase
From ioswitch2.From
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
}

func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{
From: from,
UserSpace: userSpace,
@@ -254,11 +254,11 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
type BaseReadDynNode struct {
dag.NodeBase
From ioswitch2.From
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Option types.OpenOption
}

func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace jcsypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{
From: from,
UserSpace: userSpace,
@@ -301,12 +301,12 @@ func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
type BaseWriteNode struct {
dag.NodeBase
To ioswitch2.To
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.WriteOption
}

func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{
To: to,
UserSpace: userSpace,


+ 6
- 6
common/pkgs/ioswitch2/ops2/bypass.go View File

@@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -17,8 +17,8 @@ func init() {

// 旁路Http读取
type GetShardHTTPRequest struct {
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
Output exec.VarID
}

@@ -64,11 +64,11 @@ func (o *GetShardHTTPRequest) String() string {
// 旁路Http读取
type GetShardHTTPRequestNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
}

func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardHTTPRequestNode {
func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace jcstypes.UserSpaceDetail, fileHash jcstypes.FileHash) *GetShardHTTPRequestNode {
node := &GetShardHTTPRequestNode{
UserSpace: userSpace,
FileHash: fileHash,


+ 7
- 7
common/pkgs/ioswitch2/ops2/ec.go View File

@@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -146,7 +146,7 @@ func (o *ECMultiply) String() string {
}

type CallECMultiplier struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Coef [][]byte
Inputs []exec.VarID
Outputs []exec.VarID
@@ -203,12 +203,12 @@ func (o *CallECMultiplier) String() string {

type ECMultiplyNode struct {
dag.NodeBase
EC jcsypes.ECRedundancy
EC jcstypes.ECRedundancy
InputIndexes []int
OutputIndexes []int
}

func (b *GraphNodeBuilder) NewECMultiply(ec jcsypes.ECRedundancy) *ECMultiplyNode {
func (b *GraphNodeBuilder) NewECMultiply(ec jcstypes.ECRedundancy) *ECMultiplyNode {
node := &ECMultiplyNode{
EC: ec,
}
@@ -257,13 +257,13 @@ func (t *ECMultiplyNode) GenerateOp() (exec.Op, error) {

type CallECMultiplierNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
EC jcsypes.ECRedundancy
UserSpace jcstypes.UserSpaceDetail
EC jcstypes.ECRedundancy
InputIndexes []int
OutputIndexes []int
}

func (b *GraphNodeBuilder) NewCallECMultiplier(userSpace jcsypes.UserSpaceDetail) *CallECMultiplierNode {
func (b *GraphNodeBuilder) NewCallECMultiplier(userSpace jcstypes.UserSpaceDetail) *CallECMultiplierNode {
node := &CallECMultiplierNode{
UserSpace: userSpace,
}


+ 7
- 7
common/pkgs/ioswitch2/ops2/multipart.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -40,7 +40,7 @@ func (v *UploadedPartInfoValue) Clone() exec.VarValue {
}

type MultipartInitiator struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
UploadArgs exec.VarID
UploadedParts []exec.VarID
FileOutput exec.VarID // 分片上传之后的临时文件的路径
@@ -99,7 +99,7 @@ func (o *MultipartInitiator) String() string {
}

type MultipartUpload struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
UploadArgs exec.VarID
UploadResult exec.VarID
PartStream exec.VarID
@@ -149,10 +149,10 @@ func (o *MultipartUpload) String() string {

type MultipartInitiatorNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
}

func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace jcsypes.UserSpaceDetail) *MultipartInitiatorNode {
func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace jcstypes.UserSpaceDetail) *MultipartInitiatorNode {
node := &MultipartInitiatorNode{
UserSpace: userSpace,
}
@@ -194,12 +194,12 @@ func (n *MultipartInitiatorNode) GenerateOp() (exec.Op, error) {

type MultipartUploadNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
PartNumber int
PartSize int64
}

func (b *GraphNodeBuilder) NewMultipartUpload(userSpace jcsypes.UserSpaceDetail, partNumber int, partSize int64) *MultipartUploadNode {
func (b *GraphNodeBuilder) NewMultipartUpload(userSpace jcstypes.UserSpaceDetail, partNumber int, partSize int64) *MultipartUploadNode {
node := &MultipartUploadNode{
UserSpace: userSpace,
PartNumber: partNumber,


+ 17
- 17
common/pkgs/ioswitch2/ops2/s2s.go View File

@@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -15,10 +15,10 @@ func init() {
}

type S2STransfer struct {
SrcSpace jcsypes.UserSpaceDetail
SrcPath jcsypes.JPath
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
SrcSpace jcstypes.UserSpaceDetail
SrcPath jcstypes.JPath
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
Output exec.VarID
}

@@ -55,10 +55,10 @@ func (o *S2STransfer) String() string {
}

type S2STransferDyn struct {
SrcSpace jcsypes.UserSpaceDetail
SrcSpace jcstypes.UserSpaceDetail
SrcFileInfo exec.VarID
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
Output exec.VarID
}

@@ -101,13 +101,13 @@ func (o *S2STransferDyn) String() string {

type S2STransferNode struct {
dag.NodeBase
SrcSpace jcsypes.UserSpaceDetail
SrcPath jcsypes.JPath
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
SrcSpace jcstypes.UserSpaceDetail
SrcPath jcstypes.JPath
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
}

func (b *GraphNodeBuilder) NewS2STransfer(srcSpace jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstSpace jcsypes.UserSpaceDetail, dstPath jcsypes.JPath) *S2STransferNode {
func (b *GraphNodeBuilder) NewS2STransfer(srcSpace jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstSpace jcstypes.UserSpaceDetail, dstPath jcstypes.JPath) *S2STransferNode {
n := &S2STransferNode{
SrcSpace: srcSpace,
SrcPath: srcPath,
@@ -139,12 +139,12 @@ func (n *S2STransferNode) GenerateOp() (exec.Op, error) {

type S2STransferDynNode struct {
dag.NodeBase
SrcSpace jcsypes.UserSpaceDetail
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
SrcSpace jcstypes.UserSpaceDetail
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
}

func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace jcsypes.UserSpaceDetail, dstSpace jcsypes.UserSpaceDetail, dstPath jcsypes.JPath) *S2STransferDynNode {
func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace jcstypes.UserSpaceDetail, dstSpace jcstypes.UserSpaceDetail, dstPath jcstypes.JPath) *S2STransferDynNode {
n := &S2STransferDynNode{
SrcSpace: srcSpace,
DstSpace: dstSpace,


+ 9
- 9
common/pkgs/ioswitch2/ops2/shard_store.go View File

@@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -15,8 +15,8 @@ func init() {
}

type GetShardInfo struct {
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
ShardInfo exec.VarID
}

@@ -46,7 +46,7 @@ func (o *GetShardInfo) String() string {
}

type StoreShard struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
FileInfo exec.VarID
ShardInfo exec.VarID
}
@@ -84,11 +84,11 @@ func (o *StoreShard) String() string {

type GetShardInfoNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
}

func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardInfoNode {
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcstypes.UserSpaceDetail, fileHash jcstypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{
UserSpace: userSpace,
FileHash: fileHash,
@@ -116,11 +116,11 @@ func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {

type StoreShardNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
ShardInfoKey string
}

func (b *GraphNodeBuilder) NewStoreShard(userSpace jcsypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcstypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{
UserSpace: userSpace,
ShardInfoKey: shardInfoKey,


+ 4
- 5
common/pkgs/ioswitch2/parser/gen/generator.go View File

@@ -12,8 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

// 检查使用不同编码时参数是否设置到位
@@ -385,17 +384,17 @@ func buildToNode(ctx *state.GenerateState, t ioswitch2.To) (ops2.ToNode, error)
}
}

func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil {
n.Env().ToEnvDriver(true)
return nil
}

switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)

case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)

default:


+ 4
- 5
common/pkgs/ioswitch2/parser/opt/utils.go View File

@@ -5,21 +5,20 @@ import (

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil {
n.Env().ToEnvDriver(true)
return nil
}

switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)

case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)

default:


+ 2
- 2
common/pkgs/ioswitch2/plans/complete_multipart.go View File

@@ -8,10 +8,10 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func CompleteMultipart(blocks []jcsypes.ObjectBlock, blockSpaces []jcsypes.UserSpaceDetail, targetSpace jcsypes.UserSpaceDetail, shardInfoKey string, blder *exec.PlanBuilder) error {
func CompleteMultipart(blocks []jcstypes.ObjectBlock, blockSpaces []jcstypes.UserSpaceDetail, targetSpace jcstypes.UserSpaceDetail, shardInfoKey string, blder *exec.PlanBuilder) error {
da := ops2.NewGraphNodeBuilder()

sizes := make([]int64, len(blocks))


+ 4
- 5
common/pkgs/ioswitch2/plans/utils.go View File

@@ -5,21 +5,20 @@ import (

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil {
n.Env().ToEnvDriver(true)
return nil
}

switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)

case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)

default:


+ 7
- 7
common/pkgs/ioswitchlrc/fromto.go View File

@@ -4,7 +4,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type From interface {
@@ -39,12 +39,12 @@ func (f *FromDriver) GetDataIndex() int {
}

type FromNode struct {
FileHash jcsypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcstypes.FileHash
UserSpace jcstypes.UserSpaceDetail
DataIndex int
}

func NewFromStorage(fileHash jcsypes.FileHash, space jcsypes.UserSpaceDetail, dataIndex int) *FromNode {
func NewFromStorage(fileHash jcstypes.FileHash, space jcstypes.UserSpaceDetail, dataIndex int) *FromNode {
return &FromNode{
FileHash: fileHash,
DataIndex: dataIndex,
@@ -88,14 +88,14 @@ func (t *ToDriver) GetRange() math2.Range {
}

type ToNode struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
DataIndex int
Range math2.Range
FileHashStoreKey string
Option types.WriteOption
}

func NewToStorage(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string) *ToNode {
func NewToStorage(space jcstypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string) *ToNode {
return &ToNode{
UserSpace: space,
DataIndex: dataIndex,
@@ -103,7 +103,7 @@ func NewToStorage(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey
}
}

func NewToStorageWithRange(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string, rng math2.Range) *ToNode {
func NewToStorageWithRange(space jcstypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string, rng math2.Range) *ToNode {
return &ToNode{
UserSpace: space,
DataIndex: dataIndex,


+ 3
- 3
common/pkgs/ioswitchlrc/hub_worker.go View File

@@ -7,7 +7,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

// var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.WorkerInfo](
@@ -15,8 +15,8 @@ import (
// )))

type HubWorker struct {
Hub cortypes.Hub
Address cortypes.GRPCAddressInfo
Hub jcstypes.Hub
Address jcstypes.GRPCAddressInfo
}

func (w *HubWorker) NewClient() (exec.WorkerClient, error) {


+ 14
- 14
common/pkgs/ioswitchlrc/ops2/base_store.go View File

@@ -12,7 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -23,8 +23,8 @@ func init() {

type BaseRead struct {
Output exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
}

@@ -67,7 +67,7 @@ func (o *BaseRead) String() string {
}

type BaseReadDyn struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Output exec.VarID
Path exec.VarID
Option types.OpenOption
@@ -118,8 +118,8 @@ func (o *BaseReadDyn) String() string {

type BaseWrite struct {
Input exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
WriteResult exec.VarID
Option types.WriteOption
}
@@ -164,12 +164,12 @@ func (o *BaseWrite) String() string {
type BaseReadNode struct {
dag.NodeBase
From ioswitchlrc.From
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption
}

func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{
From: from,
UserSpace: userSpace,
@@ -205,11 +205,11 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
type BaseReadDynNode struct {
dag.NodeBase
From ioswitchlrc.From
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Option types.OpenOption
}

func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace jcsypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{
From: from,
UserSpace: userSpace,
@@ -252,12 +252,12 @@ func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
type BaseWriteNode struct {
dag.NodeBase
To ioswitchlrc.To
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.WriteOption
}

func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{
To: to,
UserSpace: userSpace,


+ 5
- 5
common/pkgs/ioswitchlrc/ops2/ec.go View File

@@ -12,7 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -114,12 +114,12 @@ func (o *GalMultiply) String() string {

type LRCConstructAnyNode struct {
dag.NodeBase
LRC jcsypes.LRCRedundancy
LRC jcstypes.LRCRedundancy
InputIndexes []int
OutputIndexes []int
}

func (b *GraphNodeBuilder) NewLRCConstructAny(lrc jcsypes.LRCRedundancy) *LRCConstructAnyNode {
func (b *GraphNodeBuilder) NewLRCConstructAny(lrc jcstypes.LRCRedundancy) *LRCConstructAnyNode {
node := &LRCConstructAnyNode{
LRC: lrc,
}
@@ -168,11 +168,11 @@ func (t *LRCConstructAnyNode) GenerateOp() (exec.Op, error) {

type LRCConstructGroupNode struct {
dag.NodeBase
LRC jcsypes.LRCRedundancy
LRC jcstypes.LRCRedundancy
TargetBlockIndex int
}

func (b *GraphNodeBuilder) NewLRCConstructGroup(lrc jcsypes.LRCRedundancy) *LRCConstructGroupNode {
func (b *GraphNodeBuilder) NewLRCConstructGroup(lrc jcstypes.LRCRedundancy) *LRCConstructGroupNode {
node := &LRCConstructGroupNode{
LRC: lrc,
}


+ 9
- 9
common/pkgs/ioswitchlrc/ops2/shard_store.go View File

@@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
@@ -15,8 +15,8 @@ func init() {
}

type GetShardInfo struct {
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
ShardInfo exec.VarID
}

@@ -46,7 +46,7 @@ func (o *GetShardInfo) String() string {
}

type StoreShard struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
FileInfo exec.VarID
ShardInfo exec.VarID
}
@@ -84,11 +84,11 @@ func (o *StoreShard) String() string {

type GetShardInfoNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
}

func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardInfoNode {
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcstypes.UserSpaceDetail, fileHash jcstypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{
UserSpace: userSpace,
FileHash: fileHash,
@@ -116,11 +116,11 @@ func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {

type StoreShardNode struct {
dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
ShardInfoKey string
}

func (b *GraphNodeBuilder) NewStoreShard(userSpace jcsypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcstypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{
UserSpace: userSpace,
ShardInfoKey: shardInfoKey,


+ 5
- 5
common/pkgs/ioswitchlrc/parser/generator.go View File

@@ -9,11 +9,11 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/ops2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type GenerateContext struct {
LRC jcsypes.LRCRedundancy
LRC jcstypes.LRCRedundancy
DAG *ops2.GraphNodeBuilder
To []ioswitchlrc.To
ToNodes map[ioswitchlrc.To]ops2.ToNode
@@ -27,7 +27,7 @@ func Encode(fr ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder)
}

ctx := GenerateContext{
LRC: jcsypes.DefaultLRCRedundancy,
LRC: jcstypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(),
To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),
@@ -124,7 +124,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
// 提供数据块+编码块中的k个块,重建任意块,包括完整文件。
func ReconstructAny(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error {
ctx := GenerateContext{
LRC: jcsypes.DefaultLRCRedundancy,
LRC: jcstypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(),
To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),
@@ -245,7 +245,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
// 输入同一组的多个块,恢复出剩下缺少的一个块。
func ReconstructGroup(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error {
ctx := GenerateContext{
LRC: jcsypes.DefaultLRCRedundancy,
LRC: jcstypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(),
To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),


+ 4
- 5
common/pkgs/ioswitchlrc/parser/utils.go View File

@@ -5,21 +5,20 @@ import (

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil {
n.Env().ToEnvDriver(true)
return nil
}

switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)

case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)

default:


+ 4
- 4
common/pkgs/publock/reqbuilder/user_space.go View File

@@ -5,7 +5,7 @@ import (

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/lockprovider"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type UserSpaceLockReqBuilder struct {
@@ -15,7 +15,7 @@ type UserSpaceLockReqBuilder struct {
func (b *LockRequestBuilder) UserSpace() *UserSpaceLockReqBuilder {
return &UserSpaceLockReqBuilder{LockRequestBuilder: b}
}
func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcsypes.UserSpaceID) *UserSpaceLockReqBuilder {
func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcstypes.UserSpaceID) *UserSpaceLockReqBuilder {
b.locks = append(b.locks, types.Lock{
Path: b.makePath(spaceID),
Name: lockprovider.UserSpaceBuzyLock,
@@ -24,7 +24,7 @@ func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcsypes.UserSpaceID) *UserSpaceLo
return b
}

func (b *UserSpaceLockReqBuilder) GC(spaceID jcsypes.UserSpaceID) *UserSpaceLockReqBuilder {
func (b *UserSpaceLockReqBuilder) GC(spaceID jcstypes.UserSpaceID) *UserSpaceLockReqBuilder {
b.locks = append(b.locks, types.Lock{
Path: b.makePath(spaceID),
Name: lockprovider.UserSpaceGCLock,
@@ -33,6 +33,6 @@ func (b *UserSpaceLockReqBuilder) GC(spaceID jcsypes.UserSpaceID) *UserSpaceLock
return b
}

func (b *UserSpaceLockReqBuilder) makePath(hubID jcsypes.UserSpaceID) []string {
func (b *UserSpaceLockReqBuilder) makePath(hubID jcstypes.UserSpaceID) []string {
return []string{lockprovider.UserSpaceLockPathPrefix, strconv.FormatInt(int64(hubID), 10)}
}

+ 7
- 7
common/pkgs/rpc/auth.go View File

@@ -5,7 +5,7 @@ import (
"fmt"
"strconv"

cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@@ -27,8 +27,8 @@ const (
)

type AccessTokenAuthInfo struct {
UserID cortypes.UserID
AccessTokenID cortypes.AccessTokenID
UserID jcstypes.UserID
AccessTokenID jcstypes.AccessTokenID
Nonce string
Signature string
}
@@ -125,8 +125,8 @@ func (s *ServerBase) authUnary(
}

authInfo := AccessTokenAuthInfo{
UserID: cortypes.UserID(userID),
AccessTokenID: cortypes.AccessTokenID(accessTokenIDs[0]),
UserID: jcstypes.UserID(userID),
AccessTokenID: jcstypes.AccessTokenID(accessTokenIDs[0]),
Nonce: nonce[0],
Signature: signature[0],
}
@@ -200,8 +200,8 @@ func (s *ServerBase) authStream(
}

authInfo := AccessTokenAuthInfo{
UserID: cortypes.UserID(userID),
AccessTokenID: cortypes.AccessTokenID(accessTokenIDs[0]),
UserID: jcstypes.UserID(userID),
AccessTokenID: jcstypes.AccessTokenID(accessTokenIDs[0]),
Nonce: nonce[0],
Signature: signature[0],
}


+ 15
- 15
common/pkgs/rpc/coordinator/hub.go View File

@@ -4,7 +4,7 @@ import (
context "context"

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type HubService interface {
@@ -18,18 +18,18 @@ type HubService interface {
}

type GetHubConfig struct {
HubID cortypes.HubID `json:"hubID"`
HubID jcstypes.HubID `json:"hubID"`
}
type GetHubConfigResp struct {
Hub cortypes.Hub `json:"hub"`
Hub jcstypes.Hub `json:"hub"`
}

func ReqGetHubConfig(hubID cortypes.HubID) *GetHubConfig {
func ReqGetHubConfig(hubID jcstypes.HubID) *GetHubConfig {
return &GetHubConfig{
HubID: hubID,
}
}
func RespGetHubConfig(hub cortypes.Hub) *GetHubConfigResp {
func RespGetHubConfig(hub jcstypes.Hub) *GetHubConfigResp {
return &GetHubConfigResp{
Hub: hub,
}
@@ -43,23 +43,23 @@ func (s *Server) GetHubConfig(ctx context.Context, req *rpc.Request) (*rpc.Respo

// 获取指定节点的信息。如果HubIDs为nil,则返回所有Hub
type GetHubs struct {
HubIDs []cortypes.HubID `json:"hubIDs"`
HubIDs []jcstypes.HubID `json:"hubIDs"`
}
type GetHubsResp struct {
Hubs []*cortypes.Hub `json:"hubs"`
Hubs []*jcstypes.Hub `json:"hubs"`
}

func NewGetHubs(hubIDs []cortypes.HubID) *GetHubs {
func NewGetHubs(hubIDs []jcstypes.HubID) *GetHubs {
return &GetHubs{
HubIDs: hubIDs,
}
}
func NewGetHubsResp(hubs []*cortypes.Hub) *GetHubsResp {
func NewGetHubsResp(hubs []*jcstypes.Hub) *GetHubsResp {
return &GetHubsResp{
Hubs: hubs,
}
}
func (r *GetHubsResp) GetHub(id cortypes.HubID) *cortypes.Hub {
func (r *GetHubsResp) GetHub(id jcstypes.HubID) *jcstypes.Hub {
for _, n := range r.Hubs {
if n.HubID == id {
return n
@@ -78,18 +78,18 @@ func (s *Server) GetHubs(ctx context.Context, req *rpc.Request) (*rpc.Response,
// 获取节点连通性信息

type GetHubConnectivities struct {
HubIDs []cortypes.HubID `json:"hubIDs"`
HubIDs []jcstypes.HubID `json:"hubIDs"`
}
type GetHubConnectivitiesResp struct {
Connectivities []cortypes.HubConnectivity `json:"hubs"`
Connectivities []jcstypes.HubConnectivity `json:"hubs"`
}

func ReqGetHubConnectivities(hubIDs []cortypes.HubID) *GetHubConnectivities {
func ReqGetHubConnectivities(hubIDs []jcstypes.HubID) *GetHubConnectivities {
return &GetHubConnectivities{
HubIDs: hubIDs,
}
}
func RespGetHubConnectivities(cons []cortypes.HubConnectivity) *GetHubConnectivitiesResp {
func RespGetHubConnectivities(cons []jcstypes.HubConnectivity) *GetHubConnectivitiesResp {
return &GetHubConnectivitiesResp{
Connectivities: cons,
}
@@ -103,7 +103,7 @@ func (s *Server) GetHubConnectivities(ctx context.Context, req *rpc.Request) (*r

// 上报节点连通性信息
type ReportHubConnectivity struct {
Connecttivities []cortypes.HubConnectivity
Connecttivities []jcstypes.HubConnectivity
}
type ReportHubConnectivityResp struct {
}


+ 3
- 3
common/pkgs/rpc/coordinator/storage.go View File

@@ -4,7 +4,7 @@ import (
context "context"

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type StorageService interface {
@@ -13,10 +13,10 @@ type StorageService interface {

// 为指定的Storage选择一个适合通信的Hub
type SelectStorageHub struct {
Storages []cortypes.StorageType
Storages []jcstypes.StorageType
}
type SelectStorageHubResp struct {
Hubs []*cortypes.Hub
Hubs []*jcstypes.Hub
}

var _ = TokenAuth(Coordinator_SelectStorageHub_FullMethodName)


+ 7
- 7
common/pkgs/rpc/coordinator/user.go View File

@@ -4,7 +4,7 @@ import (
context "context"

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type UserService interface {
@@ -23,7 +23,7 @@ type UserLogin struct {
Password string
}
type UserLoginResp struct {
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
PrivateKey string
}

@@ -42,7 +42,7 @@ func (s *Server) UserLogin(ctx context.Context, req *rpc.Request) (*rpc.Response
// 客户端刷新Token,原始Token会继续有效。
type UserRefreshToken struct{}
type UserRefreshTokenResp struct {
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
PrivateKey string
}

@@ -76,12 +76,12 @@ func (s *Server) UserLogout(ctx context.Context, req *rpc.Request) (*rpc.Respons

// Hub服务加载AccessToken
type HubLoadAccessToken struct {
HubID cortypes.HubID
UserID cortypes.UserID
TokenID cortypes.AccessTokenID
HubID jcstypes.HubID
UserID jcstypes.UserID
TokenID jcstypes.AccessTokenID
}
type HubLoadAccessTokenResp struct {
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
}

func (c *Client) HubLoadAccessToken(ctx context.Context, msg *HubLoadAccessToken) (*HubLoadAccessTokenResp, *rpc.CodeError) {


+ 5
- 5
common/pkgs/rpc/hub/cache.go View File

@@ -4,7 +4,7 @@ package hubrpc
import (
"context"

jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
)

@@ -15,10 +15,10 @@ type CacheSvc interface {

// 获取Cache中文件列表
type CheckCache struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
}
type CheckCacheResp struct {
FileHashes []jcsypes.FileHash
FileHashes []jcstypes.FileHash
}

func (c *Client) CheckCache(ctx context.Context, req *CheckCache) (*CheckCacheResp, *rpc.CodeError) {
@@ -33,8 +33,8 @@ func (s *Server) CheckCache(ctx context.Context, req *rpc.Request) (*rpc.Respons

// 清理Cache中不用的文件
type CacheGC struct {
UserSpace jcsypes.UserSpaceDetail
Availables []jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
Availables []jcstypes.FileHash
}
type CacheGCResp struct{}



+ 3
- 3
common/pkgs/rpc/hub/user.go View File

@@ -4,7 +4,7 @@ import (
context "context"

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type UserSvc interface {
@@ -13,8 +13,8 @@ type UserSvc interface {

// 通知用户的Token登出
type NotifyUserAccessTokenInvalid struct {
UserID cortypes.UserID
TokenID cortypes.AccessTokenID
UserID jcstypes.UserID
TokenID jcstypes.AccessTokenID
}
type NotifyUserAccessTokenInvalidResp struct{}



+ 3
- 3
common/pkgs/rpc/hub/user_space.go View File

@@ -4,7 +4,7 @@ package hubrpc
import (
"context"

jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)
@@ -16,7 +16,7 @@ type UserSpaceSvc interface {

// 列出指定BaseStore的指定位置内的所有文件
type BaseStoreListAll struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Path string
}
type BaseStoreListAllResp struct {
@@ -35,7 +35,7 @@ func (s *Server) BaseStoreListAll(ctx context.Context, req *rpc.Request) (*rpc.R

// 批量在指定BaseStore中创建文件夹
type BaseStoreMkdirs struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Pathes []string
}



+ 8
- 8
common/pkgs/servicestats/hub_strorage_transfer.go View File

@@ -6,22 +6,22 @@ import (
"time"

"gitlink.org.cn/cloudream/common/utils/math2"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type HubStorageTransferStats struct {
data HubStorageTransferStatsData
fromHubID cortypes.HubID
fromHubID jcstypes.HubID
lock *sync.Mutex
}

type HubStorageTransferStatsData struct {
Entries map[cortypes.StorageID]*HubStorageTransferStatsEntry
Entries map[jcstypes.StorageID]*HubStorageTransferStatsEntry
StartTime time.Time
}

type HubStorageTransferStatsEntry struct {
DestStorageID cortypes.StorageID
DestStorageID jcstypes.StorageID

OutputBytes int64
MaxOutputBytes int64
@@ -36,7 +36,7 @@ type HubStorageTransferStatsEntry struct {
SuccessInput int64
}

func (s *HubStorageTransferStats) RecordUpload(dstStorageID cortypes.StorageID, transferBytes int64, isSuccess bool) {
func (s *HubStorageTransferStats) RecordUpload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) {
s.lock.Lock()
defer s.lock.Unlock()

@@ -58,7 +58,7 @@ func (s *HubStorageTransferStats) RecordUpload(dstStorageID cortypes.StorageID,
e.TotalOutput++
}

func (s *HubStorageTransferStats) RecordDownload(dstStorageID cortypes.StorageID, transferBytes int64, isSuccess bool) {
func (s *HubStorageTransferStats) RecordDownload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) {
s.lock.Lock()
defer s.lock.Unlock()

@@ -83,7 +83,7 @@ func (s *HubStorageTransferStats) Reset() time.Time {
s.lock.Lock()
defer s.lock.Unlock()

s.data.Entries = make(map[cortypes.StorageID]*HubStorageTransferStatsEntry)
s.data.Entries = make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry)
s.data.StartTime = time.Now()
return s.data.StartTime
}
@@ -93,7 +93,7 @@ func (s *HubStorageTransferStats) DumpData() HubStorageTransferStatsData {
defer s.lock.Unlock()

data := s.data
data.Entries = make(map[cortypes.StorageID]*HubStorageTransferStatsEntry)
data.Entries = make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry)
for k, v := range s.data.Entries {
v2 := *v
data.Entries[k] = &v2


+ 8
- 8
common/pkgs/servicestats/hub_transfter.go View File

@@ -6,22 +6,22 @@ import (
"time"

"gitlink.org.cn/cloudream/common/utils/math2"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type HubTransferStats struct {
data HubTransferStatsData
fromHubID cortypes.HubID
fromHubID jcstypes.HubID
lock *sync.Mutex
}

type HubTransferStatsData struct {
Entries map[cortypes.HubID]*HubTransferStatsEntry
Entries map[jcstypes.HubID]*HubTransferStatsEntry
StartTime time.Time
}

type HubTransferStatsEntry struct {
DestHubID cortypes.HubID
DestHubID jcstypes.HubID

OutputBytes int64
MaxOutputBytes int64
@@ -36,7 +36,7 @@ type HubTransferStatsEntry struct {
SuccessInput int64
}

func (s *HubTransferStats) RecordOutput(dstHubID cortypes.HubID, transferBytes int64, isSuccess bool) {
func (s *HubTransferStats) RecordOutput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) {
s.lock.Lock()
defer s.lock.Unlock()

@@ -58,7 +58,7 @@ func (s *HubTransferStats) RecordOutput(dstHubID cortypes.HubID, transferBytes i
e.TotalOutput++
}

func (s *HubTransferStats) RecordInput(dstHubID cortypes.HubID, transferBytes int64, isSuccess bool) {
func (s *HubTransferStats) RecordInput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) {
s.lock.Lock()
defer s.lock.Unlock()

@@ -85,7 +85,7 @@ func (s *HubTransferStats) Reset() time.Time {
defer s.lock.Unlock()

s.data.StartTime = time.Now()
s.data.Entries = make(map[cortypes.HubID]*HubTransferStatsEntry)
s.data.Entries = make(map[jcstypes.HubID]*HubTransferStatsEntry)
return s.data.StartTime
}

@@ -94,7 +94,7 @@ func (s *HubTransferStats) DumpData() HubTransferStatsData {
defer s.lock.Unlock()

data := s.data
data.Entries = make(map[cortypes.HubID]*HubTransferStatsEntry)
data.Entries = make(map[jcstypes.HubID]*HubTransferStatsEntry)
for k, v := range s.data.Entries {
v2 := *v
data.Entries[k] = &v2


+ 5
- 5
common/pkgs/servicestats/service_stats.go View File

@@ -4,7 +4,7 @@ import (
"sync"
"time"

cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type StatsHost struct {
@@ -14,24 +14,24 @@ type StatsHost struct {
HubStorageTransfer *HubStorageTransferStats
}

func (h *StatsHost) SetupHubTransfer(fromHubID cortypes.HubID) {
func (h *StatsHost) SetupHubTransfer(fromHubID jcstypes.HubID) {
h.HubTransfer = &HubTransferStats{
fromHubID: fromHubID,
lock: &sync.Mutex{},
data: HubTransferStatsData{
StartTime: time.Now(),
Entries: make(map[cortypes.HubID]*HubTransferStatsEntry),
Entries: make(map[jcstypes.HubID]*HubTransferStatsEntry),
},
}
}

func (h *StatsHost) SetupHubStorageTransfer(fromHubID cortypes.HubID) {
func (h *StatsHost) SetupHubStorageTransfer(fromHubID jcstypes.HubID) {
h.HubStorageTransfer = &HubStorageTransferStats{
fromHubID: fromHubID,
lock: &sync.Mutex{},
data: HubStorageTransferStatsData{
StartTime: time.Now(),
Entries: make(map[cortypes.StorageID]*HubStorageTransferStatsEntry),
Entries: make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry),
},
}
}

+ 4
- 5
common/pkgs/storage/efile/ec_multiplier.go View File

@@ -10,14 +10,13 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type ECMultiplier struct {
blder *builder
url string
feat *cortypes.ECMultiplierFeature
feat *jcstypes.ECMultiplierFeature
outputs []string
}

@@ -98,9 +97,9 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
for i, data := range r.Data {
ret[i] = types.FileInfo{
// TODO 要确认一下output的格式
Path: jcsypes.PathFromJcsPathString(m.outputs[i]),
Path: jcstypes.PathFromJcsPathString(m.outputs[i]),
Size: data.Size,
Hash: jcsypes.NewFullHashFromString(data.Sha256),
Hash: jcstypes.NewFullHashFromString(data.Sha256),
}
}



+ 7
- 8
common/pkgs/storage/efile/efile.go View File

@@ -10,12 +10,11 @@ import (
"gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
reg.RegisterBuilder[*cortypes.EFileType](func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*jcstypes.EFileType](func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{
detail: detail,
}
@@ -24,15 +23,15 @@ func init() {

type builder struct {
types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
token string
tokenLock sync.Mutex
getTokenTime time.Time
}

func (b *builder) getToken() (string, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.EFileType)
cred := b.detail.UserSpace.Credential.(*cortypes.EFileCred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.EFileType)
cred := b.detail.UserSpace.Credential.(*jcstypes.EFileCred)

b.tokenLock.Lock()
defer b.tokenLock.Unlock()
@@ -91,12 +90,12 @@ func (b *builder) getToken() (string, error) {
}

func (b *builder) CreateECMultiplier(typeOnly bool) (types.ECMultiplier, error) {
feat := types.FindFeature[*cortypes.ECMultiplierFeature](b.detail)
feat := types.FindFeature[*jcstypes.ECMultiplierFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature ECMultiplier not found")
}

cred, ok := b.detail.UserSpace.Credential.(*cortypes.EFileCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.EFileCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for efile storage", b.detail.UserSpace.Credential)
}


+ 2
- 2
common/pkgs/storage/factory/factory.go View File

@@ -6,12 +6,12 @@ import (
_ "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

// 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder,
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查)
func GetBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage)

ctor, ok := reg.StorageBuilders[typ]


+ 4
- 5
common/pkgs/storage/factory/reg/reg.go View File

@@ -5,23 +5,22 @@ import (

"gitlink.org.cn/cloudream/common/utils/reflect2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type BuilderCtor func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder
type BuilderCtor func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder

var StorageBuilders = make(map[reflect.Type]BuilderCtor)

// 注册针对指定存储服务类型的Builder
func RegisterBuilder[T cortypes.StorageType](ctor BuilderCtor) {
func RegisterBuilder[T jcstypes.StorageType](ctor BuilderCtor) {
StorageBuilders[reflect2.TypeOf[T]()] = ctor
}

// 注:此函数只给storage包内部使用,外部包请使用外层的factory.GetBuilder
// 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder,
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查)
func GetBuilderInternal(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilderInternal(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage)

ctor, ok := StorageBuilders[typ]


+ 8
- 8
common/pkgs/storage/local/base_store.go View File

@@ -9,22 +9,22 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type BaseStore struct {
root string
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
}

func NewBaseStore(root string, detail *jcsypes.UserSpaceDetail) (*BaseStore, error) {
func NewBaseStore(root string, detail *jcstypes.UserSpaceDetail) (*BaseStore, error) {
return &BaseStore{
root: root,
detail: detail,
}, nil
}

func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
log := s.getLogger()

absObjPath := filepath.Join(s.root, pat.String())
@@ -58,11 +58,11 @@ func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOp
return types.FileInfo{
Path: pat,
Size: counter.Count(),
Hash: jcsypes.NewFullHash(hasher.Sum()),
Hash: jcstypes.NewFullHash(hasher.Sum()),
}, nil
}

func (s *BaseStore) Read(objPath jcsypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
func (s *BaseStore) Read(objPath jcstypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
absObjPath := filepath.Join(s.root, objPath.JoinOSPath())
file, err := os.Open(absObjPath)
if err != nil {
@@ -86,7 +86,7 @@ func (s *BaseStore) Read(objPath jcsypes.JPath, opt types.OpenOption) (io.ReadCl
return ret, nil
}

func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
func (s *BaseStore) Mkdir(path jcstypes.JPath) error {
absObjPath := filepath.Join(s.root, path.JoinOSPath())
err := os.MkdirAll(absObjPath, 0755)
if err != nil {
@@ -96,7 +96,7 @@ func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
return nil
}

func (s *BaseStore) ReadDir(pat jcsypes.JPath) types.DirReader {
func (s *BaseStore) ReadDir(pat jcstypes.JPath) types.DirReader {
return &DirReader{
absRootPath: filepath.Join(s.root, pat.JoinOSPath()),
rootJPath: pat.Clone(),


+ 4
- 4
common/pkgs/storage/local/dir_reader.go View File

@@ -6,14 +6,14 @@ import (
"path/filepath"

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type DirReader struct {
// 完整的根路径(包括ReadDir的path参数),比如包括了盘符
absRootPath string
// ReadDir函数传递进来的path参数
rootJPath jcsypes.JPath
rootJPath jcstypes.JPath
init bool
curEntries []dirEntry
}
@@ -42,7 +42,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {

for _, e := range es {
r.curEntries = append(r.curEntries, dirEntry{
dir: jcsypes.JPath{},
dir: jcstypes.JPath{},
entry: e,
})
}
@@ -103,7 +103,7 @@ func (r *DirReader) Close() {
}

type dirEntry struct {
dir jcsypes.JPath
dir jcstypes.JPath
entry os.DirEntry
}



+ 9
- 10
common/pkgs/storage/local/local.go View File

@@ -5,12 +5,11 @@ import (

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
reg.RegisterBuilder[*cortypes.LocalType](func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*jcstypes.LocalType](func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{
detail: detail,
}
@@ -19,7 +18,7 @@ func init() {

type builder struct {
types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
}

func (b *builder) FeatureDesc() types.FeatureDesc {
@@ -27,7 +26,7 @@ func (b *builder) FeatureDesc() types.FeatureDesc {
}

func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
cred, ok := b.detail.UserSpace.Credential.(*cortypes.LocalCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.LocalCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential)
}
@@ -40,7 +39,7 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
}

func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
cred, ok := b.detail.UserSpace.Credential.(*cortypes.LocalCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.LocalCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential)
}
@@ -53,9 +52,9 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
}

func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
feat := types.FindFeature[*cortypes.MultipartUploadFeature](b.detail)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.MultipartUploadFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
}

if typeOnly {
@@ -68,9 +67,9 @@ func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
}

func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) {
feat := types.FindFeature[*cortypes.S2STransferFeature](b.detail)
feat := types.FindFeature[*jcstypes.S2STransferFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.S2STransferFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.S2STransferFeature{})
}

if typeOnly {


+ 6
- 7
common/pkgs/storage/local/multipart_upload.go View File

@@ -13,14 +13,13 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type Multiparter struct {
detail *jcsypes.UserSpaceDetail
localStg *cortypes.LocalCred
feat *cortypes.MultipartUploadFeature
detail *jcstypes.UserSpaceDetail
localStg *jcstypes.LocalCred
feat *jcstypes.MultipartUploadFeature
}

func (*Multiparter) MinPartSize() int64 {
@@ -80,7 +79,7 @@ type MultipartTask struct {
absTempDir string // 应该要是绝对路径
tempFileName string
tempPartsDir string
joinedFileJPath jcsypes.JPath
joinedFileJPath jcstypes.JPath
absJoinedFilePath string
uploadID string
}
@@ -118,7 +117,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
return types.FileInfo{
Path: i.joinedFileJPath,
Size: size,
Hash: jcsypes.NewFullHash(h),
Hash: jcstypes.NewFullHash(h),
}, nil
}



+ 9
- 10
common/pkgs/storage/local/s2s.go View File

@@ -7,24 +7,23 @@ import (
"path/filepath"

"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type S2STransfer struct {
feat *cortypes.S2STransferFeature
detail *jcsypes.UserSpaceDetail
localStg *cortypes.LocalCred
dstPath jcsypes.JPath
feat *jcstypes.S2STransferFeature
detail *jcstypes.UserSpaceDetail
localStg *jcstypes.LocalCred
dstPath jcstypes.JPath
}

// 只有同一个机器的存储之间才可以进行数据直传
func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
if types.FindFeature[*cortypes.S2STransferFeature](dst) == nil {
func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
if types.FindFeature[*jcstypes.S2STransferFeature](dst) == nil {
return false
}

_, ok := src.UserSpace.Storage.(*cortypes.LocalType)
_, ok := src.UserSpace.Storage.(*jcstypes.LocalType)
if !ok {
return false
}
@@ -37,7 +36,7 @@ func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
}

// 执行数据直传
func (s *S2STransfer) Transfer(ctx context.Context, src *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstPath jcsypes.JPath) (types.FileInfo, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (types.FileInfo, error) {
s.dstPath = dstPath

copy, err := os.OpenFile(filepath.Join(s.localStg.RootDir, s.dstPath.JoinOSPath()), os.O_WRONLY|os.O_CREATE, 0644)


+ 12
- 12
common/pkgs/storage/local/shard_store.go View File

@@ -10,18 +10,18 @@ import (

"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type ShardStore struct {
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
stgRoot string
storeAbsRoot string
lock sync.Mutex
done chan any
}

func NewShardStore(root string, detail *jcsypes.UserSpaceDetail) (*ShardStore, error) {
func NewShardStore(root string, detail *jcstypes.UserSpaceDetail) (*ShardStore, error) {
storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.JoinOSPath(), types.ShardStoreWorkingDir))
if err != nil {
return nil, fmt.Errorf("get abs root: %w", err)
@@ -43,7 +43,7 @@ func (s *ShardStore) Stop() {
s.getLogger().Infof("component stop")
}

func (s *ShardStore) Store(path jcsypes.JPath, hash jcsypes.FileHash, size int64) (types.FileInfo, error) {
func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (types.FileInfo, error) {
fullTempPath := filepath.Join(s.stgRoot, path.JoinOSPath())

s.lock.Lock()
@@ -81,7 +81,7 @@ func (s *ShardStore) Store(path jcsypes.JPath, hash jcsypes.FileHash, size int64
}, nil
}

func (s *ShardStore) Info(hash jcsypes.FileHash) (types.FileInfo, error) {
func (s *ShardStore) Info(hash jcstypes.FileHash) (types.FileInfo, error) {
s.lock.Lock()
defer s.lock.Unlock()

@@ -118,7 +118,7 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
return err
}

fileHash, err := jcsypes.ParseHash(filepath.Base(info.Name()))
fileHash, err := jcstypes.ParseHash(filepath.Base(info.Name()))
if err != nil {
return nil
}
@@ -137,11 +137,11 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
return infos, nil
}

func (s *ShardStore) GC(avaiables []jcsypes.FileHash) error {
func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error {
s.lock.Lock()
defer s.lock.Unlock()

avais := make(map[jcsypes.FileHash]bool)
avais := make(map[jcstypes.FileHash]bool)
for _, hash := range avaiables {
avais[hash] = true
}
@@ -162,7 +162,7 @@ func (s *ShardStore) GC(avaiables []jcsypes.FileHash) error {
return err
}

fileHash, err := jcsypes.ParseHash(filepath.Base(info.Name()))
fileHash, err := jcstypes.ParseHash(filepath.Base(info.Name()))
if err != nil {
return nil
}
@@ -199,14 +199,14 @@ func (s *ShardStore) getLogger() logger.Logger {
return logger.WithField("ShardStore", "Local").WithField("Storage", s.detail.UserSpace.Storage.String())
}

func (s *ShardStore) getFileDirFromHash(hash jcsypes.FileHash) string {
func (s *ShardStore) getFileDirFromHash(hash jcstypes.FileHash) string {
return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2))
}

func (s *ShardStore) getFilePathFromHash(hash jcsypes.FileHash) string {
func (s *ShardStore) getFilePathFromHash(hash jcstypes.FileHash) string {
return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2), string(hash))
}

func (s *ShardStore) getJPathFromHash(hash jcsypes.FileHash) jcsypes.JPath {
func (s *ShardStore) getJPathFromHash(hash jcstypes.FileHash) jcstypes.JPath {
return s.detail.UserSpace.WorkingDir.ConcatCompsNew(types.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
}

+ 17
- 18
common/pkgs/storage/obs/obs.go View File

@@ -9,20 +9,19 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
s3stg "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
reg.RegisterBuilder[*cortypes.OBSType](newBuilder)
reg.RegisterBuilder[*jcstypes.OBSType](newBuilder)
}

type builder struct {
types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
}

func newBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func newBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{
detail: detail,
}
@@ -33,8 +32,8 @@ func (b *builder) FeatureDesc() types.FeatureDesc {
}

func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
}
@@ -52,8 +51,8 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
}

func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
}
@@ -70,7 +69,7 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
return s3stg.NewBaseStore(b.detail, cli, bucket, s3stg.BaseStoreOption{UseAWSSha256: false})
}

func createClient(stgType *cortypes.OBSType, cred *cortypes.OBSCred) (*s3.Client, string, error) {
func createClient(stgType *jcstypes.OBSType, cred *jcstypes.OBSCred) (*s3.Client, string, error) {
awsConfig := aws.Config{}

cre := aws.Credentials{
@@ -91,17 +90,17 @@ func createClient(stgType *cortypes.OBSType, cred *cortypes.OBSCred) (*s3.Client
}

func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
feat := types.FindFeature[*cortypes.MultipartUploadFeature](b.detail)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.MultipartUploadFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
}

if typeOnly {
return (*s3stg.Multiparter)(nil), nil
}

cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
}
@@ -120,17 +119,17 @@ func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
}

func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
feat := types.FindFeature[*cortypes.S2STransferFeature](b.detail)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
feat := types.FindFeature[*jcstypes.S2STransferFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.S2STransferFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.S2STransferFeature{})
}

if typeOnly {
return (*S2STransfer)(nil), nil
}

cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
}


+ 9
- 10
common/pkgs/storage/obs/obs_test.go View File

@@ -5,40 +5,39 @@ import (
"testing"

. "github.com/smartystreets/goconvey/convey"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func Test_S2S(t *testing.T) {
Convey("OBS", t, func() {
s2s := S2STransfer{
stgType: &cortypes.OBSType{
stgType: &jcstypes.OBSType{
Region: "cn-north-4",
Endpoint: "obs.cn-north-4.myhuaweicloud.com",
Bucket: "pcm3-bucket3",
ProjectID: "",
},
cred: &cortypes.OBSCred{
cred: &jcstypes.OBSCred{
AK: "",
SK: "",
},
feat: &cortypes.S2STransferFeature{},
feat: &jcstypes.S2STransferFeature{},
}

_, err := s2s.Transfer(context.TODO(), &jcsypes.UserSpaceDetail{
UserSpace: jcsypes.UserSpace{
Storage: &cortypes.OBSType{
_, err := s2s.Transfer(context.TODO(), &jcstypes.UserSpaceDetail{
UserSpace: jcstypes.UserSpace{
Storage: &jcstypes.OBSType{
Region: "cn-north-4",
Endpoint: "obs.cn-north-4.myhuaweicloud.com",
Bucket: "pcm2-bucket2",
ProjectID: "",
},
Credential: &cortypes.OBSCred{
Credential: &jcstypes.OBSCred{
AK: "",
SK: "",
},
},
}, jcsypes.PathFromComps("test_data/test03.txt"), jcsypes.PathFromComps("atest.txt"))
}, jcstypes.PathFromComps("test_data/test03.txt"), jcstypes.PathFromComps("atest.txt"))
defer s2s.Close()

So(err, ShouldEqual, nil)


+ 12
- 13
common/pkgs/storage/obs/s2s.go View File

@@ -15,20 +15,19 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2"
stgs3 "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type S2STransfer struct {
detail *jcsypes.UserSpaceDetail
stgType *cortypes.OBSType
cred *cortypes.OBSCred
feat *cortypes.S2STransferFeature
detail *jcstypes.UserSpaceDetail
stgType *jcstypes.OBSType
cred *jcstypes.OBSCred
feat *jcstypes.S2STransferFeature
taskID *int64
omsCli *oms.OmsClient
}

func NewS2STransfer(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType, cred *cortypes.OBSCred, feat *cortypes.S2STransferFeature) *S2STransfer {
func NewS2STransfer(detail *jcstypes.UserSpaceDetail, stgType *jcstypes.OBSType, cred *jcstypes.OBSCred, feat *jcstypes.S2STransferFeature) *S2STransfer {
return &S2STransfer{
detail: detail,
stgType: stgType,
@@ -38,13 +37,13 @@ func NewS2STransfer(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType,
}

// 判断是否能从指定的源存储中直传到当前存储的目的路径
func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
req := makeRequest(src, jcsypes.JPath{})
func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
req := makeRequest(src, jcstypes.JPath{})
return req != nil
}

// 执行数据直传。返回传输后的文件路径
func (s *S2STransfer) Transfer(ctx context.Context, src *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstPath jcsypes.JPath) (types.FileInfo, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (types.FileInfo, error) {
req := makeRequest(src, srcPath)
if req == nil {
return types.FileInfo{}, fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage)
@@ -178,12 +177,12 @@ func (s *S2STransfer) Close() {
}
}

func makeRequest(srcStg *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath) *model.SrcNodeReq {
func makeRequest(srcStg *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath) *model.SrcNodeReq {
switch srcType := srcStg.UserSpace.Storage.(type) {
case *cortypes.OBSType:
case *jcstypes.OBSType:
cloudType := "HuaweiCloud"

cred, ok := srcStg.UserSpace.Credential.(*cortypes.OBSCred)
cred, ok := srcStg.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok {
return nil
}


+ 5
- 6
common/pkgs/storage/obs/shard_store.go View File

@@ -5,17 +5,16 @@ import (
"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type ShardStore struct {
*s3.ShardStore
stgType *cortypes.OBSType
cred *cortypes.OBSCred
stgType *jcstypes.OBSType
cred *jcstypes.OBSCred
}

func NewShardStore(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType, cred *cortypes.OBSCred, s3Cli *awss3.Client, bkt string) (*ShardStore, error) {
func NewShardStore(detail *jcstypes.UserSpaceDetail, stgType *jcstypes.OBSType, cred *jcstypes.OBSCred, s3Cli *awss3.Client, bkt string) (*ShardStore, error) {
sd := ShardStore{
stgType: stgType,
cred: cred,
@@ -32,7 +31,7 @@ func NewShardStore(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType, c
return &sd, nil
}

func (s *ShardStore) MakeHTTPReadRequest(fileHash jcsypes.FileHash) (types.HTTPRequest, error) {
func (s *ShardStore) MakeHTTPReadRequest(fileHash jcstypes.FileHash) (types.HTTPRequest, error) {
cli, err := obs.New(s.cred.AK, s.cred.SK, s.stgType.Endpoint)
if err != nil {
return types.HTTPRequest{}, err


+ 10
- 11
common/pkgs/storage/pool/pool.go View File

@@ -6,12 +6,11 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type userSpace struct {
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
store types.ShardStore
}

@@ -23,8 +22,8 @@ func (u *userSpace) Drop() {
}

type userSpaceKey struct {
UserID cortypes.UserID
UserSpaceID jcsypes.UserSpaceID
UserID jcstypes.UserID
UserSpaceID jcstypes.UserSpaceID
}

type Pool struct {
@@ -40,7 +39,7 @@ func NewPool() *Pool {
}
}

func (p *Pool) Drop(userID cortypes.UserID, spaceID jcsypes.UserSpaceID) {
func (p *Pool) Drop(userID jcstypes.UserID, spaceID jcstypes.UserSpaceID) {
p.lock.Lock()
defer p.lock.Unlock()

@@ -56,7 +55,7 @@ func (p *Pool) Drop(userID cortypes.UserID, spaceID jcsypes.UserSpaceID) {
delete(p.spaces, key)
}

func (p *Pool) GetShardStore(spaceDetail *jcsypes.UserSpaceDetail) (types.ShardStore, error) {
func (p *Pool) GetShardStore(spaceDetail *jcstypes.UserSpaceDetail) (types.ShardStore, error) {
p.lock.Lock()
defer p.lock.Unlock()

@@ -91,18 +90,18 @@ func (p *Pool) GetShardStore(spaceDetail *jcsypes.UserSpaceDetail) (types.ShardS
return space.store, nil
}

func (p *Pool) GetBaseStore(spaceDetail *jcsypes.UserSpaceDetail) (types.BaseStore, error) {
func (p *Pool) GetBaseStore(spaceDetail *jcstypes.UserSpaceDetail) (types.BaseStore, error) {
return factory.GetBuilder(spaceDetail).CreateBaseStore(false)
}

func (p *Pool) GetMultiparter(spaceDetail *jcsypes.UserSpaceDetail) (types.Multiparter, error) {
func (p *Pool) GetMultiparter(spaceDetail *jcstypes.UserSpaceDetail) (types.Multiparter, error) {
return factory.GetBuilder(spaceDetail).CreateMultiparter(false)
}

func (p *Pool) GetS2STransfer(spaceDetail *jcsypes.UserSpaceDetail) (types.S2STransfer, error) {
func (p *Pool) GetS2STransfer(spaceDetail *jcstypes.UserSpaceDetail) (types.S2STransfer, error) {
return factory.GetBuilder(spaceDetail).CreateS2STransfer(false)
}

func (p *Pool) GetECMultiplier(spaceDetail *jcsypes.UserSpaceDetail) (types.ECMultiplier, error) {
func (p *Pool) GetECMultiplier(spaceDetail *jcstypes.UserSpaceDetail) (types.ECMultiplier, error) {
return factory.GetBuilder(spaceDetail).CreateECMultiplier(false)
}

+ 9
- 9
common/pkgs/storage/s3/base_store.go View File

@@ -16,7 +16,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

const (
@@ -24,7 +24,7 @@ const (
)

type BaseStore struct {
Detail *jcsypes.UserSpaceDetail
Detail *jcstypes.UserSpaceDetail
Bucket string
cli *s3.Client
opt BaseStoreOption
@@ -34,7 +34,7 @@ type BaseStoreOption struct {
UseAWSSha256 bool // 能否直接使用AWS提供的SHA256校验,如果不行,则使用本地计算。默认使用本地计算。
}

func NewBaseStore(detail *jcsypes.UserSpaceDetail, cli *s3.Client, bkt string, opt BaseStoreOption) (*BaseStore, error) {
func NewBaseStore(detail *jcstypes.UserSpaceDetail, cli *s3.Client, bkt string, opt BaseStoreOption) (*BaseStore, error) {
return &BaseStore{
Detail: detail,
Bucket: bkt,
@@ -43,7 +43,7 @@ func NewBaseStore(detail *jcsypes.UserSpaceDetail, cli *s3.Client, bkt string, o
}, nil
}

func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
key := pat
meta := make(map[string]string)
if opt.ModTime.IsZero() {
@@ -81,7 +81,7 @@ func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOp

return types.FileInfo{
Path: key,
Hash: jcsypes.NewFullHash(hash),
Hash: jcstypes.NewFullHash(hash),
Size: counter.Count(),
}, nil
}
@@ -99,12 +99,12 @@ func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOp

return types.FileInfo{
Path: key,
Hash: jcsypes.NewFullHash(hashStr.Sum()),
Hash: jcstypes.NewFullHash(hashStr.Sum()),
Size: counter.Count(),
}, nil
}

func (s *BaseStore) Read(pat jcsypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
func (s *BaseStore) Read(pat jcstypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
key := pat

input := &s3.GetObjectInput{
@@ -128,7 +128,7 @@ func (s *BaseStore) Read(pat jcsypes.JPath, opt types.OpenOption) (io.ReadCloser
return resp.Body, nil
}

func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
func (s *BaseStore) Mkdir(path jcstypes.JPath) error {
_, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(s.Bucket),
Key: aws.String(path.String() + "/"),
@@ -137,7 +137,7 @@ func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
return err
}

func (s *BaseStore) ReadDir(path jcsypes.JPath) types.DirReader {
func (s *BaseStore) ReadDir(path jcstypes.JPath) types.DirReader {
return &DirReader{
cli: s.cli,
bucket: s.Bucket,


+ 3
- 3
common/pkgs/storage/s3/dir_reader.go View File

@@ -7,13 +7,13 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type DirReader struct {
cli *s3.Client
bucket string
rootPath jcsypes.JPath
rootPath jcstypes.JPath
marker *string
curInfos []types.DirEntry
eof bool
@@ -39,7 +39,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {
}

for _, obj := range resp.Contents {
key := jcsypes.PathFromJcsPathString(*obj.Key)
key := jcstypes.PathFromJcsPathString(*obj.Key)

r.curInfos = append(r.curInfos, types.DirEntry{
Path: key,


+ 7
- 8
common/pkgs/storage/s3/multipart_upload.go View File

@@ -12,18 +12,17 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

type Multiparter struct {
detail *jcsypes.UserSpaceDetail
feat *cortypes.MultipartUploadFeature
detail *jcstypes.UserSpaceDetail
feat *jcstypes.MultipartUploadFeature
bucket string
cli *s3.Client
}

func NewMultiparter(detail *jcsypes.UserSpaceDetail, feat *cortypes.MultipartUploadFeature, bkt string, cli *s3.Client) *Multiparter {
func NewMultiparter(detail *jcstypes.UserSpaceDetail, feat *jcstypes.MultipartUploadFeature, bkt string, cli *s3.Client) *Multiparter {
return &Multiparter{
detail: detail,
feat: feat,
@@ -89,9 +88,9 @@ func (m *Multiparter) UploadPart(ctx context.Context, init types.MultipartInitSt
type MultipartTask struct {
cli *s3.Client
bucket string
tempDir jcsypes.JPath
tempDir jcstypes.JPath
tempFileName string
tempFilePath jcsypes.JPath
tempFilePath jcstypes.JPath
uploadID string
}

@@ -140,7 +139,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
return types.FileInfo{}, err
}

hash := jcsypes.CalculateCompositeHash(partHashes)
hash := jcstypes.CalculateCompositeHash(partHashes)

return types.FileInfo{
Path: i.tempFilePath,


+ 13
- 14
common/pkgs/storage/s3/s3.go View File

@@ -8,20 +8,19 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
)

func init() {
reg.RegisterBuilder[*cortypes.S3Type](newBuilder)
reg.RegisterBuilder[*jcstypes.S3Type](newBuilder)
}

type builder struct {
types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
}

func newBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func newBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{
detail: detail,
}
@@ -32,8 +31,8 @@ func (b *builder) FeatureDesc() types.FeatureDesc {
}

func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*cortypes.S3Cred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for s3 storage", b.detail.UserSpace.Credential)
}
@@ -51,8 +50,8 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
}

func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*cortypes.S3Cred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for s3 storage", b.detail.UserSpace.Credential)
}
@@ -69,7 +68,7 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
return NewBaseStore(b.detail, cli, bkt, BaseStoreOption{UseAWSSha256: false})
}

func createClient(stgType *cortypes.S3Type, cred *cortypes.S3Cred) (*s3.Client, string, error) {
func createClient(stgType *jcstypes.S3Type, cred *jcstypes.S3Cred) (*s3.Client, string, error) {
awsConfig := aws.Config{}

if cred.AK != "" && cred.SK != "" {
@@ -92,13 +91,13 @@ func createClient(stgType *cortypes.S3Type, cred *cortypes.S3Cred) (*s3.Client,
}

func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.S3Type)
feat := types.FindFeature[*cortypes.MultipartUploadFeature](b.detail)
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.MultipartUploadFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
}

s3Cred, ok := b.detail.UserSpace.Credential.(*cortypes.S3Cred)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for s3 base store", b.detail.UserSpace.Credential)
}


Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save