Browse Source

coor的types移动到common

master
Sydonian 4 months ago
parent
commit
dd23f68926
100 changed files with 871 additions and 891 deletions
  1. +5
    -5
      client/internal/accessstat/access_stat.go
  2. +3
    -3
      client/internal/accesstoken/accesstoken.go
  3. +10
    -10
      client/internal/cmdline/migrate.go
  4. +2
    -2
      client/internal/cmdline/test.go
  5. +23
    -23
      client/internal/downloader/downloader.go
  6. +4
    -4
      client/internal/downloader/lrc_strip_iterator.go
  7. +3
    -3
      client/internal/downloader/strategy/selector.go
  8. +6
    -6
      client/internal/http/v1/object.go
  9. +7
    -7
      client/internal/http/v1/package.go
  10. +4
    -4
      client/internal/http/v1/presigned.go
  11. +7
    -7
      client/internal/http/v1/space_syncer.go
  12. +2
    -2
      client/internal/http/v1/user_space.go
  13. +9
    -9
      client/internal/metacache/connectivity.go
  14. +8
    -8
      client/internal/metacache/hubmeta.go
  15. +2
    -2
      client/internal/metacache/user_space_meta.go
  16. +4
    -4
      client/internal/mount/mount_linux.go
  17. +4
    -4
      client/internal/mount/mount_win.go
  18. +7
    -7
      client/internal/mount/vfs/cache/cache.go
  19. +3
    -3
      client/internal/mount/vfs/cache/file.go
  20. +17
    -17
      client/internal/mount/vfs/fuse.go
  21. +2
    -2
      client/internal/mount/vfs/fuse_bucket.go
  22. +2
    -2
      client/internal/mount/vfs/fuse_dir.go
  23. +3
    -3
      client/internal/mount/vfs/fuse_file.go
  24. +2
    -2
      client/internal/mount/vfs/fuse_root.go
  25. +4
    -4
      client/internal/repl/load.go
  26. +4
    -4
      client/internal/repl/lsp.go
  27. +26
    -26
      client/internal/services/user_space.go
  28. +6
    -6
      client/internal/services/utils.go
  29. +7
    -7
      client/internal/spacesyncer/execute_diff.go
  30. +3
    -3
      client/internal/spacesyncer/filter.go
  31. +16
    -16
      client/internal/spacesyncer/space_syncer.go
  32. +3
    -3
      client/internal/spacesyncer/trigger.go
  33. +6
    -6
      client/internal/speedstats/speedstats.go
  34. +9
    -9
      client/internal/ticktock/change_redundancy.go
  35. +9
    -9
      client/internal/ticktock/check_shardstore.go
  36. +88
    -89
      client/internal/ticktock/redundancy_recover.go
  37. +52
    -52
      client/internal/ticktock/redundancy_shrink.go
  38. +13
    -13
      client/internal/uploader/uploader.go
  39. +9
    -10
      client/internal/uploader/user_space_upload.go
  40. +7
    -7
      client/sdk/api/v1/bucket.go
  41. +24
    -24
      client/sdk/api/v1/package.go
  42. +22
    -22
      client/sdk/api/v1/presigned.go
  43. +13
    -13
      client/sdk/api/v1/space_syncer.go
  44. +2
    -2
      client/sdk/api/v1/storage_test.go
  45. +2
    -2
      client/sdk/api/v1/system.go
  46. +26
    -27
      client/sdk/api/v1/user_space.go
  47. +1
    -1
      common/globals/globals.go
  48. +2
    -2
      common/globals/utils.go
  49. +6
    -6
      common/pkgs/accesstoken/accesstoken.go
  50. +10
    -10
      common/pkgs/connectivity/collector.go
  51. +15
    -15
      common/pkgs/ioswitch2/fromto.go
  52. +4
    -4
      common/pkgs/ioswitch2/http_hub_worker.go
  53. +4
    -4
      common/pkgs/ioswitch2/hub_worker.go
  54. +14
    -14
      common/pkgs/ioswitch2/ops2/base_store.go
  55. +6
    -6
      common/pkgs/ioswitch2/ops2/bypass.go
  56. +7
    -7
      common/pkgs/ioswitch2/ops2/ec.go
  57. +7
    -7
      common/pkgs/ioswitch2/ops2/multipart.go
  58. +17
    -17
      common/pkgs/ioswitch2/ops2/s2s.go
  59. +9
    -9
      common/pkgs/ioswitch2/ops2/shard_store.go
  60. +4
    -5
      common/pkgs/ioswitch2/parser/gen/generator.go
  61. +4
    -5
      common/pkgs/ioswitch2/parser/opt/utils.go
  62. +2
    -2
      common/pkgs/ioswitch2/plans/complete_multipart.go
  63. +4
    -5
      common/pkgs/ioswitch2/plans/utils.go
  64. +7
    -7
      common/pkgs/ioswitchlrc/fromto.go
  65. +3
    -3
      common/pkgs/ioswitchlrc/hub_worker.go
  66. +14
    -14
      common/pkgs/ioswitchlrc/ops2/base_store.go
  67. +5
    -5
      common/pkgs/ioswitchlrc/ops2/ec.go
  68. +9
    -9
      common/pkgs/ioswitchlrc/ops2/shard_store.go
  69. +5
    -5
      common/pkgs/ioswitchlrc/parser/generator.go
  70. +4
    -5
      common/pkgs/ioswitchlrc/parser/utils.go
  71. +4
    -4
      common/pkgs/publock/reqbuilder/user_space.go
  72. +7
    -7
      common/pkgs/rpc/auth.go
  73. +15
    -15
      common/pkgs/rpc/coordinator/hub.go
  74. +3
    -3
      common/pkgs/rpc/coordinator/storage.go
  75. +7
    -7
      common/pkgs/rpc/coordinator/user.go
  76. +5
    -5
      common/pkgs/rpc/hub/cache.go
  77. +3
    -3
      common/pkgs/rpc/hub/user.go
  78. +3
    -3
      common/pkgs/rpc/hub/user_space.go
  79. +8
    -8
      common/pkgs/servicestats/hub_strorage_transfer.go
  80. +8
    -8
      common/pkgs/servicestats/hub_transfter.go
  81. +5
    -5
      common/pkgs/servicestats/service_stats.go
  82. +4
    -5
      common/pkgs/storage/efile/ec_multiplier.go
  83. +7
    -8
      common/pkgs/storage/efile/efile.go
  84. +2
    -2
      common/pkgs/storage/factory/factory.go
  85. +4
    -5
      common/pkgs/storage/factory/reg/reg.go
  86. +8
    -8
      common/pkgs/storage/local/base_store.go
  87. +4
    -4
      common/pkgs/storage/local/dir_reader.go
  88. +9
    -10
      common/pkgs/storage/local/local.go
  89. +6
    -7
      common/pkgs/storage/local/multipart_upload.go
  90. +9
    -10
      common/pkgs/storage/local/s2s.go
  91. +12
    -12
      common/pkgs/storage/local/shard_store.go
  92. +17
    -18
      common/pkgs/storage/obs/obs.go
  93. +9
    -10
      common/pkgs/storage/obs/obs_test.go
  94. +12
    -13
      common/pkgs/storage/obs/s2s.go
  95. +5
    -6
      common/pkgs/storage/obs/shard_store.go
  96. +10
    -11
      common/pkgs/storage/pool/pool.go
  97. +9
    -9
      common/pkgs/storage/s3/base_store.go
  98. +3
    -3
      common/pkgs/storage/s3/dir_reader.go
  99. +7
    -8
      common/pkgs/storage/s3/multipart_upload.go
  100. +13
    -14
      common/pkgs/storage/s3/s3.go

+ 5
- 5
client/internal/accessstat/access_stat.go View File

@@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async" "gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type AccessStatEventChan = async.UnboundChannel[AccessStatEvent] type AccessStatEventChan = async.UnboundChannel[AccessStatEvent]
@@ -30,9 +30,9 @@ type AccessStat struct {
} }


type entryKey struct { type entryKey struct {
objID jcsypes.ObjectID
pkgID jcsypes.PackageID
spaceID jcsypes.UserSpaceID
objID jcstypes.ObjectID
pkgID jcstypes.PackageID
spaceID jcstypes.UserSpaceID
} }


func NewAccessStat(cfg Config, db *db.DB) *AccessStat { func NewAccessStat(cfg Config, db *db.DB) *AccessStat {
@@ -44,7 +44,7 @@ func NewAccessStat(cfg Config, db *db.DB) *AccessStat {
} }
} }


func (p *AccessStat) AddAccessCounter(objID jcsypes.ObjectID, pkgID jcsypes.PackageID, spaceID jcsypes.UserSpaceID, value float64) {
func (p *AccessStat) AddAccessCounter(objID jcstypes.ObjectID, pkgID jcstypes.PackageID, spaceID jcstypes.UserSpaceID, value float64) {
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()




+ 3
- 3
client/internal/accesstoken/accesstoken.go View File

@@ -15,7 +15,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/accesstoken" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/accesstoken"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator" corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type KeeperEvent interface { type KeeperEvent interface {
@@ -30,7 +30,7 @@ type ExitEvent struct {
type Keeper struct { type Keeper struct {
cfg Config cfg Config
enabled bool enabled bool
token cortypes.UserAccessToken
token jcstypes.UserAccessToken
priKey ed25519.PrivateKey priKey ed25519.PrivateKey
lock sync.RWMutex lock sync.RWMutex
done chan any done chan any
@@ -144,7 +144,7 @@ func (k *Keeper) Stop() {
} }
} }


func (k *Keeper) GetToken() cortypes.UserAccessToken {
func (k *Keeper) GetToken() jcstypes.UserAccessToken {
k.lock.RLock() k.lock.RLock()
defer k.lock.RUnlock() defer k.lock.RUnlock()




+ 10
- 10
client/internal/cmdline/migrate.go View File

@@ -6,7 +6,7 @@ import (


"github.com/spf13/cobra" "github.com/spf13/cobra"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/config" "gitlink.org.cn/cloudream/jcs-pub/client/internal/config"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/driver/mysql" "gorm.io/driver/mysql"
"gorm.io/gorm" "gorm.io/gorm"
) )
@@ -38,15 +38,15 @@ func migrate(configPath string) {
} }
db = db.Set("gorm:table_options", "CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci") db = db.Set("gorm:table_options", "CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci")


migrateOne(db, jcsypes.Bucket{})
migrateOne(db, jcsypes.ObjectAccessStat{})
migrateOne(db, jcsypes.ObjectBlock{})
migrateOne(db, jcsypes.Object{})
migrateOne(db, jcsypes.PackageAccessStat{})
migrateOne(db, jcsypes.Package{})
migrateOne(db, jcsypes.PinnedObject{})
migrateOne(db, jcsypes.UserSpace{})
migrateOne(db, jcsypes.SpaceSyncTask{})
migrateOne(db, jcstypes.Bucket{})
migrateOne(db, jcstypes.ObjectAccessStat{})
migrateOne(db, jcstypes.ObjectBlock{})
migrateOne(db, jcstypes.Object{})
migrateOne(db, jcstypes.PackageAccessStat{})
migrateOne(db, jcstypes.Package{})
migrateOne(db, jcstypes.PinnedObject{})
migrateOne(db, jcstypes.UserSpace{})
migrateOne(db, jcstypes.SpaceSyncTask{})


fmt.Println("migrate success") fmt.Println("migrate success")
} }


+ 2
- 2
client/internal/cmdline/test.go View File

@@ -27,7 +27,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap" "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


@@ -61,7 +61,7 @@ func doTest(svc *services.Service) {


ft = ioswitch2.NewFromTo() ft = ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore("Full1AE5436AF72D8EF93923486E0E167315CEF0C91898064DADFAC22216FFBC5E3D", *space1, ioswitch2.RawStream())) ft.AddFrom(ioswitch2.NewFromShardstore("Full1AE5436AF72D8EF93923486E0E167315CEF0C91898064DADFAC22216FFBC5E3D", *space1, ioswitch2.RawStream()))
ft.AddTo(ioswitch2.NewToBaseStore(*space2, jcsypes.PathFromComps("test3.txt")))
ft.AddTo(ioswitch2.NewToBaseStore(*space2, jcstypes.PathFromComps("test3.txt")))
plans := exec.NewPlanBuilder() plans := exec.NewPlanBuilder()
parser.Parse(ft, plans) parser.Parse(ft, plans)
fmt.Println(plans) fmt.Println(plans)


+ 23
- 23
client/internal/downloader/downloader.go View File

@@ -11,7 +11,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats" "gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


const ( const (
@@ -21,18 +21,18 @@ const (
type DownloadIterator = iterator.Iterator[*Downloading] type DownloadIterator = iterator.Iterator[*Downloading]


type DownloadReqeust struct { type DownloadReqeust struct {
ObjectID jcsypes.ObjectID
ObjectID jcstypes.ObjectID
Offset int64 Offset int64
Length int64 Length int64
} }


type downloadReqeust2 struct { type downloadReqeust2 struct {
Detail *jcsypes.ObjectDetail
Detail *jcstypes.ObjectDetail
Raw DownloadReqeust Raw DownloadReqeust
} }


type Downloading struct { type Downloading struct {
Object *jcsypes.Object
Object *jcstypes.Object
File io.ReadCloser // 文件流,如果文件不存在,那么为nil File io.ReadCloser // 文件流,如果文件不存在,那么为nil
Request DownloadReqeust Request DownloadReqeust
} }
@@ -65,7 +65,7 @@ func NewDownloader(cfg Config, conn *connectivity.Collector, stgPool *pool.Pool,
} }


func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator { func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
objIDs := make([]jcsypes.ObjectID, len(reqs))
objIDs := make([]jcstypes.ObjectID, len(reqs))
for i, req := range reqs { for i, req := range reqs {
objIDs[i] = req.ObjectID objIDs[i] = req.ObjectID
} }
@@ -79,7 +79,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return iterator.FuseError[*Downloading](fmt.Errorf("request to db: %w", err)) return iterator.FuseError[*Downloading](fmt.Errorf("request to db: %w", err))
} }


detailsMap := make(map[jcsypes.ObjectID]*jcsypes.ObjectDetail)
detailsMap := make(map[jcstypes.ObjectID]*jcstypes.ObjectDetail)
for _, detail := range objDetails { for _, detail := range objDetails {
d := detail d := detail
detailsMap[detail.Object.ObjectID] = &d detailsMap[detail.Object.ObjectID] = &d
@@ -96,7 +96,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return NewDownloadObjectIterator(d, req2s) return NewDownloadObjectIterator(d, req2s)
} }


func (d *Downloader) DownloadObjectByDetail(detail jcsypes.ObjectDetail, off int64, length int64) (*Downloading, error) {
func (d *Downloader) DownloadObjectByDetail(detail jcstypes.ObjectDetail, off int64, length int64) (*Downloading, error) {
req2s := []downloadReqeust2{{ req2s := []downloadReqeust2{{
Detail: &detail, Detail: &detail,
Raw: DownloadReqeust{ Raw: DownloadReqeust{
@@ -110,56 +110,56 @@ func (d *Downloader) DownloadObjectByDetail(detail jcsypes.ObjectDetail, off int
return iter.MoveNext() return iter.MoveNext()
} }


func (d *Downloader) DownloadPackage(pkgID jcsypes.PackageID, prefix string) (jcsypes.Package, DownloadIterator, error) {
pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (jcsypes.Package, []jcsypes.ObjectDetail, error) {
func (d *Downloader) DownloadPackage(pkgID jcstypes.PackageID, prefix string) (jcstypes.Package, DownloadIterator, error) {
pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (jcstypes.Package, []jcstypes.ObjectDetail, error) {
pkg, err := d.db.Package().GetByID(tx, pkgID) pkg, err := d.db.Package().GetByID(tx, pkgID)
if err != nil { if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
} }


var details []jcsypes.ObjectDetail
var details []jcstypes.ObjectDetail
if prefix != "" { if prefix != "" {
objs, err := d.db.Object().GetWithPathPrefix(tx, pkgID, prefix) objs, err := d.db.Object().GetWithPathPrefix(tx, pkgID, prefix)
if err != nil { if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
} }


objIDs := make([]jcsypes.ObjectID, len(objs))
objIDs := make([]jcstypes.ObjectID, len(objs))
for i, obj := range objs { for i, obj := range objs {
objIDs[i] = obj.ObjectID objIDs[i] = obj.ObjectID
} }


allBlocks, err := d.db.ObjectBlock().BatchGetByObjectID(tx, objIDs) allBlocks, err := d.db.ObjectBlock().BatchGetByObjectID(tx, objIDs)
if err != nil { if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
} }


allPinnedObjs, err := d.db.PinnedObject().BatchGetByObjectID(tx, objIDs) allPinnedObjs, err := d.db.PinnedObject().BatchGetByObjectID(tx, objIDs)
if err != nil { if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err


} }
details = make([]jcsypes.ObjectDetail, 0, len(objs))
details = make([]jcstypes.ObjectDetail, 0, len(objs))
for _, obj := range objs { for _, obj := range objs {
detail := jcsypes.ObjectDetail{
detail := jcstypes.ObjectDetail{
Object: obj, Object: obj,
} }
details = append(details, detail) details = append(details, detail)
} }


jcsypes.DetailsFillObjectBlocks(details, allBlocks)
jcsypes.DetailsFillPinnedAt(details, allPinnedObjs)
jcstypes.DetailsFillObjectBlocks(details, allBlocks)
jcstypes.DetailsFillPinnedAt(details, allPinnedObjs)
} else { } else {
details, err = d.db.Object().GetPackageObjectDetails(tx, pkgID) details, err = d.db.Object().GetPackageObjectDetails(tx, pkgID)
if err != nil { if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
} }
} }


return pkg, details, nil return pkg, details, nil
}) })
if err != nil { if err != nil {
return jcsypes.Package{}, nil, err
return jcstypes.Package{}, nil, err
} }


req2s := make([]downloadReqeust2, len(details)) req2s := make([]downloadReqeust2, len(details))
@@ -180,11 +180,11 @@ func (d *Downloader) DownloadPackage(pkgID jcsypes.PackageID, prefix string) (jc


type ObjectECStrip struct { type ObjectECStrip struct {
Data []byte Data []byte
ObjectFileHash jcsypes.FileHash // 添加这条缓存时,Object的FileHash
ObjectFileHash jcstypes.FileHash // 添加这条缓存时,Object的FileHash
} }


type ECStripKey struct { type ECStripKey struct {
ObjectID jcsypes.ObjectID
ObjectID jcstypes.ObjectID
StripIndex int64 StripIndex int64
} }




+ 4
- 4
client/internal/downloader/lrc_strip_iterator.go View File

@@ -12,14 +12,14 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type LRCStripIterator struct { type LRCStripIterator struct {
downloader *Downloader downloader *Downloader
object jcsypes.Object
object jcstypes.Object
blocks []downloadBlock blocks []downloadBlock
red jcsypes.LRCRedundancy
red jcstypes.LRCRedundancy
curStripIndex int64 curStripIndex int64
cache *StripCache cache *StripCache
dataChan chan dataChanEntry dataChan chan dataChanEntry
@@ -28,7 +28,7 @@ type LRCStripIterator struct {
inited bool inited bool
} }


func NewLRCStripIterator(downloder *Downloader, object jcsypes.Object, blocks []downloadBlock, red jcsypes.LRCRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *LRCStripIterator {
func NewLRCStripIterator(downloder *Downloader, object jcstypes.Object, blocks []downloadBlock, red jcstypes.LRCRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *LRCStripIterator {
if maxPrefetch <= 0 { if maxPrefetch <= 0 {
maxPrefetch = 1 maxPrefetch = 1
} }


+ 3
- 3
client/internal/downloader/strategy/selector.go View File

@@ -12,13 +12,13 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache"
"gitlink.org.cn/cloudream/jcs-pub/common/consts" "gitlink.org.cn/cloudream/jcs-pub/common/consts"
"gitlink.org.cn/cloudream/jcs-pub/common/types" "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type Request struct { type Request struct {
Detail types.ObjectDetail Detail types.ObjectDetail
Range math2.Range Range math2.Range
DestLocation cortypes.Location
DestLocation jcstypes.Location
} }


type Strategy interface { type Strategy interface {
@@ -113,7 +113,7 @@ type downloadBlock struct {
type request2 struct { type request2 struct {
Detail types.ObjectDetail Detail types.ObjectDetail
Range math2.Range Range math2.Range
DestLocation cortypes.Location
DestLocation jcstypes.Location
} }


func (s *Selector) selectForNoneOrRep(req request2) (Strategy, error) { func (s *Selector) selectForNoneOrRep(req request2) (Strategy, error) {


+ 6
- 6
client/internal/http/v1/object.go View File

@@ -19,7 +19,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type ObjectService struct { type ObjectService struct {
@@ -113,9 +113,9 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
return return
} }


copyToPath := make([]jcsypes.JPath, 0, len(info.CopyToPath))
copyToPath := make([]jcstypes.JPath, 0, len(info.CopyToPath))
for _, p := range info.CopyToPath { for _, p := range info.CopyToPath {
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcstypes.PathFromJcsPathString(p))
} }


up, err := s.svc.Uploader.BeginUpdate(info.PackageID, info.Affinity, info.CopyTo, copyToPath) up, err := s.svc.Uploader.BeginUpdate(info.PackageID, info.Affinity, info.CopyTo, copyToPath)
@@ -147,7 +147,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
} }
path = filepath.ToSlash(path) path = filepath.ToSlash(path)


err = up.Upload(jcsypes.PathFromJcsPathString(path), file)
err = up.Upload(jcstypes.PathFromJcsPathString(path), file)
if err != nil { if err != nil {
log.Warnf("uploading file: %s", err.Error()) log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err))) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
@@ -163,7 +163,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
return return
} }


uploadeds := make([]jcsypes.Object, len(pathes))
uploadeds := make([]jcstypes.Object, len(pathes))
for i := range pathes { for i := range pathes {
uploadeds[i] = ret.Objects[pathes[i]] uploadeds[i] = ret.Objects[pathes[i]]
} }
@@ -398,7 +398,7 @@ func (s *ObjectService) DeleteByPath(ctx *gin.Context) {
return return
} }


err = s.svc.ObjectSvc().Delete([]jcsypes.ObjectID{resp.Objects[0].ObjectID})
err = s.svc.ObjectSvc().Delete([]jcstypes.ObjectID{resp.Objects[0].ObjectID})
if err != nil { if err != nil {
log.Warnf("deleting objects: %s", err.Error()) log.Warnf("deleting objects: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "delete objects failed")) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "delete objects failed"))


+ 7
- 7
client/internal/http/v1/package.go View File

@@ -21,7 +21,7 @@ import (
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -151,9 +151,9 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
return return
} }


copyToPath := make([]jcsypes.JPath, 0, len(info.CopyToPath))
copyToPath := make([]jcstypes.JPath, 0, len(info.CopyToPath))
for _, p := range info.CopyToPath { for _, p := range info.CopyToPath {
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcstypes.PathFromJcsPathString(p))
} }


up, err := s.svc.Uploader.BeginCreateUpload(info.BucketID, info.Name, info.CopyTo, copyToPath) up, err := s.svc.Uploader.BeginCreateUpload(info.BucketID, info.Name, info.CopyTo, copyToPath)
@@ -184,7 +184,7 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
} }
path = filepath.ToSlash(path) path = filepath.ToSlash(path)


err = up.Upload(jcsypes.PathFromJcsPathString(path), file)
err = up.Upload(jcstypes.PathFromJcsPathString(path), file)
if err != nil { if err != nil {
log.Warnf("uploading file: %s", err.Error()) log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err))) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
@@ -200,7 +200,7 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
return return
} }


objs := make([]jcsypes.Object, len(pathes))
objs := make([]jcstypes.Object, len(pathes))
for i := range pathes { for i := range pathes {
objs[i] = ret.Objects[pathes[i]] objs[i] = ret.Objects[pathes[i]]
} }
@@ -233,7 +233,7 @@ func (s *PackageService) Download(ctx *gin.Context) {
} }
} }


func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg jcsypes.Package, iter downloader.DownloadIterator) {
func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg jcstypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download") log := logger.WithField("HTTP", "Package.Download")


ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".zip") ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".zip")
@@ -276,7 +276,7 @@ func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownloa
} }
} }


func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg jcsypes.Package, iter downloader.DownloadIterator) {
func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg jcstypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download") log := logger.WithField("HTTP", "Package.Download")


ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".tar") ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".tar")


+ 4
- 4
client/internal/http/v1/presigned.go View File

@@ -15,7 +15,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type PresignedService struct { type PresignedService struct {
@@ -156,9 +156,9 @@ func (s *PresignedService) ObjectUpload(ctx *gin.Context) {
return return
} }


copyToPath := make([]jcsypes.JPath, 0, len(req.CopyToPath))
copyToPath := make([]jcstypes.JPath, 0, len(req.CopyToPath))
for _, p := range req.CopyToPath { for _, p := range req.CopyToPath {
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcstypes.PathFromJcsPathString(p))
} }


up, err := s.svc.Uploader.BeginUpdate(req.PackageID, req.Affinity, req.CopyTo, copyToPath) up, err := s.svc.Uploader.BeginUpdate(req.PackageID, req.Affinity, req.CopyTo, copyToPath)
@@ -171,7 +171,7 @@ func (s *PresignedService) ObjectUpload(ctx *gin.Context) {


path := filepath.ToSlash(req.Path) path := filepath.ToSlash(req.Path)


err = up.Upload(jcsypes.PathFromJcsPathString(path), ctx.Request.Body)
err = up.Upload(jcstypes.PathFromJcsPathString(path), ctx.Request.Body)
if err != nil { if err != nil {
log.Warnf("uploading file: %s", err.Error()) log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", req.Path, err))) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", req.Path, err)))


+ 7
- 7
client/internal/http/v1/space_syncer.go View File

@@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type SpaceSyncerService struct { type SpaceSyncerService struct {
@@ -41,21 +41,21 @@ func (s *SpaceSyncerService) CreateTask(ctx *gin.Context) {
return return
} }


dests := make([]jcsypes.SpaceSyncDest, 0, len(req.DestUserSpaceIDs))
dests := make([]jcstypes.SpaceSyncDest, 0, len(req.DestUserSpaceIDs))
for _, id := range req.DestUserSpaceIDs { for _, id := range req.DestUserSpaceIDs {
dests = append(dests, jcsypes.SpaceSyncDest{
DestUserSpaceID: jcsypes.UserSpaceID(id),
DestPath: jcsypes.PathFromJcsPathString(req.DestPathes[0]),
dests = append(dests, jcstypes.SpaceSyncDest{
DestUserSpaceID: jcstypes.UserSpaceID(id),
DestPath: jcstypes.PathFromJcsPathString(req.DestPathes[0]),
}) })
} }


info, err := s.svc.SpaceSyncer.CreateTask(jcsypes.SpaceSyncTask{
info, err := s.svc.SpaceSyncer.CreateTask(jcstypes.SpaceSyncTask{
Trigger: req.Trigger, Trigger: req.Trigger,
Mode: req.Mode, Mode: req.Mode,
Filters: req.Filters, Filters: req.Filters,
Options: req.Options, Options: req.Options,
SrcUserSpaceID: req.SrcUserSpaceID, SrcUserSpaceID: req.SrcUserSpaceID,
SrcPath: jcsypes.PathFromJcsPathString(req.SrcPath),
SrcPath: jcstypes.PathFromJcsPathString(req.SrcPath),
Dests: dests, Dests: dests,
}) })
if err != nil { if err != nil {


+ 2
- 2
client/internal/http/v1/user_space.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type UserSpaceService struct { type UserSpaceService struct {
@@ -52,7 +52,7 @@ func (s *UserSpaceService) CreatePackage(ctx *gin.Context) {
return return
} }


pkg, err := s.svc.Uploader.UserSpaceUpload(req.UserSpaceID, jcsypes.PathFromJcsPathString(req.Path), req.BucketID, req.Name, req.SpaceAffinity)
pkg, err := s.svc.Uploader.UserSpaceUpload(req.UserSpaceID, jcstypes.PathFromJcsPathString(req.Path), req.BucketID, req.Name, req.SpaceAffinity)
if err != nil { if err != nil {
log.Warnf("userspace create package: %s", err.Error()) log.Warnf("userspace create package: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("userspace create package: %v", err))) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("userspace create package: %v", err)))


+ 9
- 9
client/internal/metacache/connectivity.go View File

@@ -8,12 +8,12 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator" corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func (m *MetaCacheHost) AddConnectivity() *Connectivity { func (m *MetaCacheHost) AddConnectivity() *Connectivity {
cache := &Connectivity{ cache := &Connectivity{
entries: make(map[cortypes.HubID]*ConnectivityEntry),
entries: make(map[jcstypes.HubID]*ConnectivityEntry),
} }


m.caches = append(m.caches, cache) m.caches = append(m.caches, cache)
@@ -22,10 +22,10 @@ func (m *MetaCacheHost) AddConnectivity() *Connectivity {


type Connectivity struct { type Connectivity struct {
lock sync.RWMutex lock sync.RWMutex
entries map[cortypes.HubID]*ConnectivityEntry
entries map[jcstypes.HubID]*ConnectivityEntry
} }


func (c *Connectivity) Get(from cortypes.HubID, to cortypes.HubID) *time.Duration {
func (c *Connectivity) Get(from jcstypes.HubID, to jcstypes.HubID) *time.Duration {
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
c.lock.RLock() c.lock.RLock()
entry, ok := c.entries[from] entry, ok := c.entries[from]
@@ -60,12 +60,12 @@ func (c *Connectivity) ClearOutdated() {
} }
} }


func (c *Connectivity) load(hubID cortypes.HubID) {
func (c *Connectivity) load(hubID jcstypes.HubID) {
coorCli := stgglb.CoordinatorRPCPool.Get() coorCli := stgglb.CoordinatorRPCPool.Get()


defer coorCli.Release() defer coorCli.Release()


get, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]cortypes.HubID{hubID}))
get, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]jcstypes.HubID{hubID}))
if cerr != nil { if cerr != nil {
logger.Warnf("get hub connectivities: %v", cerr) logger.Warnf("get hub connectivities: %v", cerr)
return return
@@ -76,7 +76,7 @@ func (c *Connectivity) load(hubID cortypes.HubID) {


ce := &ConnectivityEntry{ ce := &ConnectivityEntry{
From: hubID, From: hubID,
To: make(map[cortypes.HubID]cortypes.HubConnectivity),
To: make(map[jcstypes.HubID]jcstypes.HubConnectivity),
UpdateTime: time.Now(), UpdateTime: time.Now(),
} }


@@ -88,7 +88,7 @@ func (c *Connectivity) load(hubID cortypes.HubID) {
} }


type ConnectivityEntry struct { type ConnectivityEntry struct {
From cortypes.HubID
To map[cortypes.HubID]cortypes.HubConnectivity
From jcstypes.HubID
To map[jcstypes.HubID]jcstypes.HubConnectivity
UpdateTime time.Time UpdateTime time.Time
} }

+ 8
- 8
client/internal/metacache/hubmeta.go View File

@@ -7,12 +7,12 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator" corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func (m *MetaCacheHost) AddHubMeta() *HubMeta { func (m *MetaCacheHost) AddHubMeta() *HubMeta {
meta := &HubMeta{} meta := &HubMeta{}
meta.cache = NewSimpleMetaCache(SimpleMetaCacheConfig[cortypes.HubID, cortypes.Hub]{
meta.cache = NewSimpleMetaCache(SimpleMetaCacheConfig[jcstypes.HubID, jcstypes.Hub]{
Getter: meta.load, Getter: meta.load,
Expire: time.Minute * 5, Expire: time.Minute * 5,
}) })
@@ -22,10 +22,10 @@ func (m *MetaCacheHost) AddHubMeta() *HubMeta {
} }


type HubMeta struct { type HubMeta struct {
cache *SimpleMetaCache[cortypes.HubID, cortypes.Hub]
cache *SimpleMetaCache[jcstypes.HubID, jcstypes.Hub]
} }


func (h *HubMeta) Get(hubID cortypes.HubID) *cortypes.Hub {
func (h *HubMeta) Get(hubID jcstypes.HubID) *jcstypes.Hub {
v, ok := h.cache.Get(hubID) v, ok := h.cache.Get(hubID)
if ok { if ok {
return &v return &v
@@ -33,9 +33,9 @@ func (h *HubMeta) Get(hubID cortypes.HubID) *cortypes.Hub {
return nil return nil
} }


func (h *HubMeta) GetMany(hubIDs []cortypes.HubID) []*cortypes.Hub {
func (h *HubMeta) GetMany(hubIDs []jcstypes.HubID) []*jcstypes.Hub {
vs, oks := h.cache.GetMany(hubIDs) vs, oks := h.cache.GetMany(hubIDs)
ret := make([]*cortypes.Hub, len(vs))
ret := make([]*jcstypes.Hub, len(vs))
for i := range vs { for i := range vs {
if oks[i] { if oks[i] {
ret[i] = &vs[i] ret[i] = &vs[i]
@@ -48,8 +48,8 @@ func (h *HubMeta) ClearOutdated() {
h.cache.ClearOutdated() h.cache.ClearOutdated()
} }


func (h *HubMeta) load(keys []cortypes.HubID) ([]cortypes.Hub, []bool) {
vs := make([]cortypes.Hub, len(keys))
func (h *HubMeta) load(keys []jcstypes.HubID) ([]jcstypes.Hub, []bool) {
vs := make([]jcstypes.Hub, len(keys))
oks := make([]bool, len(keys)) oks := make([]bool, len(keys))


coorCli := stgglb.CoordinatorRPCPool.Get() coorCli := stgglb.CoordinatorRPCPool.Get()


+ 2
- 2
client/internal/metacache/user_space_meta.go View File

@@ -8,7 +8,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator" corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
"gitlink.org.cn/cloudream/jcs-pub/common/types" "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func (m *MetaCacheHost) AddStorageMeta() *UserSpaceMeta { func (m *MetaCacheHost) AddStorageMeta() *UserSpaceMeta {
@@ -78,7 +78,7 @@ func (s *UserSpaceMeta) load(keys []types.UserSpaceID) ([]types.UserSpaceDetail,
coorCli := stgglb.CoordinatorRPCPool.Get() coorCli := stgglb.CoordinatorRPCPool.Get()
defer coorCli.Release() defer coorCli.Release()


stgs := make([]cortypes.StorageType, len(spaces))
stgs := make([]jcstypes.StorageType, len(spaces))
for i := range spaces { for i := range spaces {
stgs[i] = spaces[i].Storage stgs[i] = spaces[i].Storage
} }


+ 4
- 4
client/internal/mount/mount_linux.go View File

@@ -12,7 +12,7 @@ import (
fuse2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" fuse2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type Mount struct { type Mount struct {
@@ -107,14 +107,14 @@ func (m *Mount) StartReclaimSpace() {
m.vfs.ReclaimSpace() m.vfs.ReclaimSpace()
} }


func (m *Mount) NotifyObjectInvalid(obj jcsypes.Object) {
func (m *Mount) NotifyObjectInvalid(obj jcstypes.Object) {


} }


func (m *Mount) NotifyPackageInvalid(pkg jcsypes.Package) {
func (m *Mount) NotifyPackageInvalid(pkg jcstypes.Package) {


} }


func (m *Mount) NotifyBucketInvalid(bkt jcsypes.Bucket) {
func (m *Mount) NotifyBucketInvalid(bkt jcstypes.Bucket) {


} }

+ 4
- 4
client/internal/mount/mount_win.go View File

@@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type Mount struct { type Mount struct {
@@ -43,14 +43,14 @@ func (m *Mount) StartReclaimSpace() {


} }


func (m *Mount) NotifyObjectInvalid(obj jcsypes.Object) {
func (m *Mount) NotifyObjectInvalid(obj jcstypes.Object) {


} }


func (m *Mount) NotifyPackageInvalid(pkg jcsypes.Package) {
func (m *Mount) NotifyPackageInvalid(pkg jcstypes.Package) {


} }


func (m *Mount) NotifyBucketInvalid(bkt jcsypes.Bucket) {
func (m *Mount) NotifyBucketInvalid(bkt jcstypes.Bucket) {


} }

+ 7
- 7
client/internal/mount/vfs/cache/cache.go View File

@@ -21,7 +21,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type CacheEntry interface { type CacheEntry interface {
@@ -196,7 +196,7 @@ func (c *Cache) CreateFile(pathComps []string) *CacheFile {
// 尝试加载缓存文件,如果文件不存在,则使用obj的信息创建一个新缓存文件,而如果obj为nil,那么会返回nil。 // 尝试加载缓存文件,如果文件不存在,则使用obj的信息创建一个新缓存文件,而如果obj为nil,那么会返回nil。
// //
// 记得使用Release减少引用计数 // 记得使用Release减少引用计数
func (c *Cache) LoadFile(pathComps []string, obj *jcsypes.Object) *CacheFile {
func (c *Cache) LoadFile(pathComps []string, obj *jcstypes.Object) *CacheFile {
c.lock.Lock() c.lock.Lock()
defer c.lock.Unlock() defer c.lock.Unlock()


@@ -490,7 +490,7 @@ func (c *Cache) Move(pathComps []string, newPathComps []string) error {
type syncPackage struct { type syncPackage struct {
bktName string bktName string
pkgName string pkgName string
pkg jcsypes.Package
pkg jcstypes.Package
upObjs []*uploadingObject upObjs []*uploadingObject
} }


@@ -917,7 +917,7 @@ func (c *Cache) doUpdatingOnly(pkgs []*syncPackage) {
pathes := make([]string, 0, len(p.upObjs)) pathes := make([]string, 0, len(p.upObjs))
modTimes := make([]time.Time, 0, len(p.upObjs)) modTimes := make([]time.Time, 0, len(p.upObjs))
for _, obj := range p.upObjs { for _, obj := range p.upObjs {
pathes = append(pathes, jcsypes.JoinObjectPath(obj.pathComps[2:]...))
pathes = append(pathes, jcstypes.JoinObjectPath(obj.pathComps[2:]...))
modTimes = append(modTimes, obj.modTime) modTimes = append(modTimes, obj.modTime)
} }


@@ -1008,7 +1008,7 @@ func (c *Cache) doUploading(pkgs []*syncPackage) {


counter := io2.Counter(&rd) counter := io2.Counter(&rd)


err = upder.Upload(jcsypes.PathFromComps(o.pathComps[2:]...), counter, uploader.UploadOption{
err = upder.Upload(jcstypes.PathFromComps(o.pathComps[2:]...), counter, uploader.UploadOption{
CreateTime: o.modTime, CreateTime: o.modTime,
}) })
if err != nil { if err != nil {
@@ -1036,8 +1036,8 @@ func (c *Cache) doUploading(pkgs []*syncPackage) {
continue continue
} }


oldPath := jcsypes.JoinObjectPath(o.pathComps[2:]...)
newPath := jcsypes.JoinObjectPath(o.cache.pathComps[2:]...)
oldPath := jcstypes.JoinObjectPath(o.pathComps[2:]...)
newPath := jcstypes.JoinObjectPath(o.cache.pathComps[2:]...)


if o.isDeleted { if o.isDeleted {
upder.CancelObject(oldPath) upder.CancelObject(oldPath)


+ 3
- 3
client/internal/mount/vfs/cache/file.go View File

@@ -14,7 +14,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type CacheLevel int type CacheLevel int
@@ -102,7 +102,7 @@ type CacheFile struct {
cache *Cache cache *Cache
pathComps []string pathComps []string
info FileInfo info FileInfo
remoteObj *jcsypes.Object
remoteObj *jcstypes.Object
rwLock *sync.RWMutex rwLock *sync.RWMutex
readers []*CacheFileHandle readers []*CacheFileHandle
writers []*CacheFileHandle writers []*CacheFileHandle
@@ -262,7 +262,7 @@ func loadCacheFile(cache *Cache, pathComps []string) (*CacheFile, error) {
return ch, nil return ch, nil
} }


func newCacheFileFromObject(cache *Cache, pathComps []string, obj *jcsypes.Object) (*CacheFile, error) {
func newCacheFileFromObject(cache *Cache, pathComps []string, obj *jcstypes.Object) (*CacheFile, error) {
metaPath := cache.GetCacheMetaPath(pathComps...) metaPath := cache.GetCacheMetaPath(pathComps...)
dataPath := cache.GetCacheDataPath(pathComps...) dataPath := cache.GetCacheDataPath(pathComps...)




+ 17
- 17
client/internal/mount/vfs/fuse.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -35,7 +35,7 @@ func child(vfs *Vfs, ctx context.Context, parent FuseNode, name string) (fuse.Fs
return nil return nil
} }


objPath := jcsypes.JoinObjectPath(childPathComps[2:]...)
objPath := jcstypes.JoinObjectPath(childPathComps[2:]...)
obj, err := d.Object().GetByPath(tx, pkg.PackageID, objPath) obj, err := d.Object().GetByPath(tx, pkg.PackageID, objPath)
if err == nil { if err == nil {
ret = newFileFromObject(vfs, childPathComps, obj) ret = newFileFromObject(vfs, childPathComps, obj)
@@ -45,7 +45,7 @@ func child(vfs *Vfs, ctx context.Context, parent FuseNode, name string) (fuse.Fs
return err return err
} }


err = d.Object().HasObjectWithPrefix(tx, pkg.PackageID, objPath+jcsypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, pkg.PackageID, objPath+jcstypes.ObjectPathSeparator)
if err == nil { if err == nil {
dir := vfs.cache.LoadDir(childPathComps, &cache.CreateDirOption{ dir := vfs.cache.LoadDir(childPathComps, &cache.CreateDirOption{
ModTime: time.Now(), ModTime: time.Now(),
@@ -98,10 +98,10 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
return err return err
} }


objPath := jcsypes.JoinObjectPath(myPathComps[2:]...)
objPath := jcstypes.JoinObjectPath(myPathComps[2:]...)
objPrefix := objPath objPrefix := objPath
if objPath != "" { if objPath != "" {
objPrefix += jcsypes.ObjectPathSeparator
objPrefix += jcstypes.ObjectPathSeparator
} }


objs, coms, err := d.Object().GetByPrefixGrouped(tx, pkg.PackageID, objPrefix) objs, coms, err := d.Object().GetByPrefixGrouped(tx, pkg.PackageID, objPrefix)
@@ -110,8 +110,8 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
} }


for _, dir := range coms { for _, dir := range coms {
dir = strings.TrimSuffix(dir, jcsypes.ObjectPathSeparator)
pathComps := lo2.AppendNew(myPathComps, jcsypes.BaseName(dir))
dir = strings.TrimSuffix(dir, jcstypes.ObjectPathSeparator)
pathComps := lo2.AppendNew(myPathComps, jcstypes.BaseName(dir))


cd := vfs.cache.LoadDir(pathComps, &cache.CreateDirOption{ cd := vfs.cache.LoadDir(pathComps, &cache.CreateDirOption{
ModTime: time.Now(), ModTime: time.Now(),
@@ -124,7 +124,7 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
} }


for _, obj := range objs { for _, obj := range objs {
pathComps := lo2.AppendNew(myPathComps, jcsypes.BaseName(obj.Path))
pathComps := lo2.AppendNew(myPathComps, jcstypes.BaseName(obj.Path))
file := newFileFromObject(vfs, pathComps, obj) file := newFileFromObject(vfs, pathComps, obj)
dbEntries[file.Name()] = file dbEntries[file.Name()] = file
} }
@@ -179,14 +179,14 @@ func newFile(vfs *Vfs, ctx context.Context, name string, parent FuseNode, flags


func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) error { func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) error {
pathComps := lo2.AppendNew(parent.PathComps(), name) pathComps := lo2.AppendNew(parent.PathComps(), name)
joinedPath := jcsypes.JoinObjectPath(pathComps[2:]...)
joinedPath := jcstypes.JoinObjectPath(pathComps[2:]...)
d := vfs.db d := vfs.db


// TODO 生成系统事件 // TODO 生成系统事件
return vfs.db.DoTx(func(tx db.SQLContext) error { return vfs.db.DoTx(func(tx db.SQLContext) error {
pkg, err := d.Package().GetByFullName(tx, pathComps[0], pathComps[1]) pkg, err := d.Package().GetByFullName(tx, pathComps[0], pathComps[1])
if err == nil { if err == nil {
err := d.Object().HasObjectWithPrefix(tx, pkg.PackageID, joinedPath+jcsypes.ObjectPathSeparator)
err := d.Object().HasObjectWithPrefix(tx, pkg.PackageID, joinedPath+jcstypes.ObjectPathSeparator)
if err == nil { if err == nil {
return fuse.ErrNotEmpty return fuse.ErrNotEmpty
} }
@@ -211,7 +211,7 @@ func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) er
func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode, newName string, newParent FuseNode) error { func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode, newName string, newParent FuseNode) error {
newParentPath := newParent.PathComps() newParentPath := newParent.PathComps()
newChildPath := lo2.AppendNew(newParentPath, newName) newChildPath := lo2.AppendNew(newParentPath, newName)
newChildPathJoined := jcsypes.JoinObjectPath(newChildPath[2:]...)
newChildPathJoined := jcstypes.JoinObjectPath(newChildPath[2:]...)


// 不允许移动任何内容到Package层级以上 // 不允许移动任何内容到Package层级以上
if len(newParentPath) < 2 { if len(newParentPath) < 2 {
@@ -219,7 +219,7 @@ func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode
} }


oldChildPath := lo2.AppendNew(oldParent.PathComps(), oldName) oldChildPath := lo2.AppendNew(oldParent.PathComps(), oldName)
oldChildPathJoined := jcsypes.JoinObjectPath(oldChildPath[2:]...)
oldChildPathJoined := jcstypes.JoinObjectPath(oldChildPath[2:]...)


// 先更新远程,再更新本地,因为远程使用事务更新,可以回滚,而本地不行 // 先更新远程,再更新本地,因为远程使用事务更新,可以回滚,而本地不行
return vfs.db.DoTx(func(tx db.SQLContext) error { return vfs.db.DoTx(func(tx db.SQLContext) error {
@@ -259,7 +259,7 @@ func moveRemote(vfs *Vfs, tx db.SQLContext, oldChildPath []string, newParentPath
return fuse.ErrExists return fuse.ErrExists
} }


err = d.Object().HasObjectWithPrefix(tx, newPkg.PackageID, newChildPathJoined+jcsypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, newPkg.PackageID, newChildPathJoined+jcstypes.ObjectPathSeparator)
if err == nil { if err == nil {
return fuse.ErrExists return fuse.ErrExists
} }
@@ -283,17 +283,17 @@ func moveRemote(vfs *Vfs, tx db.SQLContext, oldChildPath []string, newParentPath
oldObj.PackageID = newPkg.PackageID oldObj.PackageID = newPkg.PackageID
oldObj.Path = newChildPathJoined oldObj.Path = newChildPathJoined


return d.Object().BatchUpdate(tx, []jcsypes.Object{oldObj})
return d.Object().BatchUpdate(tx, []jcstypes.Object{oldObj})
} }
if err != gorm.ErrRecordNotFound { if err != gorm.ErrRecordNotFound {
return err return err
} }


err = d.Object().HasObjectWithPrefix(tx, oldPkg.PackageID, oldChildPathJoined+jcsypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, oldPkg.PackageID, oldChildPathJoined+jcstypes.ObjectPathSeparator)
if err == nil { if err == nil {
return d.Object().MoveByPrefix(tx, return d.Object().MoveByPrefix(tx,
oldPkg.PackageID, oldChildPathJoined+jcsypes.ObjectPathSeparator,
newPkg.PackageID, newChildPathJoined+jcsypes.ObjectPathSeparator,
oldPkg.PackageID, oldChildPathJoined+jcstypes.ObjectPathSeparator,
newPkg.PackageID, newChildPathJoined+jcstypes.ObjectPathSeparator,
) )
} }
if err == gorm.ErrRecordNotFound { if err == gorm.ErrRecordNotFound {


+ 2
- 2
client/internal/mount/vfs/fuse_bucket.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -117,7 +117,7 @@ func (r *FuseBucket) listChildren() ([]fuse.FsEntry, error) {
return nil, err return nil, err
} }


pkgMap := make(map[string]*jcsypes.Package)
pkgMap := make(map[string]*jcstypes.Package)
for _, pkg := range pkgs { for _, pkg := range pkgs {
p := pkg p := pkg
pkgMap[pkg.Name] = &p pkgMap[pkg.Name] = &p


+ 2
- 2
client/internal/mount/vfs/fuse_dir.go View File

@@ -8,7 +8,7 @@ import (
db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db" db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -108,7 +108,7 @@ func (r *FuseDir) loadCacheDir() *cache.CacheDir {
return err return err
} }


err = r.vfs.db.Object().HasObjectWithPrefix(tx, pkg.PackageID, jcsypes.JoinObjectPath(r.pathComps[2:]...))
err = r.vfs.db.Object().HasObjectWithPrefix(tx, pkg.PackageID, jcstypes.JoinObjectPath(r.pathComps[2:]...))
if err == nil { if err == nil {
createOpt = &cache.CreateDirOption{ createOpt = &cache.CreateDirOption{
ModTime: time.Now(), ModTime: time.Now(),


+ 3
- 3
client/internal/mount/vfs/fuse_file.go View File

@@ -6,7 +6,7 @@ import (


"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -28,7 +28,7 @@ func newFileFromCache(info cache.CacheEntryInfo, vfs *Vfs) *FuseFileNode {
} }
} }


func newFileFromObject(vfs *Vfs, pathComps []string, obj jcsypes.Object) *FuseFileNode {
func newFileFromObject(vfs *Vfs, pathComps []string, obj jcstypes.Object) *FuseFileNode {
return &FuseFileNode{ return &FuseFileNode{
vfs: vfs, vfs: vfs,
pathComps: pathComps, pathComps: pathComps,
@@ -117,7 +117,7 @@ func (n *FuseFileNode) loadCacheFile() *cache.CacheFile {
return n.vfs.cache.LoadFile(n.pathComps, nil) return n.vfs.cache.LoadFile(n.pathComps, nil)
} }


cdsObj, err := n.vfs.db.Object().GetByFullPath(n.vfs.db.DefCtx(), n.pathComps[0], n.pathComps[1], jcsypes.JoinObjectPath(n.pathComps[2:]...))
cdsObj, err := n.vfs.db.Object().GetByFullPath(n.vfs.db.DefCtx(), n.pathComps[0], n.pathComps[1], jcstypes.JoinObjectPath(n.pathComps[2:]...))
if err == nil { if err == nil {
file := n.vfs.cache.LoadFile(n.pathComps, &cdsObj) file := n.vfs.cache.LoadFile(n.pathComps, &cdsObj)
if file == nil { if file == nil {


+ 2
- 2
client/internal/mount/vfs/fuse_root.go View File

@@ -8,7 +8,7 @@ import (
db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db" db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -104,7 +104,7 @@ func (r *FuseRoot) listChildren() ([]fuse.FsEntry, error) {
return nil, err return nil, err
} }


bktMap := make(map[string]*jcsypes.Bucket)
bktMap := make(map[string]*jcstypes.Bucket)
for _, bkt := range bkts { for _, bkt := range bkts {
b := bkt b := bkt
bktMap[bkt.Name] = &b bktMap[bkt.Name] = &b


+ 4
- 4
client/internal/repl/load.go View File

@@ -8,7 +8,7 @@ import (
"time" "time"


"github.com/spf13/cobra" "github.com/spf13/cobra"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -31,7 +31,7 @@ func init() {
fmt.Printf("Invalid user space ID: %s\n", args[1]) fmt.Printf("Invalid user space ID: %s\n", args[1])
} }


loadByID(cmdCtx, jcsypes.PackageID(pkgID), jcsypes.UserSpaceID(userSpaceID), args[2])
loadByID(cmdCtx, jcstypes.PackageID(pkgID), jcstypes.UserSpaceID(userSpaceID), args[2])
} else { } else {
loadByPath(cmdCtx, args[0], args[1], args[2]) loadByPath(cmdCtx, args[0], args[1], args[2])
} }
@@ -42,7 +42,7 @@ func init() {
} }


func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) { func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) {
comps := strings.Split(strings.Trim(pkgPath, jcsypes.ObjectPathSeparator), jcsypes.ObjectPathSeparator)
comps := strings.Split(strings.Trim(pkgPath, jcstypes.ObjectPathSeparator), jcstypes.ObjectPathSeparator)
if len(comps) != 2 { if len(comps) != 2 {
fmt.Printf("Package path must be in format of <bucket>/<package>") fmt.Printf("Package path must be in format of <bucket>/<package>")
return return
@@ -63,7 +63,7 @@ func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath
loadByID(cmdCtx, pkg.PackageID, stg.StorageID, rootPath) loadByID(cmdCtx, pkg.PackageID, stg.StorageID, rootPath)
} }


func loadByID(cmdCtx *CommandContext, pkgID jcsypes.PackageID, stgID jcsypes.StorageID, rootPath string) {
func loadByID(cmdCtx *CommandContext, pkgID jcstypes.PackageID, stgID jcstypes.StorageID, rootPath string) {
startTime := time.Now() startTime := time.Now()


err := cmdCtx.Cmdline.Svc.StorageSvc().LoadPackage(pkgID, stgID, rootPath) err := cmdCtx.Cmdline.Svc.StorageSvc().LoadPackage(pkgID, stgID, rootPath)


+ 4
- 4
client/internal/repl/lsp.go View File

@@ -7,7 +7,7 @@ import (


"github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra" "github.com/spf13/cobra"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -26,7 +26,7 @@ func init() {
return return
} }


lspOneByID(cmdCtx, jcsypes.PackageID(id))
lspOneByID(cmdCtx, jcstypes.PackageID(id))
} else { } else {
lspByPath(cmdCtx, args[0]) lspByPath(cmdCtx, args[0])
} }
@@ -40,7 +40,7 @@ func init() {
func lspByPath(cmdCtx *CommandContext, path string) { func lspByPath(cmdCtx *CommandContext, path string) {
db2 := cmdCtx.repl.db db2 := cmdCtx.repl.db


comps := strings.Split(strings.Trim(path, jcsypes.ObjectPathSeparator), jcsypes.ObjectPathSeparator)
comps := strings.Split(strings.Trim(path, jcstypes.ObjectPathSeparator), jcstypes.ObjectPathSeparator)
if len(comps) != 2 { if len(comps) != 2 {
fmt.Printf("Package path must be in format of <bucket>/<package>") fmt.Printf("Package path must be in format of <bucket>/<package>")
return return
@@ -58,7 +58,7 @@ func lspByPath(cmdCtx *CommandContext, path string) {
fmt.Println(wr.Render()) fmt.Println(wr.Render())
} }


func lspOneByID(cmdCtx *CommandContext, id jcsypes.PackageID) {
func lspOneByID(cmdCtx *CommandContext, id jcstypes.PackageID) {
db2 := cmdCtx.repl.db db2 := cmdCtx.repl.db


pkg, err := db2.Package().GetByID(db2.DefCtx(), id) pkg, err := db2.Package().GetByID(db2.DefCtx(), id)


+ 26
- 26
client/internal/services/user_space.go View File

@@ -8,7 +8,7 @@ import (
"github.com/samber/lo" "github.com/samber/lo"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"


"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
@@ -31,41 +31,41 @@ func (svc *Service) UserSpaceSvc() *UserSpaceService {
return &UserSpaceService{Service: svc} return &UserSpaceService{Service: svc}
} }


func (svc *UserSpaceService) Get(userspaceID jcsypes.UserSpaceID) (jcsypes.UserSpace, error) {
func (svc *UserSpaceService) Get(userspaceID jcstypes.UserSpaceID) (jcstypes.UserSpace, error) {
return svc.DB.UserSpace().GetByID(svc.DB.DefCtx(), userspaceID) return svc.DB.UserSpace().GetByID(svc.DB.DefCtx(), userspaceID)
} }


func (svc *UserSpaceService) GetByName(name string) (jcsypes.UserSpace, error) {
func (svc *UserSpaceService) GetByName(name string) (jcstypes.UserSpace, error) {
return svc.DB.UserSpace().GetByName(svc.DB.DefCtx(), name) return svc.DB.UserSpace().GetByName(svc.DB.DefCtx(), name)
} }


func (svc *UserSpaceService) GetAll() ([]jcsypes.UserSpace, error) {
func (svc *UserSpaceService) GetAll() ([]jcstypes.UserSpace, error) {
return svc.DB.UserSpace().GetAll(svc.DB.DefCtx()) return svc.DB.UserSpace().GetAll(svc.DB.DefCtx())
} }


func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpaceCreateResp, *ecode.CodeError) { func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpaceCreateResp, *ecode.CodeError) {
db2 := svc.DB db2 := svc.DB
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcsypes.UserSpace, error) {
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcstypes.UserSpace, error) {
space, err := db2.UserSpace().GetByName(tx, req.Name) space, err := db2.UserSpace().GetByName(tx, req.Name)
if err == nil { if err == nil {
return jcsypes.UserSpace{}, gorm.ErrDuplicatedKey
return jcstypes.UserSpace{}, gorm.ErrDuplicatedKey
} }
if err != gorm.ErrRecordNotFound { if err != gorm.ErrRecordNotFound {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
} }


space = jcsypes.UserSpace{
space = jcstypes.UserSpace{
Name: req.Name, Name: req.Name,
Storage: req.Storage, Storage: req.Storage,
Credential: req.Credential, Credential: req.Credential,
ShardStore: req.ShardStore, ShardStore: req.ShardStore,
Features: req.Features, Features: req.Features,
WorkingDir: jcsypes.PathFromJcsPathString(req.WorkingDir),
WorkingDir: jcstypes.PathFromJcsPathString(req.WorkingDir),
Revision: 0, Revision: 0,
} }
err = db2.UserSpace().Create(tx, &space) err = db2.UserSpace().Create(tx, &space)
if err != nil { if err != nil {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
} }
return space, nil return space, nil
}) })
@@ -80,19 +80,19 @@ func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpa


func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpaceUpdateResp, *ecode.CodeError) { func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpaceUpdateResp, *ecode.CodeError) {
db2 := svc.DB db2 := svc.DB
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcsypes.UserSpace, error) {
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcstypes.UserSpace, error) {
space, err := db2.UserSpace().GetByID(tx, req.UserSpaceID) space, err := db2.UserSpace().GetByID(tx, req.UserSpaceID)
if err != nil { if err != nil {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
} }


if space.Name != req.Name { if space.Name != req.Name {
_, err = db2.UserSpace().GetByName(tx, req.Name) _, err = db2.UserSpace().GetByName(tx, req.Name)
if err == nil { if err == nil {
return jcsypes.UserSpace{}, gorm.ErrDuplicatedKey
return jcstypes.UserSpace{}, gorm.ErrDuplicatedKey
} }
if err != gorm.ErrRecordNotFound { if err != gorm.ErrRecordNotFound {
return jcsypes.UserSpace{}, err
return jcstypes.UserSpace{}, err
} }
} }


@@ -110,7 +110,7 @@ func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpa
} }


// 通知元数据缓存无效 // 通知元数据缓存无效
svc.UserSpaceMeta.Drop([]jcsypes.UserSpaceID{req.UserSpaceID})
svc.UserSpaceMeta.Drop([]jcstypes.UserSpaceID{req.UserSpaceID})


// 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理 // 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理
svc.StgPool.Drop(stgglb.UserID, space.UserSpaceID) svc.StgPool.Drop(stgglb.UserID, space.UserSpaceID)
@@ -155,7 +155,7 @@ func (svc *UserSpaceService) Delete(req cliapi.UserSpaceDelete) (*cliapi.UserSpa
} }


// 通知元数据缓存无效 // 通知元数据缓存无效
svc.UserSpaceMeta.Drop([]jcsypes.UserSpaceID{req.UserSpaceID})
svc.UserSpaceMeta.Drop([]jcstypes.UserSpaceID{req.UserSpaceID})


// 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理 // 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理
svc.StgPool.Drop(stgglb.UserID, req.UserSpaceID) svc.StgPool.Drop(stgglb.UserID, req.UserSpaceID)
@@ -166,13 +166,13 @@ func (svc *UserSpaceService) Delete(req cliapi.UserSpaceDelete) (*cliapi.UserSpa
} }


func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTestResp, *ecode.CodeError) { func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTestResp, *ecode.CodeError) {
detail := jcsypes.UserSpaceDetail{
detail := jcstypes.UserSpaceDetail{
UserID: stgglb.UserID, UserID: stgglb.UserID,
UserSpace: jcsypes.UserSpace{
UserSpace: jcstypes.UserSpace{
Name: "test", Name: "test",
Storage: req.Storage, Storage: req.Storage,
Credential: req.Credential, Credential: req.Credential,
WorkingDir: jcsypes.PathFromJcsPathString(req.WorikingDir),
WorkingDir: jcstypes.PathFromJcsPathString(req.WorikingDir),
}, },
} }
blder := factory.GetBuilder(&detail) blder := factory.GetBuilder(&detail)
@@ -189,7 +189,7 @@ func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTe
return &cliapi.UserSpaceTestResp{}, nil return &cliapi.UserSpaceTestResp{}, nil
} }


func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, userspaceID jcsypes.UserSpaceID, rootPath string) error {
func (svc *UserSpaceService) DownloadPackage(packageID jcstypes.PackageID, userspaceID jcstypes.UserSpaceID, rootPath string) error {
destSpace := svc.UserSpaceMeta.Get(userspaceID) destSpace := svc.UserSpaceMeta.Get(userspaceID)
if destSpace == nil { if destSpace == nil {
return fmt.Errorf("userspace not found: %d", userspaceID) return fmt.Errorf("userspace not found: %d", userspaceID)
@@ -208,10 +208,10 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
} }
defer mutex.Unlock() defer mutex.Unlock()


rootJPath := jcsypes.PathFromJcsPathString(rootPath)
rootJPath := jcstypes.PathFromJcsPathString(rootPath)


dIndex := 0 dIndex := 0
var pinned []jcsypes.PinnedObject
var pinned []jcstypes.PinnedObject
for dIndex < len(details) { for dIndex < len(details) {
plans := exec.NewPlanBuilder() plans := exec.NewPlanBuilder()
for i := 0; i < 10 && dIndex < len(details); i++ { for i := 0; i < 10 && dIndex < len(details); i++ {
@@ -252,7 +252,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
return fmt.Errorf("unsupported download strategy: %T", strg) return fmt.Errorf("unsupported download strategy: %T", strg)
} }


objPath := jcsypes.PathFromJcsPathString(details[dIndex].Object.Path)
objPath := jcstypes.PathFromJcsPathString(details[dIndex].Object.Path)
dstPath := rootJPath.ConcatNew(objPath) dstPath := rootJPath.ConcatNew(objPath)


newDstSpace := *destSpace newDstSpace := *destSpace
@@ -265,7 +265,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
// 顺便保存到同存储服务的分片存储中 // 顺便保存到同存储服务的分片存储中
if destSpace.UserSpace.ShardStore != nil { if destSpace.UserSpace.ShardStore != nil {
ft.AddTo(ioswitch2.NewToShardStore(newDstSpace, ioswitch2.RawStream(), "")) ft.AddTo(ioswitch2.NewToShardStore(newDstSpace, ioswitch2.RawStream(), ""))
pinned = append(pinned, jcsypes.PinnedObject{
pinned = append(pinned, jcstypes.PinnedObject{
ObjectID: details[dIndex].Object.ObjectID, ObjectID: details[dIndex].Object.ObjectID,
UserSpaceID: destSpace.UserSpace.UserSpaceID, UserSpaceID: destSpace.UserSpace.UserSpaceID,
CreateTime: time.Now(), CreateTime: time.Now(),
@@ -297,7 +297,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
} }


err = svc.DB.DoTx(func(tx db.SQLContext) error { err = svc.DB.DoTx(func(tx db.SQLContext) error {
objIDs := make([]jcsypes.ObjectID, len(pinned))
objIDs := make([]jcstypes.ObjectID, len(pinned))
for i, obj := range pinned { for i, obj := range pinned {
objIDs[i] = obj.ObjectID objIDs[i] = obj.ObjectID
} }
@@ -307,7 +307,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, usersp
return err return err
} }


pinned = lo.Filter(pinned, func(p jcsypes.PinnedObject, idx int) bool { return avaiIDs[p.ObjectID] })
pinned = lo.Filter(pinned, func(p jcstypes.PinnedObject, idx int) bool { return avaiIDs[p.ObjectID] })
return svc.DB.PinnedObject().BatchTryCreate(svc.DB.DefCtx(), pinned) return svc.DB.PinnedObject().BatchTryCreate(svc.DB.DefCtx(), pinned)
}) })
if err != nil { if err != nil {


+ 6
- 6
client/internal/services/utils.go View File

@@ -1,22 +1,22 @@
package services package services


import ( import (
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap" "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


func getBlockTypeFromRed(red jcsypes.Redundancy) string {
func getBlockTypeFromRed(red jcstypes.Redundancy) string {
switch red.(type) { switch red.(type) {
case *jcsypes.NoneRedundancy:
case *jcstypes.NoneRedundancy:
return datamap.BlockTypeRaw return datamap.BlockTypeRaw


case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
return datamap.BlockTypeEC return datamap.BlockTypeEC


case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
return datamap.BlockTypeEC return datamap.BlockTypeEC


case *jcsypes.SegmentRedundancy:
case *jcstypes.SegmentRedundancy:
return datamap.BlockTypeSegment return datamap.BlockTypeSegment
} }
return "" return ""


+ 7
- 7
client/internal/spacesyncer/execute_diff.go View File

@@ -12,10 +12,10 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDiff) {
func executeDiff(syncer *SpaceSyncer, task *task, mode *jcstypes.SpaceSyncModeDiff) {
log := logger.WithField("Mod", logMod).WithField("TaskID", task.Task.TaskID) log := logger.WithField("Mod", logMod).WithField("TaskID", task.Task.TaskID)


startTime := time.Now() startTime := time.Now()
@@ -112,7 +112,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDif
} }


var willSync []stgtypes.DirEntry var willSync []stgtypes.DirEntry
var willMkdirs []jcsypes.JPath
var willMkdirs []jcstypes.JPath


dirTree.Iterate(func(path []string, node *trie.Node[srcDstDirEntry], isWordNode bool) trie.VisitCtrl { dirTree.Iterate(func(path []string, node *trie.Node[srcDstDirEntry], isWordNode bool) trie.VisitCtrl {
if node.Value.src == nil { if node.Value.src == nil {
@@ -123,7 +123,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDif
if node.Value.src.IsDir { if node.Value.src.IsDir {
if node.Value.dst == nil { if node.Value.dst == nil {
if node.IsEmpty() { if node.IsEmpty() {
willMkdirs = append(willMkdirs, jcsypes.PathFromComps(path...))
willMkdirs = append(willMkdirs, jcstypes.PathFromComps(path...))
} }
} }
} else { } else {
@@ -193,7 +193,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDif
} }
} }


func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *stgtypes.DirEntry) {
func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcstypes.JPath, e *stgtypes.DirEntry) {
var ptr = &tree.Root var ptr = &tree.Root
for _, c := range path.Comps() { for _, c := range path.Comps() {
if ptr.Value.src != nil && ptr.Value.src.IsDir { if ptr.Value.src != nil && ptr.Value.src.IsDir {
@@ -205,7 +205,7 @@ func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *s
ptr.Value.src = e ptr.Value.src = e
} }


func diffCreateDstNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *stgtypes.DirEntry) {
func diffCreateDstNode(tree *trie.Trie[srcDstDirEntry], path jcstypes.JPath, e *stgtypes.DirEntry) {
var ptr = &tree.Root var ptr = &tree.Root
for _, c := range path.Comps() { for _, c := range path.Comps() {
if ptr.Value.src != nil && ptr.Value.src.IsDir { if ptr.Value.src != nil && ptr.Value.src.IsDir {
@@ -227,7 +227,7 @@ type srcDstDirEntry struct {
dst *stgtypes.DirEntry dst *stgtypes.DirEntry
} }


func cmpFile(diff *jcsypes.SpaceSyncModeDiff, src, dst *stgtypes.DirEntry) bool {
func cmpFile(diff *jcstypes.SpaceSyncModeDiff, src, dst *stgtypes.DirEntry) bool {
if diff.IncludeSize && src.Size != dst.Size { if diff.IncludeSize && src.Size != dst.Size {
return false return false
} }


+ 3
- 3
client/internal/spacesyncer/filter.go View File

@@ -2,7 +2,7 @@ package spacesyncer


import ( import (
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type FilterFn func(info stgtypes.DirEntry) bool type FilterFn func(info stgtypes.DirEntry) bool
@@ -11,7 +11,7 @@ func buildFilter(task *task) FilterFn {
var fns []FilterFn var fns []FilterFn
for _, f := range task.Task.Filters { for _, f := range task.Task.Filters {
switch f := f.(type) { switch f := f.(type) {
case *jcsypes.SpaceSyncFilterSize:
case *jcstypes.SpaceSyncFilterSize:
fns = append(fns, filterSize(f)) fns = append(fns, filterSize(f))
} }
} }
@@ -26,7 +26,7 @@ func buildFilter(task *task) FilterFn {
} }
} }


func filterSize(filter *jcsypes.SpaceSyncFilterSize) FilterFn {
func filterSize(filter *jcstypes.SpaceSyncFilterSize) FilterFn {
return func(info stgtypes.DirEntry) bool { return func(info stgtypes.DirEntry) bool {
if filter.MinSize > 0 && info.Size < filter.MinSize { if filter.MinSize > 0 && info.Size < filter.MinSize {
return false return false


+ 16
- 16
client/internal/spacesyncer/space_syncer.go View File

@@ -10,7 +10,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache"
stgpool "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" stgpool "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


const ( const (
@@ -26,7 +26,7 @@ type SpaceSyncer struct {
stgPool *stgpool.Pool stgPool *stgpool.Pool
spaceMeta *metacache.UserSpaceMeta spaceMeta *metacache.UserSpaceMeta
lock sync.Mutex lock sync.Mutex
tasks map[jcsypes.SpaceSyncTaskID]*task
tasks map[jcstypes.SpaceSyncTaskID]*task
} }


func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *SpaceSyncer { func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *SpaceSyncer {
@@ -34,7 +34,7 @@ func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *
db: db, db: db,
stgPool: stgPool, stgPool: stgPool,
spaceMeta: spaceMeta, spaceMeta: spaceMeta,
tasks: make(map[jcsypes.SpaceSyncTaskID]*task),
tasks: make(map[jcstypes.SpaceSyncTaskID]*task),
} }
} }


@@ -50,7 +50,7 @@ func (s *SpaceSyncer) Start() *async.UnboundChannel[SpaceSyncerEvent] {
if err != nil { if err != nil {
log.Warnf("load task from db: %v", err) log.Warnf("load task from db: %v", err)
} else { } else {
var rms []jcsypes.SpaceSyncTaskID
var rms []jcstypes.SpaceSyncTaskID
for _, t := range allTask { for _, t := range allTask {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
tsk := task{ tsk := task{
@@ -60,14 +60,14 @@ func (s *SpaceSyncer) Start() *async.UnboundChannel[SpaceSyncerEvent] {
} }


switch tr := t.Trigger.(type) { switch tr := t.Trigger.(type) {
case *jcsypes.SpaceSyncTriggerOnce:
case *jcstypes.SpaceSyncTriggerOnce:
// Once类型的任务没有执行完也不执行了 // Once类型的任务没有执行完也不执行了
rms = append(rms, t.TaskID) rms = append(rms, t.TaskID)


case *jcsypes.SpaceSyncTriggerInterval:
case *jcstypes.SpaceSyncTriggerInterval:
triggerInterval(s, &tsk, tr) triggerInterval(s, &tsk, tr)


case *jcsypes.SpaceSyncTriggerAt:
case *jcstypes.SpaceSyncTriggerAt:
triggerAt(s, &tsk, tr) triggerAt(s, &tsk, tr)
} }


@@ -95,10 +95,10 @@ func (s *SpaceSyncer) Stop() {
t.CancelFn() t.CancelFn()
} }


s.tasks = make(map[jcsypes.SpaceSyncTaskID]*task)
s.tasks = make(map[jcstypes.SpaceSyncTaskID]*task)
} }


func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
func (s *SpaceSyncer) CreateTask(t jcstypes.SpaceSyncTask) (*TaskInfo, error) {
log := logger.WithField("Mod", logMod) log := logger.WithField("Mod", logMod)


d := s.db d := s.db
@@ -126,13 +126,13 @@ func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
s.lock.Unlock() s.lock.Unlock()


switch tr := t.Trigger.(type) { switch tr := t.Trigger.(type) {
case *jcsypes.SpaceSyncTriggerOnce:
case *jcstypes.SpaceSyncTriggerOnce:
triggerOnce(s, &tsk) triggerOnce(s, &tsk)


case *jcsypes.SpaceSyncTriggerInterval:
case *jcstypes.SpaceSyncTriggerInterval:
triggerInterval(s, &tsk, tr) triggerInterval(s, &tsk, tr)


case *jcsypes.SpaceSyncTriggerAt:
case *jcstypes.SpaceSyncTriggerAt:
triggerAt(s, &tsk, tr) triggerAt(s, &tsk, tr)
} }


@@ -143,7 +143,7 @@ func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
}, nil }, nil
} }


func (s *SpaceSyncer) CancelTask(taskID jcsypes.SpaceSyncTaskID) {
func (s *SpaceSyncer) CancelTask(taskID jcstypes.SpaceSyncTaskID) {
log := logger.WithField("Mod", logMod) log := logger.WithField("Mod", logMod)


s.lock.Lock() s.lock.Lock()
@@ -166,7 +166,7 @@ func (s *SpaceSyncer) CancelTask(taskID jcsypes.SpaceSyncTaskID) {
log.Infof("task %v canceled", taskID) log.Infof("task %v canceled", taskID)
} }


func (s *SpaceSyncer) GetTask(taskID jcsypes.SpaceSyncTaskID) *jcsypes.SpaceSyncTask {
func (s *SpaceSyncer) GetTask(taskID jcstypes.SpaceSyncTaskID) *jcstypes.SpaceSyncTask {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


@@ -181,11 +181,11 @@ func (s *SpaceSyncer) GetTask(taskID jcsypes.SpaceSyncTaskID) *jcsypes.SpaceSync
} }


type TaskInfo struct { type TaskInfo struct {
Task jcsypes.SpaceSyncTask
Task jcstypes.SpaceSyncTask
} }


type task struct { type task struct {
Task jcsypes.SpaceSyncTask
Task jcstypes.SpaceSyncTask
Context context.Context Context context.Context
CancelFn func() CancelFn func()
} }

+ 3
- 3
client/internal/spacesyncer/trigger.go View File

@@ -5,7 +5,7 @@ import (


"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func triggerOnce(syncer *SpaceSyncer, task *task) { func triggerOnce(syncer *SpaceSyncer, task *task) {
@@ -31,7 +31,7 @@ func triggerOnce(syncer *SpaceSyncer, task *task) {
}() }()
} }


func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSyncTriggerInterval) {
func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcstypes.SpaceSyncTriggerInterval) {
go func() { go func() {
log := logger.WithField("Mod", logMod) log := logger.WithField("Mod", logMod)


@@ -66,7 +66,7 @@ func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSync
}() }()
} }


func triggerAt(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSyncTriggerAt) {
func triggerAt(syncer *SpaceSyncer, task *task, trigger *jcstypes.SpaceSyncTriggerAt) {
go func() { go func() {
log := logger.WithField("Mod", logMod) log := logger.WithField("Mod", logMod)




+ 6
- 6
client/internal/speedstats/speedstats.go View File

@@ -5,7 +5,7 @@ import (
"sync" "sync"
"time" "time"


jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


const ( const (
@@ -121,9 +121,9 @@ func (p *SpeedStats) ShouldAtClient(size int64) bool {
return v < prob return v < prob
} }


func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
return jcsypes.SpeedStatsStatus{
Below100M: []jcsypes.SpeedStatsStatusEntry{
func (p *SpeedStats) DumpStatus() jcstypes.SpeedStatsStatus {
return jcstypes.SpeedStatsStatus{
Below100M: []jcstypes.SpeedStatsStatusEntry{
{ {
TotalSize: p.stats100M[0].TotalSize, TotalSize: p.stats100M[0].TotalSize,
TotalTime: p.stats100M[0].TotalTime, TotalTime: p.stats100M[0].TotalTime,
@@ -137,7 +137,7 @@ func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
LastSpeed: p.stats100M[1].LastSpeed, LastSpeed: p.stats100M[1].LastSpeed,
}, },
}, },
Below1G: []jcsypes.SpeedStatsStatusEntry{
Below1G: []jcstypes.SpeedStatsStatusEntry{
{ {
TotalSize: p.stats1G[0].TotalSize, TotalSize: p.stats1G[0].TotalSize,
TotalTime: p.stats1G[0].TotalTime, TotalTime: p.stats1G[0].TotalTime,
@@ -151,7 +151,7 @@ func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
LastSpeed: p.stats1G[1].LastSpeed, LastSpeed: p.stats1G[1].LastSpeed,
}, },
}, },
Above1G: []jcsypes.SpeedStatsStatusEntry{
Above1G: []jcstypes.SpeedStatsStatusEntry{
{ {
TotalSize: p.statsAbove1G[0].TotalSize, TotalSize: p.statsAbove1G[0].TotalSize,
TotalTime: p.statsAbove1G[0].TotalTime, TotalTime: p.statsAbove1G[0].TotalTime,


+ 9
- 9
client/internal/ticktock/change_redundancy.go View File

@@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/reflect2" "gitlink.org.cn/cloudream/common/utils/reflect2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap" "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


@@ -34,7 +34,7 @@ func (j *ChangeRedundancy) Execute(t *TickTock) {


ctx := &changeRedundancyContext{ ctx := &changeRedundancyContext{
ticktock: t, ticktock: t,
allUserSpaces: make(map[jcsypes.UserSpaceID]*userSpaceUsageInfo),
allUserSpaces: make(map[jcstypes.UserSpaceID]*userSpaceUsageInfo),
} }


spaceIDs, err := t.db.UserSpace().GetAllIDs(t.db.DefCtx()) spaceIDs, err := t.db.UserSpace().GetAllIDs(t.db.DefCtx())
@@ -58,7 +58,7 @@ func (j *ChangeRedundancy) Execute(t *TickTock) {
return return
} }


lastPkgID := jcsypes.PackageID(0)
lastPkgID := jcstypes.PackageID(0)


loop: loop:
for { for {
@@ -90,16 +90,16 @@ loop:


type changeRedundancyContext struct { type changeRedundancyContext struct {
ticktock *TickTock ticktock *TickTock
allUserSpaces map[jcsypes.UserSpaceID]*userSpaceUsageInfo
mostBlockStgIDs []jcsypes.UserSpaceID
allUserSpaces map[jcstypes.UserSpaceID]*userSpaceUsageInfo
mostBlockStgIDs []jcstypes.UserSpaceID
} }


type userSpaceUsageInfo struct { type userSpaceUsageInfo struct {
UserSpace *jcsypes.UserSpaceDetail
UserSpace *jcstypes.UserSpaceDetail
AccessAmount float64 AccessAmount float64
} }


func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.PackageDetail) error {
func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcstypes.PackageDetail) error {
log := logger.WithType[ChangeRedundancy]("TickTock") log := logger.WithType[ChangeRedundancy]("TickTock")
db2 := ctx.ticktock.db db2 := ctx.ticktock.db


@@ -121,7 +121,7 @@ func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.P
info.AccessAmount = stat.Amount info.AccessAmount = stat.Amount
} }


lastObjID := jcsypes.ObjectID(0)
lastObjID := jcstypes.ObjectID(0)
for { for {
objs, err := db.DoTx31(db2, db2.Object().BatchGetDetailsPaged, pkg.Package.PackageID, lastObjID, BatchGetObjectDetailCount) objs, err := db.DoTx31(db2, db2.Object().BatchGetDetailsPaged, pkg.Package.PackageID, lastObjID, BatchGetObjectDetailCount)
if err != nil { if err != nil {
@@ -139,7 +139,7 @@ func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.P


ctx.mostBlockStgIDs = j.summaryRepObjectBlockUserSpaces(ctx, objs, 2) ctx.mostBlockStgIDs = j.summaryRepObjectBlockUserSpaces(ctx, objs, 2)


var willShrinks []jcsypes.ObjectDetail
var willShrinks []jcstypes.ObjectDetail


for _, obj := range objs { for _, obj := range objs {
newRed, selectedSpaces := j.chooseRedundancy(ctx, obj) newRed, selectedSpaces := j.chooseRedundancy(ctx, obj)


+ 9
- 9
client/internal/ticktock/check_shardstore.go View File

@@ -10,7 +10,7 @@ import (


"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


// CheckShardStore 代表一个用于处理代理缓存检查事件的结构体 // CheckShardStore 代表一个用于处理代理缓存检查事件的结构体
@@ -52,8 +52,8 @@ func (j *CheckShardStore) Execute(t *TickTock) {
} }
} }


func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail) error {
// addr, ok := space.RecommendHub.Address.(*cortypes.GRPCAddressInfo)
func (j *CheckShardStore) checkOne(t *TickTock, space *jcstypes.UserSpaceDetail) error {
// addr, ok := space.RecommendHub.Address.(*jcstypes.GRPCAddressInfo)
// if !ok { // if !ok {
// return fmt.Errorf("master of user space %v has no grpc address", space.UserSpace) // return fmt.Errorf("master of user space %v has no grpc address", space.UserSpace)
// } // }
@@ -80,9 +80,9 @@ func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail)
return fmt.Errorf("listing all files: %w", err) return fmt.Errorf("listing all files: %w", err)
} }


fileHashes := lo.Map(infos, func(info stgtypes.FileInfo, _ int) jcsypes.FileHash { return info.Hash })
fileHashes := lo.Map(infos, func(info stgtypes.FileInfo, _ int) jcstypes.FileHash { return info.Hash })


realFileHashes := lo.SliceToMap(fileHashes, func(hash jcsypes.FileHash) (jcsypes.FileHash, bool) { return hash, true })
realFileHashes := lo.SliceToMap(fileHashes, func(hash jcstypes.FileHash) (jcstypes.FileHash, bool) { return hash, true })


// 在事务中执行缓存更新操作 // 在事务中执行缓存更新操作
t.db.DoTx(func(tx db.SQLContext) error { t.db.DoTx(func(tx db.SQLContext) error {
@@ -95,7 +95,7 @@ func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail)
} }


// checkPinnedObject 对比PinnedObject表,若实际文件不存在,则进行删除操作 // checkPinnedObject 对比PinnedObject表,若实际文件不存在,则进行删除操作
func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *jcsypes.UserSpaceDetail, realFileHashes map[jcsypes.FileHash]bool) {
func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *jcstypes.UserSpaceDetail, realFileHashes map[jcstypes.FileHash]bool) {
log := logger.WithType[CheckShardStore]("TickTock") log := logger.WithType[CheckShardStore]("TickTock")


objs, err := t.db.PinnedObject().GetObjectsByUserSpaceID(tx, space.UserSpace.UserSpaceID) objs, err := t.db.PinnedObject().GetObjectsByUserSpaceID(tx, space.UserSpace.UserSpaceID)
@@ -104,7 +104,7 @@ func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *
return return
} }


var rms []jcsypes.ObjectID
var rms []jcstypes.ObjectID
for _, c := range objs { for _, c := range objs {
if realFileHashes[c.FileHash] { if realFileHashes[c.FileHash] {
continue continue
@@ -121,7 +121,7 @@ func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *
} }


// checkObjectBlock 对比ObjectBlock表,若实际文件不存在,则进行删除操作 // checkObjectBlock 对比ObjectBlock表,若实际文件不存在,则进行删除操作
func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *jcsypes.UserSpaceDetail, realFileHashes map[jcsypes.FileHash]bool) {
func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *jcstypes.UserSpaceDetail, realFileHashes map[jcstypes.FileHash]bool) {
log := logger.WithType[CheckShardStore]("TickTock") log := logger.WithType[CheckShardStore]("TickTock")


blocks, err := t.db.ObjectBlock().GetByUserSpaceID(tx, space.UserSpace.UserSpaceID) blocks, err := t.db.ObjectBlock().GetByUserSpaceID(tx, space.UserSpace.UserSpaceID)
@@ -130,7 +130,7 @@ func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *j
return return
} }


var rms []jcsypes.FileHash
var rms []jcstypes.FileHash
for _, b := range blocks { for _, b := range blocks {
if realFileHashes[b.FileHash] { if realFileHashes[b.FileHash] {
continue continue


+ 88
- 89
client/internal/ticktock/redundancy_recover.go View File

@@ -16,56 +16,55 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
lrcparser "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser" lrcparser "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap" "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail) (jcsypes.Redundancy, []*userSpaceUsageInfo) {
func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail) (jcstypes.Redundancy, []*userSpaceUsageInfo) {
switch obj.Object.Redundancy.(type) { switch obj.Object.Redundancy.(type) {
case *jcsypes.NoneRedundancy:
case *jcstypes.NoneRedundancy:
if obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold { if obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold {
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcsypes.DefaultECRedundancy)
return &jcsypes.DefaultECRedundancy, newStgs
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
return &jcstypes.DefaultECRedundancy, newStgs
} }


return &jcsypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)


case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
if obj.Object.Size >= ctx.ticktock.cfg.ECFileSizeThreshold { if obj.Object.Size >= ctx.ticktock.cfg.ECFileSizeThreshold {
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcsypes.DefaultECRedundancy)
return &jcsypes.DefaultECRedundancy, newStgs
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcstypes.DefaultECRedundancy)
return &jcstypes.DefaultECRedundancy, newStgs
} }


newSpaces := t.rechooseUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
newSpaces := t.rechooseUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
for _, s := range newSpaces { for _, s := range newSpaces {
if !obj.ContainsBlock(0, s.UserSpace.UserSpace.UserSpaceID) && !obj.ContainsPinned(s.UserSpace.UserSpace.UserSpaceID) { if !obj.ContainsBlock(0, s.UserSpace.UserSpace.UserSpaceID) && !obj.ContainsPinned(s.UserSpace.UserSpace.UserSpaceID) {
return &jcsypes.DefaultRepRedundancy, newSpaces
return &jcstypes.DefaultRepRedundancy, newSpaces
} }
} }


return nil, nil return nil, nil


case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold { if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold {
return &jcsypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
return &jcstypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcstypes.DefaultRepRedundancy)
} }


newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &jcsypes.DefaultECRedundancy)
newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &jcstypes.DefaultECRedundancy)
for i, s := range newSpaces { for i, s := range newSpaces {
if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) { if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
return &jcsypes.DefaultECRedundancy, newSpaces
return &jcstypes.DefaultECRedundancy, newSpaces
} }
} }


return nil, nil return nil, nil


case *jcsypes.LRCRedundancy:
newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &jcsypes.DefaultLRCRedundancy)
case *jcstypes.LRCRedundancy:
newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &jcstypes.DefaultLRCRedundancy)


for i, s := range newLRCStgs { for i, s := range newLRCStgs {
if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) { if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
return &jcsypes.DefaultLRCRedundancy, newLRCStgs
return &jcstypes.DefaultLRCRedundancy, newLRCStgs
} }
} }


@@ -74,7 +73,7 @@ func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jc
return nil, nil return nil, nil
} }


func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, newRed jcsypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, newRed jcstypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
log := logger.WithType[ChangeRedundancy]("TickTock") log := logger.WithType[ChangeRedundancy]("TickTock")


var updating *db.UpdatingObjectRedundancy var updating *db.UpdatingObjectRedundancy
@@ -82,48 +81,48 @@ func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj
var err error var err error


switch srcRed := obj.Object.Redundancy.(type) { switch srcRed := obj.Object.Redundancy.(type) {
case *jcsypes.NoneRedundancy:
case *jcstypes.NoneRedundancy:
switch newRed := newRed.(type) { switch newRed := newRed.(type) {
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep")
updating, evt, err = t.noneToRep(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.noneToRep(ctx, obj, newRed, selectedUserSpaces)


case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec")
updating, evt, err = t.noneToEC(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.noneToEC(ctx, obj, newRed, selectedUserSpaces)


case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc")
updating, evt, err = t.noneToLRC(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.noneToLRC(ctx, obj, newRed, selectedUserSpaces)


case *jcsypes.SegmentRedundancy:
case *jcstypes.SegmentRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment")
updating, evt, err = t.noneToSeg(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.noneToSeg(ctx, obj, newRed, selectedUserSpaces)
} }


case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
switch newRed := newRed.(type) { switch newRed := newRed.(type) {
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
updating, evt, err = t.repToRep(ctx, obj, srcRed, selectedUserSpaces) updating, evt, err = t.repToRep(ctx, obj, srcRed, selectedUserSpaces)


case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec")
updating, evt, err = t.repToEC(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.repToEC(ctx, obj, newRed, selectedUserSpaces)
} }


case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
switch newRed := newRed.(type) { switch newRed := newRed.(type) {
case *jcsypes.RepRedundancy:
case *jcstypes.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep")
updating, evt, err = t.ecToRep(ctx, obj, srcRed, newRed, selectedUserSpaces) updating, evt, err = t.ecToRep(ctx, obj, srcRed, newRed, selectedUserSpaces)


case *jcsypes.ECRedundancy:
case *jcstypes.ECRedundancy:
updating, evt, err = t.ecToEC(ctx, obj, srcRed, newRed, selectedUserSpaces) updating, evt, err = t.ecToEC(ctx, obj, srcRed, newRed, selectedUserSpaces)
} }


case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
switch newRed := newRed.(type) { switch newRed := newRed.(type) {
case *jcsypes.LRCRedundancy:
case *jcstypes.LRCRedundancy:
updating, evt, err = t.lrcToLRC(ctx, obj, srcRed, newRed, selectedUserSpaces) updating, evt, err = t.lrcToLRC(ctx, obj, srcRed, newRed, selectedUserSpaces)
} }
} }
@@ -132,16 +131,16 @@ func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj
} }


// 统计每个对象块所在的节点,选出块最多的不超过userspaceCnt个节点 // 统计每个对象块所在的节点,选出块最多的不超过userspaceCnt个节点
func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []jcsypes.ObjectDetail, userspaceCnt int) []jcsypes.UserSpaceID {
func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []jcstypes.ObjectDetail, userspaceCnt int) []jcstypes.UserSpaceID {
type stgBlocks struct { type stgBlocks struct {
UserSpaceID jcsypes.UserSpaceID
UserSpaceID jcstypes.UserSpaceID
Count int Count int
} }


stgBlocksMap := make(map[jcsypes.UserSpaceID]*stgBlocks)
stgBlocksMap := make(map[jcstypes.UserSpaceID]*stgBlocks)
for _, obj := range objs { for _, obj := range objs {
shouldUseEC := obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold shouldUseEC := obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold
if _, ok := obj.Object.Redundancy.(*jcsypes.RepRedundancy); ok && !shouldUseEC {
if _, ok := obj.Object.Redundancy.(*jcstypes.RepRedundancy); ok && !shouldUseEC {
for _, block := range obj.Blocks { for _, block := range obj.Blocks {
if _, ok := stgBlocksMap[block.UserSpaceID]; !ok { if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
stgBlocksMap[block.UserSpaceID] = &stgBlocks{ stgBlocksMap[block.UserSpaceID] = &stgBlocks{
@@ -159,14 +158,14 @@ func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancy
return right.Count - left.Count return right.Count - left.Count
}) })


ids := lo.Map(userspaces, func(item *stgBlocks, idx int) jcsypes.UserSpaceID { return item.UserSpaceID })
ids := lo.Map(userspaces, func(item *stgBlocks, idx int) jcstypes.UserSpaceID { return item.UserSpaceID })
if len(ids) > userspaceCnt { if len(ids) > userspaceCnt {
ids = ids[:userspaceCnt] ids = ids[:userspaceCnt]
} }
return ids return ids
} }


func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *jcsypes.RepRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int { sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount) return sort2.Cmp(right.AccessAmount, left.AccessAmount)
}) })
@@ -174,7 +173,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContex
return t.chooseSoManyUserSpaces(red.RepCount, sortedUserSpaces) return t.chooseSoManyUserSpaces(red.RepCount, sortedUserSpaces)
} }


func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *jcsypes.ECRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int { sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount) return sort2.Cmp(right.AccessAmount, left.AccessAmount)
}) })
@@ -182,7 +181,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext
return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces) return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces)
} }


func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *jcsypes.LRCRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int { sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount) return sort2.Cmp(right.AccessAmount, left.AccessAmount)
}) })
@@ -198,7 +197,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForSeg(ctx *changeRedundancyContex
return t.chooseSoManyUserSpaces(segCount, sortedUserSpaces) return t.chooseSoManyUserSpaces(segCount, sortedUserSpaces)
} }


func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *jcsypes.RepRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *jcstypes.RepRedundancy) []*userSpaceUsageInfo {
type rechooseUserSpace struct { type rechooseUserSpace struct {
*userSpaceUsageInfo *userSpaceUsageInfo
HasBlock bool HasBlock bool
@@ -233,7 +232,7 @@ func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext
return t.chooseSoManyUserSpaces(red.RepCount, lo.Map(sortedStgs, func(userspace *rechooseUserSpace, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo })) return t.chooseSoManyUserSpaces(red.RepCount, lo.Map(sortedStgs, func(userspace *rechooseUserSpace, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
} }


func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy) []*userSpaceUsageInfo {
type rechooseStg struct { type rechooseStg struct {
*userSpaceUsageInfo *userSpaceUsageInfo
CachedBlockIndex int CachedBlockIndex int
@@ -269,7 +268,7 @@ func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext,
return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo })) return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
} }


func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.LRCRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy) []*userSpaceUsageInfo {
type rechooseStg struct { type rechooseStg struct {
*userSpaceUsageInfo *userSpaceUsageInfo
CachedBlockIndex int CachedBlockIndex int
@@ -322,7 +321,7 @@ func (t *ChangeRedundancy) chooseSoManyUserSpaces(count int, stgs []*userSpaceUs
var chosen []*userSpaceUsageInfo var chosen []*userSpaceUsageInfo
for len(chosen) < count { for len(chosen) < count {
// 在每一轮内都选不同地区的节点,如果节点数不够,那么就再来一轮 // 在每一轮内都选不同地区的节点,如果节点数不够,那么就再来一轮
chosenLocations := make(map[cortypes.Location]bool)
chosenLocations := make(map[jcstypes.Location]bool)
for i, stg := range extendStgs { for i, stg := range extendStgs {
if stg == nil { if stg == nil {
continue continue
@@ -341,7 +340,7 @@ func (t *ChangeRedundancy) chooseSoManyUserSpaces(count int, stgs []*userSpaceUs
return chosen return chosen
} }


func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
} }
@@ -352,7 +351,7 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.O
} }


// 如果选择的备份节点都是同一个,那么就只要上传一次 // 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })


ft := ioswitch2.NewFromTo() ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream())) ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
@@ -374,11 +373,11 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange
for i, stg := range uploadStgs { for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: 0, Index: 0,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID, UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@@ -412,7 +411,7 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.O
}, nil }, nil
} }


func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
} }
@@ -441,12 +440,12 @@ func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.Ob
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var evtTargetBlocks []datamap.Block var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer var evtBlockTrans []datamap.DataTransfer
for i := 0; i < red.N; i++ { for i := 0; i < red.N; i++ {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID, UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@@ -494,7 +493,7 @@ func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.Ob
}, nil }, nil
} }


func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
} }
@@ -522,12 +521,12 @@ func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var evtTargetBlocks []datamap.Block var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer var evtBlockTrans []datamap.DataTransfer
for i := 0; i < red.N; i++ { for i := 0; i < red.N; i++ {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID, UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@@ -576,7 +575,7 @@ func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.O
nil nil
} }


func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
} }
@@ -587,7 +586,7 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.O
} }


// 如果选择的备份节点都是同一个,那么就只要上传一次 // 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })


ft := ioswitch2.NewFromTo() ft := ioswitch2.NewFromTo()
ft.SegmentParam = red ft.SegmentParam = red
@@ -610,12 +609,12 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var evtTargetBlocks []datamap.Block var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer var evtBlockTrans []datamap.DataTransfer
for i, stg := range uploadStgs { for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID, UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@@ -664,7 +663,7 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.O
nil nil
} }


func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
} }
@@ -675,7 +674,7 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.Ob
} }


// 如果选择的备份节点都是同一个,那么就只要上传一次 // 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })


ft := ioswitch2.NewFromTo() ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream())) ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
@@ -697,11 +696,11 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.Ob
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock
var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange
for i, stg := range uploadStgs { for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: 0, Index: 0,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID, UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@@ -737,14 +736,14 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.Ob
nil nil
} }


func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, red *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
return t.noneToEC(ctx, obj, red, uploadUserSpaces) return t.noneToEC(ctx, obj, red, uploadUserSpaces)
} }


func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.ECRedundancy, tarRed *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcsypes.GrouppedObjectBlock
func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcstypes.GrouppedObjectBlock
var chosenBlockIndexes []int var chosenBlockIndexes []int
var chosenBlockStg []jcsypes.UserSpaceDetail
var chosenBlockStg []jcstypes.UserSpaceDetail
for _, block := range obj.GroupBlocks() { for _, block := range obj.GroupBlocks() {
if len(block.UserSpaceIDs) > 0 { if len(block.UserSpaceIDs) > 0 {
// TODO 考虑选择最优的节点 // TODO 考虑选择最优的节点
@@ -768,7 +767,7 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.Obj
} }


// 如果选择的备份节点都是同一个,那么就只要上传一次 // 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcstypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })


planBlder := exec.NewPlanBuilder() planBlder := exec.NewPlanBuilder()
ft := ioswitch2.NewFromTo() ft := ioswitch2.NewFromTo()
@@ -795,11 +794,11 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.Obj
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []jcsypes.ObjectBlock
var blocks []jcstypes.ObjectBlock


for i := range uploadStgs { for i := range uploadStgs {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, jcsypes.ObjectBlock{
blocks = append(blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: 0, Index: 0,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID, UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@@ -866,11 +865,11 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.Obj
nil nil
} }


func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.ECRedundancy, tarRed *jcsypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.ECRedundancy, tarRed *jcstypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
grpBlocks := obj.GroupBlocks() grpBlocks := obj.GroupBlocks()


var chosenBlocks []jcsypes.GrouppedObjectBlock
var chosenBlockStg []jcsypes.UserSpaceDetail
var chosenBlocks []jcstypes.GrouppedObjectBlock
var chosenBlockStg []jcstypes.UserSpaceDetail
for _, block := range grpBlocks { for _, block := range grpBlocks {
if len(block.UserSpaceIDs) > 0 { if len(block.UserSpaceIDs) > 0 {
stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]] stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
@@ -909,16 +908,16 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.Obje
}) })
} }


var newBlocks []jcsypes.ObjectBlock
var newBlocks []jcstypes.ObjectBlock
shouldUpdateBlocks := false shouldUpdateBlocks := false
for i, stg := range uploadUserSpaces { for i, stg := range uploadUserSpaces {
newBlock := jcsypes.ObjectBlock{
newBlock := jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID, UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
} }


grp, ok := lo.Find(grpBlocks, func(grp jcsypes.GrouppedObjectBlock) bool { return grp.Index == i })
grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })


// 如果新选中的节点已经记录在Block表中,那么就不需要任何变更 // 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
if ok && lo.Contains(grp.UserSpaceIDs, stg.UserSpace.UserSpace.UserSpaceID) { if ok && lo.Contains(grp.UserSpaceIDs, stg.UserSpace.UserSpace.UserSpaceID) {
@@ -988,7 +987,7 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.Obje


var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange
for _, block := range obj.Blocks { for _, block := range obj.Blocks {
keep := lo.ContainsBy(newBlocks, func(newBlock jcsypes.ObjectBlock) bool {
keep := lo.ContainsBy(newBlocks, func(newBlock jcstypes.ObjectBlock) bool {
return newBlock.Index == block.Index && newBlock.UserSpaceID == block.UserSpaceID return newBlock.Index == block.Index && newBlock.UserSpaceID == block.UserSpaceID
}) })
if !keep { if !keep {
@@ -1019,7 +1018,7 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.Obje
nil nil
} }


func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.LRCRedundancy, tarRed *jcsypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, srcRed *jcstypes.LRCRedundancy, tarRed *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {


blocksGrpByIndex := obj.GroupBlocks() blocksGrpByIndex := obj.GroupBlocks()


@@ -1062,8 +1061,8 @@ func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcsypes.Ob
/* /*
TODO2 修复这一块的代码 TODO2 修复这一块的代码


func (t *ChangeRedundancy) groupReconstructLRC(obj jcsypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []jcsypes.GrouppedObjectBlock, red *jcsypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
grped := make(map[int]jcsypes.GrouppedObjectBlock)
func (t *ChangeRedundancy) groupReconstructLRC(obj jcstypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
grped := make(map[int]jcstypes.GrouppedObjectBlock)
for _, b := range grpedBlocks { for _, b := range grpedBlocks {
grped[b.Index] = b grped[b.Index] = b
} }
@@ -1098,9 +1097,9 @@ TODO2 修复这一块的代码
return nil, fmt.Errorf("executing io plan: %w", err) return nil, fmt.Errorf("executing io plan: %w", err)
} }


var newBlocks []jcsypes.ObjectBlock
var newBlocks []jcstypes.ObjectBlock
for _, i := range lostBlocks { for _, i := range lostBlocks {
newBlocks = append(newBlocks, jcsypes.ObjectBlock{
newBlocks = append(newBlocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: uploadUserSpaces[i].UserSpace.UserSpace.UserSpaceID, UserSpaceID: uploadUserSpaces[i].UserSpace.UserSpace.UserSpaceID,
@@ -1109,7 +1108,7 @@ TODO2 修复这一块的代码
} }
for _, b := range grpedBlocks { for _, b := range grpedBlocks {
for _, hubID := range b.UserSpaceIDs { for _, hubID := range b.UserSpaceIDs {
newBlocks = append(newBlocks, jcsypes.ObjectBlock{
newBlocks = append(newBlocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: b.Index, Index: b.Index,
UserSpaceID: hubID, UserSpaceID: hubID,
@@ -1125,9 +1124,9 @@ TODO2 修复这一块的代码
}, nil }, nil
} }
*/ */
func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, grpBlocks []jcsypes.GrouppedObjectBlock, red *jcsypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcsypes.GrouppedObjectBlock
var chosenBlockStg []jcsypes.UserSpaceDetail
func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcstypes.ObjectDetail, grpBlocks []jcstypes.GrouppedObjectBlock, red *jcstypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcstypes.GrouppedObjectBlock
var chosenBlockStg []jcstypes.UserSpaceDetail


for _, block := range grpBlocks { for _, block := range grpBlocks {
if len(block.UserSpaceIDs) > 0 && block.Index < red.M() { if len(block.UserSpaceIDs) > 0 && block.Index < red.M() {
@@ -1154,16 +1153,16 @@ func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcsy


var froms []ioswitchlrc.From var froms []ioswitchlrc.From
var toes []ioswitchlrc.To var toes []ioswitchlrc.To
var newBlocks []jcsypes.ObjectBlock
var newBlocks []jcstypes.ObjectBlock
shouldUpdateBlocks := false shouldUpdateBlocks := false
for i, userspace := range uploadUserSpaces { for i, userspace := range uploadUserSpaces {
newBlock := jcsypes.ObjectBlock{
newBlock := jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: userspace.UserSpace.UserSpace.UserSpaceID, UserSpaceID: userspace.UserSpace.UserSpace.UserSpaceID,
} }


grp, ok := lo.Find(grpBlocks, func(grp jcsypes.GrouppedObjectBlock) bool { return grp.Index == i })
grp, ok := lo.Find(grpBlocks, func(grp jcstypes.GrouppedObjectBlock) bool { return grp.Index == i })


// 如果新选中的节点已经记录在Block表中,那么就不需要任何变更 // 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
if ok && lo.Contains(grp.UserSpaceIDs, userspace.UserSpace.UserSpace.UserSpaceID) { if ok && lo.Contains(grp.UserSpaceIDs, userspace.UserSpace.UserSpace.UserSpaceID) {


+ 52
- 52
client/internal/ticktock/redundancy_shrink.go View File

@@ -21,14 +21,14 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap" "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext, pkg jcsypes.PackageDetail, objs []jcsypes.ObjectDetail, reen *publock.Reentrant) ([]db.UpdatingObjectRedundancy, []datamap.SysEventBody, error) {
func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext, pkg jcstypes.PackageDetail, objs []jcstypes.ObjectDetail, reen *publock.Reentrant) ([]db.UpdatingObjectRedundancy, []datamap.SysEventBody, error) {
log := logger.WithType[ChangeRedundancy]("TickTock") log := logger.WithType[ChangeRedundancy]("TickTock")


var readerStgIDs []jcsypes.UserSpaceID
var readerStgIDs []jcstypes.UserSpaceID
for _, space := range execCtx.allUserSpaces { for _, space := range execCtx.allUserSpaces {
// TODO 可以考虑做成配置 // TODO 可以考虑做成配置
if space.AccessAmount >= float64(pkg.ObjectCount/2) { if space.AccessAmount >= float64(pkg.ObjectCount/2) {
@@ -37,17 +37,17 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
} }


// 只对ec和rep对象进行处理 // 只对ec和rep对象进行处理
var ecObjects []jcsypes.ObjectDetail
var repObjects []jcsypes.ObjectDetail
var ecObjects []jcstypes.ObjectDetail
var repObjects []jcstypes.ObjectDetail
for _, obj := range objs { for _, obj := range objs {
if _, ok := obj.Object.Redundancy.(*jcsypes.ECRedundancy); ok {
if _, ok := obj.Object.Redundancy.(*jcstypes.ECRedundancy); ok {
ecObjects = append(ecObjects, obj) ecObjects = append(ecObjects, obj)
} else if _, ok := obj.Object.Redundancy.(*jcsypes.RepRedundancy); ok {
} else if _, ok := obj.Object.Redundancy.(*jcstypes.RepRedundancy); ok {
repObjects = append(repObjects, obj) repObjects = append(repObjects, obj)
} }
} }


planningStgIDs := make(map[jcsypes.UserSpaceID]bool)
planningStgIDs := make(map[jcstypes.UserSpaceID]bool)


var sysEvents []datamap.SysEventBody var sysEvents []datamap.SysEventBody


@@ -80,7 +80,7 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
// 对于ec对象,则每个对象单独进行退火算法 // 对于ec对象,则每个对象单独进行退火算法
var ecObjectsUpdating []db.UpdatingObjectRedundancy var ecObjectsUpdating []db.UpdatingObjectRedundancy
for i, obj := range ecObjects { for i, obj := range ecObjects {
ecRed := obj.Object.Redundancy.(*jcsypes.ECRedundancy)
ecRed := obj.Object.Redundancy.(*jcstypes.ECRedundancy)
solu := t.startAnnealing(execCtx, readerStgIDs, annealingObject{ solu := t.startAnnealing(execCtx, readerStgIDs, annealingObject{
totalBlockCount: ecRed.N, totalBlockCount: ecRed.N,
minBlockCnt: ecRed.K, minBlockCnt: ecRed.K,
@@ -105,15 +105,15 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
return append(repObjectsUpdating, ecObjectsUpdating...), sysEvents, nil return append(repObjectsUpdating, ecObjectsUpdating...), sysEvents, nil
} }


func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcsypes.ObjectDetail) []jcsypes.UserSpaceID {
func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcstypes.ObjectDetail) []jcstypes.UserSpaceID {
type stgBlocks struct { type stgBlocks struct {
UserSpaceID jcsypes.UserSpaceID
UserSpaceID jcstypes.UserSpaceID
Count int Count int
} }


stgBlocksMap := make(map[jcsypes.UserSpaceID]*stgBlocks)
stgBlocksMap := make(map[jcstypes.UserSpaceID]*stgBlocks)
for _, obj := range objs { for _, obj := range objs {
cacheBlockStgs := make(map[jcsypes.UserSpaceID]bool)
cacheBlockStgs := make(map[jcstypes.UserSpaceID]bool)
for _, block := range obj.Blocks { for _, block := range obj.Blocks {
if _, ok := stgBlocksMap[block.UserSpaceID]; !ok { if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
stgBlocksMap[block.UserSpaceID] = &stgBlocks{ stgBlocksMap[block.UserSpaceID] = &stgBlocks{
@@ -153,17 +153,17 @@ func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcsypes.ObjectDetai
} }
} }


return lo.Map(stgs, func(item *stgBlocks, idx int) jcsypes.UserSpaceID { return item.UserSpaceID })
return lo.Map(stgs, func(item *stgBlocks, idx int) jcstypes.UserSpaceID { return item.UserSpaceID })
} }


type annealingState struct { type annealingState struct {
ctx *changeRedundancyContext ctx *changeRedundancyContext
readerStgIDs []jcsypes.UserSpaceID // 近期可能访问此对象的节点
stgsSortedByReader map[jcsypes.UserSpaceID][]stgDist // 拥有数据的节点到每个可能访问对象的节点按距离排序
object annealingObject // 进行退火的对象
blockList []objectBlock // 排序后的块分布情况
stgBlockBitmaps map[jcsypes.UserSpaceID]*bitmap.Bitmap64 // 用位图的形式表示每一个节点上有哪些块
stgCombTree combinatorialTree // 节点组合树,用于加速计算容灾度
readerStgIDs []jcstypes.UserSpaceID // 近期可能访问此对象的节点
stgsSortedByReader map[jcstypes.UserSpaceID][]stgDist // 拥有数据的节点到每个可能访问对象的节点按距离排序
object annealingObject // 进行退火的对象
blockList []objectBlock // 排序后的块分布情况
stgBlockBitmaps map[jcstypes.UserSpaceID]*bitmap.Bitmap64 // 用位图的形式表示每一个节点上有哪些块
stgCombTree combinatorialTree // 节点组合树,用于加速计算容灾度


maxScore float64 // 搜索过程中得到过的最大分数 maxScore float64 // 搜索过程中得到过的最大分数
maxScoreRmBlocks []bool // 最大分数对应的删除方案 maxScoreRmBlocks []bool // 最大分数对应的删除方案
@@ -178,30 +178,30 @@ type annealingState struct {


type objectBlock struct { type objectBlock struct {
Index int Index int
UserSpaceID jcsypes.UserSpaceID
HasEntity bool // 节点拥有实际的文件数据块
HasShadow bool // 如果节点拥有完整文件数据,那么认为这个节点拥有所有块,这些块被称为影子块
FileHash jcsypes.FileHash // 只有在拥有实际文件数据块时,这个字段才有值
Size int64 // 块大小
UserSpaceID jcstypes.UserSpaceID
HasEntity bool // 节点拥有实际的文件数据块
HasShadow bool // 如果节点拥有完整文件数据,那么认为这个节点拥有所有块,这些块被称为影子块
FileHash jcstypes.FileHash // 只有在拥有实际文件数据块时,这个字段才有值
Size int64 // 块大小
} }


type stgDist struct { type stgDist struct {
UserSpaceID jcsypes.UserSpaceID
UserSpaceID jcstypes.UserSpaceID
Distance float64 Distance float64
} }


type combinatorialTree struct { type combinatorialTree struct {
nodes []combinatorialTreeNode nodes []combinatorialTreeNode
blocksMaps map[int]bitmap.Bitmap64 blocksMaps map[int]bitmap.Bitmap64
stgIDToLocalStgID map[jcsypes.UserSpaceID]int
localStgIDToStgID []jcsypes.UserSpaceID
stgIDToLocalStgID map[jcstypes.UserSpaceID]int
localStgIDToStgID []jcstypes.UserSpaceID
} }


type annealingObject struct { type annealingObject struct {
totalBlockCount int totalBlockCount int
minBlockCnt int minBlockCnt int
pinnedAt []jcsypes.UserSpaceID
blocks []jcsypes.ObjectBlock
pinnedAt []jcstypes.UserSpaceID
blocks []jcstypes.ObjectBlock
} }


const ( const (
@@ -210,10 +210,10 @@ const (
iterActionBreak = 2 iterActionBreak = 2
) )


func newCombinatorialTree(stgBlocksMaps map[jcsypes.UserSpaceID]*bitmap.Bitmap64) combinatorialTree {
func newCombinatorialTree(stgBlocksMaps map[jcstypes.UserSpaceID]*bitmap.Bitmap64) combinatorialTree {
tree := combinatorialTree{ tree := combinatorialTree{
blocksMaps: make(map[int]bitmap.Bitmap64), blocksMaps: make(map[int]bitmap.Bitmap64),
stgIDToLocalStgID: make(map[jcsypes.UserSpaceID]int),
stgIDToLocalStgID: make(map[jcstypes.UserSpaceID]int),
} }


tree.nodes = make([]combinatorialTreeNode, (1 << len(stgBlocksMaps))) tree.nodes = make([]combinatorialTreeNode, (1 << len(stgBlocksMaps)))
@@ -271,7 +271,7 @@ func (t *combinatorialTree) GetDepth(index int) int {


// 更新某一个算力中心节点的块分布位图,同时更新它对应组合树节点的所有子节点。 // 更新某一个算力中心节点的块分布位图,同时更新它对应组合树节点的所有子节点。
// 如果更新到某个节点时,已有K个块,那么就不会再更新它的子节点 // 如果更新到某个节点时,已有K个块,那么就不会再更新它的子节点
func (t *combinatorialTree) UpdateBitmap(stgID jcsypes.UserSpaceID, mp bitmap.Bitmap64, k int) {
func (t *combinatorialTree) UpdateBitmap(stgID jcstypes.UserSpaceID, mp bitmap.Bitmap64, k int) {
t.blocksMaps[t.stgIDToLocalStgID[stgID]] = mp t.blocksMaps[t.stgIDToLocalStgID[stgID]] = mp
// 首先定义两种遍历树节点时的移动方式: // 首先定义两种遍历树节点时的移动方式:
// 1. 竖直移动(深度增加):从一个节点移动到它最左边的子节点。每移动一步,index+1 // 1. 竖直移动(深度增加):从一个节点移动到它最左边的子节点。每移动一步,index+1
@@ -416,13 +416,13 @@ type annealingSolution struct {
minAccessCost float64 // 本方案的最小访问费用 minAccessCost float64 // 本方案的最小访问费用
} }


func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerStgIDs []jcsypes.UserSpaceID, object annealingObject) annealingSolution {
func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerStgIDs []jcstypes.UserSpaceID, object annealingObject) annealingSolution {
state := &annealingState{ state := &annealingState{
ctx: ctx, ctx: ctx,
readerStgIDs: readerStgIDs, readerStgIDs: readerStgIDs,
stgsSortedByReader: make(map[jcsypes.UserSpaceID][]stgDist),
stgsSortedByReader: make(map[jcstypes.UserSpaceID][]stgDist),
object: object, object: object,
stgBlockBitmaps: make(map[jcsypes.UserSpaceID]*bitmap.Bitmap64),
stgBlockBitmaps: make(map[jcstypes.UserSpaceID]*bitmap.Bitmap64),
} }


t.initBlockList(state) t.initBlockList(state)
@@ -486,7 +486,7 @@ func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerSt
} }


func (t *ChangeRedundancy) initBlockList(ctx *annealingState) { func (t *ChangeRedundancy) initBlockList(ctx *annealingState) {
blocksMap := make(map[jcsypes.UserSpaceID][]objectBlock)
blocksMap := make(map[jcstypes.UserSpaceID][]objectBlock)


// 先生成所有的影子块 // 先生成所有的影子块
for _, pinned := range ctx.object.pinnedAt { for _, pinned := range ctx.object.pinnedAt {
@@ -680,7 +680,7 @@ func (t *ChangeRedundancy) alwaysAccept(curTemp float64, dScore float64, cooling
return v > rand.Float64() return v > rand.Float64()
} }


func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcsypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcsypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcstypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcstypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
entry := db.UpdatingObjectRedundancy{ entry := db.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
FileHash: obj.Object.FileHash, FileHash: obj.Object.FileHash,
@@ -694,8 +694,8 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *fromStg, ioswitch2.RawStream())) ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *fromStg, ioswitch2.RawStream()))


for i, f := range solu.rmBlocks { for i, f := range solu.rmBlocks {
hasCache := lo.ContainsBy(obj.Blocks, func(b jcsypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcsypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
hasCache := lo.ContainsBy(obj.Blocks, func(b jcstypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcstypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
willRm := f willRm := f


if !willRm { if !willRm {
@@ -706,7 +706,7 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s


planningHubIDs[solu.blockList[i].UserSpaceID] = true planningHubIDs[solu.blockList[i].UserSpaceID] = true
} }
entry.Blocks = append(entry.Blocks, jcsypes.ObjectBlock{
entry.Blocks = append(entry.Blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: solu.blockList[i].Index, Index: solu.blockList[i].Index,
UserSpaceID: solu.blockList[i].UserSpaceID, UserSpaceID: solu.blockList[i].UserSpaceID,
@@ -724,12 +724,12 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s
return entry return entry
} }


func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution, obj jcsypes.ObjectDetail) []datamap.SysEventBody {
func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution, obj jcstypes.ObjectDetail) []datamap.SysEventBody {
var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange


for i, f := range solu.rmBlocks { for i, f := range solu.rmBlocks {
hasCache := lo.ContainsBy(obj.Blocks, func(b jcsypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcsypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
hasCache := lo.ContainsBy(obj.Blocks, func(b jcstypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcstypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
willRm := f willRm := f


if !willRm { if !willRm {
@@ -782,7 +782,7 @@ func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution,
return []datamap.SysEventBody{transEvt, distEvt} return []datamap.SysEventBody{transEvt, distEvt}
} }


func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcsypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcsypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcstypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcstypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
entry := db.UpdatingObjectRedundancy{ entry := db.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
FileHash: obj.Object.FileHash, FileHash: obj.Object.FileHash,
@@ -790,11 +790,11 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
Redundancy: obj.Object.Redundancy, Redundancy: obj.Object.Redundancy,
} }


reconstrct := make(map[jcsypes.UserSpaceID]*[]int)
reconstrct := make(map[jcstypes.UserSpaceID]*[]int)
for i, f := range solu.rmBlocks { for i, f := range solu.rmBlocks {
block := solu.blockList[i] block := solu.blockList[i]
if !f { if !f {
entry.Blocks = append(entry.Blocks, jcsypes.ObjectBlock{
entry.Blocks = append(entry.Blocks, jcstypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: block.Index, Index: block.Index,
UserSpaceID: block.UserSpaceID, UserSpaceID: block.UserSpaceID,
@@ -815,7 +815,7 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
} }
} }


ecRed := obj.Object.Redundancy.(*jcsypes.ECRedundancy)
ecRed := obj.Object.Redundancy.(*jcstypes.ECRedundancy)


for id, idxs := range reconstrct { for id, idxs := range reconstrct {
// 依次生成每个节点上的执行计划,因为如果放到一个计划里一起生成,不能保证每个节点上的块用的都是本节点上的副本 // 依次生成每个节点上的执行计划,因为如果放到一个计划里一起生成,不能保证每个节点上的块用的都是本节点上的副本
@@ -838,10 +838,10 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
return entry return entry
} }


func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, obj jcsypes.ObjectDetail) []datamap.SysEventBody {
func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, obj jcstypes.ObjectDetail) []datamap.SysEventBody {
var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange


reconstrct := make(map[jcsypes.UserSpaceID]*[]int)
reconstrct := make(map[jcstypes.UserSpaceID]*[]int)
for i, f := range solu.rmBlocks { for i, f := range solu.rmBlocks {
block := solu.blockList[i] block := solu.blockList[i]
if !f { if !f {
@@ -917,7 +917,7 @@ func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, o
return []datamap.SysEventBody{transEvt, distEvt} return []datamap.SysEventBody{transEvt, distEvt}
} }


func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *exec.PlanBuilder, planningSpaceIDs map[jcsypes.UserSpaceID]bool, reen *publock.Reentrant) (exec.PlanResult, error) {
func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *exec.PlanBuilder, planningSpaceIDs map[jcstypes.UserSpaceID]bool, reen *publock.Reentrant) (exec.PlanResult, error) {
reqBlder := reqbuilder.NewBuilder() reqBlder := reqbuilder.NewBuilder()
for id, _ := range planningSpaceIDs { for id, _ := range planningSpaceIDs {
reqBlder.UserSpace().Buzy(id) reqBlder.UserSpace().Buzy(id)
@@ -955,7 +955,7 @@ func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *e
return ioSwRets, nil return ioSwRets, nil
} }


func (t *ChangeRedundancy) populateECObjectEntry(entry *db.UpdatingObjectRedundancy, obj jcsypes.ObjectDetail, ioRets exec.PlanResult) {
func (t *ChangeRedundancy) populateECObjectEntry(entry *db.UpdatingObjectRedundancy, obj jcstypes.ObjectDetail, ioRets exec.PlanResult) {
for i := range entry.Blocks { for i := range entry.Blocks {
if entry.Blocks[i].FileHash != "" { if entry.Blocks[i].FileHash != "" {
continue continue


+ 13
- 13
client/internal/uploader/uploader.go View File

@@ -22,7 +22,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type Uploader struct { type Uploader struct {
@@ -43,7 +43,7 @@ func NewUploader(pubLock *publock.Service, connectivity *connectivity.Collector,
} }
} }


func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpaceID, copyTo []jcsypes.UserSpaceID, copyToPath []jcsypes.JPath) (*UpdateUploader, error) {
func (u *Uploader) BeginUpdate(pkgID jcstypes.PackageID, affinity jcstypes.UserSpaceID, copyTo []jcstypes.UserSpaceID, copyToPath []jcstypes.JPath) (*UpdateUploader, error) {
spaceIDs, err := u.db.UserSpace().GetAllIDs(u.db.DefCtx()) spaceIDs, err := u.db.UserSpace().GetAllIDs(u.db.DefCtx())
if err != nil { if err != nil {
return nil, fmt.Errorf("getting user space ids: %w", err) return nil, fmt.Errorf("getting user space ids: %w", err)
@@ -84,9 +84,9 @@ func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpa
return nil, fmt.Errorf("user no available userspaces") return nil, fmt.Errorf("user no available userspaces")
} }


copyToSpaces := make([]jcsypes.UserSpaceDetail, len(copyTo))
copyToSpaces := make([]jcstypes.UserSpaceDetail, len(copyTo))
for i, spaceID := range copyTo { for i, spaceID := range copyTo {
space, ok := lo.Find(spaceDetails, func(space *jcsypes.UserSpaceDetail) bool {
space, ok := lo.Find(spaceDetails, func(space *jcstypes.UserSpaceDetail) bool {
return space.UserSpace.UserSpaceID == spaceID return space.UserSpace.UserSpaceID == spaceID
}) })
if !ok { if !ok {
@@ -118,7 +118,7 @@ func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpa
// 1. 选择设置了亲和性的节点 // 1. 选择设置了亲和性的节点
// 2. 从与当前客户端相同地域的节点中随机选一个 // 2. 从与当前客户端相同地域的节点中随机选一个
// 3. 没有的话从所有节点选择延迟最低的节点 // 3. 没有的话从所有节点选择延迟最低的节点
func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity jcsypes.UserSpaceID) UploadSpaceInfo {
func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity jcstypes.UserSpaceID) UploadSpaceInfo {
if spaceAffinity > 0 { if spaceAffinity > 0 {
aff, ok := lo.Find(spaces, func(space UploadSpaceInfo) bool { return space.Space.UserSpace.UserSpaceID == spaceAffinity }) aff, ok := lo.Find(spaces, func(space UploadSpaceInfo) bool { return space.Space.UserSpace.UserSpaceID == spaceAffinity })
if ok { if ok {
@@ -137,10 +137,10 @@ func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity j
return spaces[0] return spaces[0]
} }


func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, copyTo []jcsypes.UserSpaceID, copyToPath []jcsypes.JPath) (*CreateUploader, error) {
func (u *Uploader) BeginCreateUpload(bktID jcstypes.BucketID, pkgName string, copyTo []jcstypes.UserSpaceID, copyToPath []jcstypes.JPath) (*CreateUploader, error) {
getSpaces := u.spaceMeta.GetMany(copyTo) getSpaces := u.spaceMeta.GetMany(copyTo)


spacesStgs := make([]jcsypes.UserSpaceDetail, len(copyTo))
spacesStgs := make([]jcstypes.UserSpaceDetail, len(copyTo))
for i, stg := range getSpaces { for i, stg := range getSpaces {
if stg == nil { if stg == nil {
return nil, fmt.Errorf("storage %v not found", copyTo[i]) return nil, fmt.Errorf("storage %v not found", copyTo[i])
@@ -148,10 +148,10 @@ func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, cop
spacesStgs[i] = *stg spacesStgs[i] = *stg
} }


pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcsypes.Package, error) {
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcstypes.Package, error) {
_, err := u.db.Bucket().GetByID(tx, bktID) _, err := u.db.Bucket().GetByID(tx, bktID)
if err != nil { if err != nil {
return jcsypes.Package{}, err
return jcstypes.Package{}, err
} }


return u.db.Package().Create(u.db.DefCtx(), bktID, pkgName, time.Now()) return u.db.Package().Create(u.db.DefCtx(), bktID, pkgName, time.Now())
@@ -178,19 +178,19 @@ func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, cop
}, nil }, nil
} }


func (u *Uploader) UploadPart(objID jcsypes.ObjectID, index int, stream io.Reader) error {
func (u *Uploader) UploadPart(objID jcstypes.ObjectID, index int, stream io.Reader) error {
detail, err := u.db.Object().GetDetail(u.db.DefCtx(), objID) detail, err := u.db.Object().GetDetail(u.db.DefCtx(), objID)
if err != nil { if err != nil {
return fmt.Errorf("getting object detail: %w", err) return fmt.Errorf("getting object detail: %w", err)
} }


objDe := detail objDe := detail
_, ok := objDe.Object.Redundancy.(*jcsypes.MultipartUploadRedundancy)
_, ok := objDe.Object.Redundancy.(*jcstypes.MultipartUploadRedundancy)
if !ok { if !ok {
return fmt.Errorf("object %v is not a multipart upload", objID) return fmt.Errorf("object %v is not a multipart upload", objID)
} }


var space jcsypes.UserSpaceDetail
var space jcstypes.UserSpaceDetail
if len(objDe.Blocks) > 0 { if len(objDe.Blocks) > 0 {
cstg := u.spaceMeta.Get(objDe.Blocks[0].UserSpaceID) cstg := u.spaceMeta.Get(objDe.Blocks[0].UserSpaceID)
if cstg == nil { if cstg == nil {
@@ -272,7 +272,7 @@ func (u *Uploader) UploadPart(objID jcsypes.ObjectID, index int, stream io.Reade


shardInfo := ret.Get("shard").(*ops2.FileInfoValue) shardInfo := ret.Get("shard").(*ops2.FileInfoValue)
err = u.db.DoTx(func(tx db.SQLContext) error { err = u.db.DoTx(func(tx db.SQLContext) error {
return u.db.Object().AppendPart(tx, jcsypes.ObjectBlock{
return u.db.Object().AppendPart(tx, jcstypes.ObjectBlock{
ObjectID: objID, ObjectID: objID,
Index: index, Index: index,
UserSpaceID: space.UserSpace.UserSpaceID, UserSpaceID: space.UserSpace.UserSpaceID,


+ 9
- 10
client/internal/uploader/user_space_upload.go View File

@@ -17,20 +17,19 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator" corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcsypes.JPath, targetBktID jcsypes.BucketID, newPkgName string, uploadAffinity jcsypes.UserSpaceID) (*jcsypes.Package, error) {
func (u *Uploader) UserSpaceUpload(userSpaceID jcstypes.UserSpaceID, rootPath jcstypes.JPath, targetBktID jcstypes.BucketID, newPkgName string, uploadAffinity jcstypes.UserSpaceID) (*jcstypes.Package, error) {
srcSpace := u.spaceMeta.Get(userSpaceID) srcSpace := u.spaceMeta.Get(userSpaceID)
if srcSpace == nil { if srcSpace == nil {
return nil, fmt.Errorf("user space %d not found", userSpaceID) return nil, fmt.Errorf("user space %d not found", userSpaceID)
} }


pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcsypes.Package, error) {
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcstypes.Package, error) {
_, err := u.db.Bucket().GetByID(tx, targetBktID) _, err := u.db.Bucket().GetByID(tx, targetBktID)
if err != nil { if err != nil {
return jcsypes.Package{}, err
return jcstypes.Package{}, err
} }


return u.db.Package().Create(tx, targetBktID, newPkgName, time.Now()) return u.db.Package().Create(tx, targetBktID, newPkgName, time.Now())
@@ -49,7 +48,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcs
} }


spaceDetails := u.spaceMeta.GetMany(spaceIDs) spaceDetails := u.spaceMeta.GetMany(spaceIDs)
spaceDetails = lo.Filter(spaceDetails, func(e *jcsypes.UserSpaceDetail, i int) bool {
spaceDetails = lo.Filter(spaceDetails, func(e *jcstypes.UserSpaceDetail, i int) bool {
return e != nil && e.UserSpace.ShardStore != nil return e != nil && e.UserSpace.ShardStore != nil
}) })


@@ -59,13 +58,13 @@ func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcs
coorCli := stgglb.CoordinatorRPCPool.Get() coorCli := stgglb.CoordinatorRPCPool.Get()
defer coorCli.Release() defer coorCli.Release()


resp, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]cortypes.HubID{srcSpace.RecommendHub.HubID}))
resp, cerr := coorCli.GetHubConnectivities(context.Background(), corrpc.ReqGetHubConnectivities([]jcstypes.HubID{srcSpace.RecommendHub.HubID}))
if cerr != nil { if cerr != nil {
delPkg() delPkg()
return nil, fmt.Errorf("getting hub connectivities: %w", cerr.ToError()) return nil, fmt.Errorf("getting hub connectivities: %w", cerr.ToError())
} }


cons := make(map[cortypes.HubID]cortypes.HubConnectivity)
cons := make(map[jcstypes.HubID]jcstypes.HubConnectivity)
for _, c := range resp.Connectivities { for _, c := range resp.Connectivities {
cons[c.ToHubID] = c cons[c.ToHubID] = c
} }
@@ -153,7 +152,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcs
return &pkg, nil return &pkg, nil
} }


func (u *Uploader) uploadFromBaseStore(srcSpace *jcsypes.UserSpaceDetail, targetSpace *jcsypes.UserSpaceDetail, entries []types.DirEntry, rootPath jcsypes.JPath) ([]db.AddObjectEntry, error) {
func (u *Uploader) uploadFromBaseStore(srcSpace *jcstypes.UserSpaceDetail, targetSpace *jcstypes.UserSpaceDetail, entries []types.DirEntry, rootPath jcstypes.JPath) ([]db.AddObjectEntry, error) {
ft := ioswitch2.FromTo{} ft := ioswitch2.FromTo{}


for _, e := range entries { for _, e := range entries {
@@ -198,7 +197,7 @@ func (u *Uploader) uploadFromBaseStore(srcSpace *jcsypes.UserSpaceDetail, target
Size: info.Size, Size: info.Size,
FileHash: info.Hash, FileHash: info.Hash,
CreateTime: time.Now(), CreateTime: time.Now(),
UserSpaceIDs: []jcsypes.UserSpaceID{targetSpace.UserSpace.UserSpaceID},
UserSpaceIDs: []jcstypes.UserSpaceID{targetSpace.UserSpace.UserSpaceID},
}) })
} }




+ 7
- 7
client/sdk/api/v1/bucket.go View File

@@ -4,7 +4,7 @@ import (
"net/http" "net/http"


"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type BucketService struct { type BucketService struct {
@@ -18,7 +18,7 @@ func (c *Client) Bucket() *BucketService {
const BucketGetPath = "/bucket/get" const BucketGetPath = "/bucket/get"


type BucketGet struct { type BucketGet struct {
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
} }


func (r *BucketGet) MakeParam() *sdks.RequestParam { func (r *BucketGet) MakeParam() *sdks.RequestParam {
@@ -26,7 +26,7 @@ func (r *BucketGet) MakeParam() *sdks.RequestParam {
} }


type BucketGetResp struct { type BucketGetResp struct {
Bucket jcsypes.Bucket `json:"bucket"`
Bucket jcstypes.Bucket `json:"bucket"`
} }


func (r *BucketGetResp) ParseResponse(resp *http.Response) error { func (r *BucketGetResp) ParseResponse(resp *http.Response) error {
@@ -48,7 +48,7 @@ func (r *BucketGetByName) MakeParam() *sdks.RequestParam {
} }


type BucketGetByNameResp struct { type BucketGetByNameResp struct {
Bucket jcsypes.Bucket `json:"bucket"`
Bucket jcstypes.Bucket `json:"bucket"`
} }


func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error { func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error {
@@ -70,7 +70,7 @@ func (r *BucketCreate) MakeParam() *sdks.RequestParam {
} }


type BucketCreateResp struct { type BucketCreateResp struct {
Bucket jcsypes.Bucket `json:"bucket"`
Bucket jcstypes.Bucket `json:"bucket"`
} }


func (r *BucketCreateResp) ParseResponse(resp *http.Response) error { func (r *BucketCreateResp) ParseResponse(resp *http.Response) error {
@@ -84,7 +84,7 @@ func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) {
const BucketDeletePath = "/bucket/delete" const BucketDeletePath = "/bucket/delete"


type BucketDelete struct { type BucketDelete struct {
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
} }


func (r *BucketDelete) MakeParam() *sdks.RequestParam { func (r *BucketDelete) MakeParam() *sdks.RequestParam {
@@ -111,7 +111,7 @@ func (r *BucketListAll) MakeParam() *sdks.RequestParam {
} }


type BucketListAllResp struct { type BucketListAllResp struct {
Buckets []jcsypes.Bucket `json:"buckets"`
Buckets []jcstypes.Bucket `json:"buckets"`
} }


func (r *BucketListAllResp) ParseResponse(resp *http.Response) error { func (r *BucketListAllResp) ParseResponse(resp *http.Response) error {


+ 24
- 24
client/sdk/api/v1/package.go View File

@@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
"gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type PackageService struct { type PackageService struct {
@@ -27,7 +27,7 @@ func (c *Client) Package() *PackageService {
const PackageGetPath = "/package/get" const PackageGetPath = "/package/get"


type PackageGet struct { type PackageGet struct {
PackageID jcsypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
PackageID jcstypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
} }


func (r *PackageGet) MakeParam() *sdks.RequestParam { func (r *PackageGet) MakeParam() *sdks.RequestParam {
@@ -35,7 +35,7 @@ func (r *PackageGet) MakeParam() *sdks.RequestParam {
} }


type PackageGetResp struct { type PackageGetResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
} }


func (r *PackageGetResp) ParseResponse(resp *http.Response) error { func (r *PackageGetResp) ParseResponse(resp *http.Response) error {
@@ -58,7 +58,7 @@ func (r *PackageGetByFullName) MakeParam() *sdks.RequestParam {
} }


type PackageGetByFullNameResp struct { type PackageGetByFullNameResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
} }


func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error { func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error {
@@ -72,8 +72,8 @@ func (c *PackageService) GetByFullName(req PackageGetByFullName) (*PackageGetByF
const PackageCreatePath = "/package/create" const PackageCreatePath = "/package/create"


type PackageCreate struct { type PackageCreate struct {
BucketID jcsypes.BucketID `json:"bucketID"`
Name string `json:"name"`
BucketID jcstypes.BucketID `json:"bucketID"`
Name string `json:"name"`
} }


func (r *PackageCreate) MakeParam() *sdks.RequestParam { func (r *PackageCreate) MakeParam() *sdks.RequestParam {
@@ -81,7 +81,7 @@ func (r *PackageCreate) MakeParam() *sdks.RequestParam {
} }


type PackageCreateResp struct { type PackageCreateResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
} }


func (r *PackageCreateResp) ParseResponse(resp *http.Response) error { func (r *PackageCreateResp) ParseResponse(resp *http.Response) error {
@@ -99,14 +99,14 @@ type PackageCreateUpload struct {
Files UploadObjectIterator `json:"-"` Files UploadObjectIterator `json:"-"`
} }
type PackageCreateUploadInfo struct { type PackageCreateUploadInfo struct {
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
CopyTo []jcsypes.UserSpaceID `json:"copyTo"`
CopyToPath []string `json:"copyToPath"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
CopyTo []jcstypes.UserSpaceID `json:"copyTo"`
CopyToPath []string `json:"copyToPath"`
} }
type PackageCreateUploadResp struct { type PackageCreateUploadResp struct {
Package jcsypes.Package `json:"package"`
Objects []jcsypes.Object `json:"objects"`
Package jcstypes.Package `json:"package"`
Objects []jcstypes.Object `json:"objects"`
} }


func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUploadResp, error) { func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUploadResp, error) {
@@ -148,10 +148,10 @@ func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUp
const PackageDownloadPath = "/package/download" const PackageDownloadPath = "/package/download"


type PackageDownload struct { type PackageDownload struct {
PackageID jcsypes.PackageID `url:"packageID" form:"packageID" binding:"required"`
Prefix string `url:"prefix" form:"prefix"`
NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"`
Zip bool `url:"zip,omitempty" form:"zip"`
PackageID jcstypes.PackageID `url:"packageID" form:"packageID" binding:"required"`
Prefix string `url:"prefix" form:"prefix"`
NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"`
Zip bool `url:"zip,omitempty" form:"zip"`
} }


func (r *PackageDownload) MakeParam() *sdks.RequestParam { func (r *PackageDownload) MakeParam() *sdks.RequestParam {
@@ -208,7 +208,7 @@ func (c *PackageService) Download(req PackageDownload) (*DownloadingPackage, err
const PackageDeletePath = "/package/delete" const PackageDeletePath = "/package/delete"


type PackageDelete struct { type PackageDelete struct {
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
PackageID jcstypes.PackageID `json:"packageID" binding:"required"`
} }


func (r *PackageDelete) MakeParam() *sdks.RequestParam { func (r *PackageDelete) MakeParam() *sdks.RequestParam {
@@ -228,9 +228,9 @@ func (c *PackageService) Delete(req PackageDelete) error {
const PackageClonePath = "/package/clone" const PackageClonePath = "/package/clone"


type PackageClone struct { type PackageClone struct {
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
PackageID jcstypes.PackageID `json:"packageID" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
} }


func (r *PackageClone) MakeParam() *sdks.RequestParam { func (r *PackageClone) MakeParam() *sdks.RequestParam {
@@ -238,7 +238,7 @@ func (r *PackageClone) MakeParam() *sdks.RequestParam {
} }


type PackageCloneResp struct { type PackageCloneResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
} }


func (r *PackageCloneResp) ParseResponse(resp *http.Response) error { func (r *PackageCloneResp) ParseResponse(resp *http.Response) error {
@@ -252,7 +252,7 @@ func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) {
const PackageListBucketPackagesPath = "/package/listBucketPackages" const PackageListBucketPackagesPath = "/package/listBucketPackages"


type PackageListBucketPackages struct { type PackageListBucketPackages struct {
BucketID jcsypes.BucketID `form:"bucketID" url:"bucketID" binding:"required"`
BucketID jcstypes.BucketID `form:"bucketID" url:"bucketID" binding:"required"`
} }


func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam { func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam {
@@ -260,7 +260,7 @@ func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam {
} }


type PackageListBucketPackagesResp struct { type PackageListBucketPackagesResp struct {
Packages []jcsypes.Package `json:"packages"`
Packages []jcstypes.Package `json:"packages"`
} }


func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error { func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error {


+ 22
- 22
client/sdk/api/v1/presigned.go View File

@@ -7,7 +7,7 @@ import (


"github.com/google/go-querystring/query" "github.com/google/go-querystring/query"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/signer" "gitlink.org.cn/cloudream/jcs-pub/client/sdk/signer"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type PresignedService struct { type PresignedService struct {
@@ -33,10 +33,10 @@ func (c *PresignedService) ObjectListByPath(req PresignedObjectListByPath, expir
const PresignedObjectDownloadByPathPath = "/presigned/object/downloadByPath" const PresignedObjectDownloadByPathPath = "/presigned/object/downloadByPath"


type PresignedObjectDownloadByPath struct { type PresignedObjectDownloadByPath struct {
PackageID jcsypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
Path string `form:"path" url:"path" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
PackageID jcstypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
Path string `form:"path" url:"path" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
} }


func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) { func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) {
@@ -46,9 +46,9 @@ func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPat
const PresignedObjectDownloadPath = "/presigned/object/download" const PresignedObjectDownloadPath = "/presigned/object/download"


type PresignedObjectDownload struct { type PresignedObjectDownload struct {
ObjectID jcsypes.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
ObjectID jcstypes.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
} }


func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) { func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) {
@@ -58,15 +58,15 @@ func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn
const PresignedObjectUploadPath = "/presigned/object/upload" const PresignedObjectUploadPath = "/presigned/object/upload"


type PresignedObjectUpload struct { type PresignedObjectUpload struct {
PackageID jcsypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
Affinity jcsypes.UserSpaceID `form:"affinity" url:"affinity,omitempty"`
CopyTo []jcsypes.UserSpaceID `form:"copyTo" url:"copyTo,omitempty"`
CopyToPath []string `form:"copyToPath" url:"copyToPath,omitempty"`
PackageID jcstypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
Affinity jcstypes.UserSpaceID `form:"affinity" url:"affinity,omitempty"`
CopyTo []jcstypes.UserSpaceID `form:"copyTo" url:"copyTo,omitempty"`
CopyToPath []string `form:"copyToPath" url:"copyToPath,omitempty"`
} }


type PresignedObjectUploadResp struct { type PresignedObjectUploadResp struct {
Object jcsypes.Object `json:"object"`
Object jcstypes.Object `json:"object"`
} }


func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int) (string, error) { func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int) (string, error) {
@@ -76,12 +76,12 @@ func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int)
const PresignedObjectNewMultipartUploadPath = "/presigned/object/newMultipartUpload" const PresignedObjectNewMultipartUploadPath = "/presigned/object/newMultipartUpload"


type PresignedObjectNewMultipartUpload struct { type PresignedObjectNewMultipartUpload struct {
PackageID jcsypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
PackageID jcstypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
} }


type PresignedObjectNewMultipartUploadResp struct { type PresignedObjectNewMultipartUploadResp struct {
Object jcsypes.Object `json:"object"`
Object jcstypes.Object `json:"object"`
} }


func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultipartUpload, expireIn int) (string, error) { func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultipartUpload, expireIn int) (string, error) {
@@ -91,8 +91,8 @@ func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultip
const PresignedObjectUploadPartPath = "/presigned/object/uploadPart" const PresignedObjectUploadPartPath = "/presigned/object/uploadPart"


type PresignedObjectUploadPart struct { type PresignedObjectUploadPart struct {
ObjectID jcsypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Index int `form:"index" binding:"required" url:"index"`
ObjectID jcstypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Index int `form:"index" binding:"required" url:"index"`
} }


type PresignedUploadPartResp struct{} type PresignedUploadPartResp struct{}
@@ -104,12 +104,12 @@ func (c *PresignedService) ObjectUploadPart(req PresignedObjectUploadPart, expir
const PresignedObjectCompleteMultipartUploadPath = "/presigned/object/completeMultipartUpload" const PresignedObjectCompleteMultipartUploadPath = "/presigned/object/completeMultipartUpload"


type PresignedObjectCompleteMultipartUpload struct { type PresignedObjectCompleteMultipartUpload struct {
ObjectID jcsypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Indexes []int `form:"indexes" binding:"required" url:"indexes"`
ObjectID jcstypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Indexes []int `form:"indexes" binding:"required" url:"indexes"`
} }


type PresignedObjectCompleteMultipartUploadResp struct { type PresignedObjectCompleteMultipartUploadResp struct {
Object jcsypes.Object `json:"object"`
Object jcstypes.Object `json:"object"`
} }


func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectCompleteMultipartUpload, expireIn int) (string, error) { func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectCompleteMultipartUpload, expireIn int) (string, error) {


+ 13
- 13
client/sdk/api/v1/space_syncer.go View File

@@ -4,7 +4,7 @@ import (
"net/http" "net/http"


"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type SpaceSyncerService struct { type SpaceSyncerService struct {
@@ -20,14 +20,14 @@ func (c *Client) SpaceSyncer() *SpaceSyncerService {
const SpaceSyncerCreateTaskPath = "/spaceSyncer/createTask" const SpaceSyncerCreateTaskPath = "/spaceSyncer/createTask"


type SpaceSyncerCreateTask struct { type SpaceSyncerCreateTask struct {
Trigger jcsypes.SpaceSyncTrigger `json:"trigger" binding:"required"`
Mode jcsypes.SpaceSyncMode `json:"mode" binding:"required"`
Filters []jcsypes.SpaceSyncFilter `json:"filters"`
Options jcsypes.SpaceSyncOptions `json:"options" binding:"required"`
SrcUserSpaceID jcsypes.UserSpaceID `json:"srcUserSpaceID" binding:"required"`
SrcPath string `json:"srcPath"`
DestUserSpaceIDs []jcsypes.UserSpaceID `json:"destUserSpaceIDs" binding:"required"`
DestPathes []string `json:"destPathes" binding:"required"`
Trigger jcstypes.SpaceSyncTrigger `json:"trigger" binding:"required"`
Mode jcstypes.SpaceSyncMode `json:"mode" binding:"required"`
Filters []jcstypes.SpaceSyncFilter `json:"filters"`
Options jcstypes.SpaceSyncOptions `json:"options" binding:"required"`
SrcUserSpaceID jcstypes.UserSpaceID `json:"srcUserSpaceID" binding:"required"`
SrcPath string `json:"srcPath"`
DestUserSpaceIDs []jcstypes.UserSpaceID `json:"destUserSpaceIDs" binding:"required"`
DestPathes []string `json:"destPathes" binding:"required"`
} }


func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam { func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam {
@@ -35,7 +35,7 @@ func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam {
} }


type SpaceSyncerCreateTaskResp struct { type SpaceSyncerCreateTaskResp struct {
Task jcsypes.SpaceSyncTask `json:"task"`
Task jcstypes.SpaceSyncTask `json:"task"`
} }


func (r *SpaceSyncerCreateTaskResp) ParseResponse(resp *http.Response) error { func (r *SpaceSyncerCreateTaskResp) ParseResponse(resp *http.Response) error {
@@ -49,7 +49,7 @@ func (c *SpaceSyncerService) CreateTask(req SpaceSyncerCreateTask) (*SpaceSyncer
const SpaceSyncerGetTaskPath = "/spaceSyncer/getTask" const SpaceSyncerGetTaskPath = "/spaceSyncer/getTask"


type SpaceSyncerGetTask struct { type SpaceSyncerGetTask struct {
TaskID jcsypes.SpaceSyncTaskID `url:"taskID" binding:"required"`
TaskID jcstypes.SpaceSyncTaskID `url:"taskID" binding:"required"`
} }


func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam { func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam {
@@ -57,7 +57,7 @@ func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam {
} }


type SpaceSyncerGetTaskResp struct { type SpaceSyncerGetTaskResp struct {
Task jcsypes.SpaceSyncTask `json:"task"`
Task jcstypes.SpaceSyncTask `json:"task"`
} }


func (r *SpaceSyncerGetTaskResp) ParseResponse(resp *http.Response) error { func (r *SpaceSyncerGetTaskResp) ParseResponse(resp *http.Response) error {
@@ -71,7 +71,7 @@ func (c *SpaceSyncerService) GetTask(req SpaceSyncerGetTask) (*SpaceSyncerGetTas
const SpaceSyncerCancelTaskPath = "/spaceSyncer/cancelTask" const SpaceSyncerCancelTaskPath = "/spaceSyncer/cancelTask"


type SpaceSyncerCancelTask struct { type SpaceSyncerCancelTask struct {
TaskID jcsypes.SpaceSyncTaskID `json:"taskID" binding:"required"`
TaskID jcstypes.SpaceSyncTaskID `json:"taskID" binding:"required"`
} }


func (r *SpaceSyncerCancelTask) MakeParam() *sdks.RequestParam { func (r *SpaceSyncerCancelTask) MakeParam() *sdks.RequestParam {


+ 2
- 2
client/sdk/api/v1/storage_test.go View File

@@ -10,7 +10,7 @@ import (
. "github.com/smartystreets/goconvey/convey" . "github.com/smartystreets/goconvey/convey"
"gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/api" "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func Test_PackageGet(t *testing.T) { func Test_PackageGet(t *testing.T) {
@@ -74,7 +74,7 @@ func Test_Object(t *testing.T) {
fileData[i] = byte(i) fileData[i] = byte(i)
} }


stgAff := jcsypes.UserSpaceID(2)
stgAff := jcstypes.UserSpaceID(2)


pkgName := uuid.NewString() pkgName := uuid.NewString()
createResp, err := cli.Package().Create(PackageCreate{ createResp, err := cli.Package().Create(PackageCreate{


+ 2
- 2
client/sdk/api/v1/system.go View File

@@ -4,7 +4,7 @@ import (
"net/http" "net/http"


"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type SystemService struct { type SystemService struct {
@@ -25,7 +25,7 @@ func (r *SystemStatus) MakeParam() *sdks.RequestParam {
} }


type SystemStatusResp struct { type SystemStatusResp struct {
SpeedStats jcsypes.SpeedStatsStatus `json:"speedStats"`
SpeedStats jcstypes.SpeedStatsStatus `json:"speedStats"`
} }


func (r *SystemStatusResp) ParseResponse(resp *http.Response) error { func (r *SystemStatusResp) ParseResponse(resp *http.Response) error {


+ 26
- 27
client/sdk/api/v1/user_space.go View File

@@ -4,8 +4,7 @@ import (
"net/http" "net/http"


"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type UserSpaceService struct { type UserSpaceService struct {
@@ -19,9 +18,9 @@ func (c *Client) UserSpace() *UserSpaceService {
const UserSpaceDownloadPackagePath = "/userSpace/downloadPackage" const UserSpaceDownloadPackagePath = "/userSpace/downloadPackage"


type UserSpaceDownloadPackageReq struct { type UserSpaceDownloadPackageReq struct {
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
RootPath string `json:"rootPath"`
PackageID jcstypes.PackageID `json:"packageID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
RootPath string `json:"rootPath"`
} }


func (r *UserSpaceDownloadPackageReq) MakeParam() *sdks.RequestParam { func (r *UserSpaceDownloadPackageReq) MakeParam() *sdks.RequestParam {
@@ -41,11 +40,11 @@ func (c *UserSpaceService) DownloadPackage(req UserSpaceDownloadPackageReq) (*Us
const UserSpaceCreatePackagePath = "/userSpace/createPackage" const UserSpaceCreatePackagePath = "/userSpace/createPackage"


type UserSpaceCreatePackageReq struct { type UserSpaceCreatePackageReq struct {
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Path string `json:"path" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
SpaceAffinity jcsypes.UserSpaceID `json:"spaceAffinity"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Path string `json:"path" binding:"required"`
BucketID jcstypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
SpaceAffinity jcstypes.UserSpaceID `json:"spaceAffinity"`
} }


func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam { func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam {
@@ -53,7 +52,7 @@ func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam {
} }


type UserSpaceCreatePackageResp struct { type UserSpaceCreatePackageResp struct {
Package jcsypes.Package `json:"package"`
Package jcstypes.Package `json:"package"`
} }


func (r *UserSpaceCreatePackageResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceCreatePackageResp) ParseResponse(resp *http.Response) error {
@@ -67,7 +66,7 @@ func (c *UserSpaceService) CreatePackage(req UserSpaceCreatePackageReq) (*UserSp
const UserSpaceGetPath = "/userSpace/get" const UserSpaceGetPath = "/userSpace/get"


type UserSpaceGet struct { type UserSpaceGet struct {
UserSpaceID jcsypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"`
} }


func (r *UserSpaceGet) MakeParam() *sdks.RequestParam { func (r *UserSpaceGet) MakeParam() *sdks.RequestParam {
@@ -75,7 +74,7 @@ func (r *UserSpaceGet) MakeParam() *sdks.RequestParam {
} }


type UserSpaceGetResp struct { type UserSpaceGetResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceGetResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceGetResp) ParseResponse(resp *http.Response) error {
@@ -97,7 +96,7 @@ func (r *UserSpaceGetByName) MakeParam() *sdks.RequestParam {
} }


type UserSpaceGetByNameResp struct { type UserSpaceGetByNameResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceGetByNameResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceGetByNameResp) ParseResponse(resp *http.Response) error {
@@ -117,7 +116,7 @@ func (r *UserSpaceGetAll) MakeParam() *sdks.RequestParam {
} }


type UserSpaceGetAllResp struct { type UserSpaceGetAllResp struct {
UserSpaces []jcsypes.UserSpace `json:"userSpaces"`
UserSpaces []jcstypes.UserSpace `json:"userSpaces"`
} }


func (r *UserSpaceGetAllResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceGetAllResp) ParseResponse(resp *http.Response) error {
@@ -134,10 +133,10 @@ const UserSpaceCreatePath = "/userSpace/create"


type UserSpaceCreate struct { type UserSpaceCreate struct {
Name string `json:"name" binding:"required"` Name string `json:"name" binding:"required"`
Storage cortypes.StorageType `json:"storage" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"`
ShardStore *cortypes.ShardStoreUserConfig `json:"shardStore"`
Features []cortypes.StorageFeature `json:"features"`
Storage jcstypes.StorageType `json:"storage" binding:"required"`
Credential jcstypes.StorageCredential `json:"credential" binding:"required"`
ShardStore *jcstypes.ShardStoreUserConfig `json:"shardStore"`
Features []jcstypes.StorageFeature `json:"features"`
WorkingDir string `json:"workingDir"` WorkingDir string `json:"workingDir"`
} }


@@ -146,7 +145,7 @@ func (r *UserSpaceCreate) MakeParam() *sdks.RequestParam {
} }


type UserSpaceCreateResp struct { type UserSpaceCreateResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceCreateResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceCreateResp) ParseResponse(resp *http.Response) error {
@@ -161,10 +160,10 @@ func (c *UserSpaceService) Create(req UserSpaceCreate) (*UserSpaceCreateResp, er
const UserSpaceUpdatePath = "/userSpace/update" const UserSpaceUpdatePath = "/userSpace/update"


type UserSpaceUpdate struct { type UserSpaceUpdate struct {
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Name string `json:"name" binding:"required"` Name string `json:"name" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"`
Features []cortypes.StorageFeature `json:"features"`
Credential jcstypes.StorageCredential `json:"credential" binding:"required"`
Features []jcstypes.StorageFeature `json:"features"`
} }


func (r *UserSpaceUpdate) MakeParam() *sdks.RequestParam { func (r *UserSpaceUpdate) MakeParam() *sdks.RequestParam {
@@ -172,7 +171,7 @@ func (r *UserSpaceUpdate) MakeParam() *sdks.RequestParam {
} }


type UserSpaceUpdateResp struct { type UserSpaceUpdateResp struct {
UserSpace jcsypes.UserSpace `json:"userSpace"`
UserSpace jcstypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceUpdateResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceUpdateResp) ParseResponse(resp *http.Response) error {
@@ -187,7 +186,7 @@ func (c *UserSpaceService) Update(req UserSpaceUpdate) (*UserSpaceUpdateResp, er
const UserSpaceDeletePath = "/userSpace/delete" const UserSpaceDeletePath = "/userSpace/delete"


type UserSpaceDelete struct { type UserSpaceDelete struct {
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
UserSpaceID jcstypes.UserSpaceID `json:"userSpaceID" binding:"required"`
} }


func (r *UserSpaceDelete) MakeParam() *sdks.RequestParam { func (r *UserSpaceDelete) MakeParam() *sdks.RequestParam {
@@ -208,8 +207,8 @@ func (c *UserSpaceService) Delete(req UserSpaceDelete) (*UserSpaceDeleteResp, er
const UserSpaceTestPath = "/userSpace/test" const UserSpaceTestPath = "/userSpace/test"


type UserSpaceTest struct { type UserSpaceTest struct {
Storage cortypes.StorageType `json:"storage" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"`
Storage jcstypes.StorageType `json:"storage" binding:"required"`
Credential jcstypes.StorageCredential `json:"credential" binding:"required"`
WorikingDir string `json:"workingDir"` WorikingDir string `json:"workingDir"`
} }




+ 1
- 1
common/globals/globals.go View File

@@ -1,7 +1,7 @@
package stgglb package stgglb


import ( import (
"gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type LocalMachineInfo struct { type LocalMachineInfo struct {


+ 2
- 2
common/globals/utils.go View File

@@ -1,9 +1,9 @@
package stgglb package stgglb


import cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
import jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"


// 根据当前节点与目标地址的距离关系,选择合适的地址 // 根据当前节点与目标地址的距离关系,选择合适的地址
func SelectGRPCAddress(hub *cortypes.Hub, addr *cortypes.GRPCAddressInfo) (string, int) {
func SelectGRPCAddress(hub *jcstypes.Hub, addr *jcstypes.GRPCAddressInfo) (string, int) {
// TODO 重新设计选择LocalIP的策略 // TODO 重新设计选择LocalIP的策略
return addr.ExternalIP, addr.ExternalGRPCPort return addr.ExternalIP, addr.ExternalGRPCPort
} }

+ 6
- 6
common/pkgs/accesstoken/accesstoken.go View File

@@ -10,7 +10,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async" "gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type CacheEvent interface { type CacheEvent interface {
@@ -23,17 +23,17 @@ type ExitEvent struct {
} }


type CacheKey struct { type CacheKey struct {
UserID cortypes.UserID
TokenID cortypes.AccessTokenID
UserID jcstypes.UserID
TokenID jcstypes.AccessTokenID
} }


var ErrTokenNotFound = fmt.Errorf("token not found") var ErrTokenNotFound = fmt.Errorf("token not found")


type AccessTokenLoader func(key CacheKey) (cortypes.UserAccessToken, error)
type AccessTokenLoader func(key CacheKey) (jcstypes.UserAccessToken, error)


type CacheEntry struct { type CacheEntry struct {
IsTokenValid bool IsTokenValid bool
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
PublicKey ed25519.PublicKey PublicKey ed25519.PublicKey
LoadedAt time.Time LoadedAt time.Time
LastUsedAt time.Time LastUsedAt time.Time
@@ -227,6 +227,6 @@ func (mc *Cache) Verify(authInfo rpc.AccessTokenAuthInfo) bool {
return ed25519.Verify(token.PublicKey, []byte(MakeStringToSign(authInfo.UserID, authInfo.AccessTokenID, authInfo.Nonce)), []byte(sig)) return ed25519.Verify(token.PublicKey, []byte(MakeStringToSign(authInfo.UserID, authInfo.AccessTokenID, authInfo.Nonce)), []byte(sig))
} }


func MakeStringToSign(userID cortypes.UserID, tokenID cortypes.AccessTokenID, nonce string) string {
func MakeStringToSign(userID jcstypes.UserID, tokenID jcstypes.AccessTokenID, nonce string) string {
return fmt.Sprintf("%v.%v.%v", userID, tokenID, nonce) return fmt.Sprintf("%v.%v.%v", userID, tokenID, nonce)
} }

+ 10
- 10
common/pkgs/connectivity/collector.go View File

@@ -11,7 +11,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator" corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub" hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type CollectorEvent interface { type CollectorEvent interface {
@@ -28,7 +28,7 @@ type CollectedEvent struct {
} }


type Connectivity struct { type Connectivity struct {
ToHubID cortypes.HubID
ToHubID jcstypes.HubID
Latency *time.Duration Latency *time.Duration
TestTime time.Time TestTime time.Time
} }
@@ -38,7 +38,7 @@ type Collector struct {
enabled bool enabled bool
collectNow chan any collectNow chan any
done chan any done chan any
connectivities map[cortypes.HubID]Connectivity
connectivities map[jcstypes.HubID]Connectivity
lock *sync.RWMutex lock *sync.RWMutex
} }


@@ -48,7 +48,7 @@ func NewEnabled(cfg Config) *Collector {
enabled: true, enabled: true,
collectNow: make(chan any, 1), collectNow: make(chan any, 1),
done: make(chan any, 1), done: make(chan any, 1),
connectivities: make(map[cortypes.HubID]Connectivity),
connectivities: make(map[jcstypes.HubID]Connectivity),
lock: &sync.RWMutex{}, lock: &sync.RWMutex{},
} }
return &rpt return &rpt
@@ -58,16 +58,16 @@ func NewDisabled() *Collector {
enabled: false, enabled: false,
collectNow: make(chan any, 1), collectNow: make(chan any, 1),
done: make(chan any, 1), done: make(chan any, 1),
connectivities: make(map[cortypes.HubID]Connectivity),
connectivities: make(map[jcstypes.HubID]Connectivity),
lock: &sync.RWMutex{}, lock: &sync.RWMutex{},
} }
} }


func (r *Collector) GetAll() map[cortypes.HubID]Connectivity {
func (r *Collector) GetAll() map[jcstypes.HubID]Connectivity {
r.lock.RLock() r.lock.RLock()
defer r.lock.RUnlock() defer r.lock.RUnlock()


ret := make(map[cortypes.HubID]Connectivity)
ret := make(map[jcstypes.HubID]Connectivity)
for k, v := range r.connectivities { for k, v := range r.connectivities {
ret[k] = v ret[k] = v
} }
@@ -170,7 +170,7 @@ func (r *Collector) testing() bool {


r.lock.Lock() r.lock.Lock()
// 删除所有hub的记录,然后重建,避免hub数量变化时导致残余数据 // 删除所有hub的记录,然后重建,避免hub数量变化时导致残余数据
r.connectivities = make(map[cortypes.HubID]Connectivity)
r.connectivities = make(map[jcstypes.HubID]Connectivity)
for _, con := range cons { for _, con := range cons {
r.connectivities[con.ToHubID] = con r.connectivities[con.ToHubID] = con
} }
@@ -179,13 +179,13 @@ func (r *Collector) testing() bool {
return true return true
} }


func (r *Collector) ping(hub cortypes.Hub) Connectivity {
func (r *Collector) ping(hub jcstypes.Hub) Connectivity {
log := logger.WithType[Collector]("").WithField("HubID", hub.HubID) log := logger.WithType[Collector]("").WithField("HubID", hub.HubID)


var ip string var ip string
var port int var port int
switch addr := hub.Address.(type) { switch addr := hub.Address.(type) {
case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
// TODO 重新设计选择LocalIP的策略 // TODO 重新设计选择LocalIP的策略
ip = addr.ExternalIP ip = addr.ExternalIP
port = addr.ExternalGRPCPort port = addr.ExternalGRPCPort


+ 15
- 15
common/pkgs/ioswitch2/fromto.go View File

@@ -4,7 +4,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type From interface { type From interface {
@@ -69,9 +69,9 @@ type FromTos []FromTo


type FromTo struct { type FromTo struct {
// 如果输入或者输出用到了EC编码的流,则需要提供EC参数。 // 如果输入或者输出用到了EC编码的流,则需要提供EC参数。
ECParam *jcsypes.ECRedundancy
ECParam *jcstypes.ECRedundancy
// 同上 // 同上
SegmentParam *jcsypes.SegmentRedundancy
SegmentParam *jcstypes.SegmentRedundancy
Froms []From Froms []From
Toes []To Toes []To
} }
@@ -110,12 +110,12 @@ func (f *FromDriver) GetStreamIndex() StreamIndex {
} }


type FromShardStore struct { type FromShardStore struct {
FileHash jcsypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcstypes.FileHash
UserSpace jcstypes.UserSpaceDetail
StreamIndex StreamIndex StreamIndex StreamIndex
} }


func NewFromShardstore(fileHash jcsypes.FileHash, space jcsypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
func NewFromShardstore(fileHash jcstypes.FileHash, space jcstypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
return &FromShardStore{ return &FromShardStore{
FileHash: fileHash, FileHash: fileHash,
UserSpace: space, UserSpace: space,
@@ -128,11 +128,11 @@ func (f *FromShardStore) GetStreamIndex() StreamIndex {
} }


type FromBaseStore struct { type FromBaseStore struct {
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
} }


func NewFromBaseStore(space jcsypes.UserSpaceDetail, path jcsypes.JPath) *FromBaseStore {
func NewFromBaseStore(space jcstypes.UserSpaceDetail, path jcstypes.JPath) *FromBaseStore {
return &FromBaseStore{ return &FromBaseStore{
UserSpace: space, UserSpace: space,
Path: path, Path: path,
@@ -177,13 +177,13 @@ func (t *ToDriver) GetRange() math2.Range {
} }


type ToShardStore struct { type ToShardStore struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
StreamIndex StreamIndex StreamIndex StreamIndex
Range math2.Range Range math2.Range
ResultStoreKey string ResultStoreKey string
} }


func NewToShardStore(space jcsypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
func NewToShardStore(space jcstypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
return &ToShardStore{ return &ToShardStore{
UserSpace: space, UserSpace: space,
StreamIndex: strIdx, StreamIndex: strIdx,
@@ -191,7 +191,7 @@ func NewToShardStore(space jcsypes.UserSpaceDetail, strIdx StreamIndex, retStore
} }
} }


func NewToShardStoreWithRange(space jcsypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
func NewToShardStoreWithRange(space jcstypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
return &ToShardStore{ return &ToShardStore{
UserSpace: space, UserSpace: space,
StreamIndex: streamIndex, StreamIndex: streamIndex,
@@ -209,12 +209,12 @@ func (t *ToShardStore) GetRange() math2.Range {
} }


type ToBaseStore struct { type ToBaseStore struct {
UserSpace jcsypes.UserSpaceDetail
ObjectPath jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
ObjectPath jcstypes.JPath
Option types.WriteOption Option types.WriteOption
} }


func NewToBaseStore(space jcsypes.UserSpaceDetail, objectPath jcsypes.JPath) *ToBaseStore {
func NewToBaseStore(space jcstypes.UserSpaceDetail, objectPath jcstypes.JPath) *ToBaseStore {
return &ToBaseStore{ return &ToBaseStore{
UserSpace: space, UserSpace: space,
ObjectPath: objectPath, ObjectPath: objectPath,


+ 4
- 4
common/pkgs/ioswitch2/http_hub_worker.go View File

@@ -9,16 +9,16 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
hubapi "gitlink.org.cn/cloudream/jcs-pub/hub/sdk/api" hubapi "gitlink.org.cn/cloudream/jcs-pub/hub/sdk/api"
) )


type HttpHubWorker struct { type HttpHubWorker struct {
Hub cortypes.Hub
Hub jcstypes.Hub
} }


func (w *HttpHubWorker) NewClient() (exec.WorkerClient, error) { func (w *HttpHubWorker) NewClient() (exec.WorkerClient, error) {
addressInfo := w.Hub.Address.(*cortypes.HttpAddressInfo)
addressInfo := w.Hub.Address.(*jcstypes.HttpAddressInfo)
baseUrl := "http://" + addressInfo.ExternalIP + ":" + strconv.Itoa(addressInfo.Port) baseUrl := "http://" + addressInfo.ExternalIP + ":" + strconv.Itoa(addressInfo.Port)
config := hubapi.Config{ config := hubapi.Config{
URL: baseUrl, URL: baseUrl,
@@ -47,7 +47,7 @@ func (w *HttpHubWorker) Equals(worker exec.WorkerInfo) bool {
} }


type HttpHubWorkerClient struct { type HttpHubWorkerClient struct {
hubID cortypes.HubID
hubID jcstypes.HubID
cli *hubapi.Client cli *hubapi.Client
} }




+ 4
- 4
common/pkgs/ioswitch2/hub_worker.go View File

@@ -11,7 +11,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub" hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.WorkerInfo]( var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.WorkerInfo](
@@ -20,8 +20,8 @@ var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.Wo
))) )))


type HubWorker struct { type HubWorker struct {
Hub cortypes.Hub
Address cortypes.GRPCAddressInfo
Hub jcstypes.Hub
Address jcstypes.GRPCAddressInfo
} }


func (w *HubWorker) NewClient() (exec.WorkerClient, error) { func (w *HubWorker) NewClient() (exec.WorkerClient, error) {
@@ -43,7 +43,7 @@ func (w *HubWorker) Equals(worker exec.WorkerInfo) bool {
} }


type HubWorkerClient struct { type HubWorkerClient struct {
hubID cortypes.HubID
hubID jcstypes.HubID
cli *hubrpc.Client cli *hubrpc.Client
} }




+ 14
- 14
common/pkgs/ioswitch2/ops2/base_store.go View File

@@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


const ( const (
@@ -43,8 +43,8 @@ func (v *BaseReadStatsValue) Clone() exec.VarValue {


type BaseRead struct { type BaseRead struct {
Output exec.VarID Output exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption Option types.OpenOption
} }


@@ -101,7 +101,7 @@ func (o *BaseRead) String() string {
} }


type BaseReadDyn struct { type BaseReadDyn struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Output exec.VarID Output exec.VarID
FileInfo exec.VarID FileInfo exec.VarID
Option types.OpenOption Option types.OpenOption
@@ -167,8 +167,8 @@ func (o *BaseReadDyn) String() string {


type BaseWrite struct { type BaseWrite struct {
Input exec.VarID Input exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
FileInfo exec.VarID FileInfo exec.VarID
Option types.WriteOption Option types.WriteOption
} }
@@ -213,12 +213,12 @@ func (o *BaseWrite) String() string {
type BaseReadNode struct { type BaseReadNode struct {
dag.NodeBase dag.NodeBase
From ioswitch2.From From ioswitch2.From
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption Option types.OpenOption
} }


func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{ node := &BaseReadNode{
From: from, From: from,
UserSpace: userSpace, UserSpace: userSpace,
@@ -254,11 +254,11 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
type BaseReadDynNode struct { type BaseReadDynNode struct {
dag.NodeBase dag.NodeBase
From ioswitch2.From From ioswitch2.From
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Option types.OpenOption Option types.OpenOption
} }


func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace jcsypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace jcstypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{ node := &BaseReadDynNode{
From: from, From: from,
UserSpace: userSpace, UserSpace: userSpace,
@@ -301,12 +301,12 @@ func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
type BaseWriteNode struct { type BaseWriteNode struct {
dag.NodeBase dag.NodeBase
To ioswitch2.To To ioswitch2.To
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.WriteOption Option types.WriteOption
} }


func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{ node := &BaseWriteNode{
To: to, To: to,
UserSpace: userSpace, UserSpace: userSpace,


+ 6
- 6
common/pkgs/ioswitch2/ops2/bypass.go View File

@@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -17,8 +17,8 @@ func init() {


// 旁路Http读取 // 旁路Http读取
type GetShardHTTPRequest struct { type GetShardHTTPRequest struct {
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
Output exec.VarID Output exec.VarID
} }


@@ -64,11 +64,11 @@ func (o *GetShardHTTPRequest) String() string {
// 旁路Http读取 // 旁路Http读取
type GetShardHTTPRequestNode struct { type GetShardHTTPRequestNode struct {
dag.NodeBase dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
} }


func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardHTTPRequestNode {
func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace jcstypes.UserSpaceDetail, fileHash jcstypes.FileHash) *GetShardHTTPRequestNode {
node := &GetShardHTTPRequestNode{ node := &GetShardHTTPRequestNode{
UserSpace: userSpace, UserSpace: userSpace,
FileHash: fileHash, FileHash: fileHash,


+ 7
- 7
common/pkgs/ioswitch2/ops2/ec.go View File

@@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -146,7 +146,7 @@ func (o *ECMultiply) String() string {
} }


type CallECMultiplier struct { type CallECMultiplier struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Coef [][]byte Coef [][]byte
Inputs []exec.VarID Inputs []exec.VarID
Outputs []exec.VarID Outputs []exec.VarID
@@ -203,12 +203,12 @@ func (o *CallECMultiplier) String() string {


type ECMultiplyNode struct { type ECMultiplyNode struct {
dag.NodeBase dag.NodeBase
EC jcsypes.ECRedundancy
EC jcstypes.ECRedundancy
InputIndexes []int InputIndexes []int
OutputIndexes []int OutputIndexes []int
} }


func (b *GraphNodeBuilder) NewECMultiply(ec jcsypes.ECRedundancy) *ECMultiplyNode {
func (b *GraphNodeBuilder) NewECMultiply(ec jcstypes.ECRedundancy) *ECMultiplyNode {
node := &ECMultiplyNode{ node := &ECMultiplyNode{
EC: ec, EC: ec,
} }
@@ -257,13 +257,13 @@ func (t *ECMultiplyNode) GenerateOp() (exec.Op, error) {


type CallECMultiplierNode struct { type CallECMultiplierNode struct {
dag.NodeBase dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
EC jcsypes.ECRedundancy
UserSpace jcstypes.UserSpaceDetail
EC jcstypes.ECRedundancy
InputIndexes []int InputIndexes []int
OutputIndexes []int OutputIndexes []int
} }


func (b *GraphNodeBuilder) NewCallECMultiplier(userSpace jcsypes.UserSpaceDetail) *CallECMultiplierNode {
func (b *GraphNodeBuilder) NewCallECMultiplier(userSpace jcstypes.UserSpaceDetail) *CallECMultiplierNode {
node := &CallECMultiplierNode{ node := &CallECMultiplierNode{
UserSpace: userSpace, UserSpace: userSpace,
} }


+ 7
- 7
common/pkgs/ioswitch2/ops2/multipart.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -40,7 +40,7 @@ func (v *UploadedPartInfoValue) Clone() exec.VarValue {
} }


type MultipartInitiator struct { type MultipartInitiator struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
UploadArgs exec.VarID UploadArgs exec.VarID
UploadedParts []exec.VarID UploadedParts []exec.VarID
FileOutput exec.VarID // 分片上传之后的临时文件的路径 FileOutput exec.VarID // 分片上传之后的临时文件的路径
@@ -99,7 +99,7 @@ func (o *MultipartInitiator) String() string {
} }


type MultipartUpload struct { type MultipartUpload struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
UploadArgs exec.VarID UploadArgs exec.VarID
UploadResult exec.VarID UploadResult exec.VarID
PartStream exec.VarID PartStream exec.VarID
@@ -149,10 +149,10 @@ func (o *MultipartUpload) String() string {


type MultipartInitiatorNode struct { type MultipartInitiatorNode struct {
dag.NodeBase dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
} }


func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace jcsypes.UserSpaceDetail) *MultipartInitiatorNode {
func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace jcstypes.UserSpaceDetail) *MultipartInitiatorNode {
node := &MultipartInitiatorNode{ node := &MultipartInitiatorNode{
UserSpace: userSpace, UserSpace: userSpace,
} }
@@ -194,12 +194,12 @@ func (n *MultipartInitiatorNode) GenerateOp() (exec.Op, error) {


type MultipartUploadNode struct { type MultipartUploadNode struct {
dag.NodeBase dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
PartNumber int PartNumber int
PartSize int64 PartSize int64
} }


func (b *GraphNodeBuilder) NewMultipartUpload(userSpace jcsypes.UserSpaceDetail, partNumber int, partSize int64) *MultipartUploadNode {
func (b *GraphNodeBuilder) NewMultipartUpload(userSpace jcstypes.UserSpaceDetail, partNumber int, partSize int64) *MultipartUploadNode {
node := &MultipartUploadNode{ node := &MultipartUploadNode{
UserSpace: userSpace, UserSpace: userSpace,
PartNumber: partNumber, PartNumber: partNumber,


+ 17
- 17
common/pkgs/ioswitch2/ops2/s2s.go View File

@@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -15,10 +15,10 @@ func init() {
} }


type S2STransfer struct { type S2STransfer struct {
SrcSpace jcsypes.UserSpaceDetail
SrcPath jcsypes.JPath
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
SrcSpace jcstypes.UserSpaceDetail
SrcPath jcstypes.JPath
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
Output exec.VarID Output exec.VarID
} }


@@ -55,10 +55,10 @@ func (o *S2STransfer) String() string {
} }


type S2STransferDyn struct { type S2STransferDyn struct {
SrcSpace jcsypes.UserSpaceDetail
SrcSpace jcstypes.UserSpaceDetail
SrcFileInfo exec.VarID SrcFileInfo exec.VarID
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
Output exec.VarID Output exec.VarID
} }


@@ -101,13 +101,13 @@ func (o *S2STransferDyn) String() string {


type S2STransferNode struct { type S2STransferNode struct {
dag.NodeBase dag.NodeBase
SrcSpace jcsypes.UserSpaceDetail
SrcPath jcsypes.JPath
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
SrcSpace jcstypes.UserSpaceDetail
SrcPath jcstypes.JPath
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
} }


func (b *GraphNodeBuilder) NewS2STransfer(srcSpace jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstSpace jcsypes.UserSpaceDetail, dstPath jcsypes.JPath) *S2STransferNode {
func (b *GraphNodeBuilder) NewS2STransfer(srcSpace jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstSpace jcstypes.UserSpaceDetail, dstPath jcstypes.JPath) *S2STransferNode {
n := &S2STransferNode{ n := &S2STransferNode{
SrcSpace: srcSpace, SrcSpace: srcSpace,
SrcPath: srcPath, SrcPath: srcPath,
@@ -139,12 +139,12 @@ func (n *S2STransferNode) GenerateOp() (exec.Op, error) {


type S2STransferDynNode struct { type S2STransferDynNode struct {
dag.NodeBase dag.NodeBase
SrcSpace jcsypes.UserSpaceDetail
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
SrcSpace jcstypes.UserSpaceDetail
DstSpace jcstypes.UserSpaceDetail
DstPath jcstypes.JPath
} }


func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace jcsypes.UserSpaceDetail, dstSpace jcsypes.UserSpaceDetail, dstPath jcsypes.JPath) *S2STransferDynNode {
func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace jcstypes.UserSpaceDetail, dstSpace jcstypes.UserSpaceDetail, dstPath jcstypes.JPath) *S2STransferDynNode {
n := &S2STransferDynNode{ n := &S2STransferDynNode{
SrcSpace: srcSpace, SrcSpace: srcSpace,
DstSpace: dstSpace, DstSpace: dstSpace,


+ 9
- 9
common/pkgs/ioswitch2/ops2/shard_store.go View File

@@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -15,8 +15,8 @@ func init() {
} }


type GetShardInfo struct { type GetShardInfo struct {
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
ShardInfo exec.VarID ShardInfo exec.VarID
} }


@@ -46,7 +46,7 @@ func (o *GetShardInfo) String() string {
} }


type StoreShard struct { type StoreShard struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
FileInfo exec.VarID FileInfo exec.VarID
ShardInfo exec.VarID ShardInfo exec.VarID
} }
@@ -84,11 +84,11 @@ func (o *StoreShard) String() string {


type GetShardInfoNode struct { type GetShardInfoNode struct {
dag.NodeBase dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
} }


func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardInfoNode {
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcstypes.UserSpaceDetail, fileHash jcstypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{ node := &GetShardInfoNode{
UserSpace: userSpace, UserSpace: userSpace,
FileHash: fileHash, FileHash: fileHash,
@@ -116,11 +116,11 @@ func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {


type StoreShardNode struct { type StoreShardNode struct {
dag.NodeBase dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
ShardInfoKey string ShardInfoKey string
} }


func (b *GraphNodeBuilder) NewStoreShard(userSpace jcsypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcstypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{ node := &StoreShardNode{
UserSpace: userSpace, UserSpace: userSpace,
ShardInfoKey: shardInfoKey, ShardInfoKey: shardInfoKey,


+ 4
- 5
common/pkgs/ioswitch2/parser/gen/generator.go View File

@@ -12,8 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


// 检查使用不同编码时参数是否设置到位 // 检查使用不同编码时参数是否设置到位
@@ -385,17 +384,17 @@ func buildToNode(ctx *state.GenerateState, t ioswitch2.To) (ops2.ToNode, error)
} }
} }


func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil { if space.RecommendHub == nil {
n.Env().ToEnvDriver(true) n.Env().ToEnvDriver(true)
return nil return nil
} }


switch addr := space.RecommendHub.Address.(type) { switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true) n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)


case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true) n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)


default: default:


+ 4
- 5
common/pkgs/ioswitch2/parser/opt/utils.go View File

@@ -5,21 +5,20 @@ import (


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil { if space.RecommendHub == nil {
n.Env().ToEnvDriver(true) n.Env().ToEnvDriver(true)
return nil return nil
} }


switch addr := space.RecommendHub.Address.(type) { switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true) n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)


case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true) n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)


default: default:


+ 2
- 2
common/pkgs/ioswitch2/plans/complete_multipart.go View File

@@ -8,10 +8,10 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func CompleteMultipart(blocks []jcsypes.ObjectBlock, blockSpaces []jcsypes.UserSpaceDetail, targetSpace jcsypes.UserSpaceDetail, shardInfoKey string, blder *exec.PlanBuilder) error {
func CompleteMultipart(blocks []jcstypes.ObjectBlock, blockSpaces []jcstypes.UserSpaceDetail, targetSpace jcstypes.UserSpaceDetail, shardInfoKey string, blder *exec.PlanBuilder) error {
da := ops2.NewGraphNodeBuilder() da := ops2.NewGraphNodeBuilder()


sizes := make([]int64, len(blocks)) sizes := make([]int64, len(blocks))


+ 4
- 5
common/pkgs/ioswitch2/plans/utils.go View File

@@ -5,21 +5,20 @@ import (


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil { if space.RecommendHub == nil {
n.Env().ToEnvDriver(true) n.Env().ToEnvDriver(true)
return nil return nil
} }


switch addr := space.RecommendHub.Address.(type) { switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true) n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)


case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true) n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)


default: default:


+ 7
- 7
common/pkgs/ioswitchlrc/fromto.go View File

@@ -4,7 +4,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type From interface { type From interface {
@@ -39,12 +39,12 @@ func (f *FromDriver) GetDataIndex() int {
} }


type FromNode struct { type FromNode struct {
FileHash jcsypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcstypes.FileHash
UserSpace jcstypes.UserSpaceDetail
DataIndex int DataIndex int
} }


func NewFromStorage(fileHash jcsypes.FileHash, space jcsypes.UserSpaceDetail, dataIndex int) *FromNode {
func NewFromStorage(fileHash jcstypes.FileHash, space jcstypes.UserSpaceDetail, dataIndex int) *FromNode {
return &FromNode{ return &FromNode{
FileHash: fileHash, FileHash: fileHash,
DataIndex: dataIndex, DataIndex: dataIndex,
@@ -88,14 +88,14 @@ func (t *ToDriver) GetRange() math2.Range {
} }


type ToNode struct { type ToNode struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
DataIndex int DataIndex int
Range math2.Range Range math2.Range
FileHashStoreKey string FileHashStoreKey string
Option types.WriteOption Option types.WriteOption
} }


func NewToStorage(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string) *ToNode {
func NewToStorage(space jcstypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string) *ToNode {
return &ToNode{ return &ToNode{
UserSpace: space, UserSpace: space,
DataIndex: dataIndex, DataIndex: dataIndex,
@@ -103,7 +103,7 @@ func NewToStorage(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey
} }
} }


func NewToStorageWithRange(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string, rng math2.Range) *ToNode {
func NewToStorageWithRange(space jcstypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string, rng math2.Range) *ToNode {
return &ToNode{ return &ToNode{
UserSpace: space, UserSpace: space,
DataIndex: dataIndex, DataIndex: dataIndex,


+ 3
- 3
common/pkgs/ioswitchlrc/hub_worker.go View File

@@ -7,7 +7,7 @@ import (
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub" hubrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/hub"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


// var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.WorkerInfo]( // var _ = serder.UseTypeUnionExternallyTagged(types.Ref(types.NewTypeUnion[exec.WorkerInfo](
@@ -15,8 +15,8 @@ import (
// ))) // )))


type HubWorker struct { type HubWorker struct {
Hub cortypes.Hub
Address cortypes.GRPCAddressInfo
Hub jcstypes.Hub
Address jcstypes.GRPCAddressInfo
} }


func (w *HubWorker) NewClient() (exec.WorkerClient, error) { func (w *HubWorker) NewClient() (exec.WorkerClient, error) {


+ 14
- 14
common/pkgs/ioswitchlrc/ops2/base_store.go View File

@@ -12,7 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -23,8 +23,8 @@ func init() {


type BaseRead struct { type BaseRead struct {
Output exec.VarID Output exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption Option types.OpenOption
} }


@@ -67,7 +67,7 @@ func (o *BaseRead) String() string {
} }


type BaseReadDyn struct { type BaseReadDyn struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Output exec.VarID Output exec.VarID
Path exec.VarID Path exec.VarID
Option types.OpenOption Option types.OpenOption
@@ -118,8 +118,8 @@ func (o *BaseReadDyn) String() string {


type BaseWrite struct { type BaseWrite struct {
Input exec.VarID Input exec.VarID
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
WriteResult exec.VarID WriteResult exec.VarID
Option types.WriteOption Option types.WriteOption
} }
@@ -164,12 +164,12 @@ func (o *BaseWrite) String() string {
type BaseReadNode struct { type BaseReadNode struct {
dag.NodeBase dag.NodeBase
From ioswitchlrc.From From ioswitchlrc.From
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.OpenOption Option types.OpenOption
} }


func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{ node := &BaseReadNode{
From: from, From: from,
UserSpace: userSpace, UserSpace: userSpace,
@@ -205,11 +205,11 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
type BaseReadDynNode struct { type BaseReadDynNode struct {
dag.NodeBase dag.NodeBase
From ioswitchlrc.From From ioswitchlrc.From
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Option types.OpenOption Option types.OpenOption
} }


func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace jcsypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace jcstypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{ node := &BaseReadDynNode{
From: from, From: from,
UserSpace: userSpace, UserSpace: userSpace,
@@ -252,12 +252,12 @@ func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
type BaseWriteNode struct { type BaseWriteNode struct {
dag.NodeBase dag.NodeBase
To ioswitchlrc.To To ioswitchlrc.To
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
UserSpace jcstypes.UserSpaceDetail
Path jcstypes.JPath
Option types.WriteOption Option types.WriteOption
} }


func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace jcstypes.UserSpaceDetail, path jcstypes.JPath, opt types.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{ node := &BaseWriteNode{
To: to, To: to,
UserSpace: userSpace, UserSpace: userSpace,


+ 5
- 5
common/pkgs/ioswitchlrc/ops2/ec.go View File

@@ -12,7 +12,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -114,12 +114,12 @@ func (o *GalMultiply) String() string {


type LRCConstructAnyNode struct { type LRCConstructAnyNode struct {
dag.NodeBase dag.NodeBase
LRC jcsypes.LRCRedundancy
LRC jcstypes.LRCRedundancy
InputIndexes []int InputIndexes []int
OutputIndexes []int OutputIndexes []int
} }


func (b *GraphNodeBuilder) NewLRCConstructAny(lrc jcsypes.LRCRedundancy) *LRCConstructAnyNode {
func (b *GraphNodeBuilder) NewLRCConstructAny(lrc jcstypes.LRCRedundancy) *LRCConstructAnyNode {
node := &LRCConstructAnyNode{ node := &LRCConstructAnyNode{
LRC: lrc, LRC: lrc,
} }
@@ -168,11 +168,11 @@ func (t *LRCConstructAnyNode) GenerateOp() (exec.Op, error) {


type LRCConstructGroupNode struct { type LRCConstructGroupNode struct {
dag.NodeBase dag.NodeBase
LRC jcsypes.LRCRedundancy
LRC jcstypes.LRCRedundancy
TargetBlockIndex int TargetBlockIndex int
} }


func (b *GraphNodeBuilder) NewLRCConstructGroup(lrc jcsypes.LRCRedundancy) *LRCConstructGroupNode {
func (b *GraphNodeBuilder) NewLRCConstructGroup(lrc jcstypes.LRCRedundancy) *LRCConstructGroupNode {
node := &LRCConstructGroupNode{ node := &LRCConstructGroupNode{
LRC: lrc, LRC: lrc,
} }


+ 9
- 9
common/pkgs/ioswitchlrc/ops2/shard_store.go View File

@@ -6,7 +6,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -15,8 +15,8 @@ func init() {
} }


type GetShardInfo struct { type GetShardInfo struct {
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
ShardInfo exec.VarID ShardInfo exec.VarID
} }


@@ -46,7 +46,7 @@ func (o *GetShardInfo) String() string {
} }


type StoreShard struct { type StoreShard struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
FileInfo exec.VarID FileInfo exec.VarID
ShardInfo exec.VarID ShardInfo exec.VarID
} }
@@ -84,11 +84,11 @@ func (o *StoreShard) String() string {


type GetShardInfoNode struct { type GetShardInfoNode struct {
dag.NodeBase dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
FileHash jcstypes.FileHash
} }


func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardInfoNode {
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcstypes.UserSpaceDetail, fileHash jcstypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{ node := &GetShardInfoNode{
UserSpace: userSpace, UserSpace: userSpace,
FileHash: fileHash, FileHash: fileHash,
@@ -116,11 +116,11 @@ func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {


type StoreShardNode struct { type StoreShardNode struct {
dag.NodeBase dag.NodeBase
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
ShardInfoKey string ShardInfoKey string
} }


func (b *GraphNodeBuilder) NewStoreShard(userSpace jcsypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcstypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{ node := &StoreShardNode{
UserSpace: userSpace, UserSpace: userSpace,
ShardInfoKey: shardInfoKey, ShardInfoKey: shardInfoKey,


+ 5
- 5
common/pkgs/ioswitchlrc/parser/generator.go View File

@@ -9,11 +9,11 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/ops2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type GenerateContext struct { type GenerateContext struct {
LRC jcsypes.LRCRedundancy
LRC jcstypes.LRCRedundancy
DAG *ops2.GraphNodeBuilder DAG *ops2.GraphNodeBuilder
To []ioswitchlrc.To To []ioswitchlrc.To
ToNodes map[ioswitchlrc.To]ops2.ToNode ToNodes map[ioswitchlrc.To]ops2.ToNode
@@ -27,7 +27,7 @@ func Encode(fr ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder)
} }


ctx := GenerateContext{ ctx := GenerateContext{
LRC: jcsypes.DefaultLRCRedundancy,
LRC: jcstypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(), DAG: ops2.NewGraphNodeBuilder(),
To: toes, To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode), ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),
@@ -124,7 +124,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
// 提供数据块+编码块中的k个块,重建任意块,包括完整文件。 // 提供数据块+编码块中的k个块,重建任意块,包括完整文件。
func ReconstructAny(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error { func ReconstructAny(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error {
ctx := GenerateContext{ ctx := GenerateContext{
LRC: jcsypes.DefaultLRCRedundancy,
LRC: jcstypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(), DAG: ops2.NewGraphNodeBuilder(),
To: toes, To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode), ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),
@@ -245,7 +245,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
// 输入同一组的多个块,恢复出剩下缺少的一个块。 // 输入同一组的多个块,恢复出剩下缺少的一个块。
func ReconstructGroup(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error { func ReconstructGroup(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error {
ctx := GenerateContext{ ctx := GenerateContext{
LRC: jcsypes.DefaultLRCRedundancy,
LRC: jcstypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(), DAG: ops2.NewGraphNodeBuilder(),
To: toes, To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode), ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),


+ 4
- 5
common/pkgs/ioswitchlrc/parser/utils.go View File

@@ -5,21 +5,20 @@ import (


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcstypes.UserSpaceDetail) error {
if space.RecommendHub == nil { if space.RecommendHub == nil {
n.Env().ToEnvDriver(true) n.Env().ToEnvDriver(true)
return nil return nil
} }


switch addr := space.RecommendHub.Address.(type) { switch addr := space.RecommendHub.Address.(type) {
case *cortypes.HttpAddressInfo:
case *jcstypes.HttpAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true) n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: *space.RecommendHub}, true)


case *cortypes.GRPCAddressInfo:
case *jcstypes.GRPCAddressInfo:
n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true) n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: *space.RecommendHub, Address: *addr}, true)


default: default:


+ 4
- 4
common/pkgs/publock/reqbuilder/user_space.go View File

@@ -5,7 +5,7 @@ import (


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/lockprovider" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/lockprovider"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type UserSpaceLockReqBuilder struct { type UserSpaceLockReqBuilder struct {
@@ -15,7 +15,7 @@ type UserSpaceLockReqBuilder struct {
func (b *LockRequestBuilder) UserSpace() *UserSpaceLockReqBuilder { func (b *LockRequestBuilder) UserSpace() *UserSpaceLockReqBuilder {
return &UserSpaceLockReqBuilder{LockRequestBuilder: b} return &UserSpaceLockReqBuilder{LockRequestBuilder: b}
} }
func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcsypes.UserSpaceID) *UserSpaceLockReqBuilder {
func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcstypes.UserSpaceID) *UserSpaceLockReqBuilder {
b.locks = append(b.locks, types.Lock{ b.locks = append(b.locks, types.Lock{
Path: b.makePath(spaceID), Path: b.makePath(spaceID),
Name: lockprovider.UserSpaceBuzyLock, Name: lockprovider.UserSpaceBuzyLock,
@@ -24,7 +24,7 @@ func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcsypes.UserSpaceID) *UserSpaceLo
return b return b
} }


func (b *UserSpaceLockReqBuilder) GC(spaceID jcsypes.UserSpaceID) *UserSpaceLockReqBuilder {
func (b *UserSpaceLockReqBuilder) GC(spaceID jcstypes.UserSpaceID) *UserSpaceLockReqBuilder {
b.locks = append(b.locks, types.Lock{ b.locks = append(b.locks, types.Lock{
Path: b.makePath(spaceID), Path: b.makePath(spaceID),
Name: lockprovider.UserSpaceGCLock, Name: lockprovider.UserSpaceGCLock,
@@ -33,6 +33,6 @@ func (b *UserSpaceLockReqBuilder) GC(spaceID jcsypes.UserSpaceID) *UserSpaceLock
return b return b
} }


func (b *UserSpaceLockReqBuilder) makePath(hubID jcsypes.UserSpaceID) []string {
func (b *UserSpaceLockReqBuilder) makePath(hubID jcstypes.UserSpaceID) []string {
return []string{lockprovider.UserSpaceLockPathPrefix, strconv.FormatInt(int64(hubID), 10)} return []string{lockprovider.UserSpaceLockPathPrefix, strconv.FormatInt(int64(hubID), 10)}
} }

+ 7
- 7
common/pkgs/rpc/auth.go View File

@@ -5,7 +5,7 @@ import (
"fmt" "fmt"
"strconv" "strconv"


cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"golang.org/x/net/context" "golang.org/x/net/context"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
@@ -27,8 +27,8 @@ const (
) )


type AccessTokenAuthInfo struct { type AccessTokenAuthInfo struct {
UserID cortypes.UserID
AccessTokenID cortypes.AccessTokenID
UserID jcstypes.UserID
AccessTokenID jcstypes.AccessTokenID
Nonce string Nonce string
Signature string Signature string
} }
@@ -125,8 +125,8 @@ func (s *ServerBase) authUnary(
} }


authInfo := AccessTokenAuthInfo{ authInfo := AccessTokenAuthInfo{
UserID: cortypes.UserID(userID),
AccessTokenID: cortypes.AccessTokenID(accessTokenIDs[0]),
UserID: jcstypes.UserID(userID),
AccessTokenID: jcstypes.AccessTokenID(accessTokenIDs[0]),
Nonce: nonce[0], Nonce: nonce[0],
Signature: signature[0], Signature: signature[0],
} }
@@ -200,8 +200,8 @@ func (s *ServerBase) authStream(
} }


authInfo := AccessTokenAuthInfo{ authInfo := AccessTokenAuthInfo{
UserID: cortypes.UserID(userID),
AccessTokenID: cortypes.AccessTokenID(accessTokenIDs[0]),
UserID: jcstypes.UserID(userID),
AccessTokenID: jcstypes.AccessTokenID(accessTokenIDs[0]),
Nonce: nonce[0], Nonce: nonce[0],
Signature: signature[0], Signature: signature[0],
} }


+ 15
- 15
common/pkgs/rpc/coordinator/hub.go View File

@@ -4,7 +4,7 @@ import (
context "context" context "context"


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type HubService interface { type HubService interface {
@@ -18,18 +18,18 @@ type HubService interface {
} }


type GetHubConfig struct { type GetHubConfig struct {
HubID cortypes.HubID `json:"hubID"`
HubID jcstypes.HubID `json:"hubID"`
} }
type GetHubConfigResp struct { type GetHubConfigResp struct {
Hub cortypes.Hub `json:"hub"`
Hub jcstypes.Hub `json:"hub"`
} }


func ReqGetHubConfig(hubID cortypes.HubID) *GetHubConfig {
func ReqGetHubConfig(hubID jcstypes.HubID) *GetHubConfig {
return &GetHubConfig{ return &GetHubConfig{
HubID: hubID, HubID: hubID,
} }
} }
func RespGetHubConfig(hub cortypes.Hub) *GetHubConfigResp {
func RespGetHubConfig(hub jcstypes.Hub) *GetHubConfigResp {
return &GetHubConfigResp{ return &GetHubConfigResp{
Hub: hub, Hub: hub,
} }
@@ -43,23 +43,23 @@ func (s *Server) GetHubConfig(ctx context.Context, req *rpc.Request) (*rpc.Respo


// 获取指定节点的信息。如果HubIDs为nil,则返回所有Hub // 获取指定节点的信息。如果HubIDs为nil,则返回所有Hub
type GetHubs struct { type GetHubs struct {
HubIDs []cortypes.HubID `json:"hubIDs"`
HubIDs []jcstypes.HubID `json:"hubIDs"`
} }
type GetHubsResp struct { type GetHubsResp struct {
Hubs []*cortypes.Hub `json:"hubs"`
Hubs []*jcstypes.Hub `json:"hubs"`
} }


func NewGetHubs(hubIDs []cortypes.HubID) *GetHubs {
func NewGetHubs(hubIDs []jcstypes.HubID) *GetHubs {
return &GetHubs{ return &GetHubs{
HubIDs: hubIDs, HubIDs: hubIDs,
} }
} }
func NewGetHubsResp(hubs []*cortypes.Hub) *GetHubsResp {
func NewGetHubsResp(hubs []*jcstypes.Hub) *GetHubsResp {
return &GetHubsResp{ return &GetHubsResp{
Hubs: hubs, Hubs: hubs,
} }
} }
func (r *GetHubsResp) GetHub(id cortypes.HubID) *cortypes.Hub {
func (r *GetHubsResp) GetHub(id jcstypes.HubID) *jcstypes.Hub {
for _, n := range r.Hubs { for _, n := range r.Hubs {
if n.HubID == id { if n.HubID == id {
return n return n
@@ -78,18 +78,18 @@ func (s *Server) GetHubs(ctx context.Context, req *rpc.Request) (*rpc.Response,
// 获取节点连通性信息 // 获取节点连通性信息


type GetHubConnectivities struct { type GetHubConnectivities struct {
HubIDs []cortypes.HubID `json:"hubIDs"`
HubIDs []jcstypes.HubID `json:"hubIDs"`
} }
type GetHubConnectivitiesResp struct { type GetHubConnectivitiesResp struct {
Connectivities []cortypes.HubConnectivity `json:"hubs"`
Connectivities []jcstypes.HubConnectivity `json:"hubs"`
} }


func ReqGetHubConnectivities(hubIDs []cortypes.HubID) *GetHubConnectivities {
func ReqGetHubConnectivities(hubIDs []jcstypes.HubID) *GetHubConnectivities {
return &GetHubConnectivities{ return &GetHubConnectivities{
HubIDs: hubIDs, HubIDs: hubIDs,
} }
} }
func RespGetHubConnectivities(cons []cortypes.HubConnectivity) *GetHubConnectivitiesResp {
func RespGetHubConnectivities(cons []jcstypes.HubConnectivity) *GetHubConnectivitiesResp {
return &GetHubConnectivitiesResp{ return &GetHubConnectivitiesResp{
Connectivities: cons, Connectivities: cons,
} }
@@ -103,7 +103,7 @@ func (s *Server) GetHubConnectivities(ctx context.Context, req *rpc.Request) (*r


// 上报节点连通性信息 // 上报节点连通性信息
type ReportHubConnectivity struct { type ReportHubConnectivity struct {
Connecttivities []cortypes.HubConnectivity
Connecttivities []jcstypes.HubConnectivity
} }
type ReportHubConnectivityResp struct { type ReportHubConnectivityResp struct {
} }


+ 3
- 3
common/pkgs/rpc/coordinator/storage.go View File

@@ -4,7 +4,7 @@ import (
context "context" context "context"


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type StorageService interface { type StorageService interface {
@@ -13,10 +13,10 @@ type StorageService interface {


// 为指定的Storage选择一个适合通信的Hub // 为指定的Storage选择一个适合通信的Hub
type SelectStorageHub struct { type SelectStorageHub struct {
Storages []cortypes.StorageType
Storages []jcstypes.StorageType
} }
type SelectStorageHubResp struct { type SelectStorageHubResp struct {
Hubs []*cortypes.Hub
Hubs []*jcstypes.Hub
} }


var _ = TokenAuth(Coordinator_SelectStorageHub_FullMethodName) var _ = TokenAuth(Coordinator_SelectStorageHub_FullMethodName)


+ 7
- 7
common/pkgs/rpc/coordinator/user.go View File

@@ -4,7 +4,7 @@ import (
context "context" context "context"


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type UserService interface { type UserService interface {
@@ -23,7 +23,7 @@ type UserLogin struct {
Password string Password string
} }
type UserLoginResp struct { type UserLoginResp struct {
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
PrivateKey string PrivateKey string
} }


@@ -42,7 +42,7 @@ func (s *Server) UserLogin(ctx context.Context, req *rpc.Request) (*rpc.Response
// 客户端刷新Token,原始Token会继续有效。 // 客户端刷新Token,原始Token会继续有效。
type UserRefreshToken struct{} type UserRefreshToken struct{}
type UserRefreshTokenResp struct { type UserRefreshTokenResp struct {
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
PrivateKey string PrivateKey string
} }


@@ -76,12 +76,12 @@ func (s *Server) UserLogout(ctx context.Context, req *rpc.Request) (*rpc.Respons


// Hub服务加载AccessToken // Hub服务加载AccessToken
type HubLoadAccessToken struct { type HubLoadAccessToken struct {
HubID cortypes.HubID
UserID cortypes.UserID
TokenID cortypes.AccessTokenID
HubID jcstypes.HubID
UserID jcstypes.UserID
TokenID jcstypes.AccessTokenID
} }
type HubLoadAccessTokenResp struct { type HubLoadAccessTokenResp struct {
Token cortypes.UserAccessToken
Token jcstypes.UserAccessToken
} }


func (c *Client) HubLoadAccessToken(ctx context.Context, msg *HubLoadAccessToken) (*HubLoadAccessTokenResp, *rpc.CodeError) { func (c *Client) HubLoadAccessToken(ctx context.Context, msg *HubLoadAccessToken) (*HubLoadAccessTokenResp, *rpc.CodeError) {


+ 5
- 5
common/pkgs/rpc/hub/cache.go View File

@@ -4,7 +4,7 @@ package hubrpc
import ( import (
"context" "context"


jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
) )


@@ -15,10 +15,10 @@ type CacheSvc interface {


// 获取Cache中文件列表 // 获取Cache中文件列表
type CheckCache struct { type CheckCache struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
} }
type CheckCacheResp struct { type CheckCacheResp struct {
FileHashes []jcsypes.FileHash
FileHashes []jcstypes.FileHash
} }


func (c *Client) CheckCache(ctx context.Context, req *CheckCache) (*CheckCacheResp, *rpc.CodeError) { func (c *Client) CheckCache(ctx context.Context, req *CheckCache) (*CheckCacheResp, *rpc.CodeError) {
@@ -33,8 +33,8 @@ func (s *Server) CheckCache(ctx context.Context, req *rpc.Request) (*rpc.Respons


// 清理Cache中不用的文件 // 清理Cache中不用的文件
type CacheGC struct { type CacheGC struct {
UserSpace jcsypes.UserSpaceDetail
Availables []jcsypes.FileHash
UserSpace jcstypes.UserSpaceDetail
Availables []jcstypes.FileHash
} }
type CacheGCResp struct{} type CacheGCResp struct{}




+ 3
- 3
common/pkgs/rpc/hub/user.go View File

@@ -4,7 +4,7 @@ import (
context "context" context "context"


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type UserSvc interface { type UserSvc interface {
@@ -13,8 +13,8 @@ type UserSvc interface {


// 通知用户的Token登出 // 通知用户的Token登出
type NotifyUserAccessTokenInvalid struct { type NotifyUserAccessTokenInvalid struct {
UserID cortypes.UserID
TokenID cortypes.AccessTokenID
UserID jcstypes.UserID
TokenID jcstypes.AccessTokenID
} }
type NotifyUserAccessTokenInvalidResp struct{} type NotifyUserAccessTokenInvalidResp struct{}




+ 3
- 3
common/pkgs/rpc/hub/user_space.go View File

@@ -4,7 +4,7 @@ package hubrpc
import ( import (
"context" "context"


jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
) )
@@ -16,7 +16,7 @@ type UserSpaceSvc interface {


// 列出指定BaseStore的指定位置内的所有文件 // 列出指定BaseStore的指定位置内的所有文件
type BaseStoreListAll struct { type BaseStoreListAll struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Path string Path string
} }
type BaseStoreListAllResp struct { type BaseStoreListAllResp struct {
@@ -35,7 +35,7 @@ func (s *Server) BaseStoreListAll(ctx context.Context, req *rpc.Request) (*rpc.R


// 批量在指定BaseStore中创建文件夹 // 批量在指定BaseStore中创建文件夹
type BaseStoreMkdirs struct { type BaseStoreMkdirs struct {
UserSpace jcsypes.UserSpaceDetail
UserSpace jcstypes.UserSpaceDetail
Pathes []string Pathes []string
} }




+ 8
- 8
common/pkgs/servicestats/hub_strorage_transfer.go View File

@@ -6,22 +6,22 @@ import (
"time" "time"


"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type HubStorageTransferStats struct { type HubStorageTransferStats struct {
data HubStorageTransferStatsData data HubStorageTransferStatsData
fromHubID cortypes.HubID
fromHubID jcstypes.HubID
lock *sync.Mutex lock *sync.Mutex
} }


type HubStorageTransferStatsData struct { type HubStorageTransferStatsData struct {
Entries map[cortypes.StorageID]*HubStorageTransferStatsEntry
Entries map[jcstypes.StorageID]*HubStorageTransferStatsEntry
StartTime time.Time StartTime time.Time
} }


type HubStorageTransferStatsEntry struct { type HubStorageTransferStatsEntry struct {
DestStorageID cortypes.StorageID
DestStorageID jcstypes.StorageID


OutputBytes int64 OutputBytes int64
MaxOutputBytes int64 MaxOutputBytes int64
@@ -36,7 +36,7 @@ type HubStorageTransferStatsEntry struct {
SuccessInput int64 SuccessInput int64
} }


func (s *HubStorageTransferStats) RecordUpload(dstStorageID cortypes.StorageID, transferBytes int64, isSuccess bool) {
func (s *HubStorageTransferStats) RecordUpload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


@@ -58,7 +58,7 @@ func (s *HubStorageTransferStats) RecordUpload(dstStorageID cortypes.StorageID,
e.TotalOutput++ e.TotalOutput++
} }


func (s *HubStorageTransferStats) RecordDownload(dstStorageID cortypes.StorageID, transferBytes int64, isSuccess bool) {
func (s *HubStorageTransferStats) RecordDownload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


@@ -83,7 +83,7 @@ func (s *HubStorageTransferStats) Reset() time.Time {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


s.data.Entries = make(map[cortypes.StorageID]*HubStorageTransferStatsEntry)
s.data.Entries = make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry)
s.data.StartTime = time.Now() s.data.StartTime = time.Now()
return s.data.StartTime return s.data.StartTime
} }
@@ -93,7 +93,7 @@ func (s *HubStorageTransferStats) DumpData() HubStorageTransferStatsData {
defer s.lock.Unlock() defer s.lock.Unlock()


data := s.data data := s.data
data.Entries = make(map[cortypes.StorageID]*HubStorageTransferStatsEntry)
data.Entries = make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry)
for k, v := range s.data.Entries { for k, v := range s.data.Entries {
v2 := *v v2 := *v
data.Entries[k] = &v2 data.Entries[k] = &v2


+ 8
- 8
common/pkgs/servicestats/hub_transfter.go View File

@@ -6,22 +6,22 @@ import (
"time" "time"


"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type HubTransferStats struct { type HubTransferStats struct {
data HubTransferStatsData data HubTransferStatsData
fromHubID cortypes.HubID
fromHubID jcstypes.HubID
lock *sync.Mutex lock *sync.Mutex
} }


type HubTransferStatsData struct { type HubTransferStatsData struct {
Entries map[cortypes.HubID]*HubTransferStatsEntry
Entries map[jcstypes.HubID]*HubTransferStatsEntry
StartTime time.Time StartTime time.Time
} }


type HubTransferStatsEntry struct { type HubTransferStatsEntry struct {
DestHubID cortypes.HubID
DestHubID jcstypes.HubID


OutputBytes int64 OutputBytes int64
MaxOutputBytes int64 MaxOutputBytes int64
@@ -36,7 +36,7 @@ type HubTransferStatsEntry struct {
SuccessInput int64 SuccessInput int64
} }


func (s *HubTransferStats) RecordOutput(dstHubID cortypes.HubID, transferBytes int64, isSuccess bool) {
func (s *HubTransferStats) RecordOutput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


@@ -58,7 +58,7 @@ func (s *HubTransferStats) RecordOutput(dstHubID cortypes.HubID, transferBytes i
e.TotalOutput++ e.TotalOutput++
} }


func (s *HubTransferStats) RecordInput(dstHubID cortypes.HubID, transferBytes int64, isSuccess bool) {
func (s *HubTransferStats) RecordInput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


@@ -85,7 +85,7 @@ func (s *HubTransferStats) Reset() time.Time {
defer s.lock.Unlock() defer s.lock.Unlock()


s.data.StartTime = time.Now() s.data.StartTime = time.Now()
s.data.Entries = make(map[cortypes.HubID]*HubTransferStatsEntry)
s.data.Entries = make(map[jcstypes.HubID]*HubTransferStatsEntry)
return s.data.StartTime return s.data.StartTime
} }


@@ -94,7 +94,7 @@ func (s *HubTransferStats) DumpData() HubTransferStatsData {
defer s.lock.Unlock() defer s.lock.Unlock()


data := s.data data := s.data
data.Entries = make(map[cortypes.HubID]*HubTransferStatsEntry)
data.Entries = make(map[jcstypes.HubID]*HubTransferStatsEntry)
for k, v := range s.data.Entries { for k, v := range s.data.Entries {
v2 := *v v2 := *v
data.Entries[k] = &v2 data.Entries[k] = &v2


+ 5
- 5
common/pkgs/servicestats/service_stats.go View File

@@ -4,7 +4,7 @@ import (
"sync" "sync"
"time" "time"


cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type StatsHost struct { type StatsHost struct {
@@ -14,24 +14,24 @@ type StatsHost struct {
HubStorageTransfer *HubStorageTransferStats HubStorageTransfer *HubStorageTransferStats
} }


func (h *StatsHost) SetupHubTransfer(fromHubID cortypes.HubID) {
func (h *StatsHost) SetupHubTransfer(fromHubID jcstypes.HubID) {
h.HubTransfer = &HubTransferStats{ h.HubTransfer = &HubTransferStats{
fromHubID: fromHubID, fromHubID: fromHubID,
lock: &sync.Mutex{}, lock: &sync.Mutex{},
data: HubTransferStatsData{ data: HubTransferStatsData{
StartTime: time.Now(), StartTime: time.Now(),
Entries: make(map[cortypes.HubID]*HubTransferStatsEntry),
Entries: make(map[jcstypes.HubID]*HubTransferStatsEntry),
}, },
} }
} }


func (h *StatsHost) SetupHubStorageTransfer(fromHubID cortypes.HubID) {
func (h *StatsHost) SetupHubStorageTransfer(fromHubID jcstypes.HubID) {
h.HubStorageTransfer = &HubStorageTransferStats{ h.HubStorageTransfer = &HubStorageTransferStats{
fromHubID: fromHubID, fromHubID: fromHubID,
lock: &sync.Mutex{}, lock: &sync.Mutex{},
data: HubStorageTransferStatsData{ data: HubStorageTransferStatsData{
StartTime: time.Now(), StartTime: time.Now(),
Entries: make(map[cortypes.StorageID]*HubStorageTransferStatsEntry),
Entries: make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry),
}, },
} }
} }

+ 4
- 5
common/pkgs/storage/efile/ec_multiplier.go View File

@@ -10,14 +10,13 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2" "gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type ECMultiplier struct { type ECMultiplier struct {
blder *builder blder *builder
url string url string
feat *cortypes.ECMultiplierFeature
feat *jcstypes.ECMultiplierFeature
outputs []string outputs []string
} }


@@ -98,9 +97,9 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
for i, data := range r.Data { for i, data := range r.Data {
ret[i] = types.FileInfo{ ret[i] = types.FileInfo{
// TODO 要确认一下output的格式 // TODO 要确认一下output的格式
Path: jcsypes.PathFromJcsPathString(m.outputs[i]),
Path: jcstypes.PathFromJcsPathString(m.outputs[i]),
Size: data.Size, Size: data.Size,
Hash: jcsypes.NewFullHashFromString(data.Sha256),
Hash: jcstypes.NewFullHashFromString(data.Sha256),
} }
} }




+ 7
- 8
common/pkgs/storage/efile/efile.go View File

@@ -10,12 +10,11 @@ import (
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
reg.RegisterBuilder[*cortypes.EFileType](func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*jcstypes.EFileType](func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{ return &builder{
detail: detail, detail: detail,
} }
@@ -24,15 +23,15 @@ func init() {


type builder struct { type builder struct {
types.EmptyBuilder types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
token string token string
tokenLock sync.Mutex tokenLock sync.Mutex
getTokenTime time.Time getTokenTime time.Time
} }


func (b *builder) getToken() (string, error) { func (b *builder) getToken() (string, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.EFileType)
cred := b.detail.UserSpace.Credential.(*cortypes.EFileCred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.EFileType)
cred := b.detail.UserSpace.Credential.(*jcstypes.EFileCred)


b.tokenLock.Lock() b.tokenLock.Lock()
defer b.tokenLock.Unlock() defer b.tokenLock.Unlock()
@@ -91,12 +90,12 @@ func (b *builder) getToken() (string, error) {
} }


func (b *builder) CreateECMultiplier(typeOnly bool) (types.ECMultiplier, error) { func (b *builder) CreateECMultiplier(typeOnly bool) (types.ECMultiplier, error) {
feat := types.FindFeature[*cortypes.ECMultiplierFeature](b.detail)
feat := types.FindFeature[*jcstypes.ECMultiplierFeature](b.detail)
if feat == nil { if feat == nil {
return nil, fmt.Errorf("feature ECMultiplier not found") return nil, fmt.Errorf("feature ECMultiplier not found")
} }


cred, ok := b.detail.UserSpace.Credential.(*cortypes.EFileCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.EFileCred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for efile storage", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for efile storage", b.detail.UserSpace.Credential)
} }


+ 2
- 2
common/pkgs/storage/factory/factory.go View File

@@ -6,12 +6,12 @@ import (
_ "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage" _ "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


// 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder, // 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder,
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查) // 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查)
func GetBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage) typ := reflect.TypeOf(detail.UserSpace.Storage)


ctor, ok := reg.StorageBuilders[typ] ctor, ok := reg.StorageBuilders[typ]


+ 4
- 5
common/pkgs/storage/factory/reg/reg.go View File

@@ -5,23 +5,22 @@ import (


"gitlink.org.cn/cloudream/common/utils/reflect2" "gitlink.org.cn/cloudream/common/utils/reflect2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type BuilderCtor func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder
type BuilderCtor func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder


var StorageBuilders = make(map[reflect.Type]BuilderCtor) var StorageBuilders = make(map[reflect.Type]BuilderCtor)


// 注册针对指定存储服务类型的Builder // 注册针对指定存储服务类型的Builder
func RegisterBuilder[T cortypes.StorageType](ctor BuilderCtor) {
func RegisterBuilder[T jcstypes.StorageType](ctor BuilderCtor) {
StorageBuilders[reflect2.TypeOf[T]()] = ctor StorageBuilders[reflect2.TypeOf[T]()] = ctor
} }


// 注:此函数只给storage包内部使用,外部包请使用外层的factory.GetBuilder // 注:此函数只给storage包内部使用,外部包请使用外层的factory.GetBuilder
// 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder, // 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder,
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查) // 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查)
func GetBuilderInternal(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilderInternal(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage) typ := reflect.TypeOf(detail.UserSpace.Storage)


ctor, ok := StorageBuilders[typ] ctor, ok := StorageBuilders[typ]


+ 8
- 8
common/pkgs/storage/local/base_store.go View File

@@ -9,22 +9,22 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type BaseStore struct { type BaseStore struct {
root string root string
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
} }


func NewBaseStore(root string, detail *jcsypes.UserSpaceDetail) (*BaseStore, error) {
func NewBaseStore(root string, detail *jcstypes.UserSpaceDetail) (*BaseStore, error) {
return &BaseStore{ return &BaseStore{
root: root, root: root,
detail: detail, detail: detail,
}, nil }, nil
} }


func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
log := s.getLogger() log := s.getLogger()


absObjPath := filepath.Join(s.root, pat.String()) absObjPath := filepath.Join(s.root, pat.String())
@@ -58,11 +58,11 @@ func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOp
return types.FileInfo{ return types.FileInfo{
Path: pat, Path: pat,
Size: counter.Count(), Size: counter.Count(),
Hash: jcsypes.NewFullHash(hasher.Sum()),
Hash: jcstypes.NewFullHash(hasher.Sum()),
}, nil }, nil
} }


func (s *BaseStore) Read(objPath jcsypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
func (s *BaseStore) Read(objPath jcstypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
absObjPath := filepath.Join(s.root, objPath.JoinOSPath()) absObjPath := filepath.Join(s.root, objPath.JoinOSPath())
file, err := os.Open(absObjPath) file, err := os.Open(absObjPath)
if err != nil { if err != nil {
@@ -86,7 +86,7 @@ func (s *BaseStore) Read(objPath jcsypes.JPath, opt types.OpenOption) (io.ReadCl
return ret, nil return ret, nil
} }


func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
func (s *BaseStore) Mkdir(path jcstypes.JPath) error {
absObjPath := filepath.Join(s.root, path.JoinOSPath()) absObjPath := filepath.Join(s.root, path.JoinOSPath())
err := os.MkdirAll(absObjPath, 0755) err := os.MkdirAll(absObjPath, 0755)
if err != nil { if err != nil {
@@ -96,7 +96,7 @@ func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
return nil return nil
} }


func (s *BaseStore) ReadDir(pat jcsypes.JPath) types.DirReader {
func (s *BaseStore) ReadDir(pat jcstypes.JPath) types.DirReader {
return &DirReader{ return &DirReader{
absRootPath: filepath.Join(s.root, pat.JoinOSPath()), absRootPath: filepath.Join(s.root, pat.JoinOSPath()),
rootJPath: pat.Clone(), rootJPath: pat.Clone(),


+ 4
- 4
common/pkgs/storage/local/dir_reader.go View File

@@ -6,14 +6,14 @@ import (
"path/filepath" "path/filepath"


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type DirReader struct { type DirReader struct {
// 完整的根路径(包括ReadDir的path参数),比如包括了盘符 // 完整的根路径(包括ReadDir的path参数),比如包括了盘符
absRootPath string absRootPath string
// ReadDir函数传递进来的path参数 // ReadDir函数传递进来的path参数
rootJPath jcsypes.JPath
rootJPath jcstypes.JPath
init bool init bool
curEntries []dirEntry curEntries []dirEntry
} }
@@ -42,7 +42,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {


for _, e := range es { for _, e := range es {
r.curEntries = append(r.curEntries, dirEntry{ r.curEntries = append(r.curEntries, dirEntry{
dir: jcsypes.JPath{},
dir: jcstypes.JPath{},
entry: e, entry: e,
}) })
} }
@@ -103,7 +103,7 @@ func (r *DirReader) Close() {
} }


type dirEntry struct { type dirEntry struct {
dir jcsypes.JPath
dir jcstypes.JPath
entry os.DirEntry entry os.DirEntry
} }




+ 9
- 10
common/pkgs/storage/local/local.go View File

@@ -5,12 +5,11 @@ import (


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
reg.RegisterBuilder[*cortypes.LocalType](func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*jcstypes.LocalType](func(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{ return &builder{
detail: detail, detail: detail,
} }
@@ -19,7 +18,7 @@ func init() {


type builder struct { type builder struct {
types.EmptyBuilder types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
} }


func (b *builder) FeatureDesc() types.FeatureDesc { func (b *builder) FeatureDesc() types.FeatureDesc {
@@ -27,7 +26,7 @@ func (b *builder) FeatureDesc() types.FeatureDesc {
} }


func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) { func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
cred, ok := b.detail.UserSpace.Credential.(*cortypes.LocalCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.LocalCred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential)
} }
@@ -40,7 +39,7 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
} }


func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) { func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
cred, ok := b.detail.UserSpace.Credential.(*cortypes.LocalCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.LocalCred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for local storage", b.detail.UserSpace.Credential)
} }
@@ -53,9 +52,9 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
} }


func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) { func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
feat := types.FindFeature[*cortypes.MultipartUploadFeature](b.detail)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil { if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.MultipartUploadFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
} }


if typeOnly { if typeOnly {
@@ -68,9 +67,9 @@ func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
} }


func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) { func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) {
feat := types.FindFeature[*cortypes.S2STransferFeature](b.detail)
feat := types.FindFeature[*jcstypes.S2STransferFeature](b.detail)
if feat == nil { if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.S2STransferFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.S2STransferFeature{})
} }


if typeOnly { if typeOnly {


+ 6
- 7
common/pkgs/storage/local/multipart_upload.go View File

@@ -13,14 +13,13 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2" "gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type Multiparter struct { type Multiparter struct {
detail *jcsypes.UserSpaceDetail
localStg *cortypes.LocalCred
feat *cortypes.MultipartUploadFeature
detail *jcstypes.UserSpaceDetail
localStg *jcstypes.LocalCred
feat *jcstypes.MultipartUploadFeature
} }


func (*Multiparter) MinPartSize() int64 { func (*Multiparter) MinPartSize() int64 {
@@ -80,7 +79,7 @@ type MultipartTask struct {
absTempDir string // 应该要是绝对路径 absTempDir string // 应该要是绝对路径
tempFileName string tempFileName string
tempPartsDir string tempPartsDir string
joinedFileJPath jcsypes.JPath
joinedFileJPath jcstypes.JPath
absJoinedFilePath string absJoinedFilePath string
uploadID string uploadID string
} }
@@ -118,7 +117,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
return types.FileInfo{ return types.FileInfo{
Path: i.joinedFileJPath, Path: i.joinedFileJPath,
Size: size, Size: size,
Hash: jcsypes.NewFullHash(h),
Hash: jcstypes.NewFullHash(h),
}, nil }, nil
} }




+ 9
- 10
common/pkgs/storage/local/s2s.go View File

@@ -7,24 +7,23 @@ import (
"path/filepath" "path/filepath"


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type S2STransfer struct { type S2STransfer struct {
feat *cortypes.S2STransferFeature
detail *jcsypes.UserSpaceDetail
localStg *cortypes.LocalCred
dstPath jcsypes.JPath
feat *jcstypes.S2STransferFeature
detail *jcstypes.UserSpaceDetail
localStg *jcstypes.LocalCred
dstPath jcstypes.JPath
} }


// 只有同一个机器的存储之间才可以进行数据直传 // 只有同一个机器的存储之间才可以进行数据直传
func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
if types.FindFeature[*cortypes.S2STransferFeature](dst) == nil {
func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
if types.FindFeature[*jcstypes.S2STransferFeature](dst) == nil {
return false return false
} }


_, ok := src.UserSpace.Storage.(*cortypes.LocalType)
_, ok := src.UserSpace.Storage.(*jcstypes.LocalType)
if !ok { if !ok {
return false return false
} }
@@ -37,7 +36,7 @@ func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
} }


// 执行数据直传 // 执行数据直传
func (s *S2STransfer) Transfer(ctx context.Context, src *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstPath jcsypes.JPath) (types.FileInfo, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (types.FileInfo, error) {
s.dstPath = dstPath s.dstPath = dstPath


copy, err := os.OpenFile(filepath.Join(s.localStg.RootDir, s.dstPath.JoinOSPath()), os.O_WRONLY|os.O_CREATE, 0644) copy, err := os.OpenFile(filepath.Join(s.localStg.RootDir, s.dstPath.JoinOSPath()), os.O_WRONLY|os.O_CREATE, 0644)


+ 12
- 12
common/pkgs/storage/local/shard_store.go View File

@@ -10,18 +10,18 @@ import (


"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type ShardStore struct { type ShardStore struct {
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
stgRoot string stgRoot string
storeAbsRoot string storeAbsRoot string
lock sync.Mutex lock sync.Mutex
done chan any done chan any
} }


func NewShardStore(root string, detail *jcsypes.UserSpaceDetail) (*ShardStore, error) {
func NewShardStore(root string, detail *jcstypes.UserSpaceDetail) (*ShardStore, error) {
storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.JoinOSPath(), types.ShardStoreWorkingDir)) storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.JoinOSPath(), types.ShardStoreWorkingDir))
if err != nil { if err != nil {
return nil, fmt.Errorf("get abs root: %w", err) return nil, fmt.Errorf("get abs root: %w", err)
@@ -43,7 +43,7 @@ func (s *ShardStore) Stop() {
s.getLogger().Infof("component stop") s.getLogger().Infof("component stop")
} }


func (s *ShardStore) Store(path jcsypes.JPath, hash jcsypes.FileHash, size int64) (types.FileInfo, error) {
func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (types.FileInfo, error) {
fullTempPath := filepath.Join(s.stgRoot, path.JoinOSPath()) fullTempPath := filepath.Join(s.stgRoot, path.JoinOSPath())


s.lock.Lock() s.lock.Lock()
@@ -81,7 +81,7 @@ func (s *ShardStore) Store(path jcsypes.JPath, hash jcsypes.FileHash, size int64
}, nil }, nil
} }


func (s *ShardStore) Info(hash jcsypes.FileHash) (types.FileInfo, error) {
func (s *ShardStore) Info(hash jcstypes.FileHash) (types.FileInfo, error) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


@@ -118,7 +118,7 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
return err return err
} }


fileHash, err := jcsypes.ParseHash(filepath.Base(info.Name()))
fileHash, err := jcstypes.ParseHash(filepath.Base(info.Name()))
if err != nil { if err != nil {
return nil return nil
} }
@@ -137,11 +137,11 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
return infos, nil return infos, nil
} }


func (s *ShardStore) GC(avaiables []jcsypes.FileHash) error {
func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


avais := make(map[jcsypes.FileHash]bool)
avais := make(map[jcstypes.FileHash]bool)
for _, hash := range avaiables { for _, hash := range avaiables {
avais[hash] = true avais[hash] = true
} }
@@ -162,7 +162,7 @@ func (s *ShardStore) GC(avaiables []jcsypes.FileHash) error {
return err return err
} }


fileHash, err := jcsypes.ParseHash(filepath.Base(info.Name()))
fileHash, err := jcstypes.ParseHash(filepath.Base(info.Name()))
if err != nil { if err != nil {
return nil return nil
} }
@@ -199,14 +199,14 @@ func (s *ShardStore) getLogger() logger.Logger {
return logger.WithField("ShardStore", "Local").WithField("Storage", s.detail.UserSpace.Storage.String()) return logger.WithField("ShardStore", "Local").WithField("Storage", s.detail.UserSpace.Storage.String())
} }


func (s *ShardStore) getFileDirFromHash(hash jcsypes.FileHash) string {
func (s *ShardStore) getFileDirFromHash(hash jcstypes.FileHash) string {
return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2)) return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2))
} }


func (s *ShardStore) getFilePathFromHash(hash jcsypes.FileHash) string {
func (s *ShardStore) getFilePathFromHash(hash jcstypes.FileHash) string {
return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2), string(hash)) return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2), string(hash))
} }


func (s *ShardStore) getJPathFromHash(hash jcsypes.FileHash) jcsypes.JPath {
func (s *ShardStore) getJPathFromHash(hash jcstypes.FileHash) jcstypes.JPath {
return s.detail.UserSpace.WorkingDir.ConcatCompsNew(types.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash)) return s.detail.UserSpace.WorkingDir.ConcatCompsNew(types.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
} }

+ 17
- 18
common/pkgs/storage/obs/obs.go View File

@@ -9,20 +9,19 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
s3stg "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3" s3stg "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
reg.RegisterBuilder[*cortypes.OBSType](newBuilder)
reg.RegisterBuilder[*jcstypes.OBSType](newBuilder)
} }


type builder struct { type builder struct {
types.EmptyBuilder types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
} }


func newBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func newBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{ return &builder{
detail: detail, detail: detail,
} }
@@ -33,8 +32,8 @@ func (b *builder) FeatureDesc() types.FeatureDesc {
} }


func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) { func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
} }
@@ -52,8 +51,8 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
} }


func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) { func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
} }
@@ -70,7 +69,7 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
return s3stg.NewBaseStore(b.detail, cli, bucket, s3stg.BaseStoreOption{UseAWSSha256: false}) return s3stg.NewBaseStore(b.detail, cli, bucket, s3stg.BaseStoreOption{UseAWSSha256: false})
} }


func createClient(stgType *cortypes.OBSType, cred *cortypes.OBSCred) (*s3.Client, string, error) {
func createClient(stgType *jcstypes.OBSType, cred *jcstypes.OBSCred) (*s3.Client, string, error) {
awsConfig := aws.Config{} awsConfig := aws.Config{}


cre := aws.Credentials{ cre := aws.Credentials{
@@ -91,17 +90,17 @@ func createClient(stgType *cortypes.OBSType, cred *cortypes.OBSCred) (*s3.Client
} }


func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) { func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
feat := types.FindFeature[*cortypes.MultipartUploadFeature](b.detail)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil { if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.MultipartUploadFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
} }


if typeOnly { if typeOnly {
return (*s3stg.Multiparter)(nil), nil return (*s3stg.Multiparter)(nil), nil
} }


cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
} }
@@ -120,17 +119,17 @@ func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
} }


func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) { func (b *builder) CreateS2STransfer(typeOnly bool) (types.S2STransfer, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.OBSType)
feat := types.FindFeature[*cortypes.S2STransferFeature](b.detail)
stgType := b.detail.UserSpace.Storage.(*jcstypes.OBSType)
feat := types.FindFeature[*jcstypes.S2STransferFeature](b.detail)
if feat == nil { if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.S2STransferFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.S2STransferFeature{})
} }


if typeOnly { if typeOnly {
return (*S2STransfer)(nil), nil return (*S2STransfer)(nil), nil
} }


cred, ok := b.detail.UserSpace.Credential.(*cortypes.OBSCred)
cred, ok := b.detail.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for obs storage", b.detail.UserSpace.Credential)
} }


+ 9
- 10
common/pkgs/storage/obs/obs_test.go View File

@@ -5,40 +5,39 @@ import (
"testing" "testing"


. "github.com/smartystreets/goconvey/convey" . "github.com/smartystreets/goconvey/convey"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func Test_S2S(t *testing.T) { func Test_S2S(t *testing.T) {
Convey("OBS", t, func() { Convey("OBS", t, func() {
s2s := S2STransfer{ s2s := S2STransfer{
stgType: &cortypes.OBSType{
stgType: &jcstypes.OBSType{
Region: "cn-north-4", Region: "cn-north-4",
Endpoint: "obs.cn-north-4.myhuaweicloud.com", Endpoint: "obs.cn-north-4.myhuaweicloud.com",
Bucket: "pcm3-bucket3", Bucket: "pcm3-bucket3",
ProjectID: "", ProjectID: "",
}, },
cred: &cortypes.OBSCred{
cred: &jcstypes.OBSCred{
AK: "", AK: "",
SK: "", SK: "",
}, },
feat: &cortypes.S2STransferFeature{},
feat: &jcstypes.S2STransferFeature{},
} }


_, err := s2s.Transfer(context.TODO(), &jcsypes.UserSpaceDetail{
UserSpace: jcsypes.UserSpace{
Storage: &cortypes.OBSType{
_, err := s2s.Transfer(context.TODO(), &jcstypes.UserSpaceDetail{
UserSpace: jcstypes.UserSpace{
Storage: &jcstypes.OBSType{
Region: "cn-north-4", Region: "cn-north-4",
Endpoint: "obs.cn-north-4.myhuaweicloud.com", Endpoint: "obs.cn-north-4.myhuaweicloud.com",
Bucket: "pcm2-bucket2", Bucket: "pcm2-bucket2",
ProjectID: "", ProjectID: "",
}, },
Credential: &cortypes.OBSCred{
Credential: &jcstypes.OBSCred{
AK: "", AK: "",
SK: "", SK: "",
}, },
}, },
}, jcsypes.PathFromComps("test_data/test03.txt"), jcsypes.PathFromComps("atest.txt"))
}, jcstypes.PathFromComps("test_data/test03.txt"), jcstypes.PathFromComps("atest.txt"))
defer s2s.Close() defer s2s.Close()


So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)


+ 12
- 13
common/pkgs/storage/obs/s2s.go View File

@@ -15,20 +15,19 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2" "gitlink.org.cn/cloudream/common/utils/os2"
stgs3 "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3" stgs3 "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type S2STransfer struct { type S2STransfer struct {
detail *jcsypes.UserSpaceDetail
stgType *cortypes.OBSType
cred *cortypes.OBSCred
feat *cortypes.S2STransferFeature
detail *jcstypes.UserSpaceDetail
stgType *jcstypes.OBSType
cred *jcstypes.OBSCred
feat *jcstypes.S2STransferFeature
taskID *int64 taskID *int64
omsCli *oms.OmsClient omsCli *oms.OmsClient
} }


func NewS2STransfer(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType, cred *cortypes.OBSCred, feat *cortypes.S2STransferFeature) *S2STransfer {
func NewS2STransfer(detail *jcstypes.UserSpaceDetail, stgType *jcstypes.OBSType, cred *jcstypes.OBSCred, feat *jcstypes.S2STransferFeature) *S2STransfer {
return &S2STransfer{ return &S2STransfer{
detail: detail, detail: detail,
stgType: stgType, stgType: stgType,
@@ -38,13 +37,13 @@ func NewS2STransfer(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType,
} }


// 判断是否能从指定的源存储中直传到当前存储的目的路径 // 判断是否能从指定的源存储中直传到当前存储的目的路径
func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
req := makeRequest(src, jcsypes.JPath{})
func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
req := makeRequest(src, jcstypes.JPath{})
return req != nil return req != nil
} }


// 执行数据直传。返回传输后的文件路径 // 执行数据直传。返回传输后的文件路径
func (s *S2STransfer) Transfer(ctx context.Context, src *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstPath jcsypes.JPath) (types.FileInfo, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (types.FileInfo, error) {
req := makeRequest(src, srcPath) req := makeRequest(src, srcPath)
if req == nil { if req == nil {
return types.FileInfo{}, fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage) return types.FileInfo{}, fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage)
@@ -178,12 +177,12 @@ func (s *S2STransfer) Close() {
} }
} }


func makeRequest(srcStg *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath) *model.SrcNodeReq {
func makeRequest(srcStg *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath) *model.SrcNodeReq {
switch srcType := srcStg.UserSpace.Storage.(type) { switch srcType := srcStg.UserSpace.Storage.(type) {
case *cortypes.OBSType:
case *jcstypes.OBSType:
cloudType := "HuaweiCloud" cloudType := "HuaweiCloud"


cred, ok := srcStg.UserSpace.Credential.(*cortypes.OBSCred)
cred, ok := srcStg.UserSpace.Credential.(*jcstypes.OBSCred)
if !ok { if !ok {
return nil return nil
} }


+ 5
- 6
common/pkgs/storage/obs/shard_store.go View File

@@ -5,17 +5,16 @@ import (
"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type ShardStore struct { type ShardStore struct {
*s3.ShardStore *s3.ShardStore
stgType *cortypes.OBSType
cred *cortypes.OBSCred
stgType *jcstypes.OBSType
cred *jcstypes.OBSCred
} }


func NewShardStore(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType, cred *cortypes.OBSCred, s3Cli *awss3.Client, bkt string) (*ShardStore, error) {
func NewShardStore(detail *jcstypes.UserSpaceDetail, stgType *jcstypes.OBSType, cred *jcstypes.OBSCred, s3Cli *awss3.Client, bkt string) (*ShardStore, error) {
sd := ShardStore{ sd := ShardStore{
stgType: stgType, stgType: stgType,
cred: cred, cred: cred,
@@ -32,7 +31,7 @@ func NewShardStore(detail *jcsypes.UserSpaceDetail, stgType *cortypes.OBSType, c
return &sd, nil return &sd, nil
} }


func (s *ShardStore) MakeHTTPReadRequest(fileHash jcsypes.FileHash) (types.HTTPRequest, error) {
func (s *ShardStore) MakeHTTPReadRequest(fileHash jcstypes.FileHash) (types.HTTPRequest, error) {
cli, err := obs.New(s.cred.AK, s.cred.SK, s.stgType.Endpoint) cli, err := obs.New(s.cred.AK, s.cred.SK, s.stgType.Endpoint)
if err != nil { if err != nil {
return types.HTTPRequest{}, err return types.HTTPRequest{}, err


+ 10
- 11
common/pkgs/storage/pool/pool.go View File

@@ -6,12 +6,11 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async" "gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type userSpace struct { type userSpace struct {
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
store types.ShardStore store types.ShardStore
} }


@@ -23,8 +22,8 @@ func (u *userSpace) Drop() {
} }


type userSpaceKey struct { type userSpaceKey struct {
UserID cortypes.UserID
UserSpaceID jcsypes.UserSpaceID
UserID jcstypes.UserID
UserSpaceID jcstypes.UserSpaceID
} }


type Pool struct { type Pool struct {
@@ -40,7 +39,7 @@ func NewPool() *Pool {
} }
} }


func (p *Pool) Drop(userID cortypes.UserID, spaceID jcsypes.UserSpaceID) {
func (p *Pool) Drop(userID jcstypes.UserID, spaceID jcstypes.UserSpaceID) {
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()


@@ -56,7 +55,7 @@ func (p *Pool) Drop(userID cortypes.UserID, spaceID jcsypes.UserSpaceID) {
delete(p.spaces, key) delete(p.spaces, key)
} }


func (p *Pool) GetShardStore(spaceDetail *jcsypes.UserSpaceDetail) (types.ShardStore, error) {
func (p *Pool) GetShardStore(spaceDetail *jcstypes.UserSpaceDetail) (types.ShardStore, error) {
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()


@@ -91,18 +90,18 @@ func (p *Pool) GetShardStore(spaceDetail *jcsypes.UserSpaceDetail) (types.ShardS
return space.store, nil return space.store, nil
} }


func (p *Pool) GetBaseStore(spaceDetail *jcsypes.UserSpaceDetail) (types.BaseStore, error) {
func (p *Pool) GetBaseStore(spaceDetail *jcstypes.UserSpaceDetail) (types.BaseStore, error) {
return factory.GetBuilder(spaceDetail).CreateBaseStore(false) return factory.GetBuilder(spaceDetail).CreateBaseStore(false)
} }


func (p *Pool) GetMultiparter(spaceDetail *jcsypes.UserSpaceDetail) (types.Multiparter, error) {
func (p *Pool) GetMultiparter(spaceDetail *jcstypes.UserSpaceDetail) (types.Multiparter, error) {
return factory.GetBuilder(spaceDetail).CreateMultiparter(false) return factory.GetBuilder(spaceDetail).CreateMultiparter(false)
} }


func (p *Pool) GetS2STransfer(spaceDetail *jcsypes.UserSpaceDetail) (types.S2STransfer, error) {
func (p *Pool) GetS2STransfer(spaceDetail *jcstypes.UserSpaceDetail) (types.S2STransfer, error) {
return factory.GetBuilder(spaceDetail).CreateS2STransfer(false) return factory.GetBuilder(spaceDetail).CreateS2STransfer(false)
} }


func (p *Pool) GetECMultiplier(spaceDetail *jcsypes.UserSpaceDetail) (types.ECMultiplier, error) {
func (p *Pool) GetECMultiplier(spaceDetail *jcstypes.UserSpaceDetail) (types.ECMultiplier, error) {
return factory.GetBuilder(spaceDetail).CreateECMultiplier(false) return factory.GetBuilder(spaceDetail).CreateECMultiplier(false)
} }

+ 9
- 9
common/pkgs/storage/s3/base_store.go View File

@@ -16,7 +16,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


const ( const (
@@ -24,7 +24,7 @@ const (
) )


type BaseStore struct { type BaseStore struct {
Detail *jcsypes.UserSpaceDetail
Detail *jcstypes.UserSpaceDetail
Bucket string Bucket string
cli *s3.Client cli *s3.Client
opt BaseStoreOption opt BaseStoreOption
@@ -34,7 +34,7 @@ type BaseStoreOption struct {
UseAWSSha256 bool // 能否直接使用AWS提供的SHA256校验,如果不行,则使用本地计算。默认使用本地计算。 UseAWSSha256 bool // 能否直接使用AWS提供的SHA256校验,如果不行,则使用本地计算。默认使用本地计算。
} }


func NewBaseStore(detail *jcsypes.UserSpaceDetail, cli *s3.Client, bkt string, opt BaseStoreOption) (*BaseStore, error) {
func NewBaseStore(detail *jcstypes.UserSpaceDetail, cli *s3.Client, bkt string, opt BaseStoreOption) (*BaseStore, error) {
return &BaseStore{ return &BaseStore{
Detail: detail, Detail: detail,
Bucket: bkt, Bucket: bkt,
@@ -43,7 +43,7 @@ func NewBaseStore(detail *jcsypes.UserSpaceDetail, cli *s3.Client, bkt string, o
}, nil }, nil
} }


func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
key := pat key := pat
meta := make(map[string]string) meta := make(map[string]string)
if opt.ModTime.IsZero() { if opt.ModTime.IsZero() {
@@ -81,7 +81,7 @@ func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOp


return types.FileInfo{ return types.FileInfo{
Path: key, Path: key,
Hash: jcsypes.NewFullHash(hash),
Hash: jcstypes.NewFullHash(hash),
Size: counter.Count(), Size: counter.Count(),
}, nil }, nil
} }
@@ -99,12 +99,12 @@ func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOp


return types.FileInfo{ return types.FileInfo{
Path: key, Path: key,
Hash: jcsypes.NewFullHash(hashStr.Sum()),
Hash: jcstypes.NewFullHash(hashStr.Sum()),
Size: counter.Count(), Size: counter.Count(),
}, nil }, nil
} }


func (s *BaseStore) Read(pat jcsypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
func (s *BaseStore) Read(pat jcstypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
key := pat key := pat


input := &s3.GetObjectInput{ input := &s3.GetObjectInput{
@@ -128,7 +128,7 @@ func (s *BaseStore) Read(pat jcsypes.JPath, opt types.OpenOption) (io.ReadCloser
return resp.Body, nil return resp.Body, nil
} }


func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
func (s *BaseStore) Mkdir(path jcstypes.JPath) error {
_, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{ _, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(s.Bucket), Bucket: aws.String(s.Bucket),
Key: aws.String(path.String() + "/"), Key: aws.String(path.String() + "/"),
@@ -137,7 +137,7 @@ func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
return err return err
} }


func (s *BaseStore) ReadDir(path jcsypes.JPath) types.DirReader {
func (s *BaseStore) ReadDir(path jcstypes.JPath) types.DirReader {
return &DirReader{ return &DirReader{
cli: s.cli, cli: s.cli,
bucket: s.Bucket, bucket: s.Bucket,


+ 3
- 3
common/pkgs/storage/s3/dir_reader.go View File

@@ -7,13 +7,13 @@ import (
"github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type DirReader struct { type DirReader struct {
cli *s3.Client cli *s3.Client
bucket string bucket string
rootPath jcsypes.JPath
rootPath jcstypes.JPath
marker *string marker *string
curInfos []types.DirEntry curInfos []types.DirEntry
eof bool eof bool
@@ -39,7 +39,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {
} }


for _, obj := range resp.Contents { for _, obj := range resp.Contents {
key := jcsypes.PathFromJcsPathString(*obj.Key)
key := jcstypes.PathFromJcsPathString(*obj.Key)


r.curInfos = append(r.curInfos, types.DirEntry{ r.curInfos = append(r.curInfos, types.DirEntry{
Path: key, Path: key,


+ 7
- 8
common/pkgs/storage/s3/multipart_upload.go View File

@@ -12,18 +12,17 @@ import (
"gitlink.org.cn/cloudream/common/utils/os2" "gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type Multiparter struct { type Multiparter struct {
detail *jcsypes.UserSpaceDetail
feat *cortypes.MultipartUploadFeature
detail *jcstypes.UserSpaceDetail
feat *jcstypes.MultipartUploadFeature
bucket string bucket string
cli *s3.Client cli *s3.Client
} }


func NewMultiparter(detail *jcsypes.UserSpaceDetail, feat *cortypes.MultipartUploadFeature, bkt string, cli *s3.Client) *Multiparter {
func NewMultiparter(detail *jcstypes.UserSpaceDetail, feat *jcstypes.MultipartUploadFeature, bkt string, cli *s3.Client) *Multiparter {
return &Multiparter{ return &Multiparter{
detail: detail, detail: detail,
feat: feat, feat: feat,
@@ -89,9 +88,9 @@ func (m *Multiparter) UploadPart(ctx context.Context, init types.MultipartInitSt
type MultipartTask struct { type MultipartTask struct {
cli *s3.Client cli *s3.Client
bucket string bucket string
tempDir jcsypes.JPath
tempDir jcstypes.JPath
tempFileName string tempFileName string
tempFilePath jcsypes.JPath
tempFilePath jcstypes.JPath
uploadID string uploadID string
} }


@@ -140,7 +139,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
return types.FileInfo{}, err return types.FileInfo{}, err
} }


hash := jcsypes.CalculateCompositeHash(partHashes)
hash := jcstypes.CalculateCompositeHash(partHashes)


return types.FileInfo{ return types.FileInfo{
Path: i.tempFilePath, Path: i.tempFilePath,


+ 13
- 14
common/pkgs/storage/s3/s3.go View File

@@ -8,20 +8,19 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
reg.RegisterBuilder[*cortypes.S3Type](newBuilder)
reg.RegisterBuilder[*jcstypes.S3Type](newBuilder)
} }


type builder struct { type builder struct {
types.EmptyBuilder types.EmptyBuilder
detail *jcsypes.UserSpaceDetail
detail *jcstypes.UserSpaceDetail
} }


func newBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
func newBuilder(detail *jcstypes.UserSpaceDetail) types.StorageBuilder {
return &builder{ return &builder{
detail: detail, detail: detail,
} }
@@ -32,8 +31,8 @@ func (b *builder) FeatureDesc() types.FeatureDesc {
} }


func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) { func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*cortypes.S3Cred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for s3 storage", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for s3 storage", b.detail.UserSpace.Credential)
} }
@@ -51,8 +50,8 @@ func (b *builder) CreateShardStore(typeOnly bool) (types.ShardStore, error) {
} }


func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) { func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*cortypes.S3Cred)
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for s3 storage", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for s3 storage", b.detail.UserSpace.Credential)
} }
@@ -69,7 +68,7 @@ func (b *builder) CreateBaseStore(typeOnly bool) (types.BaseStore, error) {
return NewBaseStore(b.detail, cli, bkt, BaseStoreOption{UseAWSSha256: false}) return NewBaseStore(b.detail, cli, bkt, BaseStoreOption{UseAWSSha256: false})
} }


func createClient(stgType *cortypes.S3Type, cred *cortypes.S3Cred) (*s3.Client, string, error) {
func createClient(stgType *jcstypes.S3Type, cred *jcstypes.S3Cred) (*s3.Client, string, error) {
awsConfig := aws.Config{} awsConfig := aws.Config{}


if cred.AK != "" && cred.SK != "" { if cred.AK != "" && cred.SK != "" {
@@ -92,13 +91,13 @@ func createClient(stgType *cortypes.S3Type, cred *cortypes.S3Cred) (*s3.Client,
} }


func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) { func (b *builder) CreateMultiparter(typeOnly bool) (types.Multiparter, error) {
stgType := b.detail.UserSpace.Storage.(*cortypes.S3Type)
feat := types.FindFeature[*cortypes.MultipartUploadFeature](b.detail)
stgType := b.detail.UserSpace.Storage.(*jcstypes.S3Type)
feat := types.FindFeature[*jcstypes.MultipartUploadFeature](b.detail)
if feat == nil { if feat == nil {
return nil, fmt.Errorf("feature %T not found", cortypes.MultipartUploadFeature{})
return nil, fmt.Errorf("feature %T not found", jcstypes.MultipartUploadFeature{})
} }


s3Cred, ok := b.detail.UserSpace.Credential.(*cortypes.S3Cred)
s3Cred, ok := b.detail.UserSpace.Credential.(*jcstypes.S3Cred)
if !ok { if !ok {
return nil, fmt.Errorf("invalid storage credential type %T for s3 base store", b.detail.UserSpace.Credential) return nil, fmt.Errorf("invalid storage credential type %T for s3 base store", b.detail.UserSpace.Credential)
} }


Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save