From 33199bfef5cb33d75de32bbbd8fc8c744b11314d Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 11 Nov 2024 16:41:27 +0800 Subject: [PATCH] =?UTF-8?q?ShardStorage=E5=92=8CSharedStorage=E5=90=88?= =?UTF-8?q?=E5=B9=B6=E5=88=B0Storage=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/services/cache.go | 2 +- client/internal/services/storage.go | 4 +-- common/models/models.go | 6 ++-- common/pkgs/db2/shard_storage.go | 25 ---------------- common/pkgs/db2/shared_storage.go | 25 ---------------- common/pkgs/db2/storage.go | 30 -------------------- common/pkgs/downloader/iterator.go | 2 +- common/pkgs/storage/mgr/create_shardstore.go | 2 +- common/pkgs/storage/mgr/mgr.go | 4 +-- coordinator/internal/cmd/migrate.go | 2 -- coordinator/internal/mq/hub.go | 22 -------------- 11 files changed, 9 insertions(+), 115 deletions(-) delete mode 100644 common/pkgs/db2/shard_storage.go delete mode 100644 common/pkgs/db2/shared_storage.go diff --git a/client/internal/services/cache.go b/client/internal/services/cache.go index 919feb2..964c2a5 100644 --- a/client/internal/services/cache.go +++ b/client/internal/services/cache.go @@ -31,7 +31,7 @@ func (svc *CacheService) StartCacheMovePackage(userID cdssdk.UserID, packageID c return 0, "", fmt.Errorf("get storage detail: %w", err) } - if getStg.Storages[0].Shard == nil { + if getStg.Storages[0].Storage.ShardStore == nil { return 0, "", fmt.Errorf("shard storage is not enabled") } diff --git a/client/internal/services/storage.go b/client/internal/services/storage.go index 4b35f55..17a5cf2 100644 --- a/client/internal/services/storage.go +++ b/client/internal/services/storage.go @@ -62,7 +62,7 @@ func (svc *StorageService) StartStorageLoadPackage(userID cdssdk.UserID, package return 0, "", fmt.Errorf("getting storage info: %w", err) } - if stgResp.Storages[0].Shard == nil { + if stgResp.Storages[0].Storage.ShardStore == nil { return 0, "", fmt.Errorf("shard storage is not enabled") } @@ -133,7 +133,7 @@ func (svc *StorageService) StartStorageCreatePackage(userID cdssdk.UserID, bucke return 0, "", fmt.Errorf("getting storage info: %w", err) } - if stgResp.Storages[0].Shard == nil { + if stgResp.Storages[0].Storage.ShardStore == nil { return 0, "", fmt.Errorf("shard storage is not enabled") } diff --git a/common/models/models.go b/common/models/models.go index b2fc530..988ae38 100644 --- a/common/models/models.go +++ b/common/models/models.go @@ -128,10 +128,8 @@ func (ObjectAccessStat) TableName() string { } type StorageDetail struct { - Storage cdssdk.Storage `json:"storage"` - MasterHub *cdssdk.Hub `json:"masterHub"` - Shard *cdssdk.ShardStorage `json:"shard"` - Shared *cdssdk.SharedStorage `json:"shared"` + Storage cdssdk.Storage `json:"storage"` + MasterHub *cdssdk.Hub `json:"masterHub"` } type ObjectStorage struct { diff --git a/common/pkgs/db2/shard_storage.go b/common/pkgs/db2/shard_storage.go deleted file mode 100644 index 64306c4..0000000 --- a/common/pkgs/db2/shard_storage.go +++ /dev/null @@ -1,25 +0,0 @@ -package db2 - -import ( - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" -) - -type ShardStorageDB struct { - *DB -} - -func (db *DB) ShardStorage() *ShardStorageDB { - return &ShardStorageDB{DB: db} -} - -func (*ShardStorageDB) GetByStorageID(ctx SQLContext, stgID cdssdk.StorageID) (cdssdk.ShardStorage, error) { - var ret cdssdk.ShardStorage - err := ctx.Table("ShardStorage").First(&ret, stgID).Error - return ret, err -} - -func (*ShardStorageDB) BatchGetByStorageIDs(ctx SQLContext, stgIDs []cdssdk.StorageID) ([]cdssdk.ShardStorage, error) { - var ret []cdssdk.ShardStorage - err := ctx.Table("ShardStorage").Find(&ret, "StorageID IN (?)", stgIDs).Error - return ret, err -} diff --git a/common/pkgs/db2/shared_storage.go b/common/pkgs/db2/shared_storage.go deleted file mode 100644 index 2317d1f..0000000 --- a/common/pkgs/db2/shared_storage.go +++ /dev/null @@ -1,25 +0,0 @@ -package db2 - -import ( - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" -) - -type SharedStorageDB struct { - *DB -} - -func (db *DB) SharedStorage() *SharedStorageDB { - return &SharedStorageDB{DB: db} -} - -func (*SharedStorageDB) GetByStorageID(ctx SQLContext, stgID cdssdk.StorageID) (cdssdk.SharedStorage, error) { - var ret cdssdk.SharedStorage - err := ctx.Table("SharedStorage").First(&ret, stgID).Error - return ret, err -} - -func (*SharedStorageDB) BatchGetByStorageIDs(ctx SQLContext, stgIDs []cdssdk.StorageID) ([]cdssdk.SharedStorage, error) { - var ret []cdssdk.SharedStorage - err := ctx.Table("SharedStorage").Find(&ret, "StorageID IN (?)", stgIDs).Error - return ret, err -} diff --git a/common/pkgs/db2/storage.go b/common/pkgs/db2/storage.go index 9d6ce03..c702b4c 100644 --- a/common/pkgs/db2/storage.go +++ b/common/pkgs/db2/storage.go @@ -88,11 +88,9 @@ func (db *StorageDB) GetHubStorages(ctx SQLContext, hubID cdssdk.HubID) ([]model func (db *StorageDB) FillDetails(ctx SQLContext, details []stgmod.StorageDetail) error { stgsMp := make(map[cdssdk.StorageID]*stgmod.StorageDetail) - stgIDs := make([]cdssdk.StorageID, 0, len(details)) var masterHubIDs []cdssdk.HubID for i := range details { stgsMp[details[i].Storage.StorageID] = &details[i] - stgIDs = append(stgIDs, details[i].Storage.StorageID) masterHubIDs = append(masterHubIDs, details[i].Storage.MasterHub) } @@ -118,33 +116,5 @@ func (db *StorageDB) FillDetails(ctx SQLContext, details []stgmod.StorageDetail) } } - // 获取分片存储 - shards, err := db.ShardStorage().BatchGetByStorageIDs(ctx, stgIDs) - if err != nil && err != gorm.ErrRecordNotFound { - return fmt.Errorf("getting shard storage: %w", err) - } - for _, shard := range shards { - stg := stgsMp[shard.StorageID] - if stg == nil { - continue - } - - stg.Shard = &shard - } - - // 获取共享存储的相关信息 - shareds, err := db.SharedStorage().BatchGetByStorageIDs(ctx, stgIDs) - if err != nil && err != gorm.ErrRecordNotFound { - return fmt.Errorf("getting shared storage: %w", err) - } - for _, shared := range shareds { - stg := stgsMp[shared.StorageID] - if stg == nil { - continue - } - - stg.Shared = &shared - } - return nil } diff --git a/common/pkgs/downloader/iterator.go b/common/pkgs/downloader/iterator.go index 1549857..cbe814a 100644 --- a/common/pkgs/downloader/iterator.go +++ b/common/pkgs/downloader/iterator.go @@ -108,7 +108,7 @@ func (i *DownloadObjectIterator) init() error { if s == nil { return fmt.Errorf("storage %v not found", stgIDs[idx]) } - if s.Shard == nil { + if s.Storage.ShardStore == nil { return fmt.Errorf("storage %v has no shard store", stgIDs[idx]) } diff --git a/common/pkgs/storage/mgr/create_shardstore.go b/common/pkgs/storage/mgr/create_shardstore.go index ba11ae5..cf1627f 100644 --- a/common/pkgs/storage/mgr/create_shardstore.go +++ b/common/pkgs/storage/mgr/create_shardstore.go @@ -10,7 +10,7 @@ import ( ) func createShardStore(detail stgmod.StorageDetail, ch *types.StorageEventChan, stg *storage) error { - switch confg := detail.Shard.Config.(type) { + switch confg := detail.Storage.ShardStore.(type) { case *cdssdk.LocalShardStorage: store, err := local.NewShardStore(detail.Storage, *confg) if err != nil { diff --git a/common/pkgs/storage/mgr/mgr.go b/common/pkgs/storage/mgr/mgr.go index 4c487c8..ffed1a5 100644 --- a/common/pkgs/storage/mgr/mgr.go +++ b/common/pkgs/storage/mgr/mgr.go @@ -48,7 +48,7 @@ func (m *Manager) InitStorage(detail stgmod.StorageDetail) error { stg := &storage{} - if detail.Shard != nil { + if detail.Storage.ShardStore != nil { err := createShardStore(detail, m.eventChan, stg) if err != nil { stopStorage(stg) @@ -56,7 +56,7 @@ func (m *Manager) InitStorage(detail stgmod.StorageDetail) error { } } - if detail.Shared != nil { + if detail.Storage.SharedStore != nil { err := createSharedStore(detail, m.eventChan, stg) if err != nil { stopStorage(stg) diff --git a/coordinator/internal/cmd/migrate.go b/coordinator/internal/cmd/migrate.go index 4d6046a..eb55f0c 100644 --- a/coordinator/internal/cmd/migrate.go +++ b/coordinator/internal/cmd/migrate.go @@ -52,8 +52,6 @@ func migrate(configPath string) { migrateOne(db, stgmod.PackageAccessStat{}) migrateOne(db, cdssdk.Package{}) migrateOne(db, cdssdk.PinnedObject{}) - migrateOne(db, cdssdk.ShardStorage{}) - migrateOne(db, cdssdk.SharedStorage{}) migrateOne(db, model.StoragePackage{}) migrateOne(db, cdssdk.Storage{}) migrateOne(db, model.UserStorage{}) diff --git a/coordinator/internal/mq/hub.go b/coordinator/internal/mq/hub.go index 3478deb..fc75cdf 100644 --- a/coordinator/internal/mq/hub.go +++ b/coordinator/internal/mq/hub.go @@ -30,33 +30,11 @@ func (svc *Service) GetHubConfig(msg *coormq.GetHubConfig) (*coormq.GetHubConfig return nil, mq.Failed(errorcode.OperationFailed, fmt.Sprintf("getting hub storages: %v", err)) } - var stgIDs []cdssdk.StorageID for _, stg := range stgs { detailsMap[stg.StorageID] = &stgmod.StorageDetail{ Storage: stg, MasterHub: &hub, } - stgIDs = append(stgIDs, stg.StorageID) - } - - shards, err := svc.db2.ShardStorage().BatchGetByStorageIDs(svc.db2.DefCtx(), stgIDs) - if err != nil { - log.Warnf("getting shard storages: %v", err) - return nil, mq.Failed(errorcode.OperationFailed, fmt.Sprintf("getting shard storages: %v", err)) - } - for _, shard := range shards { - sh := shard - detailsMap[shard.StorageID].Shard = &sh - } - - shareds, err := svc.db2.SharedStorage().BatchGetByStorageIDs(svc.db2.DefCtx(), stgIDs) - if err != nil { - log.Warnf("getting shared storages: %v", err) - return nil, mq.Failed(errorcode.OperationFailed, fmt.Sprintf("getting shared storages: %v", err)) - } - for _, shared := range shareds { - sh := shared - detailsMap[shared.StorageID].Shared = &sh } var details []stgmod.StorageDetail