Browse Source

优化数据结构

gitlink
Sydonian 10 months ago
parent
commit
384f9cd141
10 changed files with 75 additions and 13 deletions
  1. +1
    -1
      client/internal/services/cache.go
  2. +1
    -1
      client/internal/services/storage.go
  3. +4
    -4
      common/pkgs/storage/local/local.go
  4. +12
    -0
      common/pkgs/storage/local/shard_store.go
  5. +12
    -0
      common/pkgs/storage/local/shared_store.go
  6. +4
    -4
      common/pkgs/storage/s3/s3.go
  7. +12
    -0
      common/pkgs/storage/s3/shard_store.go
  8. +12
    -0
      common/pkgs/storage/s3/shared_store.go
  9. +16
    -2
      common/pkgs/storage/types/types.go
  10. +1
    -1
      common/pkgs/uploader/uploader.go

+ 1
- 1
client/internal/services/cache.go View File

@@ -32,7 +32,7 @@ func (svc *CacheService) StartCacheMovePackage(userID cdssdk.UserID, packageID c
return 0, "", fmt.Errorf("get storage detail: %w", err)
}

if !factory.GetBuilder(*getStg.Storages[0]).HasShardStore() {
if !factory.GetBuilder(*getStg.Storages[0]).ShardStoreDesc().Enabled() {
return 0, "", fmt.Errorf("shard storage is not enabled")
}



+ 1
- 1
client/internal/services/storage.go View File

@@ -105,7 +105,7 @@ func (svc *StorageService) LoadPackage(userID cdssdk.UserID, packageID cdssdk.Pa

ft.AddTo(ioswitch2.NewLoadToShared(*destStg.MasterHub, destStg.Storage, path.Join(rootPath, obj.Object.Path)))
// 顺便保存到同存储服务的分片存储中
if factory.GetBuilder(*destStg).HasShardStore() {
if factory.GetBuilder(*destStg).ShardStoreDesc().Enabled() {
ft.AddTo(ioswitch2.NewToShardStore(*destStg.MasterHub, *destStg, ioswitch2.RawStream(), ""))
pinned = append(pinned, obj.Object.ObjectID)
}


+ 4
- 4
common/pkgs/storage/local/local.go View File

@@ -58,12 +58,12 @@ func (b *builder) CreateAgent() (types.StorageAgent, error) {
return agt, nil
}

func (b *builder) HasShardStore() bool {
return b.detail.Storage.ShardStore != nil
func (b *builder) ShardStoreDesc() types.ShardStoreDesc {
return &ShardStoreDesc{builder: b}
}

func (b *builder) HasSharedStore() bool {
return b.detail.Storage.SharedStore != nil
func (b *builder) SharedStoreDesc() types.SharedStoreDesc {
return &SharedStoreDesc{builder: b}
}

func (b *builder) CreateMultiparter() (types.Multiparter, error) {


+ 12
- 0
common/pkgs/storage/local/shard_store.go View File

@@ -22,6 +22,18 @@ const (
BlocksDir = "blocks"
)

type ShardStoreDesc struct {
builder *builder
}

func (s *ShardStoreDesc) Enabled() bool {
return s.builder.detail.Storage.ShardStore != nil
}

func (s *ShardStoreDesc) HasBypassNotifier() bool {
return true
}

type ShardStore struct {
agt *agent
cfg cdssdk.LocalShardStorage


+ 12
- 0
common/pkgs/storage/local/shared_store.go View File

@@ -10,6 +10,18 @@ import (
"gitlink.org.cn/cloudream/storage/common/pkgs/storage/types"
)

type SharedStoreDesc struct {
builder *builder
}

func (d *SharedStoreDesc) Enabled() bool {
return d.builder.detail.Storage.SharedStore != nil
}

func (d *SharedStoreDesc) HasBypassNotifier() bool {
return false
}

type SharedStore struct {
agt *agent
cfg cdssdk.LocalSharedStorage


+ 4
- 4
common/pkgs/storage/s3/s3.go View File

@@ -59,12 +59,12 @@ func (b *builder) CreateAgent() (types.StorageAgent, error) {
return agt, nil
}

func (b *builder) HasShardStore() bool {
return b.detail.Storage.ShardStore != nil
func (b *builder) ShardStoreDesc() types.ShardStoreDesc {
return &ShardStoreDesc{builder: b}
}

func (b *builder) HasSharedStore() bool {
return false
func (b *builder) SharedStoreDesc() types.SharedStoreDesc {
return &SharedStoreDesc{}
}

func (b *builder) CreateMultiparter() (types.Multiparter, error) {


+ 12
- 0
common/pkgs/storage/s3/shard_store.go View File

@@ -25,6 +25,18 @@ const (
BlocksDir = "blocks"
)

type ShardStoreDesc struct {
builder *builder
}

func (s *ShardStoreDesc) Enabled() bool {
return s.builder.detail.Storage.ShardStore != nil
}

func (s *ShardStoreDesc) HasBypassNotifier() bool {
return true
}

type ShardStoreOption struct {
UseAWSSha256 bool // 能否直接使用AWS提供的SHA256校验,如果不行,则使用本地计算。默认使用本地计算。
}


+ 12
- 0
common/pkgs/storage/s3/shared_store.go View File

@@ -0,0 +1,12 @@
package s3

type SharedStoreDesc struct {
}

func (d *SharedStoreDesc) Enabled() bool {
return false
}

func (d *SharedStoreDesc) HasBypassNotifier() bool {
return false
}

+ 16
- 2
common/pkgs/storage/types/types.go View File

@@ -43,9 +43,23 @@ type StorageBuilder interface {
// 创建一个在MasterHub上长期运行的存储服务
CreateAgent() (StorageAgent, error)
// 是否支持分片存储服务
HasShardStore() bool
ShardStoreDesc() ShardStoreDesc
// 是否支持共享存储服务
HasSharedStore() bool
SharedStoreDesc() SharedStoreDesc
// 创建一个分片上传组件
CreateMultiparter() (Multiparter, error)
}

type ShardStoreDesc interface {
// 是否已启动
Enabled() bool
// 是否能旁路上传
HasBypassNotifier() bool
}

type SharedStoreDesc interface {
// 是否已启动
Enabled() bool
// 是否能旁路上传
HasBypassNotifier() bool
}

+ 1
- 1
common/pkgs/uploader/uploader.go View File

@@ -86,7 +86,7 @@ func (u *Uploader) BeginUpdate(userID cdssdk.UserID, pkgID cdssdk.PackageID, aff
if stg.MasterHub == nil {
return nil, fmt.Errorf("load to storage %v has no master hub", stgID)
}
if factory.GetBuilder(stg).HasSharedStore() {
if factory.GetBuilder(stg).ShardStoreDesc().Enabled() {
return nil, fmt.Errorf("load to storage %v has no shared store", stgID)
}



Loading…
Cancel
Save