From 7a1c6f39aba4955da5ce9caa1b08a8828784a4d5 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 3 Apr 2025 16:45:56 +0800 Subject: [PATCH] =?UTF-8?q?=E8=BF=81=E7=A7=BB=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- coordinator/types/public_storage.go | 50 ++++++++ coordinator/types/shard_storage.go | 51 ++++++++ coordinator/types/storage.go | 171 +++++++++++++++++++++++++++ coordinator/types/storage_feature.go | 112 ++++++++++++++++++ coordinator/types/types.go | 70 +++++++++++ 5 files changed, 454 insertions(+) create mode 100644 coordinator/types/public_storage.go create mode 100644 coordinator/types/shard_storage.go create mode 100644 coordinator/types/storage.go create mode 100644 coordinator/types/storage_feature.go create mode 100644 coordinator/types/types.go diff --git a/coordinator/types/public_storage.go b/coordinator/types/public_storage.go new file mode 100644 index 0000000..9ed5be3 --- /dev/null +++ b/coordinator/types/public_storage.go @@ -0,0 +1,50 @@ +package types + +/* +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/utils/serder" +) + +type PublicStoreConfig interface { + GetPublicStoreType() string + // 输出调试用的字符串，不要包含敏感信息 + String() string +} + +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[PublicStoreConfig]( + (*LocalPublicStorage)(nil), + (*S3PublicStorage)(nil), +)), "type") + +type LocalPublicStorage struct { + serder.Metadata `union:"Local"` + Type string `json:"type"` + // 调度Package时的Package的根路径 + LoadBase string `json:"loadBase"` +} + +func (s *LocalPublicStorage) GetPublicStoreType() string { + return "Local" +} + +func (s *LocalPublicStorage) String() string { + return fmt.Sprintf("Local[LoadBase=%v]", s.LoadBase) +} + +type S3PublicStorage struct { + serder.Metadata `union:"S3"` + Type string `json:"type"` + LoadBase string `json:"loadBase"` +} + +func (s *S3PublicStorage) GetPublicStoreType() string { + return "S3" +} + +func (s *S3PublicStorage) String() string { + return fmt.Sprintf("S3[LoadBase=%v]", s.LoadBase) +} +*/ diff --git a/coordinator/types/shard_storage.go b/coordinator/types/shard_storage.go new file mode 100644 index 0000000..04be462 --- /dev/null +++ b/coordinator/types/shard_storage.go @@ -0,0 +1,51 @@ +package types + +/* +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/utils/serder" +) + +// 分片存储服务的配置数据 +type ShardStoreConfig interface { + GetShardStoreType() string + // 输出调试用的字符串，不要包含敏感信息 + String() string +} + +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[ShardStoreConfig]( + (*LocalShardStorage)(nil), + (*S3ShardStorage)(nil), +)), "type") + +type LocalShardStorage struct { + serder.Metadata `union:"Local"` + Type string `json:"type"` + Root string `json:"root"` + MaxSize int64 `json:"maxSize"` +} + +func (s *LocalShardStorage) GetShardStoreType() string { + return "Local" +} + +func (s *LocalShardStorage) String() string { + return fmt.Sprintf("Local[root=%s, maxSize=%d]", s.Root, s.MaxSize) +} + +type S3ShardStorage struct { + serder.Metadata `union:"S3"` + Type string `json:"type"` + Root string `json:"root"` +} + +func (s *S3ShardStorage) GetShardStoreType() string { + return "S3" +} + +func (s *S3ShardStorage) String() string { + return fmt.Sprintf("S3[root=%s]", s.Root) +} +*/ diff --git a/coordinator/types/storage.go b/coordinator/types/storage.go new file mode 100644 index 0000000..1e4bf5f --- /dev/null +++ b/coordinator/types/storage.go @@ -0,0 +1,171 @@ +package types + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/utils/serder" +) + +type UserStorageConfig interface { + GetUserStorageConfigType() string +} + +type Storage struct { + StorageID StorageID `json:"storageID" gorm:"column:StorageID; primaryKey; type:bigint; autoIncrement;"` + Name string `json:"name" gorm:"column:Name; type:varchar(256); not null"` + // 完全管理此存储服务的Hub的ID + MasterHub HubID `json:"masterHub" gorm:"column:MasterHub; type:bigint; not null"` + // 存储服务的类型，包含地址信息以及鉴权所需数据 + Type StorageType `json:"type" gorm:"column:Type; type:json; not null; serializer:union"` + // 存储服务拥有的特别功能 + Features []StorageFeature `json:"features" gorm:"column:Features; type:json; serializer:union"` +} + +func (Storage) TableName() string { + return "Storage" +} + +func (s *Storage) String() string { + return fmt.Sprintf("%v(%v)", s.Name, s.StorageID) +} + +// 存储服务地址 +type StorageType interface { + GetStorageType() string + // 输出调试用的字符串，不要包含敏感信息 + String() string +} + +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[StorageType]( + (*MashupStorageType)(nil), + (*LocalStorageType)(nil), + (*OBSType)(nil), + (*OSSType)(nil), + (*COSType)(nil), + (*EFileType)(nil), + (*S3Type)(nil), +)), "type") + +// 多种存储服务的混合存储服务。需谨慎选择存储服务的组合，避免出Bug +type MashupStorageType struct { + serder.Metadata `union:"Mashup"` + Type string `json:"type"` + Agent StorageType `json:"agent"` // 创建Agent时，使用的存储服务类型 + Feature StorageType `json:"feature"` // 根据Feature创建组件时使用的存储服务类型 +} + +func (a *MashupStorageType) GetStorageType() string { + return "Mashup" +} + +func (a *MashupStorageType) String() string { + return "Mashup" +} + +type LocalStorageType struct { + serder.Metadata `union:"Local"` + Type string `json:"type"` +} + +func (a *LocalStorageType) GetStorageType() string { + return "Local" +} + +func (a *LocalStorageType) String() string { + return "Local" +} + +type OSSType struct { + serder.Metadata `union:"OSS"` + Type string `json:"type"` + Region string `json:"region"` + AK string `json:"accessKeyId"` + SK string `json:"secretAccessKey"` + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` +} + +func (a *OSSType) GetStorageType() string { + return "OSS" +} + +func (a *OSSType) String() string { + return "OSS" +} + +type OBSType struct { + serder.Metadata `union:"OBS"` + Type string `json:"type"` + Region string `json:"region"` + AK string `json:"accessKeyId"` + SK string `json:"secretAccessKey"` + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` + ProjectID string `json:"projectID"` +} + +func (a *OBSType) GetStorageType() string { + return "OBS" +} + +func (a *OBSType) String() string { + return "OBS" +} + +type COSType struct { + serder.Metadata `union:"COS"` + Type string `json:"type"` + Region string `json:"region"` + AK string `json:"accessKeyId"` + SK string `json:"secretAccessKey"` + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` +} + +func (a *COSType) GetStorageType() string { + return "COS" +} + +func (a *COSType) String() string { + return "COS" +} + +type EFileType struct { + serder.Metadata `union:"EFile"` + Type string `json:"type"` + TokenURL string `json:"tokenURL"` + APIURL string `json:"apiURL"` + TokenExpire int `json:"tokenExpire"` // 单位秒 + User string `json:"user"` + Password string `json:"password"` + OrgID string `json:"orgID"` + ClusterID string `json:"clusterID"` +} + +func (a *EFileType) GetStorageType() string { + return "EFile" +} + +func (a *EFileType) String() string { + return "EFile" +} + +// 通用的S3协议的存储服务 +type S3Type struct { + serder.Metadata `union:"S3"` + Type string `json:"type"` + Region string `json:"region"` + AK string `json:"accessKeyId"` + SK string `json:"secretAccessKey"` + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` +} + +func (a *S3Type) GetStorageType() string { + return "S3" +} + +func (a *S3Type) String() string { + return "S3" +} diff --git a/coordinator/types/storage_feature.go b/coordinator/types/storage_feature.go new file mode 100644 index 0000000..2afeaa5 --- /dev/null +++ b/coordinator/types/storage_feature.go @@ -0,0 +1,112 @@ +package types + +import ( + "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/utils/serder" +) + +// 存储服务特性 +type StorageFeature interface { + GetFeatureType() string + // 输出调试用的字符串，不要包含敏感信息 + String() string +} + +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[StorageFeature]( + (*TempStore)(nil), + (*BypassWriteFeature)(nil), + (*MultipartUploadFeature)(nil), + (*InternalServerlessCallFeature)(nil), + (*S2STransferFeature)(nil), + (*ECMultiplierFeature)(nil), +)), "type") + +type TempStore struct { + serder.Metadata `union:"TempStore"` + Type string `json:"type"` + TempRoot string `json:"tempRoot"` // 临时文件存放目录 +} + +func (f *TempStore) GetFeatureType() string { + return "TempStore" +} + +func (f *TempStore) String() string { + return "TempStore" +} + +// 存储服务支持被非MasterHub直接上传文件 +type BypassWriteFeature struct { + serder.Metadata `union:"BypassWrite"` + Type string `json:"type"` +} + +func (f *BypassWriteFeature) GetFeatureType() string { + return "BypassWrite" +} + +func (f *BypassWriteFeature) String() string { + return "BypassWrite" +} + +// 存储服务支持分段上传 +type MultipartUploadFeature struct { + serder.Metadata `union:"MultipartUpload"` + Type string `json:"type"` + TempDir string `json:"tempDir"` // 临时文件存放目录 + MinPartSize int64 `json:"minPartSize"` // 最小分段大小 + MaxPartSize int64 `json:"maxPartSize"` // 最大分段大小 +} + +func (f *MultipartUploadFeature) GetFeatureType() string { + return "MultipartUpload" +} + +func (f *MultipartUploadFeature) String() string { + return "MultipartUpload" +} + +// 在存储服务所在的环境中部署有内部的Serverless服务 +type InternalServerlessCallFeature struct { + serder.Metadata `union:"InternalServerlessCall"` + Type string `json:"type"` + CommandDir string `json:"commandDir"` // 存放命令文件的目录 +} + +func (f *InternalServerlessCallFeature) GetFeatureType() string { + return "InternalServerlessCall" +} + +func (f *InternalServerlessCallFeature) String() string { + return "InternalServerlessCall" +} + +// 存储服务之间直传文件 +type S2STransferFeature struct { + serder.Metadata `union:"S2STransfer"` + Type string `json:"type"` + TempDir string `json:"tempDir"` // 临时文件存放目录 +} + +func (f *S2STransferFeature) GetFeatureType() string { + return "S2STransfer" +} + +func (f *S2STransferFeature) String() string { + return "S2STransfer" +} + +// 存储服务提供了能进行EC计算的接口 +type ECMultiplierFeature struct { + serder.Metadata `union:"ECMultiplier"` + Type string `json:"type"` + TempDir string `json:"tempDir"` // 临时文件存放目录 +} + +func (f *ECMultiplierFeature) GetFeatureType() string { + return "ECMultiplier" +} + +func (f *ECMultiplierFeature) String() string { + return "ECMultiplier" +} diff --git a/coordinator/types/types.go b/coordinator/types/types.go new file mode 100644 index 0000000..a116dfc --- /dev/null +++ b/coordinator/types/types.go @@ -0,0 +1,70 @@ +package types + +import ( + "fmt" + "time" + + "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/utils/serder" +) + +type StorageID int64 + +type HubID int64 + +type LocationID int64 + +type Hub struct { + HubID HubID `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + Address HubAddressInfo `gorm:"column:Address; type:json; serializer:union" json:"address"` + LocationID LocationID `gorm:"column:LocationID; type:bigint; not null" json:"locationID"` + State string `gorm:"column:State; type:varchar(255); not null" json:"state"` + LastReportTime *time.Time `gorm:"column:LastReportTime; type:datetime" json:"lastReportTime"` +} + +func (Hub) TableName() string { + return "Hub" +} + +type HubAddressInfo interface { +} + +var HubAddressUnion = types.NewTypeUnion[HubAddressInfo]( + (*GRPCAddressInfo)(nil), + (*HttpAddressInfo)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&HubAddressUnion, "type") + +type GRPCAddressInfo struct { + serder.Metadata `union:"GRPC"` + Type string `json:"type"` + LocalIP string `json:"localIP"` + ExternalIP string `json:"externalIP"` + LocalGRPCPort int `json:"localGRPCPort"` + ExternalGRPCPort int `json:"externalGRPCPort"` +} + +type HttpAddressInfo struct { + serder.Metadata `union:"HTTP"` + Type string `json:"type"` + LocalIP string `json:"localIP"` + ExternalIP string `json:"externalIP"` + Port int `json:"port"` +} + +func (n Hub) String() string { + return fmt.Sprintf("%v(%v)", n.Name, n.HubID) +} + +type HubConnectivity struct { + FromHubID HubID `gorm:"column:FromHubID; primaryKey; type:bigint" json:"fromHubID"` + ToHubID HubID `gorm:"column:ToHubID; primaryKey; type:bigint" json:"ToHubID"` + Latency *float32 `gorm:"column:Latency; type:float" json:"latency"` + TestTime time.Time `gorm:"column:TestTime; type:datetime" json:"testTime"` +} + +func (HubConnectivity) TableName() string { + return "HubConnectivity" +}