迁移代码

coordinator/types/public_storage.go

@@ -0,0 +1,50 @@
+package types
+
+/*
+import (
+	"fmt"
+
+	"gitlink.org.cn/cloudream/common/pkgs/types"
+	"gitlink.org.cn/cloudream/common/utils/serder"
+)
+
+type PublicStoreConfig interface {
+	GetPublicStoreType() string
+	// 输出调试用的字符串，不要包含敏感信息
+	String() string
+}
+
+var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[PublicStoreConfig](
+	(*LocalPublicStorage)(nil),
+	(*S3PublicStorage)(nil),
+)), "type")
+
+type LocalPublicStorage struct {
+	serder.Metadata `union:"Local"`
+	Type            string `json:"type"`
+	// 调度Package时的Package的根路径
+	LoadBase string `json:"loadBase"`
+}
+
+func (s *LocalPublicStorage) GetPublicStoreType() string {
+	return "Local"
+}
+
+func (s *LocalPublicStorage) String() string {
+	return fmt.Sprintf("Local[LoadBase=%v]", s.LoadBase)
+}
+
+type S3PublicStorage struct {
+	serder.Metadata `union:"S3"`
+	Type            string `json:"type"`
+	LoadBase        string `json:"loadBase"`
+}
+
+func (s *S3PublicStorage) GetPublicStoreType() string {
+	return "S3"
+}
+
+func (s *S3PublicStorage) String() string {
+	return fmt.Sprintf("S3[LoadBase=%v]", s.LoadBase)
+}
+*/

coordinator/types/shard_storage.go

@@ -0,0 +1,51 @@
+package types
+
+/*
+import (
+	"fmt"
+
+	"gitlink.org.cn/cloudream/common/pkgs/types"
+	"gitlink.org.cn/cloudream/common/utils/serder"
+)
+
+// 分片存储服务的配置数据
+type ShardStoreConfig interface {
+	GetShardStoreType() string
+	// 输出调试用的字符串，不要包含敏感信息
+	String() string
+}
+
+var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[ShardStoreConfig](
+	(*LocalShardStorage)(nil),
+	(*S3ShardStorage)(nil),
+)), "type")
+
+type LocalShardStorage struct {
+	serder.Metadata `union:"Local"`
+	Type            string `json:"type"`
+	Root            string `json:"root"`
+	MaxSize         int64  `json:"maxSize"`
+}
+
+func (s *LocalShardStorage) GetShardStoreType() string {
+	return "Local"
+}
+
+func (s *LocalShardStorage) String() string {
+	return fmt.Sprintf("Local[root=%s, maxSize=%d]", s.Root, s.MaxSize)
+}
+
+type S3ShardStorage struct {
+	serder.Metadata `union:"S3"`
+	Type            string `json:"type"`
+	Root            string `json:"root"`
+}
+
+func (s *S3ShardStorage) GetShardStoreType() string {
+	return "S3"
+}
+
+func (s *S3ShardStorage) String() string {
+	return fmt.Sprintf("S3[root=%s]", s.Root)
+}
+*/

coordinator/types/storage.go

@@ -0,0 +1,171 @@
+package types
+
+import (
+	"fmt"
+
+	"gitlink.org.cn/cloudream/common/pkgs/types"
+	"gitlink.org.cn/cloudream/common/utils/serder"
+)
+
+type UserStorageConfig interface {
+	GetUserStorageConfigType() string
+}
+
+type Storage struct {
+	StorageID StorageID `json:"storageID" gorm:"column:StorageID; primaryKey; type:bigint; autoIncrement;"`
+	Name      string    `json:"name" gorm:"column:Name; type:varchar(256); not null"`
+	// 完全管理此存储服务的Hub的ID
+	MasterHub HubID `json:"masterHub" gorm:"column:MasterHub; type:bigint; not null"`
+	// 存储服务的类型，包含地址信息以及鉴权所需数据
+	Type StorageType `json:"type" gorm:"column:Type; type:json; not null; serializer:union"`
+	// 存储服务拥有的特别功能
+	Features []StorageFeature `json:"features" gorm:"column:Features; type:json; serializer:union"`
+}
+
+func (Storage) TableName() string {
+	return "Storage"
+}
+
+func (s *Storage) String() string {
+	return fmt.Sprintf("%v(%v)", s.Name, s.StorageID)
+}
+
+// 存储服务地址
+type StorageType interface {
+	GetStorageType() string
+	// 输出调试用的字符串，不要包含敏感信息
+	String() string
+}
+
+var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[StorageType](
+	(*MashupStorageType)(nil),
+	(*LocalStorageType)(nil),
+	(*OBSType)(nil),
+	(*OSSType)(nil),
+	(*COSType)(nil),
+	(*EFileType)(nil),
+	(*S3Type)(nil),
+)), "type")
+
+// 多种存储服务的混合存储服务。需谨慎选择存储服务的组合，避免出Bug
+type MashupStorageType struct {
+	serder.Metadata `union:"Mashup"`
+	Type            string      `json:"type"`
+	Agent           StorageType `json:"agent"`   // 创建Agent时，使用的存储服务类型
+	Feature         StorageType `json:"feature"` // 根据Feature创建组件时使用的存储服务类型
+}
+
+func (a *MashupStorageType) GetStorageType() string {
+	return "Mashup"
+}
+
+func (a *MashupStorageType) String() string {
+	return "Mashup"
+}
+
+type LocalStorageType struct {
+	serder.Metadata `union:"Local"`
+	Type            string `json:"type"`
+}
+
+func (a *LocalStorageType) GetStorageType() string {
+	return "Local"
+}
+
+func (a *LocalStorageType) String() string {
+	return "Local"
+}
+
+type OSSType struct {
+	serder.Metadata `union:"OSS"`
+	Type            string `json:"type"`
+	Region          string `json:"region"`
+	AK              string `json:"accessKeyId"`
+	SK              string `json:"secretAccessKey"`
+	Endpoint        string `json:"endpoint"`
+	Bucket          string `json:"bucket"`
+}
+
+func (a *OSSType) GetStorageType() string {
+	return "OSS"
+}
+
+func (a *OSSType) String() string {
+	return "OSS"
+}
+
+type OBSType struct {
+	serder.Metadata `union:"OBS"`
+	Type            string `json:"type"`
+	Region          string `json:"region"`
+	AK              string `json:"accessKeyId"`
+	SK              string `json:"secretAccessKey"`
+	Endpoint        string `json:"endpoint"`
+	Bucket          string `json:"bucket"`
+	ProjectID       string `json:"projectID"`
+}
+
+func (a *OBSType) GetStorageType() string {
+	return "OBS"
+}
+
+func (a *OBSType) String() string {
+	return "OBS"
+}
+
+type COSType struct {
+	serder.Metadata `union:"COS"`
+	Type            string `json:"type"`
+	Region          string `json:"region"`
+	AK              string `json:"accessKeyId"`
+	SK              string `json:"secretAccessKey"`
+	Endpoint        string `json:"endpoint"`
+	Bucket          string `json:"bucket"`
+}
+
+func (a *COSType) GetStorageType() string {
+	return "COS"
+}
+
+func (a *COSType) String() string {
+	return "COS"
+}
+
+type EFileType struct {
+	serder.Metadata `union:"EFile"`
+	Type            string `json:"type"`
+	TokenURL        string `json:"tokenURL"`
+	APIURL          string `json:"apiURL"`
+	TokenExpire     int    `json:"tokenExpire"` // 单位秒
+	User            string `json:"user"`
+	Password        string `json:"password"`
+	OrgID           string `json:"orgID"`
+	ClusterID       string `json:"clusterID"`
+}
+
+func (a *EFileType) GetStorageType() string {
+	return "EFile"
+}
+
+func (a *EFileType) String() string {
+	return "EFile"
+}
+
+// 通用的S3协议的存储服务
+type S3Type struct {
+	serder.Metadata `union:"S3"`
+	Type            string `json:"type"`
+	Region          string `json:"region"`
+	AK              string `json:"accessKeyId"`
+	SK              string `json:"secretAccessKey"`
+	Endpoint        string `json:"endpoint"`
+	Bucket          string `json:"bucket"`
+}
+
+func (a *S3Type) GetStorageType() string {
+	return "S3"
+}
+
+func (a *S3Type) String() string {
+	return "S3"
+}

coordinator/types/storage_feature.go

@@ -0,0 +1,112 @@
+package types
+
+import (
+	"gitlink.org.cn/cloudream/common/pkgs/types"
+	"gitlink.org.cn/cloudream/common/utils/serder"
+)
+
+// 存储服务特性
+type StorageFeature interface {
+	GetFeatureType() string
+	// 输出调试用的字符串，不要包含敏感信息
+	String() string
+}
+
+var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[StorageFeature](
+	(*TempStore)(nil),
+	(*BypassWriteFeature)(nil),
+	(*MultipartUploadFeature)(nil),
+	(*InternalServerlessCallFeature)(nil),
+	(*S2STransferFeature)(nil),
+	(*ECMultiplierFeature)(nil),
+)), "type")
+
+type TempStore struct {
+	serder.Metadata `union:"TempStore"`
+	Type            string `json:"type"`
+	TempRoot        string `json:"tempRoot"` // 临时文件存放目录
+}
+
+func (f *TempStore) GetFeatureType() string {
+	return "TempStore"
+}
+
+func (f *TempStore) String() string {
+	return "TempStore"
+}
+
+// 存储服务支持被非MasterHub直接上传文件
+type BypassWriteFeature struct {
+	serder.Metadata `union:"BypassWrite"`
+	Type            string `json:"type"`
+}
+
+func (f *BypassWriteFeature) GetFeatureType() string {
+	return "BypassWrite"
+}
+
+func (f *BypassWriteFeature) String() string {
+	return "BypassWrite"
+}
+
+// 存储服务支持分段上传
+type MultipartUploadFeature struct {
+	serder.Metadata `union:"MultipartUpload"`
+	Type            string `json:"type"`
+	TempDir         string `json:"tempDir"`     // 临时文件存放目录
+	MinPartSize     int64  `json:"minPartSize"` // 最小分段大小
+	MaxPartSize     int64  `json:"maxPartSize"` // 最大分段大小
+}
+
+func (f *MultipartUploadFeature) GetFeatureType() string {
+	return "MultipartUpload"
+}
+
+func (f *MultipartUploadFeature) String() string {
+	return "MultipartUpload"
+}
+
+// 在存储服务所在的环境中部署有内部的Serverless服务
+type InternalServerlessCallFeature struct {
+	serder.Metadata `union:"InternalServerlessCall"`
+	Type            string `json:"type"`
+	CommandDir      string `json:"commandDir"` // 存放命令文件的目录
+}
+
+func (f *InternalServerlessCallFeature) GetFeatureType() string {
+	return "InternalServerlessCall"
+}
+
+func (f *InternalServerlessCallFeature) String() string {
+	return "InternalServerlessCall"
+}
+
+// 存储服务之间直传文件
+type S2STransferFeature struct {
+	serder.Metadata `union:"S2STransfer"`
+	Type            string `json:"type"`
+	TempDir         string `json:"tempDir"` // 临时文件存放目录
+}
+
+func (f *S2STransferFeature) GetFeatureType() string {
+	return "S2STransfer"
+}
+
+func (f *S2STransferFeature) String() string {
+	return "S2STransfer"
+}
+
+// 存储服务提供了能进行EC计算的接口
+type ECMultiplierFeature struct {
+	serder.Metadata `union:"ECMultiplier"`
+	Type            string `json:"type"`
+	TempDir         string `json:"tempDir"` // 临时文件存放目录
+}
+
+func (f *ECMultiplierFeature) GetFeatureType() string {
+	return "ECMultiplier"
+}
+
+func (f *ECMultiplierFeature) String() string {
+	return "ECMultiplier"
+}

coordinator/types/types.go

@@ -0,0 +1,70 @@
+package types
+
+import (
+	"fmt"
+	"time"
+
+	"gitlink.org.cn/cloudream/common/pkgs/types"
+	"gitlink.org.cn/cloudream/common/utils/serder"
+)
+
+type StorageID int64
+
+type HubID int64
+
+type LocationID int64
+
+type Hub struct {
+	HubID          HubID          `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"`
+	Name           string         `gorm:"column:Name; type:varchar(255); not null" json:"name"`
+	Address        HubAddressInfo `gorm:"column:Address; type:json; serializer:union" json:"address"`
+	LocationID     LocationID     `gorm:"column:LocationID; type:bigint; not null" json:"locationID"`
+	State          string         `gorm:"column:State; type:varchar(255); not null" json:"state"`
+	LastReportTime *time.Time     `gorm:"column:LastReportTime; type:datetime" json:"lastReportTime"`
+}
+
+func (Hub) TableName() string {
+	return "Hub"
+}
+
+type HubAddressInfo interface {
+}
+
+var HubAddressUnion = types.NewTypeUnion[HubAddressInfo](
+	(*GRPCAddressInfo)(nil),
+	(*HttpAddressInfo)(nil),
+)
+
+var _ = serder.UseTypeUnionInternallyTagged(&HubAddressUnion, "type")
+
+type GRPCAddressInfo struct {
+	serder.Metadata  `union:"GRPC"`
+	Type             string `json:"type"`
+	LocalIP          string `json:"localIP"`
+	ExternalIP       string `json:"externalIP"`
+	LocalGRPCPort    int    `json:"localGRPCPort"`
+	ExternalGRPCPort int    `json:"externalGRPCPort"`
+}
+
+type HttpAddressInfo struct {
+	serder.Metadata `union:"HTTP"`
+	Type            string `json:"type"`
+	LocalIP         string `json:"localIP"`
+	ExternalIP      string `json:"externalIP"`
+	Port            int    `json:"port"`
+}
+
+func (n Hub) String() string {
+	return fmt.Sprintf("%v(%v)", n.Name, n.HubID)
+}
+
+type HubConnectivity struct {
+	FromHubID HubID     `gorm:"column:FromHubID; primaryKey; type:bigint" json:"fromHubID"`
+	ToHubID   HubID     `gorm:"column:ToHubID; primaryKey; type:bigint" json:"ToHubID"`
+	Latency   *float32  `gorm:"column:Latency; type:float" json:"latency"`
+	TestTime  time.Time `gorm:"column:TestTime; type:datetime" json:"testTime"`
+}
+
+func (HubConnectivity) TableName() string {
+	return "HubConnectivity"
+}