From f2535505dc41082ce6d8c608cfe3020074a4af2c Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 14 Aug 2023 15:39:36 +0800 Subject: [PATCH] =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E8=84=9A=E6=9C=AC=E5=92=8C?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=88=B0storage-common=E5=BA=93=EF=BC=9B?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- assets/confs/agent.config.json | 34 ++++++++ assets/confs/client.config.json | 28 ++++++ assets/confs/coordinator.config.json | 20 +++++ assets/confs/scanner.config.json | 31 +++++++ assets/confs/sysSetting.xml | 63 ++++++++++++++ assets/scripts/create_database.sql | 125 +++++++++++++++++++++++++++ go.mod | 3 + go.sum | 14 +++ utils/config.go | 4 +- 9 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 assets/confs/agent.config.json create mode 100644 assets/confs/client.config.json create mode 100644 assets/confs/coordinator.config.json create mode 100644 assets/confs/scanner.config.json create mode 100644 assets/confs/sysSetting.xml create mode 100644 assets/scripts/create_database.sql diff --git a/assets/confs/agent.config.json b/assets/confs/agent.config.json new file mode 100644 index 0000000..dc28967 --- /dev/null +++ b/assets/confs/agent.config.json @@ -0,0 +1,34 @@ +{ + "id": 1, + "grpcListenAddress": "127.0.0.1:5010", + "grpcPort": 5010, + "ecPacketSize": 10, + "localIP": "127.0.0.1", + "externalIP": "127.0.0.1", + "storageBaseDir": ".", + "tempFileLifetime": 3600, + "logger": { + "output": "file", + "outputFileName": "agent", + "outputDirectory": "log", + "level": "debug" + }, + "rabbitMQ": { + "address": "127.0.0.1:5672", + "account": "cloudream", + "password": "123456", + "vhost": "/" + }, + "ipfs": { + "port": 5001 + }, + "distlock": { + "etcdAddress": "127.0.0.1:2379", + "etcdUsername": "", + "etcdPassword": "", + "etcdLockAcquireTimeoutMs": 5000, + "etcdLockLeaseTimeSec": 5, + "lockRequestLeaseTimeSec": 5, + "submitLockRequestWithoutLease": true + } +} \ No newline at end of file diff --git a/assets/confs/client.config.json b/assets/confs/client.config.json new file mode 100644 index 0000000..5afdd35 --- /dev/null +++ b/assets/confs/client.config.json @@ -0,0 +1,28 @@ +{ + "grpcPort": 5010, + "ecPacketSize": 10, + "ipfsPort": 10, + "maxRepCount": 10, + "localIP": "127.0.0.1", + "externalIP": "127.0.0.1", + "logger": { + "output": "stdout", + "level": "debug" + }, + "rabbitMQ": { + "address": "127.0.0.1:5672", + "account": "cloudream", + "password": "123456", + "vhost": "/" + }, + "ipfs": null, + "distlock": { + "etcdAddress": "127.0.0.1:2379", + "etcdUsername": "", + "etcdPassword": "", + "etcdLockAcquireTimeoutMs": 5000, + "etcdLockLeaseTimeSec": 5, + "lockRequestLeaseTimeSec": 5, + "submitLockRequestWithoutLease": true + } +} \ No newline at end of file diff --git a/assets/confs/coordinator.config.json b/assets/confs/coordinator.config.json new file mode 100644 index 0000000..181e3ba --- /dev/null +++ b/assets/confs/coordinator.config.json @@ -0,0 +1,20 @@ +{ + "logger": { + "output": "file", + "outputFileName": "coordinator", + "outputDirectory": "log", + "level": "debug" + }, + "db": { + "address": "127.0.0.1:3306", + "account": "root", + "password": "123456", + "databaseName": "cloudream" + }, + "rabbitMQ": { + "address": "127.0.0.1:5672", + "account": "cloudream", + "password": "123456", + "vhost": "/" + } +} \ No newline at end of file diff --git a/assets/confs/scanner.config.json b/assets/confs/scanner.config.json new file mode 100644 index 0000000..78644ff --- /dev/null +++ b/assets/confs/scanner.config.json @@ -0,0 +1,31 @@ +{ + "minAvailableRepProportion": 0.8, + "nodeUnavailableSeconds": 300, + "logger": { + "output": "file", + "outputFileName": "scanner", + "outputDirectory": "log", + "level": "debug" + }, + "db": { + "address": "127.0.0.1:3306", + "account": "root", + "password": "123456", + "databaseName": "cloudream" + }, + "rabbitMQ": { + "address": "127.0.0.1:5672", + "account": "cloudream", + "password": "123456", + "vhost": "/" + }, + "distlock": { + "etcdAddress": "127.0.0.1:2379", + "etcdUsername": "", + "etcdPassword": "", + "etcdLockAcquireTimeoutMs": 5000, + "etcdLockLeaseTimeSec": 5, + "lockRequestLeaseTimeSec": 5, + "submitLockRequestWithoutLease": true + } +} \ No newline at end of file diff --git a/assets/confs/sysSetting.xml b/assets/confs/sysSetting.xml new file mode 100644 index 0000000..d78805b --- /dev/null +++ b/assets/confs/sysSetting.xml @@ -0,0 +1,63 @@ + + +local.addr +101.201.215.165 + + +controller.addr +101.201.215.196 + + +agents.addr +/hw-sh/123.60.146.162 +/hw-bj/120.46.183.86 +/ali/101.201.215.165 + + +agents.location +ali +hw-sh +hw-bj + + +oec.controller.thread.num +4 + + +oec.agent.thread.num +2 + + +oec.cmddist.thread.num +2 + + +packet.size +131072 + + +ec.concurrent.num +2 + + +ec.policy +rs_9_6RS96961-1 +rs_3_2RS96321-1 +edu_9_6EDU96961-1 +edu_3_2EDU32321-1 +dfc_9_4DFC941-13,2 + + +inter.inner.addr + +172.23.85.69172.23.85.71172.23.85.70 +192.168.0.69 +192.168.0.76 + + +101.201.215.196101.201.215.165101.201.214.111 +123.60.146.162 +120.46.183.86 + + + diff --git a/assets/scripts/create_database.sql b/assets/scripts/create_database.sql new file mode 100644 index 0000000..1c8379b --- /dev/null +++ b/assets/scripts/create_database.sql @@ -0,0 +1,125 @@ +drop database if exists cloudream; + +create database cloudream; + +use cloudream; + +create table Node ( + NodeID int not null auto_increment primary key comment '节点ID', + Name varchar(128) not null comment '节点名称', + LocalIP varchar(128) not null comment '节点的内网IP', + ExternalIP varchar(128) not null comment '节点的外网IP', + LocationID int not null comment '节点的地域', + State varchar(128) comment '节点的状态', + LastReportTime timestamp comment '节点上次上报时间' +) comment = '节点表'; + +insert into Node (NodeID, Name, LocalIP, ExternalIP, LocationID, State) values (0, "LocalNode", "localhost", "localhost", 0, 1); + +create table Storage ( + StorageID int not null auto_increment primary key comment '存储服务ID', + Name varchar(100) not null comment '存储服务名称', + NodeID int not null comment '存储服务所在节点的ID', + Directory varchar(4096) not null comment '存储服务所在节点的目录', + State varchar(100) comment '状态' +) comment = "存储服务表"; + +insert into Storage (StorageID, Name, NodeID, Directory, State) values (1, "HuaWei-Cloud", 1,"/" ,"Online"); + +create table NodeDelay ( + SourceNodeID int not null comment '发起检测的节点ID', + DestinationNodeID int not null comment '被检测节点的ID', + DelayInMs int not null comment '发起节点与被检测节点间延迟(毫秒)', + primary key(SourceNodeID, DestinationNodeID) +) comment = '节点延迟表'; + +create table User ( + UserID int not null primary key comment '用户ID', + Password varchar(100) not null comment '用户密码' +) comment = '用户密码表'; + +create table UserBucket ( + UserID int not null comment '用户ID', + BucketID int not null comment '用户可访问的桶ID', + primary key(UserID, BucketID) +) comment = '用户桶权限表'; +insert into UserBucket (UserID, BucketID) values (0, 1); +create table UserNode ( + UserID int not null comment '用户ID', + NodeID int not null comment '用户可使用的节点ID', + primary key(UserID, NodeID) +) comment = '用户节点权限表'; + +insert into UserNode (UserID, NodeID) values (0, 1); + +create table UserStorage ( + UserID int not null comment "用户ID", + StorageID int not null comment "存储服务ID", + primary key(UserID, StorageID) +); + +insert into UserStorage (UserID, StorageID) values (0, 1); + +create table Bucket ( + BucketID int not null auto_increment primary key comment '桶ID', + Name varchar(100) not null comment '桶名', + CreatorID int not null comment '创建者ID' +) comment = '桶表'; + +insert into Bucket (BucketID, Name, CreatorID) values (0, "bucket01", 0); + +create table Object ( + ObjectID int not null auto_increment primary key comment '对象ID', + Name varchar(100) not null comment '对象名', + BucketID int not null comment '桶ID', + State varchar(100) not null comment '对象状态', + FileSize bigint not null comment '对象大小(Byte)', + Redundancy varchar(100) not null comment '对象冗余策略' + DirName varchar(100) not null comment '对象所属文件夹' +) comment = '对象表'; + +create table ObjectRep ( + ObjectID int not null primary key comment '对象ID', + RepCount int not null comment '对象的副本数', + FileHash varchar(100) not null comment '副本哈希值' +) comment = '对象副本表'; + +create table ObjectBlock ( + BlockID int not null auto_increment primary key comment '编码块块ID', + ObjectID int not null comment '对象ID', + InnerID int not null comment '编码块在条带内的排序', + BlockHash varchar(100) not null comment '编码块哈希值' +) comment = '对象编码块表'; + +create table Cache ( + FileHash varchar(100) not null comment '编码块块ID', + NodeID int not null comment '节点ID', + State varchar(100) not null comment '状态', + CacheTime timestamp not null comment '缓存时间', + Priority int not null comment '编码块优先级', + primary key(FileHash, NodeID) +) comment = '缓存表'; + +create table StorageObject ( + ObjectID int not null comment '对象ID', + StorageID int not null comment '存储服务ID', + UserID int not null comment '调度了此文件的用户ID', + State varchar(100) not null comment '对象状态', + primary key(ObjectID, StorageID, UserID) +); + +create table Location ( + LocationID int not null auto_increment primary key comment 'ID', + Name varchar(128) not null comment '名称' +) comment = '地域表'; +insert into Location (LocationID, Name) values (1, "Local"); + +create table Ec ( + EcID int not null comment '纠删码ID', + Name varchar(128) not null comment '纠删码名称', + EcK int not null comment 'ecK', + EcN int not null comment 'ecN' +) comment = '纠删码表'; + +insert into Ec (EcID, Name, EcK, EcN) values (1, "rs_9_6", 6, 9); +insert into Ec (EcID, Name, EcK, EcN) values (2, "rs_5_3", 3, 5); \ No newline at end of file diff --git a/go.mod b/go.mod index 7f68049..3060a54 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module gitlink.org.cn/cloudream/storage-common require ( github.com/baohan10/reedsolomon v0.0.0-20230406042632-43574cac9fa7 + github.com/beevik/etree v1.2.0 + github.com/go-ping/ping v1.1.0 github.com/go-sql-driver/mysql v1.7.1 github.com/jmoiron/sqlx v1.3.5 github.com/samber/lo v1.36.0 @@ -30,6 +32,7 @@ require ( github.com/zyedidia/generic v1.2.1 // indirect golang.org/x/exp v0.0.0-20230519143937-03e91628a987 // indirect golang.org/x/net v0.8.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect diff --git a/go.sum b/go.sum index db07586..c5ba10c 100644 --- a/go.sum +++ b/go.sum @@ -2,9 +2,13 @@ github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UME github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA= github.com/baohan10/reedsolomon v0.0.0-20230406042632-43574cac9fa7 h1:wcvD6enR///dFvb9cRodx5SGbPH4G4jPjw+aVIWkAKE= github.com/baohan10/reedsolomon v0.0.0-20230406042632-43574cac9fa7/go.mod h1:rAxMF6pVaFK/s6T4gGczvloccNbtwzuYaP2Y7W6flE8= +github.com/beevik/etree v1.2.0 h1:l7WETslUG/T+xOPs47dtd6jov2Ii/8/OjCldk5fYfQw= +github.com/beevik/etree v1.2.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw= +github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= @@ -14,6 +18,7 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -62,14 +67,23 @@ github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XD github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= golang.org/x/exp v0.0.0-20230519143937-03e91628a987 h1:3xJIFvzUFbu4ls0BTBYcgbCGhA63eAOEMxIHugyXJqA= golang.org/x/exp v0.0.0-20230519143937-03e91628a987/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= diff --git a/utils/config.go b/utils/config.go index d69134e..332b851 100644 --- a/utils/config.go +++ b/utils/config.go @@ -27,7 +27,7 @@ func (r *EcConfig) GetN() int { func GetEcPolicy() *map[string]EcConfig { doc := etree.NewDocument() - if err := doc.ReadFromFile("../conf/sysSetting.xml"); err != nil { + if err := doc.ReadFromFile("../confs/sysSetting.xml"); err != nil { panic(err) } ecMap := make(map[string]EcConfig, 20) @@ -53,7 +53,7 @@ func GetEcPolicy() *map[string]EcConfig { func GetAgentIps() []string { doc := etree.NewDocument() - if err := doc.ReadFromFile("../conf/sysSetting.xml"); err != nil { + if err := doc.ReadFromFile("../confs/sysSetting.xml"); err != nil { panic(err) } root := doc.SelectElement("setting")