| @@ -164,6 +164,15 @@ func serveHTTP(configPath string, opts serveHTTPOptions) { | |||||
| evtPubChan := evtPub.Start() | evtPubChan := evtPub.Start() | ||||
| defer evtPub.Stop() | defer evtPub.Stop() | ||||
| // 系统事件监听器 | |||||
| evtWtchr, err := sysevent.NewWatcherHost(config.Cfg().SysEvent) | |||||
| if err != nil { | |||||
| logger.Errorf("new sysevent watcher host: %v", err) | |||||
| os.Exit(1) | |||||
| } | |||||
| evtWtchrChan := evtWtchr.Start() | |||||
| defer evtWtchr.Stop() | |||||
| // 连接性信息收集 | // 连接性信息收集 | ||||
| var conCol *connectivity.Collector | var conCol *connectivity.Collector | ||||
| if stgglb.StandaloneMode { | if stgglb.StandaloneMode { | ||||
| @@ -198,7 +207,7 @@ func serveHTTP(configPath string, opts serveHTTPOptions) { | |||||
| strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, spaceMeta, hubMeta, conMeta) | strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, spaceMeta, hubMeta, conMeta) | ||||
| // 下载器 | // 下载器 | ||||
| dlder := downloader.NewDownloader(config.Cfg().Downloader, conCol, stgPool, strgSel, db, spdStats) | |||||
| dlder := downloader.NewDownloader(config.Cfg().Downloader, conCol, stgPool, strgSel, db, spdStats, evtPub) | |||||
| // 上传器 | // 上传器 | ||||
| uploader := uploader.NewUploader(publock, conCol, stgPool, spaceMeta, db) | uploader := uploader.NewUploader(publock, conCol, stgPool, spaceMeta, db) | ||||
| @@ -230,7 +239,7 @@ func serveHTTP(configPath string, opts serveHTTPOptions) { | |||||
| mntChan := mnt.Start() | mntChan := mnt.Start() | ||||
| defer mnt.Stop() | defer mnt.Stop() | ||||
| svc := services.NewService(publock, dlder, acStat, uploader, strgSel, spaceMeta, db, evtPub, mnt, stgPool, spaceSync, tktk, spdStats, accToken) | |||||
| svc := services.NewService(publock, dlder, acStat, uploader, strgSel, spaceMeta, db, evtPub, evtWtchr, mnt, stgPool, spaceSync, tktk, spdStats, accToken) | |||||
| // HTTP接口 | // HTTP接口 | ||||
| httpCfgJSON := config.Cfg().HTTP | httpCfgJSON := config.Cfg().HTTP | ||||
| @@ -261,6 +270,7 @@ func serveHTTP(configPath string, opts serveHTTPOptions) { | |||||
| accTokenEvt := accTokenChan.Receive() | accTokenEvt := accTokenChan.Receive() | ||||
| evtPubEvt := evtPubChan.Receive() | evtPubEvt := evtPubChan.Receive() | ||||
| evtWtchrEvt := evtWtchrChan.Receive() | |||||
| conColEvt := conColChan.Receive() | conColEvt := conColChan.Receive() | ||||
| acStatEvt := acStatChan.Receive() | acStatEvt := acStatChan.Receive() | ||||
| spaceSyncEvt := spaceSyncChan.Receive() | spaceSyncEvt := spaceSyncChan.Receive() | ||||
| @@ -312,6 +322,26 @@ loop: | |||||
| } | } | ||||
| evtPubEvt = evtPubChan.Receive() | evtPubEvt = evtPubChan.Receive() | ||||
| case e := <-evtWtchrEvt.Chan(): | |||||
| if e.Err != nil { | |||||
| logger.Errorf("receive watcher event: %v", err) | |||||
| break loop | |||||
| } | |||||
| switch val := e.Value.(type) { | |||||
| case sysevent.WatcherExited: | |||||
| if val.Err != nil { | |||||
| logger.Errorf("sysevent watcher exited with error: %v", val.Err) | |||||
| } else { | |||||
| logger.Info("sysevent watcher exited") | |||||
| } | |||||
| break loop | |||||
| case sysevent.OtherError: | |||||
| logger.Errorf("sysevent watcher: %v", val) | |||||
| } | |||||
| evtWtchrEvt = evtWtchrChan.Receive() | |||||
| case e := <-conColEvt.Chan(): | case e := <-conColEvt.Chan(): | ||||
| if e.Err != nil { | if e.Err != nil { | ||||
| logger.Errorf("receive connectivity event: %v", err) | logger.Errorf("receive connectivity event: %v", err) | ||||
| @@ -172,6 +172,15 @@ func test(configPath string) { | |||||
| evtPubChan := evtPub.Start() | evtPubChan := evtPub.Start() | ||||
| defer evtPub.Stop() | defer evtPub.Stop() | ||||
| // 系统事件监听器 | |||||
| evtWtchr, err := sysevent.NewWatcherHost(config.Cfg().SysEvent) | |||||
| if err != nil { | |||||
| logger.Errorf("new sysevent watcher host: %v", err) | |||||
| os.Exit(1) | |||||
| } | |||||
| evtWtchrChan := evtWtchr.Start() | |||||
| defer evtWtchr.Stop() | |||||
| // 连接性信息收集 | // 连接性信息收集 | ||||
| var conCol *connectivity.Collector | var conCol *connectivity.Collector | ||||
| if stgglb.StandaloneMode { | if stgglb.StandaloneMode { | ||||
| @@ -206,7 +215,7 @@ func test(configPath string) { | |||||
| strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, stgMeta, hubMeta, conMeta) | strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, stgMeta, hubMeta, conMeta) | ||||
| // 下载器 | // 下载器 | ||||
| dlder := downloader.NewDownloader(config.Cfg().Downloader, conCol, stgPool, strgSel, db, spdStats) | |||||
| dlder := downloader.NewDownloader(config.Cfg().Downloader, conCol, stgPool, strgSel, db, spdStats, evtPub) | |||||
| // 上传器 | // 上传器 | ||||
| uploader := uploader.NewUploader(publock, conCol, stgPool, stgMeta, db) | uploader := uploader.NewUploader(publock, conCol, stgPool, stgMeta, db) | ||||
| @@ -216,7 +225,7 @@ func test(configPath string) { | |||||
| spaceSyncChan := spaceSync.Start() | spaceSyncChan := spaceSync.Start() | ||||
| defer spaceSync.Stop() | defer spaceSync.Stop() | ||||
| svc := services.NewService(publock, dlder, acStat, uploader, strgSel, stgMeta, db, evtPub, nil, stgPool, spaceSync, nil, spdStats, accToken) | |||||
| svc := services.NewService(publock, dlder, acStat, uploader, strgSel, stgMeta, db, evtPub, evtWtchr, nil, stgPool, spaceSync, nil, spdStats, accToken) | |||||
| go func() { | go func() { | ||||
| doTest(svc) | doTest(svc) | ||||
| @@ -225,6 +234,7 @@ func test(configPath string) { | |||||
| /// 开始监听各个模块的事件 | /// 开始监听各个模块的事件 | ||||
| accTokenEvt := accTokenChan.Receive() | accTokenEvt := accTokenChan.Receive() | ||||
| evtPubEvt := evtPubChan.Receive() | evtPubEvt := evtPubChan.Receive() | ||||
| evtWtchrEvt := evtWtchrChan.Receive() | |||||
| conColEvt := conColChan.Receive() | conColEvt := conColChan.Receive() | ||||
| acStatEvt := acStatChan.Receive() | acStatEvt := acStatChan.Receive() | ||||
| spaceSyncEvt := spaceSyncChan.Receive() | spaceSyncEvt := spaceSyncChan.Receive() | ||||
| @@ -272,6 +282,26 @@ loop: | |||||
| } | } | ||||
| evtPubEvt = evtPubChan.Receive() | evtPubEvt = evtPubChan.Receive() | ||||
| case e := <-evtWtchrEvt.Chan(): | |||||
| if e.Err != nil { | |||||
| logger.Errorf("receive watcher event: %v", err) | |||||
| break loop | |||||
| } | |||||
| switch val := e.Value.(type) { | |||||
| case sysevent.WatcherExited: | |||||
| if val.Err != nil { | |||||
| logger.Errorf("sysevent watcher exited with error: %v", val.Err) | |||||
| } else { | |||||
| logger.Info("sysevent watcher exited") | |||||
| } | |||||
| break loop | |||||
| case sysevent.OtherError: | |||||
| logger.Errorf("sysevent watcher: %v", val) | |||||
| } | |||||
| evtWtchrEvt = evtWtchrChan.Receive() | |||||
| case e := <-conColEvt.Chan(): | case e := <-conColEvt.Chan(): | ||||
| if e.Err != nil { | if e.Err != nil { | ||||
| logger.Errorf("receive connectivity event: %v", err) | logger.Errorf("receive connectivity event: %v", err) | ||||
| @@ -152,6 +152,15 @@ func vfsTest(configPath string, opts serveHTTPOptions) { | |||||
| evtPubChan := evtPub.Start() | evtPubChan := evtPub.Start() | ||||
| defer evtPub.Stop() | defer evtPub.Stop() | ||||
| // 系统事件监听器 | |||||
| evtWtchr, err := sysevent.NewWatcherHost(config.Cfg().SysEvent) | |||||
| if err != nil { | |||||
| logger.Errorf("new sysevent watcher host: %v", err) | |||||
| os.Exit(1) | |||||
| } | |||||
| evtWtchrChan := evtWtchr.Start() | |||||
| defer evtWtchr.Stop() | |||||
| // 连接性信息收集 | // 连接性信息收集 | ||||
| var conCol *connectivity.Collector | var conCol *connectivity.Collector | ||||
| if stgglb.StandaloneMode { | if stgglb.StandaloneMode { | ||||
| @@ -186,7 +195,7 @@ func vfsTest(configPath string, opts serveHTTPOptions) { | |||||
| strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, stgMeta, hubMeta, conMeta) | strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, stgMeta, hubMeta, conMeta) | ||||
| // 下载器 | // 下载器 | ||||
| dlder := downloader.NewDownloader(config.Cfg().Downloader, conCol, stgPool, strgSel, db, spdStats) | |||||
| dlder := downloader.NewDownloader(config.Cfg().Downloader, conCol, stgPool, strgSel, db, spdStats, evtPub) | |||||
| // 上传器 | // 上传器 | ||||
| uploader := uploader.NewUploader(publock, conCol, stgPool, stgMeta, db) | uploader := uploader.NewUploader(publock, conCol, stgPool, stgMeta, db) | ||||
| @@ -209,7 +218,7 @@ func vfsTest(configPath string, opts serveHTTPOptions) { | |||||
| mntChan := mnt.Start() | mntChan := mnt.Start() | ||||
| defer mnt.Stop() | defer mnt.Stop() | ||||
| svc := services.NewService(publock, dlder, acStat, uploader, strgSel, stgMeta, db, evtPub, mnt, stgPool, spaceSync, nil, spdStats, accToken) | |||||
| svc := services.NewService(publock, dlder, acStat, uploader, strgSel, stgMeta, db, evtPub, evtWtchr, mnt, stgPool, spaceSync, nil, spdStats, accToken) | |||||
| // HTTP接口 | // HTTP接口 | ||||
| httpCfgJSON := config.Cfg().HTTP | httpCfgJSON := config.Cfg().HTTP | ||||
| @@ -244,6 +253,7 @@ func vfsTest(configPath string, opts serveHTTPOptions) { | |||||
| accTokenEvt := accTokenChan.Receive() | accTokenEvt := accTokenChan.Receive() | ||||
| evtPubEvt := evtPubChan.Receive() | evtPubEvt := evtPubChan.Receive() | ||||
| evtWtchrEvt := evtWtchrChan.Receive() | |||||
| conColEvt := conColChan.Receive() | conColEvt := conColChan.Receive() | ||||
| acStatEvt := acStatChan.Receive() | acStatEvt := acStatChan.Receive() | ||||
| spaceSyncEvt := spaceSyncChan.Receive() | spaceSyncEvt := spaceSyncChan.Receive() | ||||
| @@ -293,6 +303,26 @@ loop: | |||||
| } | } | ||||
| evtPubEvt = evtPubChan.Receive() | evtPubEvt = evtPubChan.Receive() | ||||
| case e := <-evtWtchrEvt.Chan(): | |||||
| if e.Err != nil { | |||||
| logger.Errorf("receive watcher event: %v", err) | |||||
| break loop | |||||
| } | |||||
| switch val := e.Value.(type) { | |||||
| case sysevent.WatcherExited: | |||||
| if val.Err != nil { | |||||
| logger.Errorf("sysevent watcher exited with error: %v", val.Err) | |||||
| } else { | |||||
| logger.Info("sysevent watcher exited") | |||||
| } | |||||
| break loop | |||||
| case sysevent.OtherError: | |||||
| logger.Errorf("sysevent watcher: %v", val) | |||||
| } | |||||
| evtWtchrEvt = evtWtchrChan.Receive() | |||||
| case e := <-conColEvt.Chan(): | case e := <-conColEvt.Chan(): | ||||
| if e.Err != nil { | if e.Err != nil { | ||||
| logger.Errorf("receive connectivity event: %v", err) | logger.Errorf("receive connectivity event: %v", err) | ||||
| @@ -814,3 +814,17 @@ func (db *ObjectDB) BatchCreateByDetails(ctx SQLContext, pkgID jcstypes.PackageI | |||||
| return affectedObjs, nil | return affectedObjs, nil | ||||
| } | } | ||||
| func (*ObjectDB) Summary(ctx SQLContext) (cnt int64, size int64, err error) { | |||||
| var stats struct { | |||||
| Cnt int64 `gorm:"column:Cnt; type:bigint;"` | |||||
| Size int64 `gorm:"column:Size; type:bigint;"` | |||||
| } | |||||
| err = ctx.Table("Object").Select("Count(Size) as Cnt, Sum(Size) as Size").Find(&stats).Error | |||||
| if err != nil { | |||||
| return 0, 0, err | |||||
| } | |||||
| return stats.Cnt, stats.Size, nil | |||||
| } | |||||
| @@ -11,6 +11,7 @@ import ( | |||||
| "gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats" | "gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity" | "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" | "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent" | |||||
| jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types" | jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types" | ||||
| ) | ) | ||||
| @@ -45,9 +46,10 @@ type Downloader struct { | |||||
| selector *strategy.Selector | selector *strategy.Selector | ||||
| db *db.DB | db *db.DB | ||||
| speedStats *speedstats.SpeedStats | speedStats *speedstats.SpeedStats | ||||
| evtPub *sysevent.Publisher | |||||
| } | } | ||||
| func NewDownloader(cfg Config, conn *connectivity.Collector, stgPool *pool.Pool, sel *strategy.Selector, db *db.DB, speedStats *speedstats.SpeedStats) *Downloader { | |||||
| func NewDownloader(cfg Config, conn *connectivity.Collector, stgPool *pool.Pool, sel *strategy.Selector, db *db.DB, speedStats *speedstats.SpeedStats, evtPub *sysevent.Publisher) *Downloader { | |||||
| if cfg.MaxStripCacheCount == 0 { | if cfg.MaxStripCacheCount == 0 { | ||||
| cfg.MaxStripCacheCount = DefaultMaxStripCacheCount | cfg.MaxStripCacheCount = DefaultMaxStripCacheCount | ||||
| } | } | ||||
| @@ -61,6 +63,7 @@ func NewDownloader(cfg Config, conn *connectivity.Collector, stgPool *pool.Pool, | |||||
| selector: sel, | selector: sel, | ||||
| db: db, | db: db, | ||||
| speedStats: speedStats, | speedStats: speedStats, | ||||
| evtPub: evtPub, | |||||
| } | } | ||||
| } | } | ||||
| @@ -5,9 +5,12 @@ import ( | |||||
| "fmt" | "fmt" | ||||
| "io" | "io" | ||||
| "reflect" | "reflect" | ||||
| "time" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | "gitlink.org.cn/cloudream/common/pkgs/logger" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" | "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan/ops" | |||||
| "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap" | |||||
| "gitlink.org.cn/cloudream/common/utils/io2" | "gitlink.org.cn/cloudream/common/utils/io2" | ||||
| "gitlink.org.cn/cloudream/common/utils/math2" | "gitlink.org.cn/cloudream/common/utils/math2" | ||||
| @@ -140,19 +143,42 @@ func (i *DownloadObjectIterator) downloadDirect(req downloadReqeust2, strg strat | |||||
| exeCtx := exec.NewExecContext() | exeCtx := exec.NewExecContext() | ||||
| exec.SetValueByType(exeCtx, i.downloader.stgPool) | exec.SetValueByType(exeCtx, i.downloader.stgPool) | ||||
| exec := plans.Execute(exeCtx) | exec := plans.Execute(exeCtx) | ||||
| rd, err := exec.BeginRead(strHandle) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| counter := io2.CounterCloser(rd, nil) | |||||
| go func() { | go func() { | ||||
| startTime := time.Now() | |||||
| ret, err := exec.Wait(context.TODO()) | ret, err := exec.Wait(context.TODO()) | ||||
| if err != nil { | if err != nil { | ||||
| logger.Warnf("downloading object %v: %v", req.Raw.ObjectID, err) | logger.Warnf("downloading object %v: %v", req.Raw.ObjectID, err) | ||||
| } | } | ||||
| transBytes := int64(0) | |||||
| for _, v := range ret.GetArray(ops2.BaseReadStatsStoreKey) { | for _, v := range ret.GetArray(ops2.BaseReadStatsStoreKey) { | ||||
| v2 := v.(*ops2.BaseReadStatsValue) | v2 := v.(*ops2.BaseReadStatsValue) | ||||
| i.downloader.speedStats.Record(v2.Size, v2.Time, v2.Location.IsDriver) | |||||
| i.downloader.speedStats.Record(v2.Length, v2.ElapsedTime, v2.Location.IsDriver) | |||||
| transBytes += v2.Length | |||||
| } | } | ||||
| for _, v := range ret.GetArray(ops.SendStreamStatsStoreKey) { | |||||
| v2 := v.(*ops.SendStreamStatsValue) | |||||
| transBytes += v2.Length | |||||
| } | |||||
| i.downloader.evtPub.Publish(&datamap.BodyObjectAccessStats{ | |||||
| ObjectID: req.Raw.ObjectID, | |||||
| RequestSize: len, | |||||
| TransferAmount: transBytes, | |||||
| ElapsedTime: time.Since(startTime), | |||||
| }) | |||||
| }() | }() | ||||
| return exec.BeginRead(strHandle) | |||||
| return counter, nil | |||||
| } | } | ||||
| func (i *DownloadObjectIterator) downloadECReconstruct(req downloadReqeust2, strg strategy.ECReconstructStrategy) (io.ReadCloser, error) { | func (i *DownloadObjectIterator) downloadECReconstruct(req downloadReqeust2, strg strategy.ECReconstructStrategy) (io.ReadCloser, error) { | ||||
| @@ -187,7 +213,10 @@ func (i *DownloadObjectIterator) downloadECReconstruct(req downloadReqeust2, str | |||||
| } | } | ||||
| pr, pw := io.Pipe() | pr, pw := io.Pipe() | ||||
| counter := io2.CounterCloser(pr, nil) | |||||
| go func() { | go func() { | ||||
| startTime := time.Now() | |||||
| readPos := req.Raw.Offset | readPos := req.Raw.Offset | ||||
| totalReadLen := req.Detail.Object.Size - req.Raw.Offset | totalReadLen := req.Detail.Object.Size - req.Raw.Offset | ||||
| if req.Raw.Length >= 0 { | if req.Raw.Length >= 0 { | ||||
| @@ -196,6 +225,17 @@ func (i *DownloadObjectIterator) downloadECReconstruct(req downloadReqeust2, str | |||||
| firstStripIndex := readPos / strg.Redundancy.StripSize() | firstStripIndex := readPos / strg.Redundancy.StripSize() | ||||
| stripIter := NewStripIterator(i.downloader, req.Detail.Object, downloadBlks, strg.Redundancy, firstStripIndex, i.downloader.strips, i.downloader.cfg.ECStripPrefetchCount) | stripIter := NewStripIterator(i.downloader, req.Detail.Object, downloadBlks, strg.Redundancy, firstStripIndex, i.downloader.strips, i.downloader.cfg.ECStripPrefetchCount) | ||||
| // defer顺序不能改,因为CollectStats需要在Close之后调用 | |||||
| defer func() { | |||||
| stats := stripIter.CollectStats() | |||||
| i.downloader.evtPub.Publish(&datamap.BodyObjectAccessStats{ | |||||
| ObjectID: req.Raw.ObjectID, | |||||
| RequestSize: length, | |||||
| TransferAmount: stats.TransferredBytes, | |||||
| ElapsedTime: time.Since(startTime), | |||||
| }) | |||||
| }() | |||||
| defer stripIter.Close() | defer stripIter.Close() | ||||
| for totalReadLen > 0 { | for totalReadLen > 0 { | ||||
| @@ -224,5 +264,5 @@ func (i *DownloadObjectIterator) downloadECReconstruct(req downloadReqeust2, str | |||||
| pw.Close() | pw.Close() | ||||
| }() | }() | ||||
| return pr, nil | |||||
| return counter, nil | |||||
| } | } | ||||
| @@ -129,7 +129,7 @@ func (s *LRCStripIterator) downloading() { | |||||
| for _, v := range ret.GetArray(ops2.BaseReadStatsStoreKey) { | for _, v := range ret.GetArray(ops2.BaseReadStatsStoreKey) { | ||||
| v2 := v.(*ops2.BaseReadStatsValue) | v2 := v.(*ops2.BaseReadStatsValue) | ||||
| s.downloader.speedStats.Record(v2.Size, v2.Time, v2.Location.IsDriver) | |||||
| s.downloader.speedStats.Record(v2.Length, v2.ElapsedTime, v2.Location.IsDriver) | |||||
| } | } | ||||
| }() | }() | ||||
| defer cancel() | defer cancel() | ||||
| @@ -9,6 +9,7 @@ import ( | |||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | "gitlink.org.cn/cloudream/common/pkgs/logger" | ||||
| "gitlink.org.cn/cloudream/common/utils/math2" | "gitlink.org.cn/cloudream/common/utils/math2" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" | "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan/ops" | |||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" | "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" | "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" | "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" | ||||
| @@ -26,19 +27,25 @@ type Strip struct { | |||||
| } | } | ||||
| type StripIterator struct { | type StripIterator struct { | ||||
| downloader *Downloader | |||||
| object jcstypes.Object | |||||
| blocks []downloadBlock | |||||
| red jcstypes.ECRedundancy | |||||
| curStripIndex int64 | |||||
| cache *StripCache | |||||
| dataChan chan dataChanEntry | |||||
| downloadingDone chan any | |||||
| downloadingDoneOnce sync.Once | |||||
| inited bool | |||||
| downloadingStream io.ReadCloser | |||||
| downloadingStripIndex int64 | |||||
| downloadingPlanCtxCancel func() | |||||
| downloader *Downloader | |||||
| object jcstypes.Object | |||||
| blocks []downloadBlock | |||||
| red jcstypes.ECRedundancy | |||||
| curStripIndex int64 | |||||
| cache *StripCache | |||||
| dataChan chan dataChanEntry | |||||
| downloadingDone chan any | |||||
| downloadingDoneOnce sync.Once | |||||
| inited bool | |||||
| downloadingStream io.ReadCloser | |||||
| downloadingStripIndex int64 | |||||
| statsLock *sync.Cond | |||||
| downloadingStrips int | |||||
| transferredBytes int64 // 总传输量 | |||||
| } | |||||
| type StripDownloadStats struct { | |||||
| TransferredBytes int64 // 总传输量 | |||||
| } | } | ||||
| type dataChanEntry struct { | type dataChanEntry struct { | ||||
| @@ -61,6 +68,7 @@ func NewStripIterator(downloader *Downloader, object jcstypes.Object, blocks []d | |||||
| cache: cache, | cache: cache, | ||||
| dataChan: make(chan dataChanEntry, maxPrefetch-1), | dataChan: make(chan dataChanEntry, maxPrefetch-1), | ||||
| downloadingDone: make(chan any), | downloadingDone: make(chan any), | ||||
| statsLock: sync.NewCond(&sync.Mutex{}), | |||||
| } | } | ||||
| return iter | return iter | ||||
| @@ -115,6 +123,19 @@ func (s *StripIterator) Close() { | |||||
| }) | }) | ||||
| } | } | ||||
| // 必须在Close之后调用! | |||||
| func (s *StripIterator) CollectStats() StripDownloadStats { | |||||
| s.statsLock.L.Lock() | |||||
| for s.downloadingStrips > 0 { | |||||
| s.statsLock.Wait() | |||||
| } | |||||
| s.statsLock.L.Unlock() | |||||
| return StripDownloadStats{ | |||||
| TransferredBytes: s.transferredBytes, | |||||
| } | |||||
| } | |||||
| func (s *StripIterator) downloading(startStripIndex int64) { | func (s *StripIterator) downloading(startStripIndex int64) { | ||||
| curStripIndex := startStripIndex | curStripIndex := startStripIndex | ||||
| loop: | loop: | ||||
| @@ -177,7 +198,6 @@ loop: | |||||
| if s.downloadingStream != nil { | if s.downloadingStream != nil { | ||||
| s.downloadingStream.Close() | s.downloadingStream.Close() | ||||
| s.downloadingPlanCtxCancel() | |||||
| } | } | ||||
| close(s.dataChan) | close(s.dataChan) | ||||
| } | } | ||||
| @@ -196,7 +216,6 @@ func (s *StripIterator) readStrip(stripIndex int64, buf []byte) (int, error) { | |||||
| if s.downloadingStream == nil || s.downloadingStripIndex != stripIndex { | if s.downloadingStream == nil || s.downloadingStripIndex != stripIndex { | ||||
| if s.downloadingStream != nil { | if s.downloadingStream != nil { | ||||
| s.downloadingStream.Close() | s.downloadingStream.Close() | ||||
| s.downloadingPlanCtxCancel() | |||||
| } | } | ||||
| ft := ioswitch2.NewFromTo() | ft := ioswitch2.NewFromTo() | ||||
| @@ -221,29 +240,44 @@ func (s *StripIterator) readStrip(stripIndex int64, buf []byte) (int, error) { | |||||
| exec.SetValueByType(exeCtx, s.downloader.stgPool) | exec.SetValueByType(exeCtx, s.downloader.stgPool) | ||||
| exec := plans.Execute(exeCtx) | exec := plans.Execute(exeCtx) | ||||
| ctx, cancel := context.WithCancel(context.Background()) | |||||
| s.statsLock.L.Lock() | |||||
| s.downloadingStrips += 1 | |||||
| s.statsLock.L.Unlock() | |||||
| s.statsLock.Broadcast() | |||||
| go func() { | go func() { | ||||
| ret, err := exec.Wait(ctx) | |||||
| ret, err := exec.Wait(context.Background()) | |||||
| if err != nil { | if err != nil { | ||||
| logger.Warnf("downloading strip: %v", err) | logger.Warnf("downloading strip: %v", err) | ||||
| return | return | ||||
| } | } | ||||
| transBytes := int64(0) | |||||
| for _, v := range ret.GetArray(ops2.BaseReadStatsStoreKey) { | for _, v := range ret.GetArray(ops2.BaseReadStatsStoreKey) { | ||||
| v2 := v.(*ops2.BaseReadStatsValue) | v2 := v.(*ops2.BaseReadStatsValue) | ||||
| s.downloader.speedStats.Record(v2.Size, v2.Time, v2.Location.IsDriver) | |||||
| s.downloader.speedStats.Record(v2.Length, v2.ElapsedTime, v2.Location.IsDriver) | |||||
| transBytes += v2.Length | |||||
| } | } | ||||
| for _, v := range ret.GetArray(ops.SendStreamStatsStoreKey) { | |||||
| v2 := v.(*ops.SendStreamStatsValue) | |||||
| transBytes += v2.Length | |||||
| } | |||||
| s.statsLock.L.Lock() | |||||
| s.downloadingStrips -= 1 | |||||
| s.transferredBytes += transBytes | |||||
| s.statsLock.L.Unlock() | |||||
| s.statsLock.Broadcast() | |||||
| }() | }() | ||||
| str, err := exec.BeginRead(hd) | str, err := exec.BeginRead(hd) | ||||
| if err != nil { | if err != nil { | ||||
| cancel() | |||||
| return 0, err | return 0, err | ||||
| } | } | ||||
| s.downloadingStream = str | s.downloadingStream = str | ||||
| s.downloadingStripIndex = stripIndex | s.downloadingStripIndex = stripIndex | ||||
| s.downloadingPlanCtxCancel = cancel | |||||
| } | } | ||||
| n, err := io.ReadFull(s.downloadingStream, buf) | n, err := io.ReadFull(s.downloadingStream, buf) | ||||
| @@ -95,4 +95,6 @@ func (s *Server) InitRouters(rt gin.IRoutes, ah *auth.Auth, proxy *proxy.Cluster | |||||
| rt.GET(cliapi.PubShardsGetPath, certAuth, s.PubShards().Get) | rt.GET(cliapi.PubShardsGetPath, certAuth, s.PubShards().Get) | ||||
| rt.GET(cliapi.PubShardsExportPackagePath, certAuth, s.PubShards().ExportPackage) | rt.GET(cliapi.PubShardsExportPackagePath, certAuth, s.PubShards().ExportPackage) | ||||
| rt.POST(cliapi.PubShardsImportPackagePath, certAuth, s.PubShards().ImportPackage) | rt.POST(cliapi.PubShardsImportPackagePath, certAuth, s.PubShards().ImportPackage) | ||||
| rt.POST(cliapi.SysEventWatchPath, certAuth, s.SysEvent().Watch) | |||||
| } | } | ||||
| @@ -0,0 +1,57 @@ | |||||
| package http | |||||
| import ( | |||||
| "net/http" | |||||
| "github.com/gin-gonic/gin" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | |||||
| "gitlink.org.cn/cloudream/common/utils/serder" | |||||
| "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" | |||||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||||
| "gitlink.org.cn/cloudream/jcs-pub/common/ecode" | |||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent" | |||||
| ) | |||||
| type SysEventService struct { | |||||
| *Server | |||||
| } | |||||
| func (s *Server) SysEvent() *SysEventService { | |||||
| return &SysEventService{s} | |||||
| } | |||||
| func (s *SysEventService) Watch(ctx *gin.Context) { | |||||
| log := logger.WithField("HTTP", "SysEvent.Watch") | |||||
| _, err := types.ShouldBindJSONEx[cliapi.SysEventWatch](ctx) | |||||
| if err != nil { | |||||
| log.Warnf("binding body: %s", err.Error()) | |||||
| ctx.JSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "%v", err)) | |||||
| return | |||||
| } | |||||
| if s.svc.EvtWatcher == nil { | |||||
| ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "event watcher not configured")) | |||||
| return | |||||
| } | |||||
| ctx.Writer.Header().Set("Content-Type", "text/event-stream") | |||||
| ctx.Writer.Header().Set("Cache-Control", "no-cache") | |||||
| ctx.Writer.Header().Set("Connection", "keep-alive") | |||||
| wtchr := s.svc.EvtWatcher.AddWatcherFn(func(event sysevent.SysEvent) { | |||||
| data, err := serder.ObjectToJSONEx(event) | |||||
| if err != nil { | |||||
| log.Warnf("serializing event: %s", err.Error()) | |||||
| return | |||||
| } | |||||
| ctx.SSEvent("message", string(data)) | |||||
| ctx.Writer.Flush() | |||||
| }) | |||||
| <-ctx.Request.Context().Done() | |||||
| s.svc.EvtWatcher.RemoveWatcher(wtchr) | |||||
| log.Debugf("event watcher stopped") | |||||
| } | |||||
| @@ -27,6 +27,7 @@ type Service struct { | |||||
| UserSpaceMeta *metacache.UserSpaceMeta | UserSpaceMeta *metacache.UserSpaceMeta | ||||
| DB *db.DB | DB *db.DB | ||||
| EvtPub *sysevent.Publisher | EvtPub *sysevent.Publisher | ||||
| EvtWatcher *sysevent.WatcherHost // 为nil表示没有配置Watcher | |||||
| Mount *mount.Mount | Mount *mount.Mount | ||||
| StgPool *pool.Pool | StgPool *pool.Pool | ||||
| SpaceSyncer *spacesyncer.SpaceSyncer | SpaceSyncer *spacesyncer.SpaceSyncer | ||||
| @@ -44,6 +45,7 @@ func NewService( | |||||
| userSpaceMeta *metacache.UserSpaceMeta, | userSpaceMeta *metacache.UserSpaceMeta, | ||||
| db *db.DB, | db *db.DB, | ||||
| evtPub *sysevent.Publisher, | evtPub *sysevent.Publisher, | ||||
| evtWatcher *sysevent.WatcherHost, | |||||
| mount *mount.Mount, | mount *mount.Mount, | ||||
| stgPool *pool.Pool, | stgPool *pool.Pool, | ||||
| spaceSyncer *spacesyncer.SpaceSyncer, | spaceSyncer *spacesyncer.SpaceSyncer, | ||||
| @@ -60,6 +62,7 @@ func NewService( | |||||
| UserSpaceMeta: userSpaceMeta, | UserSpaceMeta: userSpaceMeta, | ||||
| DB: db, | DB: db, | ||||
| EvtPub: evtPub, | EvtPub: evtPub, | ||||
| EvtWatcher: evtWatcher, | |||||
| Mount: mount, | Mount: mount, | ||||
| StgPool: stgPool, | StgPool: stgPool, | ||||
| SpaceSyncer: spaceSyncer, | SpaceSyncer: spaceSyncer, | ||||
| @@ -298,7 +298,7 @@ func (svc *UserSpaceService) DownloadPackage(req cliapi.UserSpaceDownloadPackage | |||||
| for _, v := range ret.GetArray(ops2.BaseReadStatsStoreKey) { | for _, v := range ret.GetArray(ops2.BaseReadStatsStoreKey) { | ||||
| v2 := v.(*ops2.BaseReadStatsValue) | v2 := v.(*ops2.BaseReadStatsValue) | ||||
| svc.SpeedStats.Record(v2.Size, v2.Time, v2.Location.IsDriver) | |||||
| svc.SpeedStats.Record(v2.Length, v2.ElapsedTime, v2.Location.IsDriver) | |||||
| } | } | ||||
| err = svc.DB.DoTx(func(tx db.SQLContext) error { | err = svc.DB.DoTx(func(tx db.SQLContext) error { | ||||
| @@ -55,27 +55,27 @@ func New() *SpeedStats { | |||||
| } | } | ||||
| } | } | ||||
| func (p *SpeedStats) Record(size int64, time time.Duration, happenedAtClient bool) { | |||||
| func (p *SpeedStats) Record(size int64, elapsedTime time.Duration, happenedAtClient bool) { | |||||
| p.lock.Lock() | p.lock.Lock() | ||||
| defer p.lock.Unlock() | defer p.lock.Unlock() | ||||
| if size < 100*1024*1024 { | if size < 100*1024*1024 { | ||||
| if happenedAtClient { | if happenedAtClient { | ||||
| p.stats100M[0].Record(size, time) | |||||
| p.stats100M[0].Record(size, elapsedTime) | |||||
| } else { | } else { | ||||
| p.stats100M[1].Record(size, time) | |||||
| p.stats100M[1].Record(size, elapsedTime) | |||||
| } | } | ||||
| } else if size < 1024*1024*1024 { | } else if size < 1024*1024*1024 { | ||||
| if happenedAtClient { | if happenedAtClient { | ||||
| p.stats1G[0].Record(size, time) | |||||
| p.stats1G[0].Record(size, elapsedTime) | |||||
| } else { | } else { | ||||
| p.stats1G[1].Record(size, time) | |||||
| p.stats1G[1].Record(size, elapsedTime) | |||||
| } | } | ||||
| } else { | } else { | ||||
| if happenedAtClient { | if happenedAtClient { | ||||
| p.statsAbove1G[0].Record(size, time) | |||||
| p.statsAbove1G[0].Record(size, elapsedTime) | |||||
| } else { | } else { | ||||
| p.statsAbove1G[1].Record(size, time) | |||||
| p.statsAbove1G[1].Record(size, elapsedTime) | |||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| @@ -0,0 +1,62 @@ | |||||
| package ticktock | |||||
| import ( | |||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | |||||
| "gitlink.org.cn/cloudream/common/utils/reflect2" | |||||
| stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" | |||||
| jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types" | |||||
| "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap" | |||||
| ) | |||||
| type StatsLog struct { | |||||
| } | |||||
| func (j *StatsLog) Name() string { | |||||
| return reflect2.TypeNameOf[StatsLog]() | |||||
| } | |||||
| func (j *StatsLog) Execute(t *TickTock) { | |||||
| j.logGlobalObjectStats(t) | |||||
| } | |||||
| func (j *StatsLog) logGlobalObjectStats(t *TickTock) { | |||||
| log := logger.WithType[StatsLog]("Event") | |||||
| cnt, size, err := t.db.Object().Summary(t.db.DefCtx()) | |||||
| if err != nil { | |||||
| log.Warnf("get object summary: %v", err) | |||||
| return | |||||
| } | |||||
| usIDs, err := t.db.UserSpace().GetAllIDs(t.db.DefCtx()) | |||||
| if err != nil { | |||||
| log.Warnf("get user space ids: %v", err) | |||||
| return | |||||
| } | |||||
| var logUsIDs []jcstypes.UserSpaceID | |||||
| var usStats []stgtypes.ShardStoreStats | |||||
| details := t.spaceMeta.GetMany(usIDs) | |||||
| for _, d := range details { | |||||
| if d.UserSpace.ShardStore == nil { | |||||
| continue | |||||
| } | |||||
| store, err := t.stgPool.GetShardStore(d) | |||||
| if err != nil { | |||||
| log.Warnf("get shard store %v: %v", d, err) | |||||
| continue | |||||
| } | |||||
| logUsIDs = append(logUsIDs, d.UserSpace.UserSpaceID) | |||||
| usStats = append(usStats, store.Stats()) | |||||
| } | |||||
| t.evtPub.Publish(&datamap.BodyGlobalObjectStats{ | |||||
| TotalObjectCount: cnt, | |||||
| TotalDataSize: size, | |||||
| UserSpaceIDs: logUsIDs, | |||||
| UserSpaceStats: usStats, | |||||
| }) | |||||
| } | |||||
| @@ -114,4 +114,8 @@ func (t *TickTock) initJobs() { | |||||
| gocron.NewAtTime(12, 0, 0), | gocron.NewAtTime(12, 0, 0), | ||||
| gocron.NewAtTime(18, 0, 0), | gocron.NewAtTime(18, 0, 0), | ||||
| ))) | ))) | ||||
| t.addJob(&StatsLog{}, gocron.DailyJob(1, gocron.NewAtTimes( | |||||
| gocron.NewAtTime(0, 0, 0), | |||||
| ))) | |||||
| } | } | ||||
| @@ -0,0 +1,55 @@ | |||||
| package api | |||||
| import ( | |||||
| "fmt" | |||||
| "io" | |||||
| "net/http" | |||||
| "strings" | |||||
| "gitlink.org.cn/cloudream/common/sdks" | |||||
| "gitlink.org.cn/cloudream/common/utils/http2" | |||||
| "gitlink.org.cn/cloudream/common/utils/serder" | |||||
| ) | |||||
| type SysEventService struct { | |||||
| *Client | |||||
| } | |||||
| func (c *Client) SysEvent() *SysEventService { | |||||
| return &SysEventService{ | |||||
| Client: c, | |||||
| } | |||||
| } | |||||
| const SysEventWatchPath = "/sysEvent/watch" | |||||
| type SysEventWatch struct{} | |||||
| func (r *SysEventWatch) MakeParam() *sdks.RequestParam { | |||||
| return sdks.MakeJSONParam(http.MethodPost, SysEventWatchPath, r) | |||||
| } | |||||
| type SysEventWatchResp struct { | |||||
| Stream io.ReadCloser | |||||
| } | |||||
| func (r *SysEventWatchResp) ParseResponse(resp *http.Response) error { | |||||
| contType := resp.Header.Get("Content-Type") | |||||
| if strings.Contains(contType, http2.ContentTypeJSON) { | |||||
| var err error | |||||
| var r sdks.CodeDataResponse[any] | |||||
| if err = serder.JSONToObjectStreamExRaw(resp.Body, &r); err != nil { | |||||
| return fmt.Errorf("parsing response: %w", err) | |||||
| } | |||||
| return &sdks.CodeMessageError{Code: r.Code, Message: r.Message} | |||||
| } | |||||
| r.Stream = resp.Body | |||||
| return nil | |||||
| } | |||||
| func (c *SysEventService) Watch(req SysEventWatch) (*SysEventWatchResp, error) { | |||||
| return JSONAPI(&c.cfg, c.httpCli, &req, &SysEventWatchResp{}) | |||||
| } | |||||
| @@ -63,6 +63,9 @@ func (s *Executor) Run(ctx *ExecContext) (ExecutorResult, error) { | |||||
| return ExecutorResult{}, err | return ExecutorResult{}, err | ||||
| } | } | ||||
| s.lock.Lock() | |||||
| defer s.lock.Unlock() | |||||
| return ExecutorResult{Stored: s.store}, nil | return ExecutorResult{Stored: s.store}, nil | ||||
| } | } | ||||
| @@ -86,6 +86,8 @@ func (s *Worker) FindByIDContexted(ctx context.Context, id PlanID) *Executor { | |||||
| } | } | ||||
| type WorkerInfo interface { | type WorkerInfo interface { | ||||
| // Worker名称 | |||||
| Name() string | |||||
| NewClient() (WorkerClient, error) | NewClient() (WorkerClient, error) | ||||
| // 判断两个worker是否相同 | // 判断两个worker是否相同 | ||||
| Equals(worker WorkerInfo) bool | Equals(worker WorkerInfo) bool | ||||
| @@ -3,6 +3,7 @@ package ops | |||||
| import ( | import ( | ||||
| "fmt" | "fmt" | ||||
| "io" | "io" | ||||
| "time" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/future" | "gitlink.org.cn/cloudream/common/pkgs/future" | ||||
| "gitlink.org.cn/cloudream/common/utils/io2" | "gitlink.org.cn/cloudream/common/utils/io2" | ||||
| @@ -10,6 +11,10 @@ import ( | |||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" | "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" | ||||
| ) | ) | ||||
| const ( | |||||
| SendStreamStatsStoreKey = "Stats.SendStream" | |||||
| ) | |||||
| func init() { | func init() { | ||||
| exec.UseOp[*SendStream]() | exec.UseOp[*SendStream]() | ||||
| exec.UseOp[*GetStream]() | exec.UseOp[*GetStream]() | ||||
| @@ -17,10 +22,23 @@ func init() { | |||||
| exec.UseOp[*GetVar]() | exec.UseOp[*GetVar]() | ||||
| } | } | ||||
| type SendStreamStatsValue struct { | |||||
| IsSend bool | |||||
| Length int64 | |||||
| Time time.Duration | |||||
| Src exec.Location | |||||
| Dst exec.Location | |||||
| } | |||||
| func (v *SendStreamStatsValue) Clone() exec.VarValue { | |||||
| v2 := *v | |||||
| return &v2 | |||||
| } | |||||
| type SendStream struct { | type SendStream struct { | ||||
| Input exec.VarID `json:"input"` | |||||
| Send exec.VarID `json:"send"` | |||||
| Worker exec.WorkerInfo `json:"worker"` | |||||
| Input exec.VarID | |||||
| Send exec.VarID | |||||
| Worker exec.WorkerInfo | |||||
| } | } | ||||
| func (o *SendStream) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | func (o *SendStream) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | ||||
| @@ -36,12 +54,27 @@ func (o *SendStream) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||||
| } | } | ||||
| defer cli.Close() | defer cli.Close() | ||||
| counter := io2.CounterCloser(inputStr.Stream, nil) | |||||
| startTime := time.Now() | |||||
| // 发送后流的ID不同 | // 发送后流的ID不同 | ||||
| err = cli.SendStream(ctx.Context, e.Plan().ID, o.Send, inputStr.Stream) | |||||
| err = cli.SendStream(ctx.Context, e.Plan().ID, o.Send, counter) | |||||
| if err != nil { | if err != nil { | ||||
| return fmt.Errorf("sending stream: %w", err) | return fmt.Errorf("sending stream: %w", err) | ||||
| } | } | ||||
| e.Store(SendStreamStatsStoreKey, &SendStreamStatsValue{ | |||||
| IsSend: true, | |||||
| Length: counter.Count(), | |||||
| Time: time.Since(startTime), | |||||
| Src: e.Location(), | |||||
| Dst: exec.Location{ | |||||
| IsDriver: false, | |||||
| WorkerName: o.Worker.Name(), | |||||
| }, | |||||
| }) | |||||
| return nil | return nil | ||||
| } | } | ||||
| @@ -68,14 +101,29 @@ func (o *GetStream) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||||
| return fmt.Errorf("getting stream: %w", err) | return fmt.Errorf("getting stream: %w", err) | ||||
| } | } | ||||
| startTime := time.Now() | |||||
| counter := io2.CounterCloser(str, nil) | |||||
| fut := future.NewSetVoid() | fut := future.NewSetVoid() | ||||
| // 获取后送到本地的流ID是不同的 | // 获取后送到本地的流ID是不同的 | ||||
| str = io2.AfterReadClosedOnce(str, func(closer io.ReadCloser) { | |||||
| str = io2.AfterReadClosedOnce(counter, func(closer io.ReadCloser) { | |||||
| fut.SetVoid() | fut.SetVoid() | ||||
| }) | }) | ||||
| e.PutVar(o.Output, &exec.StreamValue{Stream: str}) | e.PutVar(o.Output, &exec.StreamValue{Stream: str}) | ||||
| return fut.Wait(ctx.Context) | |||||
| err = fut.Wait(ctx.Context) | |||||
| e.Store(SendStreamStatsStoreKey, &SendStreamStatsValue{ | |||||
| IsSend: false, | |||||
| Length: counter.Count(), | |||||
| Time: time.Since(startTime), | |||||
| Src: exec.Location{ | |||||
| IsDriver: false, | |||||
| WorkerName: o.Worker.Name(), | |||||
| }, | |||||
| Dst: e.Location(), | |||||
| }) | |||||
| return err | |||||
| } | } | ||||
| func (o *GetStream) String() string { | func (o *GetStream) String() string { | ||||
| @@ -17,6 +17,10 @@ type HttpHubWorker struct { | |||||
| Hub jcstypes.Hub | Hub jcstypes.Hub | ||||
| } | } | ||||
| func (w *HttpHubWorker) Name() string { | |||||
| return fmt.Sprintf("%v", w.Hub.HubID) | |||||
| } | |||||
| func (w *HttpHubWorker) NewClient() (exec.WorkerClient, error) { | func (w *HttpHubWorker) NewClient() (exec.WorkerClient, error) { | ||||
| addressInfo := w.Hub.Address.(*jcstypes.HttpAddressInfo) | addressInfo := w.Hub.Address.(*jcstypes.HttpAddressInfo) | ||||
| baseUrl := "http://" + addressInfo.ExternalIP + ":" + strconv.Itoa(addressInfo.Port) | baseUrl := "http://" + addressInfo.ExternalIP + ":" + strconv.Itoa(addressInfo.Port) | ||||
| @@ -30,7 +34,7 @@ func (w *HttpHubWorker) NewClient() (exec.WorkerClient, error) { | |||||
| return nil, err | return nil, err | ||||
| } | } | ||||
| return &HttpHubWorkerClient{hubID: w.Hub.HubID, cli: cli}, nil | |||||
| return &HttpHubWorkerClient{workerName: w.Name(), hubID: w.Hub.HubID, cli: cli}, nil | |||||
| } | } | ||||
| func (w *HttpHubWorker) String() string { | func (w *HttpHubWorker) String() string { | ||||
| @@ -47,14 +51,15 @@ func (w *HttpHubWorker) Equals(worker exec.WorkerInfo) bool { | |||||
| } | } | ||||
| type HttpHubWorkerClient struct { | type HttpHubWorkerClient struct { | ||||
| hubID jcstypes.HubID | |||||
| cli *hubapi.Client | |||||
| workerName string | |||||
| hubID jcstypes.HubID | |||||
| cli *hubapi.Client | |||||
| } | } | ||||
| func (c *HttpHubWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) (exec.ExecutorResult, error) { | func (c *HttpHubWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) (exec.ExecutorResult, error) { | ||||
| resp, err := c.cli.ExecuteIOPlan(hubapi.ExecuteIOPlanReq{ | resp, err := c.cli.ExecuteIOPlan(hubapi.ExecuteIOPlanReq{ | ||||
| Plan: plan, | Plan: plan, | ||||
| WorkerName: fmt.Sprintf("%v", c.hubID), | |||||
| WorkerName: c.workerName, | |||||
| }) | }) | ||||
| if err != nil { | if err != nil { | ||||
| return exec.ExecutorResult{}, err | return exec.ExecutorResult{}, err | ||||
| @@ -24,9 +24,13 @@ type HubWorker struct { | |||||
| Address jcstypes.GRPCAddressInfo | Address jcstypes.GRPCAddressInfo | ||||
| } | } | ||||
| func (w *HubWorker) Name() string { | |||||
| return fmt.Sprintf("%v", w.Hub.HubID) | |||||
| } | |||||
| func (w *HubWorker) NewClient() (exec.WorkerClient, error) { | func (w *HubWorker) NewClient() (exec.WorkerClient, error) { | ||||
| cli := stgglb.HubRPCPool.Get(stgglb.SelectGRPCAddress(&w.Hub, &w.Address)) | cli := stgglb.HubRPCPool.Get(stgglb.SelectGRPCAddress(&w.Hub, &w.Address)) | ||||
| return &HubWorkerClient{hubID: w.Hub.HubID, cli: cli}, nil | |||||
| return &HubWorkerClient{workerName: w.Name(), hubID: w.Hub.HubID, cli: cli}, nil | |||||
| } | } | ||||
| func (w *HubWorker) String() string { | func (w *HubWorker) String() string { | ||||
| @@ -43,12 +47,13 @@ func (w *HubWorker) Equals(worker exec.WorkerInfo) bool { | |||||
| } | } | ||||
| type HubWorkerClient struct { | type HubWorkerClient struct { | ||||
| hubID jcstypes.HubID | |||||
| cli *hubrpc.Client | |||||
| workerName string | |||||
| hubID jcstypes.HubID | |||||
| cli *hubrpc.Client | |||||
| } | } | ||||
| func (c *HubWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) (exec.ExecutorResult, error) { | func (c *HubWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) (exec.ExecutorResult, error) { | ||||
| resp, err := c.cli.ExecuteIOPlan(ctx, &hubrpc.ExecuteIOPlan{Plan: plan, WorkerName: fmt.Sprintf("%v", c.hubID)}) | |||||
| resp, err := c.cli.ExecuteIOPlan(ctx, &hubrpc.ExecuteIOPlan{Plan: plan, WorkerName: c.workerName}) | |||||
| if err != nil { | if err != nil { | ||||
| return exec.ExecutorResult{}, err.ToError() | return exec.ExecutorResult{}, err.ToError() | ||||
| } | } | ||||
| @@ -17,7 +17,7 @@ import ( | |||||
| ) | ) | ||||
| const ( | const ( | ||||
| BaseReadStatsStoreKey = "BaseReadSpeed" | |||||
| BaseReadStatsStoreKey = "Stats.BaseRead" | |||||
| ) | ) | ||||
| func init() { | func init() { | ||||
| @@ -28,16 +28,16 @@ func init() { | |||||
| } | } | ||||
| type BaseReadStatsValue struct { | type BaseReadStatsValue struct { | ||||
| Size int64 | |||||
| Time time.Duration | |||||
| Location exec.Location | |||||
| Length int64 | |||||
| ElapsedTime time.Duration | |||||
| Location exec.Location | |||||
| } | } | ||||
| func (v *BaseReadStatsValue) Clone() exec.VarValue { | func (v *BaseReadStatsValue) Clone() exec.VarValue { | ||||
| return &BaseReadStatsValue{ | return &BaseReadStatsValue{ | ||||
| Size: v.Size, | |||||
| Time: v.Time, | |||||
| Location: v.Location, | |||||
| Length: v.Length, | |||||
| ElapsedTime: v.ElapsedTime, | |||||
| Location: v.Location, | |||||
| } | } | ||||
| } | } | ||||
| @@ -72,18 +72,7 @@ func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||||
| } | } | ||||
| startTime := time.Now() | startTime := time.Now() | ||||
| counter := io2.CounterCloser(stream, func(cnt int64, err error) { | |||||
| if err != nil && err != io.EOF { | |||||
| return | |||||
| } | |||||
| // 要注意这个回调一定要在return之前调用 | |||||
| e.Store(BaseReadStatsStoreKey, &BaseReadStatsValue{ | |||||
| Size: cnt, | |||||
| Time: time.Since(startTime), | |||||
| Location: e.Location(), | |||||
| }) | |||||
| }) | |||||
| counter := io2.CounterCloser(stream, nil) | |||||
| fut := future.NewSetVoid() | fut := future.NewSetVoid() | ||||
| output := &exec.StreamValue{ | output := &exec.StreamValue{ | ||||
| @@ -93,7 +82,13 @@ func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||||
| } | } | ||||
| e.PutVar(o.Output, output) | e.PutVar(o.Output, output) | ||||
| return fut.Wait(ctx.Context) | |||||
| err = fut.Wait(ctx.Context) | |||||
| e.Store(BaseReadStatsStoreKey, &BaseReadStatsValue{ | |||||
| Length: counter.Count(), | |||||
| ElapsedTime: time.Since(startTime), | |||||
| Location: e.Location(), | |||||
| }) | |||||
| return err | |||||
| } | } | ||||
| func (o *BaseRead) String() string { | func (o *BaseRead) String() string { | ||||
| @@ -137,18 +132,7 @@ func (o *BaseReadDyn) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||||
| } | } | ||||
| startTime := time.Now() | startTime := time.Now() | ||||
| counter := io2.CounterCloser(stream, func(cnt int64, err error) { | |||||
| if err != nil && err != io.EOF { | |||||
| return | |||||
| } | |||||
| // 要注意这个回调一定要在return之前调用 | |||||
| e.Store(BaseReadStatsStoreKey, &BaseReadStatsValue{ | |||||
| Size: cnt, | |||||
| Time: time.Since(startTime), | |||||
| Location: e.Location(), | |||||
| }) | |||||
| }) | |||||
| counter := io2.CounterCloser(stream, nil) | |||||
| fut := future.NewSetVoid() | fut := future.NewSetVoid() | ||||
| output := &exec.StreamValue{ | output := &exec.StreamValue{ | ||||
| @@ -158,7 +142,13 @@ func (o *BaseReadDyn) Execute(ctx *exec.ExecContext, e *exec.Executor) error { | |||||
| } | } | ||||
| e.PutVar(o.Output, output) | e.PutVar(o.Output, output) | ||||
| return fut.Wait(ctx.Context) | |||||
| err = fut.Wait(ctx.Context) | |||||
| e.Store(BaseReadStatsStoreKey, &BaseReadStatsValue{ | |||||
| Length: counter.Count(), | |||||
| ElapsedTime: time.Since(startTime), | |||||
| Location: e.Location(), | |||||
| }) | |||||
| return err | |||||
| } | } | ||||
| func (o *BaseReadDyn) String() string { | func (o *BaseReadDyn) String() string { | ||||
| @@ -2,6 +2,7 @@ package ioswitchlrc | |||||
| import ( | import ( | ||||
| "context" | "context" | ||||
| "fmt" | |||||
| "io" | "io" | ||||
| stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" | stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" | ||||
| @@ -19,6 +20,10 @@ type HubWorker struct { | |||||
| Address jcstypes.GRPCAddressInfo | Address jcstypes.GRPCAddressInfo | ||||
| } | } | ||||
| func (w *HubWorker) Name() string { | |||||
| return fmt.Sprintf("%v", w.Hub.HubID) | |||||
| } | |||||
| func (w *HubWorker) NewClient() (exec.WorkerClient, error) { | func (w *HubWorker) NewClient() (exec.WorkerClient, error) { | ||||
| cli := stgglb.HubRPCPool.Get(stgglb.SelectGRPCAddress(&w.Hub, &w.Address)) | cli := stgglb.HubRPCPool.Get(stgglb.SelectGRPCAddress(&w.Hub, &w.Address)) | ||||
| return &HubWorkerClient{cli: cli}, nil | return &HubWorkerClient{cli: cli}, nil | ||||
| @@ -38,11 +43,12 @@ func (w *HubWorker) Equals(worker exec.WorkerInfo) bool { | |||||
| } | } | ||||
| type HubWorkerClient struct { | type HubWorkerClient struct { | ||||
| cli *hubrpc.Client | |||||
| workerName string | |||||
| cli *hubrpc.Client | |||||
| } | } | ||||
| func (c *HubWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) (exec.ExecutorResult, error) { | func (c *HubWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) (exec.ExecutorResult, error) { | ||||
| resp, err := c.cli.ExecuteIOPlan(ctx, &hubrpc.ExecuteIOPlan{Plan: plan}) | |||||
| resp, err := c.cli.ExecuteIOPlan(ctx, &hubrpc.ExecuteIOPlan{Plan: plan, WorkerName: c.workerName}) | |||||
| if err != nil { | if err != nil { | ||||
| return exec.ExecutorResult{}, err.ToError() | return exec.ExecutorResult{}, err.ToError() | ||||
| } | } | ||||
| @@ -112,7 +112,7 @@ type PubShardsStats struct { | |||||
| Password string | Password string | ||||
| } | } | ||||
| type PubShardsStatsResp struct { | type PubShardsStatsResp struct { | ||||
| Stats stgtypes.Stats | |||||
| Stats stgtypes.ShardStoreStats | |||||
| } | } | ||||
| var _ = TokenAuth(Hub_PubShardsStats_FullMethodName) | var _ = TokenAuth(Hub_PubShardsStats_FullMethodName) | ||||
| @@ -219,6 +219,7 @@ func DownloadStreamServer[Resp DownloadStreamResp, Req any, APIRet DownloadStrea | |||||
| if cerr != nil { | if cerr != nil { | ||||
| return WrapCodeError(cerr) | return WrapCodeError(cerr) | ||||
| } | } | ||||
| defer resp.GetStream().Close() | |||||
| cw := NewChunkedWriter(ret) | cw := NewChunkedWriter(ret) | ||||
| data, err := serder.ObjectToJSONEx(resp) | data, err := serder.ObjectToJSONEx(resp) | ||||
| @@ -18,22 +18,23 @@ type HubStorageTransferStats struct { | |||||
| type HubStorageTransferStatsData struct { | type HubStorageTransferStatsData struct { | ||||
| Entries map[jcstypes.StorageID]*HubStorageTransferStatsEntry | Entries map[jcstypes.StorageID]*HubStorageTransferStatsEntry | ||||
| StartTime time.Time | StartTime time.Time | ||||
| EndTime time.Time | |||||
| } | } | ||||
| type HubStorageTransferStatsEntry struct { | type HubStorageTransferStatsEntry struct { | ||||
| DestStorageID jcstypes.StorageID | DestStorageID jcstypes.StorageID | ||||
| OutputBytes int64 | |||||
| MaxOutputBytes int64 | |||||
| MinOutputBytes int64 | |||||
| TotalOutput int64 | |||||
| SuccessOutput int64 | |||||
| InputBytes int64 | |||||
| MaxInputBytes int64 | |||||
| MinInputBytes int64 | |||||
| TotalInput int64 | |||||
| SuccessInput int64 | |||||
| OutputBytes int64 | |||||
| MaxOutputBytes int64 | |||||
| MinOutputBytes int64 | |||||
| TotalOutputReqs int64 | |||||
| SuccessOutputReqs int64 | |||||
| InputBytes int64 | |||||
| MaxInputBytes int64 | |||||
| MinInputBytes int64 | |||||
| TotalInputReqs int64 | |||||
| SuccessInputReqs int64 | |||||
| } | } | ||||
| func (s *HubStorageTransferStats) RecordUpload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) { | func (s *HubStorageTransferStats) RecordUpload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) { | ||||
| @@ -53,9 +54,9 @@ func (s *HubStorageTransferStats) RecordUpload(dstStorageID jcstypes.StorageID, | |||||
| e.MaxOutputBytes = math2.Max(e.MaxOutputBytes, transferBytes) | e.MaxOutputBytes = math2.Max(e.MaxOutputBytes, transferBytes) | ||||
| e.MinOutputBytes = math2.Min(e.MinOutputBytes, transferBytes) | e.MinOutputBytes = math2.Min(e.MinOutputBytes, transferBytes) | ||||
| if isSuccess { | if isSuccess { | ||||
| e.SuccessOutput++ | |||||
| e.SuccessOutputReqs++ | |||||
| } | } | ||||
| e.TotalOutput++ | |||||
| e.TotalOutputReqs++ | |||||
| } | } | ||||
| func (s *HubStorageTransferStats) RecordDownload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) { | func (s *HubStorageTransferStats) RecordDownload(dstStorageID jcstypes.StorageID, transferBytes int64, isSuccess bool) { | ||||
| @@ -75,20 +76,11 @@ func (s *HubStorageTransferStats) RecordDownload(dstStorageID jcstypes.StorageID | |||||
| e.MaxInputBytes = math2.Max(e.MaxInputBytes, transferBytes) | e.MaxInputBytes = math2.Max(e.MaxInputBytes, transferBytes) | ||||
| e.MinInputBytes = math2.Min(e.MinInputBytes, transferBytes) | e.MinInputBytes = math2.Min(e.MinInputBytes, transferBytes) | ||||
| if isSuccess { | if isSuccess { | ||||
| e.SuccessInput++ | |||||
| e.SuccessInputReqs++ | |||||
| } | } | ||||
| } | } | ||||
| func (s *HubStorageTransferStats) Reset() time.Time { | |||||
| s.lock.Lock() | |||||
| defer s.lock.Unlock() | |||||
| s.data.Entries = make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry) | |||||
| s.data.StartTime = time.Now() | |||||
| return s.data.StartTime | |||||
| } | |||||
| func (s *HubStorageTransferStats) DumpData() HubStorageTransferStatsData { | |||||
| func (s *HubStorageTransferStats) DumpData(reset bool) HubStorageTransferStatsData { | |||||
| s.lock.Lock() | s.lock.Lock() | ||||
| defer s.lock.Unlock() | defer s.lock.Unlock() | ||||
| @@ -98,6 +90,12 @@ func (s *HubStorageTransferStats) DumpData() HubStorageTransferStatsData { | |||||
| v2 := *v | v2 := *v | ||||
| data.Entries[k] = &v2 | data.Entries[k] = &v2 | ||||
| } | } | ||||
| data.EndTime = time.Now() | |||||
| if reset { | |||||
| s.data.Entries = make(map[jcstypes.StorageID]*HubStorageTransferStatsEntry) | |||||
| s.data.StartTime = time.Now() | |||||
| } | |||||
| return data | return data | ||||
| } | } | ||||
| @@ -18,22 +18,23 @@ type HubTransferStats struct { | |||||
| type HubTransferStatsData struct { | type HubTransferStatsData struct { | ||||
| Entries map[jcstypes.HubID]*HubTransferStatsEntry | Entries map[jcstypes.HubID]*HubTransferStatsEntry | ||||
| StartTime time.Time | StartTime time.Time | ||||
| EndTime time.Time | |||||
| } | } | ||||
| type HubTransferStatsEntry struct { | type HubTransferStatsEntry struct { | ||||
| DestHubID jcstypes.HubID | DestHubID jcstypes.HubID | ||||
| OutputBytes int64 | |||||
| MaxOutputBytes int64 | |||||
| MinOutputBytes int64 | |||||
| TotalOutput int64 | |||||
| SuccessOutput int64 | |||||
| OutputBytes int64 | |||||
| MaxOutputBytes int64 | |||||
| MinOutputBytes int64 | |||||
| OutputReqs int64 // 发送数据的请求数量 | |||||
| SuccessOutputReqs int64 // 发送成功的请求数量 | |||||
| InputBytes int64 | |||||
| MaxInputBytes int64 | |||||
| MinInputBytes int64 | |||||
| TotalInput int64 | |||||
| SuccessInput int64 | |||||
| InputBytes int64 | |||||
| MaxInputBytes int64 | |||||
| MinInputBytes int64 | |||||
| TotalInputReqs int64 // 下载数据的请求数量 | |||||
| SuccessInputReqs int64 // 下载成功的请求数量 | |||||
| } | } | ||||
| func (s *HubTransferStats) RecordOutput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) { | func (s *HubTransferStats) RecordOutput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) { | ||||
| @@ -53,9 +54,9 @@ func (s *HubTransferStats) RecordOutput(dstHubID jcstypes.HubID, transferBytes i | |||||
| e.MaxOutputBytes = math2.Max(e.MaxOutputBytes, transferBytes) | e.MaxOutputBytes = math2.Max(e.MaxOutputBytes, transferBytes) | ||||
| e.MinOutputBytes = math2.Min(e.MinOutputBytes, transferBytes) | e.MinOutputBytes = math2.Min(e.MinOutputBytes, transferBytes) | ||||
| if isSuccess { | if isSuccess { | ||||
| e.SuccessOutput++ | |||||
| e.SuccessOutputReqs++ | |||||
| } | } | ||||
| e.TotalOutput++ | |||||
| e.OutputReqs++ | |||||
| } | } | ||||
| func (s *HubTransferStats) RecordInput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) { | func (s *HubTransferStats) RecordInput(dstHubID jcstypes.HubID, transferBytes int64, isSuccess bool) { | ||||
| @@ -75,21 +76,12 @@ func (s *HubTransferStats) RecordInput(dstHubID jcstypes.HubID, transferBytes in | |||||
| e.MaxInputBytes = math2.Max(e.MaxInputBytes, transferBytes) | e.MaxInputBytes = math2.Max(e.MaxInputBytes, transferBytes) | ||||
| e.MinInputBytes = math2.Min(e.MinInputBytes, transferBytes) | e.MinInputBytes = math2.Min(e.MinInputBytes, transferBytes) | ||||
| if isSuccess { | if isSuccess { | ||||
| e.SuccessInput++ | |||||
| e.SuccessInputReqs++ | |||||
| } | } | ||||
| e.TotalInput++ | |||||
| e.TotalInputReqs++ | |||||
| } | } | ||||
| func (s *HubTransferStats) Reset() time.Time { | |||||
| s.lock.Lock() | |||||
| defer s.lock.Unlock() | |||||
| s.data.StartTime = time.Now() | |||||
| s.data.Entries = make(map[jcstypes.HubID]*HubTransferStatsEntry) | |||||
| return s.data.StartTime | |||||
| } | |||||
| func (s *HubTransferStats) DumpData() HubTransferStatsData { | |||||
| func (s *HubTransferStats) DumpData(reset bool) HubTransferStatsData { | |||||
| s.lock.Lock() | s.lock.Lock() | ||||
| defer s.lock.Unlock() | defer s.lock.Unlock() | ||||
| @@ -99,5 +91,12 @@ func (s *HubTransferStats) DumpData() HubTransferStatsData { | |||||
| v2 := *v | v2 := *v | ||||
| data.Entries[k] = &v2 | data.Entries[k] = &v2 | ||||
| } | } | ||||
| data.EndTime = time.Now() | |||||
| if reset { | |||||
| s.data.StartTime = time.Now() | |||||
| s.data.Entries = make(map[jcstypes.HubID]*HubTransferStatsEntry) | |||||
| } | |||||
| return data | return data | ||||
| } | } | ||||
| @@ -207,11 +207,11 @@ func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error { | |||||
| return nil | return nil | ||||
| } | } | ||||
| func (s *ShardStore) Stats() stgtypes.Stats { | |||||
| func (s *ShardStore) Stats() stgtypes.ShardStoreStats { | |||||
| s.lock.Lock() | s.lock.Lock() | ||||
| defer s.lock.Unlock() | defer s.lock.Unlock() | ||||
| return stgtypes.Stats{ | |||||
| return stgtypes.ShardStoreStats{ | |||||
| Status: stgtypes.StatusOK, | Status: stgtypes.StatusOK, | ||||
| FileCount: s.totalShardsCnt, | FileCount: s.totalShardsCnt, | ||||
| TotalSize: s.detail.UserSpace.ShardStore.MaxSize, | TotalSize: s.detail.UserSpace.ShardStore.MaxSize, | ||||
| @@ -88,13 +88,13 @@ func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error { | |||||
| return nil | return nil | ||||
| } | } | ||||
| func (s *ShardStore) Stats() stgtypes.Stats { | |||||
| func (s *ShardStore) Stats() stgtypes.ShardStoreStats { | |||||
| resp, cerr := s.hubCli.PubShardsStats(context.Background(), &hubrpc.PubShardsStats{ | resp, cerr := s.hubCli.PubShardsStats(context.Background(), &hubrpc.PubShardsStats{ | ||||
| PubShardsID: s.stgType.PubShardsID, | PubShardsID: s.stgType.PubShardsID, | ||||
| Password: s.stgType.Password, | Password: s.stgType.Password, | ||||
| }) | }) | ||||
| if cerr != nil { | if cerr != nil { | ||||
| return stgtypes.Stats{} | |||||
| return stgtypes.ShardStoreStats{} | |||||
| } | } | ||||
| return resp.Stats | return resp.Stats | ||||
| } | } | ||||
| @@ -195,11 +195,11 @@ func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error { | |||||
| return nil | return nil | ||||
| } | } | ||||
| func (s *ShardStore) Stats() stgtypes.Stats { | |||||
| func (s *ShardStore) Stats() stgtypes.ShardStoreStats { | |||||
| s.lock.Lock() | s.lock.Lock() | ||||
| defer s.lock.Unlock() | defer s.lock.Unlock() | ||||
| return stgtypes.Stats{ | |||||
| return stgtypes.ShardStoreStats{ | |||||
| Status: stgtypes.StatusOK, | Status: stgtypes.StatusOK, | ||||
| FileCount: s.totalShardsCnt, | FileCount: s.totalShardsCnt, | ||||
| TotalSize: s.Detail.UserSpace.ShardStore.MaxSize, | TotalSize: s.Detail.UserSpace.ShardStore.MaxSize, | ||||
| @@ -244,11 +244,11 @@ func (s *ShardStore) GC(avaiables []jcstypes.FileHash) error { | |||||
| return nil | return nil | ||||
| } | } | ||||
| func (s *ShardStore) Stats() stgtypes.Stats { | |||||
| func (s *ShardStore) Stats() stgtypes.ShardStoreStats { | |||||
| s.lock.Lock() | s.lock.Lock() | ||||
| defer s.lock.Unlock() | defer s.lock.Unlock() | ||||
| return stgtypes.Stats{ | |||||
| return stgtypes.ShardStoreStats{ | |||||
| Status: stgtypes.StatusOK, | Status: stgtypes.StatusOK, | ||||
| FileCount: s.totalShardsCnt, | FileCount: s.totalShardsCnt, | ||||
| TotalSize: s.Detail.UserSpace.ShardStore.MaxSize, | TotalSize: s.Detail.UserSpace.ShardStore.MaxSize, | ||||
| @@ -18,30 +18,22 @@ type ShardStore interface { | |||||
| // 垃圾清理。只保留availables中的文件,删除其他文件 | // 垃圾清理。只保留availables中的文件,删除其他文件 | ||||
| GC(avaiables []jcstypes.FileHash) error | GC(avaiables []jcstypes.FileHash) error | ||||
| // 获得存储系统信息 | // 获得存储系统信息 | ||||
| Stats() Stats | |||||
| Stats() ShardStoreStats | |||||
| } | } | ||||
| type Stats struct { | |||||
| type ShardStoreStats struct { | |||||
| // 存储服务状态,如果状态正常,此值应该是StatusOK | // 存储服务状态,如果状态正常,此值应该是StatusOK | ||||
| Status Status | |||||
| Status ShardStoreStatus `json:"status"` | |||||
| // 文件总数 | // 文件总数 | ||||
| FileCount int64 | |||||
| FileCount int64 `json:"fileCount"` | |||||
| // 存储空间总大小 | // 存储空间总大小 | ||||
| TotalSize int64 | |||||
| TotalSize int64 `json:"totalSize"` | |||||
| // 已使用的存储空间大小,可以超过存储空间总大小 | // 已使用的存储空间大小,可以超过存储空间总大小 | ||||
| UsedSize int64 | |||||
| UsedSize int64 `json:"usedSize"` | |||||
| // 描述信息,用于调试 | // 描述信息,用于调试 | ||||
| Description string | |||||
| Description string `json:"description"` | |||||
| } | } | ||||
| type Status interface { | |||||
| String() string | |||||
| } | |||||
| type OKStatus struct{} | |||||
| func (s *OKStatus) String() string { | |||||
| return "OK" | |||||
| } | |||||
| type ShardStoreStatus string | |||||
| var StatusOK = &OKStatus{} | |||||
| var StatusOK ShardStoreStatus = "OK" | |||||
| @@ -70,6 +70,7 @@ func NewPublisher(cfg Config, thisSource Source) (*Publisher, error) { | |||||
| } | } | ||||
| pub := &Publisher{ | pub := &Publisher{ | ||||
| cfg: cfg, | |||||
| connection: connection, | connection: connection, | ||||
| channel: channel, | channel: channel, | ||||
| eventChan: async.NewUnboundChannel[SysEvent](), | eventChan: async.NewUnboundChannel[SysEvent](), | ||||
| @@ -21,6 +21,7 @@ type WatcherExited struct { | |||||
| } | } | ||||
| type WatcherHost struct { | type WatcherHost struct { | ||||
| cfg Config | |||||
| watchers []Watcher | watchers []Watcher | ||||
| lock sync.Mutex | lock sync.Mutex | ||||
| connection *amqp.Connection | connection *amqp.Connection | ||||
| @@ -29,6 +30,12 @@ type WatcherHost struct { | |||||
| } | } | ||||
| func NewWatcherHost(cfg Config) (*WatcherHost, error) { | func NewWatcherHost(cfg Config) (*WatcherHost, error) { | ||||
| if !cfg.Enabled { | |||||
| return &WatcherHost{ | |||||
| cfg: cfg, | |||||
| }, nil | |||||
| } | |||||
| config := amqp.Config{ | config := amqp.Config{ | ||||
| Vhost: cfg.VHost, | Vhost: cfg.VHost, | ||||
| } | } | ||||
| @@ -80,6 +87,7 @@ func NewWatcherHost(cfg Config) (*WatcherHost, error) { | |||||
| } | } | ||||
| wat := &WatcherHost{ | wat := &WatcherHost{ | ||||
| cfg: cfg, | |||||
| connection: connection, | connection: connection, | ||||
| channel: channel, | channel: channel, | ||||
| recvChan: recvChan, | recvChan: recvChan, | ||||
| @@ -92,6 +100,10 @@ func (w *WatcherHost) Start() *async.UnboundChannel[WatcherEvent] { | |||||
| ch := async.NewUnboundChannel[WatcherEvent]() | ch := async.NewUnboundChannel[WatcherEvent]() | ||||
| go func() { | go func() { | ||||
| if !w.cfg.Enabled { | |||||
| return | |||||
| } | |||||
| defer ch.Close() | defer ch.Close() | ||||
| defer w.channel.Close() | defer w.channel.Close() | ||||
| defer w.connection.Close() | defer w.connection.Close() | ||||
| @@ -119,6 +131,19 @@ func (w *WatcherHost) Start() *async.UnboundChannel[WatcherEvent] { | |||||
| return ch | return ch | ||||
| } | } | ||||
| func (w *WatcherHost) Stop() { | |||||
| if !w.cfg.Enabled { | |||||
| return | |||||
| } | |||||
| w.channel.Close() | |||||
| w.connection.Close() | |||||
| } | |||||
| func (w *WatcherHost) Enabled() bool { | |||||
| return w.cfg.Enabled | |||||
| } | |||||
| func (w *WatcherHost) AddWatcher(watcher Watcher) { | func (w *WatcherHost) AddWatcher(watcher Watcher) { | ||||
| w.lock.Lock() | w.lock.Lock() | ||||
| defer w.lock.Unlock() | defer w.lock.Unlock() | ||||
| @@ -6,6 +6,7 @@ import ( | |||||
| "gitlink.org.cn/cloudream/common/pkgs/types" | "gitlink.org.cn/cloudream/common/pkgs/types" | ||||
| "gitlink.org.cn/cloudream/common/utils/serder" | "gitlink.org.cn/cloudream/common/utils/serder" | ||||
| stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" | |||||
| jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types" | jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types" | ||||
| ) | ) | ||||
| @@ -98,10 +99,12 @@ var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[SysEven | |||||
| (*BodyNewUserSpace)(nil), | (*BodyNewUserSpace)(nil), | ||||
| (*BodyUserSpaceUpdated)(nil), | (*BodyUserSpaceUpdated)(nil), | ||||
| (*BodyUserSpaceDeleted)(nil), | (*BodyUserSpaceDeleted)(nil), | ||||
| (*BodyGlobalObjectStats)(nil), | |||||
| // (*BodyStorageStats)(nil), | // (*BodyStorageStats)(nil), | ||||
| // (*BodyHubTransferStats)(nil), | |||||
| // (*BodyHubStorageTransferStats)(nil), | |||||
| (*BodyHubTransferStats)(nil), | |||||
| (*BodyHubStorageTransferStats)(nil), | |||||
| (*BodyObjectAccessStats)(nil), | |||||
| (*BodyBlockTransfer)(nil), | (*BodyBlockTransfer)(nil), | ||||
| (*BodyBlockDistribution)(nil), | (*BodyBlockDistribution)(nil), | ||||
| @@ -225,6 +228,25 @@ func (b *BodyStorageStats) GetBodyType() string { | |||||
| func (b *BodyStorageStats) OnUnionSerializing() { | func (b *BodyStorageStats) OnUnionSerializing() { | ||||
| b.Type = b.GetBodyType() | b.Type = b.GetBodyType() | ||||
| } | } | ||||
| */ | |||||
| // 全局总数据量的统计信息 | |||||
| type BodyGlobalObjectStats struct { | |||||
| serder.Metadata `union:"GlobalObjectStats"` | |||||
| Type string `json:"type"` | |||||
| TotalObjectCount int64 `json:"totalObjectCount"` | |||||
| TotalDataSize int64 `json:"totalDataSize"` | |||||
| UserSpaceIDs []jcstypes.UserSpaceID `json:"userSpaceIDs"` | |||||
| UserSpaceStats []stgtypes.ShardStoreStats `json:"userSpaceStats"` | |||||
| } | |||||
| func (b *BodyGlobalObjectStats) GetBodyType() string { | |||||
| return "GlobalObjectStats" | |||||
| } | |||||
| func (b *BodyGlobalObjectStats) OnUnionSerializing() { | |||||
| b.Type = b.GetBodyType() | |||||
| } | |||||
| // Hub数据传输统计信息的事件 | // Hub数据传输统计信息的事件 | ||||
| type BodyHubTransferStats struct { | type BodyHubTransferStats struct { | ||||
| @@ -232,7 +254,7 @@ type BodyHubTransferStats struct { | |||||
| Type string `json:"type"` | Type string `json:"type"` | ||||
| SourceHubID jcstypes.HubID `json:"sourceHubID"` | SourceHubID jcstypes.HubID `json:"sourceHubID"` | ||||
| TargetHubID jcstypes.HubID `json:"targetHubID"` | TargetHubID jcstypes.HubID `json:"targetHubID"` | ||||
| Send DataTrans `json:"send"` | |||||
| Send TransferStats `json:"send"` | |||||
| StartTimestamp time.Time `json:"startTimestamp"` | StartTimestamp time.Time `json:"startTimestamp"` | ||||
| EndTimestamp time.Time `json:"endTimestamp"` | EndTimestamp time.Time `json:"endTimestamp"` | ||||
| } | } | ||||
| @@ -245,13 +267,13 @@ func (b *BodyHubTransferStats) OnUnionSerializing() { | |||||
| b.Type = b.GetBodyType() | b.Type = b.GetBodyType() | ||||
| } | } | ||||
| type DataTrans struct { | |||||
| TotalTransfer int64 `json:"totalTransfer"` | |||||
| type TransferStats struct { | |||||
| Amount int64 `json:"amount"` | |||||
| RequestCount int64 `json:"requestCount"` | RequestCount int64 `json:"requestCount"` | ||||
| FailedRequestCount int64 `json:"failedRequestCount"` | FailedRequestCount int64 `json:"failedRequestCount"` | ||||
| AvgTransfer int64 `json:"avgTransfer"` | |||||
| MaxTransfer int64 `json:"maxTransfer"` | |||||
| MinTransfer int64 `json:"minTransfer"` | |||||
| RequestAverage int64 `json:"requestAverage"` | |||||
| RequestMax int64 `json:"requestMax"` | |||||
| RequestMin int64 `json:"requestMin"` | |||||
| } | } | ||||
| // Hub和Storage数据传输统计信息的事件 | // Hub和Storage数据传输统计信息的事件 | ||||
| @@ -260,8 +282,8 @@ type BodyHubStorageTransferStats struct { | |||||
| Type string `json:"type"` | Type string `json:"type"` | ||||
| HubID jcstypes.HubID `json:"hubID"` | HubID jcstypes.HubID `json:"hubID"` | ||||
| StorageID jcstypes.StorageID `json:"storageID"` | StorageID jcstypes.StorageID `json:"storageID"` | ||||
| Send DataTrans `json:"send"` | |||||
| Receive DataTrans `json:"receive"` | |||||
| Send TransferStats `json:"send"` | |||||
| Receive TransferStats `json:"receive"` | |||||
| StartTimestamp time.Time `json:"startTimestamp"` | StartTimestamp time.Time `json:"startTimestamp"` | ||||
| EndTimestamp time.Time `json:"endTimestamp"` | EndTimestamp time.Time `json:"endTimestamp"` | ||||
| } | } | ||||
| @@ -273,7 +295,24 @@ func (b *BodyHubStorageTransferStats) GetBodyType() string { | |||||
| func (b *BodyHubStorageTransferStats) OnUnionSerializing() { | func (b *BodyHubStorageTransferStats) OnUnionSerializing() { | ||||
| b.Type = b.GetBodyType() | b.Type = b.GetBodyType() | ||||
| } | } | ||||
| */ | |||||
| // 对象读取或调度时产生的传输事件 | |||||
| type BodyObjectAccessStats struct { | |||||
| serder.Metadata `union:"ObjectAccessStats"` | |||||
| Type string `json:"type"` | |||||
| ObjectID jcstypes.ObjectID `json:"objectID"` | |||||
| RequestSize int64 `json:"requestSize"` | |||||
| TransferAmount int64 `json:"transferAmount"` | |||||
| ElapsedTime time.Duration `json:"elapsedTime"` | |||||
| } | |||||
| func (b *BodyObjectAccessStats) GetBodyType() string { | |||||
| return "ObjectAccessStats" | |||||
| } | |||||
| func (b *BodyObjectAccessStats) OnUnionSerializing() { | |||||
| b.Type = b.GetBodyType() | |||||
| } | |||||
| // 块传输的事件 | // 块传输的事件 | ||||
| type BodyBlockTransfer struct { | type BodyBlockTransfer struct { | ||||
| @@ -81,6 +81,7 @@ require ( | |||||
| github.com/robfig/cron/v3 v3.0.1 // indirect | github.com/robfig/cron/v3 v3.0.1 // indirect | ||||
| github.com/rogpeppe/go-internal v1.12.0 // indirect | github.com/rogpeppe/go-internal v1.12.0 // indirect | ||||
| github.com/tjfoc/gmsm v1.4.1 // indirect | github.com/tjfoc/gmsm v1.4.1 // indirect | ||||
| github.com/tmaxmax/go-sse v0.11.0 // indirect | |||||
| github.com/twitchyliquid64/golang-asm v0.15.1 // indirect | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect | ||||
| go.mongodb.org/mongo-driver v1.12.0 // indirect | go.mongodb.org/mongo-driver v1.12.0 // indirect | ||||
| golang.org/x/arch v0.8.0 // indirect | golang.org/x/arch v0.8.0 // indirect | ||||
| @@ -329,6 +329,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf | |||||
| github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= | ||||
| github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= | github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= | ||||
| github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= | github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= | ||||
| github.com/tmaxmax/go-sse v0.11.0 h1:nogmJM6rJUoOLoAwEKeQe5XlVpt9l7N82SS1jI7lWFg= | |||||
| github.com/tmaxmax/go-sse v0.11.0/go.mod h1:u/2kZQR1tyngo1lKaNCj1mJmhXGZWS1Zs5yiSOD+Eg8= | |||||
| github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= | github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= | ||||
| github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= | ||||
| github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= | ||||
| @@ -121,7 +121,7 @@ func serve(configPath string, opts serveOptions) { | |||||
| defer evtPub.Stop() | defer evtPub.Stop() | ||||
| // 初始化定时任务执行器 | // 初始化定时任务执行器 | ||||
| tktk := ticktock.New(config.Cfg().TickTock, config.Cfg().ID, stgPool) | |||||
| tktk := ticktock.New(config.Cfg().TickTock, config.Cfg().ID, stgPool, evtPub) | |||||
| tktk.Start() | tktk.Start() | ||||
| defer tktk.Stop() | defer tktk.Stop() | ||||
| @@ -90,16 +90,19 @@ func (s *LoadedStore) GC(userID jcstypes.UserID, fileHashes []jcstypes.FileHash) | |||||
| }) | }) | ||||
| } | } | ||||
| func (s *LoadedStore) GetUserStats(userID jcstypes.UserID) stgtypes.Stats { | |||||
| cnt := int64(0) | |||||
| size := int64(0) | |||||
| s.ClientFileHashDB.Table("Shard").Select("Count(Hash), Sum(Size)").Find(&cnt) | |||||
| func (s *LoadedStore) GetUserStats(userID jcstypes.UserID) stgtypes.ShardStoreStats { | |||||
| var stats struct { | |||||
| Cnt int64 `gorm:"column:Cnt; type:bigint;"` | |||||
| Size int64 `gorm:"column:Size; type:bigint;"` | |||||
| } | |||||
| s.ClientFileHashDB.Table("Shard").Select("Count(Hash) as Cnt, Sum(Size) as Size").Find(&stats) | |||||
| return stgtypes.Stats{ | |||||
| return stgtypes.ShardStoreStats{ | |||||
| Status: stgtypes.StatusOK, | Status: stgtypes.StatusOK, | ||||
| FileCount: cnt, | |||||
| FileCount: stats.Cnt, | |||||
| TotalSize: s.Config.ShardStore.MaxSize, | TotalSize: s.Config.ShardStore.MaxSize, | ||||
| UsedSize: size, | |||||
| UsedSize: stats.Size, | |||||
| } | } | ||||
| } | } | ||||
| @@ -0,0 +1,69 @@ | |||||
| package ticktock | |||||
| import ( | |||||
| "gitlink.org.cn/cloudream/common/utils/reflect2" | |||||
| stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" | |||||
| "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap" | |||||
| ) | |||||
| type StatsLog struct { | |||||
| } | |||||
| func (j *StatsLog) Name() string { | |||||
| return reflect2.TypeNameOf[StatsLog]() | |||||
| } | |||||
| func (j *StatsLog) Execute(t *TickTock) { | |||||
| j.logTransferStats(t) | |||||
| } | |||||
| func (j *StatsLog) logTransferStats(t *TickTock) { | |||||
| // log := logger.WithType[StatsLog]("Event") | |||||
| if stgglb.Stats.HubTransfer != nil { | |||||
| data := stgglb.Stats.HubTransfer.DumpData(true) | |||||
| for k, v := range data.Entries { | |||||
| t.evtPub.Publish(&datamap.BodyHubTransferStats{ | |||||
| SourceHubID: t.myHubID, | |||||
| TargetHubID: k, | |||||
| Send: datamap.TransferStats{ | |||||
| Amount: v.OutputBytes, | |||||
| RequestCount: v.OutputReqs, | |||||
| FailedRequestCount: v.OutputReqs - v.SuccessOutputReqs, | |||||
| RequestAverage: v.OutputBytes / v.OutputReqs, | |||||
| RequestMax: v.MaxOutputBytes, | |||||
| RequestMin: v.MinOutputBytes, | |||||
| }, | |||||
| StartTimestamp: data.StartTime, | |||||
| EndTimestamp: data.EndTime, | |||||
| }) | |||||
| } | |||||
| } | |||||
| if stgglb.Stats.HubStorageTransfer != nil { | |||||
| data := stgglb.Stats.HubStorageTransfer.DumpData(true) | |||||
| for k, v := range data.Entries { | |||||
| t.evtPub.Publish(&datamap.BodyHubStorageTransferStats{ | |||||
| HubID: t.myHubID, | |||||
| StorageID: k, | |||||
| Send: datamap.TransferStats{ | |||||
| Amount: v.OutputBytes, | |||||
| RequestCount: v.TotalOutputReqs, | |||||
| FailedRequestCount: v.TotalOutputReqs - v.SuccessOutputReqs, | |||||
| RequestAverage: v.OutputBytes / v.TotalOutputReqs, | |||||
| RequestMax: v.MaxOutputBytes, | |||||
| RequestMin: v.MinOutputBytes, | |||||
| }, | |||||
| Receive: datamap.TransferStats{ | |||||
| Amount: v.InputBytes, | |||||
| RequestCount: v.TotalInputReqs, | |||||
| FailedRequestCount: v.TotalInputReqs - v.SuccessInputReqs, | |||||
| RequestAverage: v.InputBytes / v.TotalInputReqs, | |||||
| RequestMax: v.MaxInputBytes, | |||||
| RequestMin: v.MinInputBytes, | |||||
| }, | |||||
| StartTimestamp: data.StartTime, | |||||
| EndTimestamp: data.EndTime, | |||||
| }) | |||||
| } | |||||
| } | |||||
| } | |||||
| @@ -6,6 +6,7 @@ import ( | |||||
| "github.com/go-co-op/gocron/v2" | "github.com/go-co-op/gocron/v2" | ||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | "gitlink.org.cn/cloudream/common/pkgs/logger" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" | "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" | ||||
| "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent" | |||||
| jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types" | jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types" | ||||
| ) | ) | ||||
| @@ -26,9 +27,10 @@ type TickTock struct { | |||||
| myHubID jcstypes.HubID | myHubID jcstypes.HubID | ||||
| stgPool *pool.Pool | stgPool *pool.Pool | ||||
| evtPub *sysevent.Publisher | |||||
| } | } | ||||
| func New(cfg Config, myHubID jcstypes.HubID, stgPool *pool.Pool) *TickTock { | |||||
| func New(cfg Config, myHubID jcstypes.HubID, stgPool *pool.Pool, evtPub *sysevent.Publisher) *TickTock { | |||||
| sch, _ := gocron.NewScheduler() | sch, _ := gocron.NewScheduler() | ||||
| t := &TickTock{ | t := &TickTock{ | ||||
| cfg: cfg, | cfg: cfg, | ||||
| @@ -37,6 +39,7 @@ func New(cfg Config, myHubID jcstypes.HubID, stgPool *pool.Pool) *TickTock { | |||||
| myHubID: myHubID, | myHubID: myHubID, | ||||
| stgPool: stgPool, | stgPool: stgPool, | ||||
| evtPub: evtPub, | |||||
| } | } | ||||
| t.initJobs() | t.initJobs() | ||||
| return t | return t | ||||
| @@ -74,4 +77,8 @@ func (t *TickTock) addJob(job Job, duration gocron.JobDefinition) { | |||||
| func (t *TickTock) initJobs() { | func (t *TickTock) initJobs() { | ||||
| t.addJob(&TestHubConnectivities{myHubID: t.myHubID}, gocron.DurationJob(t.cfg.TestHubConnectivitiesInterval)) | t.addJob(&TestHubConnectivities{myHubID: t.myHubID}, gocron.DurationJob(t.cfg.TestHubConnectivitiesInterval)) | ||||
| t.addJob(&StatsLog{}, gocron.DailyJob(1, gocron.NewAtTimes( | |||||
| gocron.NewAtTime(0, 0, 0), | |||||
| ))) | |||||
| } | } | ||||
| @@ -0,0 +1,15 @@ | |||||
| package sysevent | |||||
| import ( | |||||
| "github.com/spf13/cobra" | |||||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/admin" | |||||
| ) | |||||
| var SysEventCmd = &cobra.Command{ | |||||
| Use: "sysevent", | |||||
| Aliases: []string{"sevt"}, | |||||
| } | |||||
| func init() { | |||||
| admin.AdminCmd.AddCommand(SysEventCmd) | |||||
| } | |||||
| @@ -0,0 +1,45 @@ | |||||
| package sysevent | |||||
| import ( | |||||
| "bytes" | |||||
| "encoding/json" | |||||
| "fmt" | |||||
| "github.com/spf13/cobra" | |||||
| se "github.com/tmaxmax/go-sse" | |||||
| cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" | |||||
| "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd" | |||||
| ) | |||||
| func init() { | |||||
| var opt watchOpt | |||||
| cmd := cobra.Command{ | |||||
| Use: "watch", | |||||
| Args: cobra.ExactArgs(0), | |||||
| RunE: func(c *cobra.Command, args []string) error { | |||||
| ctx := cmd.GetCmdCtx(c) | |||||
| return watch(c, ctx, opt, args) | |||||
| }, | |||||
| } | |||||
| SysEventCmd.AddCommand(&cmd) | |||||
| } | |||||
| type watchOpt struct { | |||||
| } | |||||
| func watch(c *cobra.Command, ctx *cmd.CommandContext, opt watchOpt, args []string) error { | |||||
| resp, err := ctx.Client.SysEvent().Watch(cliapi.SysEventWatch{}) | |||||
| if err != nil { | |||||
| return fmt.Errorf("begin watch : %v", err) | |||||
| } | |||||
| defer resp.Stream.Close() | |||||
| se.Read(resp.Stream, nil)(func(e se.Event, err error) bool { | |||||
| buf := bytes.NewBuffer(nil) | |||||
| json.Indent(buf, []byte(e.Data), "", " ") | |||||
| fmt.Println(buf.String()) | |||||
| return true | |||||
| }) | |||||
| return nil | |||||
| } | |||||
| @@ -2,6 +2,7 @@ package all | |||||
| import ( | import ( | ||||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/admin" | _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/admin" | ||||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/admin/sysevent" | |||||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/admin/ticktock" | _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/admin/ticktock" | ||||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/bucket" | _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/bucket" | ||||
| _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/geto" | _ "gitlink.org.cn/cloudream/jcs-pub/jcsctl/cmd/geto" | ||||