package services import ( "context" "fmt" "strconv" "time" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan/ops" jcstypes "gitlink.org.cn/cloudream/jcs-pub/common/types" "gitlink.org.cn/cloudream/jcs-pub/common/types/datamap" "gorm.io/gorm" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader/strategy" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" "gitlink.org.cn/cloudream/jcs-pub/common/ecode" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory" ) type UserSpaceService struct { *Service } func (svc *Service) UserSpaceSvc() *UserSpaceService { return &UserSpaceService{Service: svc} } func (svc *UserSpaceService) Get(userspaceID jcstypes.UserSpaceID) (jcstypes.UserSpace, error) { return svc.DB.UserSpace().GetByID(svc.DB.DefCtx(), userspaceID) } func (svc *UserSpaceService) GetByName(name string) (jcstypes.UserSpace, error) { return svc.DB.UserSpace().GetByName(svc.DB.DefCtx(), name) } func (svc *UserSpaceService) GetAll() ([]jcstypes.UserSpace, error) { return svc.DB.UserSpace().GetAll(svc.DB.DefCtx()) } func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpaceCreateResp, *ecode.CodeError) { db2 := svc.DB space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcstypes.UserSpace, error) { space, err := db2.UserSpace().GetByName(tx, req.Name) if err == nil { return jcstypes.UserSpace{}, gorm.ErrDuplicatedKey } if err != gorm.ErrRecordNotFound { return jcstypes.UserSpace{}, err } space = jcstypes.UserSpace{ Name: req.Name, Storage: req.Storage, Credential: req.Credential, ShardStore: req.ShardStore, Features: req.Features, WorkingDir: jcstypes.PathFromJcsPathString(req.WorkingDir), Revision: 0, } err = db2.UserSpace().Create(tx, &space) if err != nil { return jcstypes.UserSpace{}, err } return space, nil }) if err == gorm.ErrDuplicatedKey { return nil, ecode.New(ecode.DataExists, "user space name already exists") } if err != nil { return nil, ecode.Newf(ecode.OperationFailed, "%v", err) } return &cliapi.UserSpaceCreateResp{UserSpace: space}, nil } func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpaceUpdateResp, *ecode.CodeError) { db2 := svc.DB space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcstypes.UserSpace, error) { space, err := db2.UserSpace().GetByID(tx, req.UserSpaceID) if err != nil { return jcstypes.UserSpace{}, err } if space.Name != req.Name { _, err = db2.UserSpace().GetByName(tx, req.Name) if err == nil { return jcstypes.UserSpace{}, gorm.ErrDuplicatedKey } if err != gorm.ErrRecordNotFound { return jcstypes.UserSpace{}, err } } space.Name = req.Name space.Credential = req.Credential space.Features = req.Features space.Revision += 1 return space, db2.UserSpace().UpdateColumns(tx, space, "Name", "Credential", "Features", "Revision") }) if err == gorm.ErrDuplicatedKey { return nil, ecode.New(ecode.DataExists, "user space name already exists") } if err != nil { return nil, ecode.Newf(ecode.OperationFailed, "%v", err) } // 通知元数据缓存无效 svc.UserSpaceMeta.Drop([]jcstypes.UserSpaceID{req.UserSpaceID}) // 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理。还有集群模式 svc.StgPool.Drop(stgglb.UserID, space.UserSpaceID) // TODO 考虑加锁再进行操作 return &cliapi.UserSpaceUpdateResp{UserSpace: space}, nil } func (svc *UserSpaceService) Delete(req cliapi.UserSpaceDelete) (*cliapi.UserSpaceDeleteResp, *ecode.CodeError) { db2 := svc.DB err := db2.DoTx(func(tx db.SQLContext) error { err := db2.UserSpace().DeleteByID(tx, req.UserSpaceID) if err != nil { return err } err = db2.ObjectBlock().DeleteByUserSpaceID(tx, req.UserSpaceID) if err != nil { return err } err = db2.PinnedObject().DeleteByUserSpaceID(tx, req.UserSpaceID) if err != nil { return err } err = db2.ObjectAccessStat().DeleteByUserSpaceID(tx, req.UserSpaceID) if err != nil { return err } err = db2.PackageAccessStat().DeleteByUserSpaceID(tx, req.UserSpaceID) if err != nil { return err } return nil }) if err != nil { return nil, ecode.Newf(ecode.OperationFailed, "%v", err) } // 通知元数据缓存无效 svc.UserSpaceMeta.Drop([]jcstypes.UserSpaceID{req.UserSpaceID}) // 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理 svc.StgPool.Drop(stgglb.UserID, req.UserSpaceID) // TODO 考虑加锁再进行操作,并且增加机制打断已经在进行的操作。 return &cliapi.UserSpaceDeleteResp{}, nil } func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTestResp, *ecode.CodeError) { detail := jcstypes.UserSpaceDetail{ UserID: stgglb.UserID, UserSpace: jcstypes.UserSpace{ Name: "test", Storage: req.Storage, Credential: req.Credential, WorkingDir: jcstypes.PathFromJcsPathString(req.WorikingDir), }, } blder := factory.GetBuilder(&detail) baseStore, err := blder.CreateBaseStore(false) if err != nil { return nil, ecode.Newf(ecode.OperationFailed, "%v", err) } err = baseStore.Test() if err != nil { return nil, ecode.Newf(ecode.OperationFailed, "%v", err) } return &cliapi.UserSpaceTestResp{}, nil } func (svc *UserSpaceService) DownloadPackage(req cliapi.UserSpaceDownloadPackage) error { destSpace := svc.UserSpaceMeta.Get(req.UserSpaceID) if destSpace == nil { return fmt.Errorf("userspace not found: %d", req.UserSpaceID) } details, err := db.DoTx11(svc.DB, svc.DB.Object().GetPackageObjectDetails, req.PackageID) if err != nil { return err } mutex, err := svc.PubLock.BeginMutex(). UserSpace().Buzy(req.UserSpaceID).End(). Lock() if err != nil { return fmt.Errorf("acquire locks failed, err: %w", err) } defer mutex.Unlock() rootJPath := jcstypes.PathFromJcsPathString(req.RootPath) concy := req.Concurrency if concy == 0 { concy = 5 } dIndex := 0 var pinned []jcstypes.PinnedObject for dIndex < len(details) { plans := exec.NewPlanBuilder() for i := 0; i < concy && dIndex < len(details); i++ { strg, err := svc.StrategySelector.Select(strategy.Request{ Detail: details[dIndex], DestLocation: destSpace.UserSpace.Storage.GetLocation(), }) if err != nil { return fmt.Errorf("select download strategy: %w", err) } shouldAtClient := svc.SpeedStats.ShouldAtClient(details[dIndex].Object.Size) ft := ioswitch2.NewFromTo() switch strg := strg.(type) { case *strategy.DirectStrategy: if shouldAtClient && strg.UserSpace.RecommendHub != nil { newSpace := strg.UserSpace newSpace.RecommendHub = nil ft.AddFrom(ioswitch2.NewFromShardstore(strg.Detail.Object.FileHash, newSpace, ioswitch2.RawStream())) } else { ft.AddFrom(ioswitch2.NewFromShardstore(strg.Detail.Object.FileHash, strg.UserSpace, ioswitch2.RawStream())) } case *strategy.ECReconstructStrategy: for i, b := range strg.Blocks { if shouldAtClient && strg.UserSpaces[i].RecommendHub != nil { newSpace := strg.UserSpaces[i] newSpace.RecommendHub = nil ft.AddFrom(ioswitch2.NewFromShardstore(b.FileHash, newSpace, ioswitch2.ECStream(b.Index))) } else { ft.AddFrom(ioswitch2.NewFromShardstore(b.FileHash, strg.UserSpaces[i], ioswitch2.ECStream(b.Index))) } ft.ECParam = &strg.Redundancy } default: return fmt.Errorf("unsupported download strategy: %T", strg) } objPath := jcstypes.PathFromJcsPathString(details[dIndex].Object.Path) dstPath := rootJPath.ConcatNew(objPath) newDstSpace := *destSpace if shouldAtClient { newDstSpace.RecommendHub = nil } ft.AddTo(ioswitch2.NewToBaseStore(newDstSpace, dstPath).WithRange(math2.NewRange(0, details[dIndex].Object.Size))) // 顺便保存到同存储服务的分片存储中 if req.SaveToShards && destSpace.UserSpace.ShardStore != nil { ft.AddTo(ioswitch2.NewToShardStore(newDstSpace, ioswitch2.RawStream(), "").WithRange(math2.NewRange(0, details[dIndex].Object.Size))) pinned = append(pinned, jcstypes.PinnedObject{ ObjectID: details[dIndex].Object.ObjectID, UserSpaceID: destSpace.UserSpace.UserSpaceID, CreateTime: time.Now(), }) } // 单独统计每一个对象的下载速度信息 ft.StatsCtx = fmt.Sprintf("%v", dIndex) err = parser.Parse(ft, plans) if err != nil { return fmt.Errorf("parse plan: %w", err) } dIndex++ } // 记录访问统计 for _, obj := range details { svc.AccessStat.AddAccessCounter(obj.Object.ObjectID, req.PackageID, req.UserSpaceID, 1) } exeCtx := exec.NewExecContext() exec.SetValueByType(exeCtx, svc.StgPool) drv := plans.Execute(exeCtx) ret, err := drv.Wait(context.Background()) if err != nil { return err } // 统计下载速度 trans := make(map[int]int64) elapseds := make(map[int]time.Duration) for _, v := range ret.GetArray(ops2.BaseReadStatsStoreKey) { v2 := v.(*ops2.BaseReadStatsValue) svc.SpeedStats.Record(v2.Length, v2.ElapsedTime, v2.Location.IsDriver) idx, _ := strconv.Atoi(v2.StatsCtx) trans[idx] += v2.Length elapseds[idx] = v2.ElapsedTime } for _, v := range ret.GetArray(ops.SendStreamStatsStoreKey) { v2 := v.(*ops.SendStreamStatsValue) idx, _ := strconv.Atoi(v2.StatsCtx) trans[idx] += v2.Length } for idx, len := range trans { svc.EvtPub.Publish(&datamap.BodyObjectAccessStats{ ObjectID: details[idx].Object.ObjectID, RequestSize: details[idx].Object.Size, TransferAmount: len, ElapsedTime: elapseds[idx], }) } err = svc.DB.DoTx(func(tx db.SQLContext) error { objIDs := make([]jcstypes.ObjectID, len(pinned)) for i, obj := range pinned { objIDs[i] = obj.ObjectID } avaiIDs, err := svc.DB.Object().BatchTestObjectID(tx, objIDs) if err != nil { return err } pinned = lo.Filter(pinned, func(p jcstypes.PinnedObject, idx int) bool { return avaiIDs[p.ObjectID] }) return svc.DB.PinnedObject().BatchTryCreate(svc.DB.DefCtx(), pinned) }) if err != nil { logger.Warnf("create pinned objects: %v", err) } } return nil }