package services import ( "context" "fmt" "path" "strings" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/trie" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader/strategy" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" hubmq "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/mq/hub" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" ) type UserSpaceService struct { *Service } func (svc *Service) UserSpaceSvc() *UserSpaceService { return &UserSpaceService{Service: svc} } func (svc *UserSpaceService) Get(userspaceID clitypes.UserSpaceID) (clitypes.UserSpace, error) { return svc.DB.UserSpace().GetByID(svc.DB.DefCtx(), userspaceID) } func (svc *UserSpaceService) GetByName(name string) (clitypes.UserSpace, error) { return svc.DB.UserSpace().GetByName(svc.DB.DefCtx(), name) } func (svc *UserSpaceService) LoadPackage(packageID clitypes.PackageID, userspaceID clitypes.UserSpaceID, rootPath string) error { coorCli, err := stgglb.CoordinatorMQPool.Acquire() if err != nil { return fmt.Errorf("new coordinator client: %w", err) } defer stgglb.CoordinatorMQPool.Release(coorCli) destStg := svc.UserSpaceMeta.Get(userspaceID) if destStg == nil { return fmt.Errorf("userspace not found: %d", userspaceID) } if destStg.MasterHub == nil { return fmt.Errorf("userspace %v has no master hub", userspaceID) } details, err := db.DoTx11(svc.DB, svc.DB.Object().GetPackageObjectDetails, packageID) if err != nil { return err } var pinned []clitypes.ObjectID plans := exec.NewPlanBuilder() for _, obj := range details { strg, err := svc.StrategySelector.Select(strategy.Request{ Detail: obj, DestHub: destStg.MasterHub.HubID, }) if err != nil { return fmt.Errorf("select download strategy: %w", err) } ft := ioswitch2.NewFromTo() switch strg := strg.(type) { case *strategy.DirectStrategy: ft.AddFrom(ioswitch2.NewFromShardstore(strg.Detail.Object.FileHash, *strg.UserSpace.MasterHub, strg.UserSpace, ioswitch2.RawStream())) case *strategy.ECReconstructStrategy: for i, b := range strg.Blocks { ft.AddFrom(ioswitch2.NewFromShardstore(b.FileHash, *strg.UserSpaces[i].MasterHub, strg.UserSpaces[i], ioswitch2.ECStream(b.Index))) ft.ECParam = &strg.Redundancy } default: return fmt.Errorf("unsupported download strategy: %T", strg) } ft.AddTo(ioswitch2.NewToPublicStore(*destStg.MasterHub, *destStg, path.Join(rootPath, obj.Object.Path))) // 顺便保存到同存储服务的分片存储中 if destStg.UserSpace.ShardStore != nil { ft.AddTo(ioswitch2.NewToShardStore(*destStg.MasterHub, *destStg, ioswitch2.RawStream(), "")) pinned = append(pinned, obj.Object.ObjectID) } err = parser.Parse(ft, plans) if err != nil { return fmt.Errorf("parse plan: %w", err) } } mutex, err := reqbuilder.NewBuilder(). Shard().Buzy(userspaceID). MutexLock(svc.PubLock) if err != nil { return fmt.Errorf("acquire locks failed, err: %w", err) } defer mutex.Unlock() // 记录访问统计 for _, obj := range details { svc.AccessStat.AddAccessCounter(obj.Object.ObjectID, packageID, userspaceID, 1) } drv := plans.Execute(exec.NewExecContext()) _, err = drv.Wait(context.Background()) if err != nil { return err } return nil } func (svc *UserSpaceService) SpaceToSpace(srcSpaceID clitypes.UserSpaceID, srcPath string, dstSpaceID clitypes.UserSpaceID, dstPath string) (clitypes.SpaceToSpaceResult, error) { srcSpace := svc.UserSpaceMeta.Get(srcSpaceID) if srcSpace == nil { return clitypes.SpaceToSpaceResult{}, fmt.Errorf("source userspace not found: %d", srcSpaceID) } if srcSpace.MasterHub == nil { return clitypes.SpaceToSpaceResult{}, fmt.Errorf("source userspace %v has no master hub", srcSpaceID) } srcSpaceCli, err := stgglb.HubMQPool.Acquire(srcSpace.MasterHub.HubID) if err != nil { return clitypes.SpaceToSpaceResult{}, fmt.Errorf("new source userspace client: %w", err) } defer stgglb.HubMQPool.Release(srcSpaceCli) dstSpace := svc.UserSpaceMeta.Get(dstSpaceID) if dstSpace == nil { return clitypes.SpaceToSpaceResult{}, fmt.Errorf("destination userspace not found: %d", dstSpaceID) } if dstSpace.MasterHub == nil { return clitypes.SpaceToSpaceResult{}, fmt.Errorf("destination userspace %v has no master hub", dstSpaceID) } dstSpaceCli, err := stgglb.HubMQPool.Acquire(dstSpace.MasterHub.HubID) if err != nil { return clitypes.SpaceToSpaceResult{}, fmt.Errorf("new destination userspace client: %w", err) } defer stgglb.HubMQPool.Release(dstSpaceCli) srcPath = strings.Trim(srcPath, cdssdk.ObjectPathSeparator) dstPath = strings.Trim(dstPath, cdssdk.ObjectPathSeparator) if srcPath == "" { return clitypes.SpaceToSpaceResult{}, fmt.Errorf("source path is empty") } if dstPath == "" { return clitypes.SpaceToSpaceResult{}, fmt.Errorf("destination path is empty") } listAllResp, err := srcSpaceCli.PublicStoreListAll(&hubmq.PublicStoreListAll{ UserSpace: *srcSpace, Path: srcPath, }) if err != nil { return clitypes.SpaceToSpaceResult{}, fmt.Errorf("list all from source userspace: %w", err) } srcPathComps := clitypes.SplitObjectPath(srcPath) srcDirCompLen := len(srcPathComps) - 1 entryTree := trie.NewTrie[*types.PublicStoreEntry]() for _, e := range listAllResp.Entries { pa, ok := strings.CutSuffix(e.Path, clitypes.ObjectPathSeparator) comps := clitypes.SplitObjectPath(pa) e.Path = pa e2 := e entryTree.CreateWords(comps[srcDirCompLen:]).Value = &e2 e2.IsDir = e2.IsDir || ok } entryTree.Iterate(func(path []string, node *trie.Node[*types.PublicStoreEntry], isWordNode bool) trie.VisitCtrl { if node.Value == nil { return trie.VisitContinue } if node.Value.IsDir && len(node.WordNexts) > 0 { node.Value = nil return trie.VisitContinue } if !node.Value.IsDir && len(node.WordNexts) == 0 { node.WordNexts = nil } return trie.VisitContinue }) var filePathes []string var dirPathes []string entryTree.Iterate(func(path []string, node *trie.Node[*types.PublicStoreEntry], isWordNode bool) trie.VisitCtrl { if node.Value == nil { return trie.VisitContinue } if node.Value.IsDir { dirPathes = append(dirPathes, node.Value.Path) } else { filePathes = append(filePathes, node.Value.Path) } return trie.VisitContinue }) var success []string var failed []string for _, f := range filePathes { newPath := strings.Replace(f, srcPath, dstPath, 1) ft := ioswitch2.NewFromTo() ft.AddFrom(ioswitch2.NewFromPublicStore(*srcSpace.MasterHub, *srcSpace, f)) ft.AddTo(ioswitch2.NewToPublicStore(*dstSpace.MasterHub, *dstSpace, newPath)) plans := exec.NewPlanBuilder() err = parser.Parse(ft, plans) if err != nil { failed = append(failed, f) logger.Warnf("s2s: parse plan of file %v: %v", f, err) continue } _, err = plans.Execute(exec.NewExecContext()).Wait(context.Background()) if err != nil { failed = append(failed, f) logger.Warnf("s2s: execute plan of file %v: %v", f, err) continue } success = append(success, f) } newDirPathes := make([]string, 0, len(dirPathes)) for i := range dirPathes { newDirPathes = append(newDirPathes, strings.Replace(dirPathes[i], srcPath, dstPath, 1)) } mkdirResp, err := dstSpaceCli.PublicStoreMkdirs(&hubmq.PublicStoreMkdirs{ UserSpace: *dstSpace, Pathes: newDirPathes, }) if err != nil { failed = append(failed, dirPathes...) logger.Warnf("s2s: mkdirs to destination userspace: %v", err) } else { for i := range dirPathes { if mkdirResp.Successes[i] { success = append(success, dirPathes[i]) } else { failed = append(failed, dirPathes[i]) } } } return clitypes.SpaceToSpaceResult{ Success: success, Failed: failed, }, nil }