package spacesyncer import ( "context" "io" "time" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/trie" "gitlink.org.cn/cloudream/common/utils/math2" clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" ) func executeDiff(syncer *SpaceSyncer, task *task, mode *clitypes.SpaceSyncModeDiff) { log := logger.WithField("Mod", logMod).WithField("TaskID", task.Task.TaskID) startTime := time.Now() log.Infof("begin full sync task") defer func() { log.Infof("full sync task finished, time: %v", time.Since(startTime)) }() srcSpace := syncer.spaceMeta.Get(task.Task.SrcUserSpaceID) if srcSpace == nil { log.Warnf("src space %v not found", task.Task.SrcUserSpaceID) return } if len(task.Task.Dests) > 1 { log.Warnf("diff mode only support one dest now") } dstSpace := syncer.spaceMeta.Get(task.Task.Dests[0].DestUserSpaceID) if dstSpace == nil { log.Warnf("dest space %v not found", task.Task.Dests[0].DestUserSpaceID) return } srcBase, err := syncer.stgPool.GetBaseStore(srcSpace) if err != nil { log.Warnf("get src base store error: %v", err) return } dstBase, err := syncer.stgPool.GetBaseStore(dstSpace) if err != nil { log.Warnf("get dst base store error: %v", err) return } filter := buildFilter(task) srcReader := srcBase.ReadDir(task.Task.SrcPath) dstReader := dstBase.ReadDir(task.Task.Dests[0].DestPath) dirTree := trie.NewTrie[srcDstDirEntry]() for { e, err := srcReader.Next() if err == io.EOF { break } if err != nil { log.Warnf("read src dir: %v", err) return } if !filter(e) { continue } rela := e.Path.Clone() rela.DropFrontN(task.Task.SrcPath.Len()) ne := e ne.Path = rela.Clone() if !filter(ne) { continue } diffCreateSrcNode(dirTree, rela, &e) } for { e, err := dstReader.Next() if err == io.EOF { break } if err != nil { log.Warnf("read dst dir: %v", err) return } if !filter(e) { continue } rela := e.Path.Clone() rela.DropFrontN(task.Task.Dests[0].DestPath.Len()) ne := e ne.Path = rela.Clone() if !filter(ne) { continue } diffCreateDstNode(dirTree, rela, &e) } var willSync []stgtypes.DirEntry var willMkdirs []clitypes.JPath dirTree.Iterate(func(path []string, node *trie.Node[srcDstDirEntry], isWordNode bool) trie.VisitCtrl { if node.Value.src == nil { // 目前不支持删除多余文件 return trie.VisitContinue } if node.Value.src.IsDir { if node.Value.dst == nil { if node.IsEmpty() { willMkdirs = append(willMkdirs, clitypes.PathFromComps(path...)) } } } else { if node.Value.dst == nil { // 目标路径不存在(不是文件也不是目录),需要同步 if node.IsEmpty() { willSync = append(willSync, *node.Value.src) } } else if !node.Value.dst.IsDir { // 目标路径是个文件,但文件指纹不同,需要同步 if !cmpFile(mode, node.Value.src, node.Value.dst) { willSync = append(willSync, *node.Value.src) } } // 目标路径是个目录,则不进行同步 } return trie.VisitContinue }) willSyncCnt := len(willSync) for len(willSync) > 0 { syncs := willSync[:math2.Min(len(willSync), 50)] willSync = willSync[len(syncs):] ft := ioswitch2.NewFromTo() for _, s := range syncs { ft.AddFrom(ioswitch2.NewFromBaseStore(*srcSpace, s.Path)) rela := s.Path.Clone() rela.DropFrontN(task.Task.SrcPath.Len()) dstPath := task.Task.Dests[0].DestPath.ConcatNew(rela) to := ioswitch2.NewToBaseStore(*dstSpace, dstPath) to.Option.ModTime = s.ModTime ft.AddTo(to) } planBld := exec.NewPlanBuilder() err := parser.Parse(ft, planBld) if err != nil { log.Warnf("parse fromto: %v", err) return } execCtx := exec.NewWithContext(task.Context) exec.SetValueByType(execCtx, syncer.stgPool) _, err = planBld.Execute(execCtx).Wait(context.Background()) if err != nil { log.Warnf("execute plan: %v", err) return } } log.Infof("%v files synced", willSyncCnt) if !task.Task.Options.NoEmptyDirectories && len(willMkdirs) > 0 { for _, p := range willMkdirs { rela := p.Clone() rela.DropFrontN(task.Task.SrcPath.Len()) dstPath := task.Task.Dests[0].DestPath.ConcatNew(rela) err := dstBase.Mkdir(dstPath) if err != nil { log.Warnf("mkdir: %v", err) continue } } } } func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path clitypes.JPath, e *stgtypes.DirEntry) { var ptr = &tree.Root for _, c := range path.Comps() { if ptr.Value.src != nil && ptr.Value.src.IsDir { ptr.Value.src = nil } ptr = ptr.Create(c) } ptr.Value.src = e } func diffCreateDstNode(tree *trie.Trie[srcDstDirEntry], path clitypes.JPath, e *stgtypes.DirEntry) { var ptr = &tree.Root for _, c := range path.Comps() { if ptr.Value.src != nil && ptr.Value.src.IsDir { ptr.Value.src = nil } if ptr.Value.dst != nil && ptr.Value.dst.IsDir { ptr.Value.dst = nil } ptr = ptr.Create(c) } ptr.Value.dst = e } type srcDstDirEntry struct { src *stgtypes.DirEntry dst *stgtypes.DirEntry } func cmpFile(diff *clitypes.SpaceSyncModeDiff, src, dst *stgtypes.DirEntry) bool { if diff.IncludeSize && src.Size != dst.Size { return false } if diff.IncludeModTime && src.ModTime != dst.ModTime { return false } return true }