Browse Source

优化代码

master
Sydonian 3 months ago
parent
commit
b3eb3248e0
13 changed files with 52 additions and 56 deletions
  1. +1
    -1
      client/internal/http/v1/pub_shards.go
  2. +4
    -4
      client/internal/spacesyncer/execute_diff.go
  3. +2
    -2
      client/internal/spacesyncer/execute_full.go
  4. +2
    -2
      client/internal/uploader/user_space_upload.go
  5. +4
    -4
      common/pkgs/storage/local/base_store.go
  6. +1
    -1
      common/pkgs/storage/local/dir_reader.go
  7. +2
    -2
      common/pkgs/storage/local/multipart_upload.go
  8. +2
    -2
      common/pkgs/storage/local/s2s.go
  9. +3
    -3
      common/pkgs/storage/local/shard_store.go
  10. +5
    -5
      common/pkgs/storage/rclone/shard_store.go
  11. +1
    -1
      common/pkgs/storage/types/utils.go
  12. +22
    -26
      common/types/path.go
  13. +3
    -3
      jcsctl/cmd/putp/file_iterator.go

+ 1
- 1
client/internal/http/v1/pub_shards.go View File

@@ -98,7 +98,7 @@ func (s *PubShardsService) Join(ctx *gin.Context) {
Credential: resp.PubShards.Credential,
ShardStore: &resp.PubShards.ShardStore,
Features: resp.PubShards.Features,
WorkingDir: resp.PubShards.WorkingDir.ConcatCompsNew("parts", fmt.Sprintf("%v", s.svc.AccToken.GetToken().UserID)).String(),
WorkingDir: resp.PubShards.WorkingDir.PushNew("parts", fmt.Sprintf("%v", s.svc.AccToken.GetToken().UserID)).String(),
})
if cerr2 != nil {
ctx.JSON(http.StatusOK, types.Failed(ecode.ErrorCode(cerr2.Code), cerr2.Message))


+ 4
- 4
client/internal/spacesyncer/execute_diff.go View File

@@ -74,7 +74,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcstypes.SpaceSyncModeDi
}

rela := e.Path.Clone()
rela.DropFrontN(task.Task.SrcPath.Len())
rela.PopFrontN(task.Task.SrcPath.Len())

ne := e
ne.Path = rela.Clone()
@@ -100,7 +100,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcstypes.SpaceSyncModeDi
}

rela := e.Path.Clone()
rela.DropFrontN(task.Task.Dests[0].DestPath.Len())
rela.PopFrontN(task.Task.Dests[0].DestPath.Len())

ne := e
ne.Path = rela.Clone()
@@ -154,7 +154,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcstypes.SpaceSyncModeDi
for _, s := range syncs {
ft.AddFrom(ioswitch2.NewFromBaseStore(*srcSpace, s.Path))
rela := s.Path.Clone()
rela.DropFrontN(task.Task.SrcPath.Len())
rela.PopFrontN(task.Task.SrcPath.Len())
dstPath := task.Task.Dests[0].DestPath.ConcatNew(rela)
to := ioswitch2.NewToBaseStore(*dstSpace, dstPath)
to.Option.ModTime = s.ModTime
@@ -182,7 +182,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *jcstypes.SpaceSyncModeDi
if !task.Task.Options.NoEmptyDirectories && len(willMkdirs) > 0 {
for _, p := range willMkdirs {
rela := p.Clone()
rela.DropFrontN(task.Task.SrcPath.Len())
rela.PopFrontN(task.Task.SrcPath.Len())
dstPath := task.Task.Dests[0].DestPath.ConcatNew(rela)
err := dstBase.Mkdir(dstPath)
if err != nil {


+ 2
- 2
client/internal/spacesyncer/execute_full.go View File

@@ -72,7 +72,7 @@ func executeFull(syncer *SpaceSyncer, task *task) {
}

rela := e.Path.Clone()
rela.DropFrontN(task.Task.SrcPath.Len())
rela.PopFrontN(task.Task.SrcPath.Len())

ne := e
ne.Path = rela.Clone()
@@ -145,7 +145,7 @@ func executeFull(syncer *SpaceSyncer, task *task) {
for i, base := range dstBases {
if base != nil {
dirPath := task.Task.Dests[i].DestPath.Clone()
dirPath.ConcatComps(path)
dirPath.Push(path...)
err := base.Mkdir(dirPath)
if err != nil {
log.Warnf("mkdir %v at user space %v: %v", dirPath, dstSpaces[i].String(), err)


+ 2
- 2
client/internal/uploader/user_space_upload.go View File

@@ -185,10 +185,10 @@ func (u *Uploader) uploadFromBaseStore(srcSpace *jcstypes.UserSpaceDetail, targe
}

pat := e.Path.Clone()
pat.DropFrontN(rootPath.Len() - 1)
pat.PopFrontN(rootPath.Len() - 1)
// 如果对象路径和RootPath相同(即RootPath是一个文件),则用文件名作为对象Path
if pat.Len() > 1 {
pat.DropFrontN(1)
pat.PopFrontN(1)
}

info := ret.Get(e.Path.String()).(*ops2.FileInfoValue)


+ 4
- 4
common/pkgs/storage/local/base_store.go View File

@@ -63,7 +63,7 @@ func (s *BaseStore) Write(pat jcstypes.JPath, stream io.Reader, opt stgtypes.Wri
}

func (s *BaseStore) Read(objPath jcstypes.JPath, opt stgtypes.OpenOption) (io.ReadCloser, error) {
absObjPath := filepath.Join(s.root, objPath.JoinOSPath())
absObjPath := filepath.Join(s.root, objPath.OSString())
file, err := os.Open(absObjPath)
if err != nil {
return nil, err
@@ -87,7 +87,7 @@ func (s *BaseStore) Read(objPath jcstypes.JPath, opt stgtypes.OpenOption) (io.Re
}

func (s *BaseStore) Mkdir(path jcstypes.JPath) error {
absObjPath := filepath.Join(s.root, path.JoinOSPath())
absObjPath := filepath.Join(s.root, path.OSString())
err := os.MkdirAll(absObjPath, 0755)
if err != nil {
return err
@@ -98,7 +98,7 @@ func (s *BaseStore) Mkdir(path jcstypes.JPath) error {

func (s *BaseStore) ReadDir(pat jcstypes.JPath) stgtypes.DirReader {
return &DirReader{
absRootPath: filepath.Join(s.root, pat.JoinOSPath()),
absRootPath: filepath.Join(s.root, pat.OSString()),
rootJPath: pat.Clone(),
}
}
@@ -106,7 +106,7 @@ func (s *BaseStore) ReadDir(pat jcstypes.JPath) stgtypes.DirReader {
func (s *BaseStore) CleanTemps() {
log := s.getLogger()

tempDir := filepath.Join(s.root, s.detail.UserSpace.WorkingDir.JoinOSPath(), stgtypes.TempWorkingDir)
tempDir := filepath.Join(s.root, s.detail.UserSpace.WorkingDir.OSString(), stgtypes.TempWorkingDir)
entries, err := os.ReadDir(tempDir)
if err != nil {
log.Warnf("read temp dir: %v", err)


+ 1
- 1
common/pkgs/storage/local/dir_reader.go View File

@@ -57,7 +57,7 @@ func (r *DirReader) Next() (stgtypes.DirEntry, error) {
r.curEntries = r.curEntries[1:]

if entry.entry.IsDir() {
es, err := os.ReadDir(filepath.Join(r.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name()))
es, err := os.ReadDir(filepath.Join(r.absRootPath, entry.dir.OSString(), entry.entry.Name()))
if err != nil {
return stgtypes.DirEntry{}, err
}


+ 2
- 2
common/pkgs/storage/local/multipart_upload.go View File

@@ -31,7 +31,7 @@ func (*Multiparter) MaxPartSize() int64 {
}

func (m *Multiparter) Initiate(ctx context.Context) (stgtypes.MultipartTask, error) {
tempDir := filepath.Join(m.localStg.RootDir, m.detail.UserSpace.WorkingDir.JoinOSPath(), stgtypes.TempWorkingDir)
tempDir := filepath.Join(m.localStg.RootDir, m.detail.UserSpace.WorkingDir.OSString(), stgtypes.TempWorkingDir)
absTempDir, err := filepath.Abs(tempDir)
if err != nil {
return nil, fmt.Errorf("get abs temp dir %v: %v", tempDir, err)
@@ -51,7 +51,7 @@ func (m *Multiparter) Initiate(ctx context.Context) (stgtypes.MultipartTask, err
absTempDir: absTempDir,
tempFileName: tempFileName,
tempPartsDir: tempPartsDir,
joinedFileJPath: m.detail.UserSpace.WorkingDir.ConcatCompsNew(stgtypes.TempWorkingDir, tempFileName+".joined"),
joinedFileJPath: m.detail.UserSpace.WorkingDir.PushNew(stgtypes.TempWorkingDir, tempFileName+".joined"),
absJoinedFilePath: absJoinedFilePath,
uploadID: tempPartsDir,
}, nil


+ 2
- 2
common/pkgs/storage/local/s2s.go View File

@@ -39,13 +39,13 @@ func (*S2STransfer) CanTransfer(src, dst *jcstypes.UserSpaceDetail) bool {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcstypes.UserSpaceDetail, srcPath jcstypes.JPath, dstPath jcstypes.JPath) (stgtypes.FileInfo, error) {
s.dstPath = dstPath

copy, err := os.OpenFile(filepath.Join(s.localStg.RootDir, s.dstPath.JoinOSPath()), os.O_WRONLY|os.O_CREATE, 0644)
copy, err := os.OpenFile(filepath.Join(s.localStg.RootDir, s.dstPath.OSString()), os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return stgtypes.FileInfo{}, err
}
defer copy.Close()

srcFile, err := os.Open(filepath.Join(s.localStg.RootDir, srcPath.JoinOSPath()))
srcFile, err := os.Open(filepath.Join(s.localStg.RootDir, srcPath.OSString()))
if err != nil {
return stgtypes.FileInfo{}, err
}


+ 3
- 3
common/pkgs/storage/local/shard_store.go View File

@@ -22,7 +22,7 @@ type ShardStore struct {
}

func NewShardStore(root string, detail *jcstypes.UserSpaceDetail) (*ShardStore, error) {
storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.JoinOSPath(), stgtypes.ShardStoreWorkingDir))
storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.OSString(), stgtypes.ShardStoreWorkingDir))
if err != nil {
return nil, fmt.Errorf("get abs root: %w", err)
}
@@ -44,7 +44,7 @@ func (s *ShardStore) Stop() {
}

func (s *ShardStore) Store(path jcstypes.JPath, hash jcstypes.FileHash, size int64) (stgtypes.FileInfo, error) {
fullTempPath := filepath.Join(s.stgRoot, path.JoinOSPath())
fullTempPath := filepath.Join(s.stgRoot, path.OSString())

s.lock.Lock()
defer s.lock.Unlock()
@@ -208,5 +208,5 @@ func (s *ShardStore) getFilePathFromHash(hash jcstypes.FileHash) string {
}

func (s *ShardStore) getJPathFromHash(hash jcstypes.FileHash) jcstypes.JPath {
return s.detail.UserSpace.WorkingDir.ConcatCompsNew(stgtypes.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
return s.detail.UserSpace.WorkingDir.PushNew(stgtypes.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
}

+ 5
- 5
common/pkgs/storage/rclone/shard_store.go View File

@@ -250,21 +250,21 @@ func (s *ShardStore) getLogger() logger.Logger {
}

func (s *ShardStore) getFileDirFromHash(hash jcstypes.FileHash) string {
return s.Detail.UserSpace.WorkingDir.ConcatCompsNew(stgtypes.ShardStoreWorkingDir, hash.GetHashPrefix(2)).String()
return s.Detail.UserSpace.WorkingDir.PushNew(stgtypes.ShardStoreWorkingDir, hash.GetHashPrefix(2)).String()
}

func (s *ShardStore) getFilePathFromHash(hash jcstypes.FileHash) string {
return s.Detail.UserSpace.WorkingDir.ConcatCompsNew(stgtypes.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash)).String()
return s.Detail.UserSpace.WorkingDir.PushNew(stgtypes.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash)).String()
}

func (s *ShardStore) getJPathFromHash(hash jcstypes.FileHash) jcstypes.JPath {
return s.Detail.UserSpace.WorkingDir.ConcatCompsNew(stgtypes.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
return s.Detail.UserSpace.WorkingDir.PushNew(stgtypes.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
}

func (s *ShardStore) getStoreRoot() string {
return s.Detail.UserSpace.WorkingDir.ConcatCompsNew(stgtypes.ShardStoreWorkingDir).String()
return s.Detail.UserSpace.WorkingDir.PushNew(stgtypes.ShardStoreWorkingDir).String()
}

func (s *ShardStore) getStoreRootJPath() jcstypes.JPath {
return s.Detail.UserSpace.WorkingDir.ConcatCompsNew(stgtypes.ShardStoreWorkingDir)
return s.Detail.UserSpace.WorkingDir.PushNew(stgtypes.ShardStoreWorkingDir)
}

+ 1
- 1
common/pkgs/storage/types/utils.go View File

@@ -19,6 +19,6 @@ func FindFeature[T jcstypes.StorageFeature](detail *jcstypes.UserSpaceDetail) T
func MakeTempDirPath(detail *jcstypes.UserSpaceDetail, comps ...string) jcstypes.JPath {
p := detail.UserSpace.WorkingDir.Clone()
p.Push(TempWorkingDir)
p.ConcatComps(comps)
p.Push(comps...)
return p
}

+ 22
- 26
common/types/path.go View File

@@ -31,8 +31,14 @@ func (p *JPath) LastComp() string {
return p.comps[len(p.comps)-1]
}

func (p *JPath) Push(comp string) {
p.comps = append(p.comps, comp)
func (p *JPath) Push(comp ...string) {
p.comps = append(p.comps, comp...)
}

func (p *JPath) PushNew(comp ...string) JPath {
clone := p.Clone()
clone.Push(comp...)
return clone
}

func (p *JPath) Pop() string {
@@ -44,7 +50,19 @@ func (p *JPath) Pop() string {
return comp
}

func (p *JPath) SplitParent() JPath {
func (p *JPath) PopFrontN(cnt int) {
if cnt >= len(p.comps) {
p.comps = nil
return
}
if cnt <= 0 {
return
}

p.comps = p.comps[cnt:]
}

func (p *JPath) CutParent() JPath {
if len(p.comps) <= 1 {
return JPath{}
}
@@ -69,18 +87,6 @@ func (p *JPath) CopyParent() JPath {
return parent
}

func (p *JPath) DropFrontN(cnt int) {
if cnt >= len(p.comps) {
p.comps = nil
return
}
if cnt <= 0 {
return
}

p.comps = p.comps[cnt:]
}

func (p *JPath) Concat(other JPath) {
p.comps = append(p.comps, other.comps...)
}
@@ -91,16 +97,6 @@ func (p *JPath) ConcatNew(other JPath) JPath {
return clone
}

func (p *JPath) ConcatComps(comps []string) {
p.comps = append(p.comps, comps...)
}

func (p *JPath) ConcatCompsNew(comps ...string) JPath {
clone := p.Clone()
clone.ConcatComps(comps)
return clone
}

func (p *JPath) Clone() JPath {
clone := JPath{
comps: make([]string, len(p.comps)),
@@ -109,7 +105,7 @@ func (p *JPath) Clone() JPath {
return clone
}

func (p *JPath) JoinOSPath() string {
func (p *JPath) OSString() string {
return filepath.Join(p.comps...)
}



+ 3
- 3
jcsctl/cmd/putp/file_iterator.go View File

@@ -48,7 +48,7 @@ func (i *FileIterator) MoveNext() (*cliapi.UploadingObject, error) {
i.curEntries = i.curEntries[1:]

if entry.entry.IsDir() {
es, err := os.ReadDir(filepath.Join(i.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name()))
es, err := os.ReadDir(filepath.Join(i.absRootPath, entry.dir.OSString(), entry.entry.Name()))
if err != nil {
return nil, err
}
@@ -65,7 +65,7 @@ func (i *FileIterator) MoveNext() (*cliapi.UploadingObject, error) {
continue
}

file, err := os.Open(filepath.Join(i.absRootPath, entry.dir.JoinOSPath(), entry.entry.Name()))
file, err := os.Open(filepath.Join(i.absRootPath, entry.dir.OSString(), entry.entry.Name()))
if err != nil {
return nil, err
}
@@ -78,7 +78,7 @@ func (i *FileIterator) MoveNext() (*cliapi.UploadingObject, error) {
i.fileCount++

jpath := i.jpathRoot.ConcatNew(entry.dir)
path := jpath.ConcatCompsNew(entry.entry.Name()).String()
path := jpath.PushNew(entry.entry.Name()).String()

now := time.Now()
if !i.lastStartTime.IsZero() {


Loading…
Cancel
Save