JCS-pub/common/pkgs/storage/local/shard_store.go

package local

import (
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"sync"

	"gitlink.org.cn/cloudream/common/pkgs/logger"
	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
)

type ShardStore struct {
	detail       *clitypes.UserSpaceDetail
	stgRoot      string
	storeAbsRoot string
	lock         sync.Mutex
	done         chan any
}

func NewShardStore(root string, detail *clitypes.UserSpaceDetail) (*ShardStore, error) {
	storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir, types.ShardStoreWorkingDir))
	if err != nil {
		return nil, fmt.Errorf("get abs root: %w", err)
	}

	return &ShardStore{
		detail:       detail,
		stgRoot:      root,
		storeAbsRoot: storeAbsRoot,
		done:         make(chan any, 1),
	}, nil
}

func (s *ShardStore) Start(ch *types.StorageEventChan) {
	s.getLogger().Infof("component start, root: %v, max size: %v", s.storeAbsRoot, s.detail.UserSpace.ShardStore.MaxSize)
}

func (s *ShardStore) Stop() {
	s.getLogger().Infof("component stop")
}

func (s *ShardStore) Store(path string, hash clitypes.FileHash, size int64) (types.FileInfo, error) {
	fullTempPath := filepath.Join(s.stgRoot, path)

	s.lock.Lock()
	defer s.lock.Unlock()

	log := s.getLogger()

	log.Debugf("%v bypass uploaded, size: %v, hash: %v", fullTempPath, size, hash)

	blockDir := s.getFileDirFromHash(hash)
	err := os.MkdirAll(blockDir, 0755)
	if err != nil {
		log.Warnf("make block dir %v: %v", blockDir, err)
		return types.FileInfo{}, fmt.Errorf("making block dir: %w", err)
	}

	newPath := filepath.Join(blockDir, string(hash))
	_, err = os.Stat(newPath)
	if os.IsNotExist(err) {
		err = os.Rename(fullTempPath, newPath)
		if err != nil {
			log.Warnf("rename %v to %v: %v", fullTempPath, newPath, err)
			return types.FileInfo{}, fmt.Errorf("rename file: %w", err)
		}

	} else if err != nil {
		log.Warnf("get file %v stat: %v", newPath, err)
		return types.FileInfo{}, fmt.Errorf("get file stat: %w", err)
	}

	return types.FileInfo{
		Hash: hash,
		Size: size,
		Path: s.getSlashFilePathFromHash(hash),
	}, nil
}

func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
	s.lock.Lock()
	defer s.lock.Unlock()

	filePath := s.getFilePathFromHash(hash)
	info, err := os.Stat(filePath)
	if err != nil {
		return types.FileInfo{}, err
	}

	return types.FileInfo{
		Hash: hash,
		Size: info.Size(),
		Path: s.getSlashFilePathFromHash(hash),
	}, nil
}

func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
	s.lock.Lock()
	defer s.lock.Unlock()

	var infos []types.FileInfo

	err := filepath.WalkDir(s.storeAbsRoot, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}

		if d.IsDir() {
			return nil
		}

		info, err := d.Info()
		if err != nil {
			return err
		}

		fileHash, err := clitypes.ParseHash(filepath.Base(info.Name()))
		if err != nil {
			return nil
		}

		infos = append(infos, types.FileInfo{
			Hash: fileHash,
			Size: info.Size(),
			Path: s.getSlashFilePathFromHash(fileHash),
		})
		return nil
	})
	if err != nil && !errors.Is(err, os.ErrNotExist) {
		return nil, err
	}

	return infos, nil
}

func (s *ShardStore) GC(avaiables []clitypes.FileHash) error {
	s.lock.Lock()
	defer s.lock.Unlock()

	avais := make(map[clitypes.FileHash]bool)
	for _, hash := range avaiables {
		avais[hash] = true
	}

	cnt := 0

	err := filepath.WalkDir(s.storeAbsRoot, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}

		if d.IsDir() {
			return nil
		}

		info, err := d.Info()
		if err != nil {
			return err
		}

		fileHash, err := clitypes.ParseHash(filepath.Base(info.Name()))
		if err != nil {
			return nil
		}

		if !avais[fileHash] {
			err = os.Remove(path)
			if err != nil {
				s.getLogger().Warnf("remove file %v: %v", path, err)
			} else {
				cnt++
			}
		}

		return nil
	})
	if err != nil && !errors.Is(err, os.ErrNotExist) {
		return err
	}

	s.getLogger().Infof("purge %d files", cnt)

	// TODO 无法保证原子性，所以删除失败只打日志
	return nil
}

func (s *ShardStore) Stats() types.Stats {
	// TODO 统计本地存储的相关信息
	return types.Stats{
		Status: types.StatusOK,
	}
}

func (s *ShardStore) getLogger() logger.Logger {
	return logger.WithField("ShardStore", "Local").WithField("Storage", s.detail.UserSpace.Storage.String())
}

func (s *ShardStore) getFileDirFromHash(hash clitypes.FileHash) string {
	return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2))
}

func (s *ShardStore) getFilePathFromHash(hash clitypes.FileHash) string {
	return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2), string(hash))
}

func (s *ShardStore) getSlashFilePathFromHash(hash clitypes.FileHash) string {
	return types.PathJoin(s.detail.UserSpace.WorkingDir, types.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
}