package local import ( "errors" "fmt" "io/fs" "os" "path/filepath" "sync" "gitlink.org.cn/cloudream/common/pkgs/logger" clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" ) type ShardStore struct { detail *clitypes.UserSpaceDetail stgRoot string storeAbsRoot string lock sync.Mutex done chan any } func NewShardStore(root string, detail *clitypes.UserSpaceDetail) (*ShardStore, error) { storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir, types.ShardStoreWorkingDir)) if err != nil { return nil, fmt.Errorf("get abs root: %w", err) } return &ShardStore{ detail: detail, stgRoot: root, storeAbsRoot: storeAbsRoot, done: make(chan any, 1), }, nil } func (s *ShardStore) Start(ch *types.StorageEventChan) { s.getLogger().Infof("component start, root: %v, max size: %v", s.storeAbsRoot, s.detail.UserSpace.ShardStore.MaxSize) } func (s *ShardStore) Stop() { s.getLogger().Infof("component stop") } func (s *ShardStore) Store(path string, hash clitypes.FileHash, size int64) (types.FileInfo, error) { fullTempPath := filepath.Join(s.stgRoot, path) s.lock.Lock() defer s.lock.Unlock() log := s.getLogger() log.Debugf("%v bypass uploaded, size: %v, hash: %v", fullTempPath, size, hash) blockDir := s.getFileDirFromHash(hash) err := os.MkdirAll(blockDir, 0755) if err != nil { log.Warnf("make block dir %v: %v", blockDir, err) return types.FileInfo{}, fmt.Errorf("making block dir: %w", err) } newPath := filepath.Join(blockDir, string(hash)) _, err = os.Stat(newPath) if os.IsNotExist(err) { err = os.Rename(fullTempPath, newPath) if err != nil { log.Warnf("rename %v to %v: %v", fullTempPath, newPath, err) return types.FileInfo{}, fmt.Errorf("rename file: %w", err) } } else if err != nil { log.Warnf("get file %v stat: %v", newPath, err) return types.FileInfo{}, fmt.Errorf("get file stat: %w", err) } return types.FileInfo{ Hash: hash, Size: size, Path: s.getSlashFilePathFromHash(hash), }, nil } func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) { s.lock.Lock() defer s.lock.Unlock() filePath := s.getFilePathFromHash(hash) info, err := os.Stat(filePath) if err != nil { return types.FileInfo{}, err } return types.FileInfo{ Hash: hash, Size: info.Size(), Path: s.getSlashFilePathFromHash(hash), }, nil } func (s *ShardStore) ListAll() ([]types.FileInfo, error) { s.lock.Lock() defer s.lock.Unlock() var infos []types.FileInfo err := filepath.WalkDir(s.storeAbsRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } info, err := d.Info() if err != nil { return err } fileHash, err := clitypes.ParseHash(filepath.Base(info.Name())) if err != nil { return nil } infos = append(infos, types.FileInfo{ Hash: fileHash, Size: info.Size(), Path: s.getSlashFilePathFromHash(fileHash), }) return nil }) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, err } return infos, nil } func (s *ShardStore) GC(avaiables []clitypes.FileHash) error { s.lock.Lock() defer s.lock.Unlock() avais := make(map[clitypes.FileHash]bool) for _, hash := range avaiables { avais[hash] = true } cnt := 0 err := filepath.WalkDir(s.storeAbsRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } info, err := d.Info() if err != nil { return err } fileHash, err := clitypes.ParseHash(filepath.Base(info.Name())) if err != nil { return nil } if !avais[fileHash] { err = os.Remove(path) if err != nil { s.getLogger().Warnf("remove file %v: %v", path, err) } else { cnt++ } } return nil }) if err != nil && !errors.Is(err, os.ErrNotExist) { return err } s.getLogger().Infof("purge %d files", cnt) // TODO 无法保证原子性,所以删除失败只打日志 return nil } func (s *ShardStore) Stats() types.Stats { // TODO 统计本地存储的相关信息 return types.Stats{ Status: types.StatusOK, } } func (s *ShardStore) getLogger() logger.Logger { return logger.WithField("ShardStore", "Local").WithField("Storage", s.detail.UserSpace.Storage.String()) } func (s *ShardStore) getFileDirFromHash(hash clitypes.FileHash) string { return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2)) } func (s *ShardStore) getFilePathFromHash(hash clitypes.FileHash) string { return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2), string(hash)) } func (s *ShardStore) getSlashFilePathFromHash(hash clitypes.FileHash) string { return types.PathJoin(s.detail.UserSpace.WorkingDir, types.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash)) }