Browse Source

'0412-新增了注释'

gitlink
JeshuaRen 1 year ago
parent
commit
125558b1e9
27 changed files with 3938 additions and 272 deletions
  1. +2
    -4
      agent/internal/task/create_package.go
  2. +2
    -0
      client/internal/cmdline/package.go
  3. +48
    -9
      client/internal/http/bucket.go
  4. +90
    -50
      client/internal/http/object.go
  5. +64
    -58
      client/internal/http/package.go
  6. +17
    -13
      client/internal/http/server.go
  7. +19
    -14
      client/internal/services/bucket.go
  8. +84
    -7
      client/internal/services/object.go
  9. +35
    -10
      client/internal/services/package.go
  10. +19
    -6
      common/pkgs/cmd/upload_objects.go
  11. +15
    -0
      common/pkgs/db/bucket.go
  12. +1
    -1
      common/pkgs/db/model/model.go
  13. +54
    -16
      common/pkgs/db/object.go
  14. +20
    -0
      common/pkgs/db/object_block.go
  15. +14
    -0
      common/pkgs/db/package.go
  16. +20
    -0
      common/pkgs/db/pinned_object.go
  17. +1
    -2
      common/pkgs/iterator/download_object_iterator.go
  18. +33
    -3
      common/pkgs/mq/coordinator/bucket.go
  19. +131
    -13
      common/pkgs/mq/coordinator/object.go
  20. +40
    -5
      common/pkgs/mq/coordinator/package.go
  21. +19
    -3
      coordinator/internal/mq/bucket.go
  22. +383
    -7
      coordinator/internal/mq/object.go
  23. +28
    -16
      coordinator/internal/mq/package.go
  24. +473
    -0
      log/agent.log
  25. +2291
    -0
      log/scanner.log
  26. +22
    -22
      scanner/internal/event/check_package_redundancy.go
  27. +13
    -13
      scanner/internal/event/clean_pinned.go

+ 2
- 4
agent/internal/task/create_package.go View File

@@ -84,8 +84,7 @@ func (t *CreatePackage) Execute(task *task.Task[TaskContext], ctx TaskContext, c
return
}

// 上传包内的对象
uploadRet, err := cmd.NewUploadObjects(t.userID, createResp.PackageID, t.objIter, t.nodeAffinity).Execute(&cmd.UploadObjectsContext{
uploadRet, err := cmd.NewUploadObjects(t.userID, createResp.Package.PackageID, t.objIter, t.nodeAffinity).Execute(&cmd.UploadObjectsContext{
Distlock: ctx.distlock,
Connectivity: ctx.connectivity,
})
@@ -99,8 +98,7 @@ func (t *CreatePackage) Execute(task *task.Task[TaskContext], ctx TaskContext, c
return
}

// 保存上传结果到任务结果中
t.Result.PackageID = createResp.PackageID
t.Result.PackageID = createResp.Package.PackageID
t.Result.Objects = uploadRet.Objects

// 完成任务并设置移除延迟


+ 2
- 0
client/internal/cmdline/package.go View File

@@ -211,7 +211,9 @@ func init() {

commands.MustAdd(PackageDeletePackage, "pkg", "delete")

// 查询package缓存到哪些节点
commands.MustAdd(PackageGetCachedNodes, "pkg", "cached")

// 查询package调度到哪些节点
commands.MustAdd(PackageGetLoadedNodes, "pkg", "loaded")
}

+ 48
- 9
client/internal/http/bucket.go View File

@@ -21,13 +21,32 @@ func (s *Server) Bucket() *BucketService {
}
}

// Create 创建一个新的存储桶
// ctx *gin.Context: Gin框架的上下文对象,用于处理HTTP请求和响应
func (s *BucketService) GetByName(ctx *gin.Context) {
log := logger.WithField("HTTP", "Bucket.GetByName")

var req cdssdk.BucketGetByName
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding query: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

bucket, err := s.svc.BucketSvc().GetBucketByName(req.UserID, req.Name)
if err != nil {
log.Warnf("getting bucket by name: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get bucket by name failed"))
return
}

ctx.JSON(http.StatusOK, OK(cdssdk.BucketGetByNameResp{
Bucket: bucket,
}))
}

func (s *BucketService) Create(ctx *gin.Context) {
log := logger.WithField("HTTP", "Bucket.Create")

var req cdssdk.BucketCreateReq
// 尝试从HTTP请求绑定JSON请求体到结构体
var req cdssdk.BucketCreate
if err := ctx.ShouldBindJSON(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
// 绑定失败,返回错误信息
@@ -35,8 +54,7 @@ func (s *BucketService) Create(ctx *gin.Context) {
return
}

// 调用服务层方法,创建存储桶
bucketID, err := s.svc.BucketSvc().CreateBucket(req.UserID, req.BucketName)
bucket, err := s.svc.BucketSvc().CreateBucket(req.UserID, req.Name)
if err != nil {
log.Warnf("creating bucket: %s", err.Error())
// 创建存储桶失败,返回错误信息
@@ -46,7 +64,7 @@ func (s *BucketService) Create(ctx *gin.Context) {

// 创建存储桶成功,返回成功响应
ctx.JSON(http.StatusOK, OK(cdssdk.BucketCreateResp{
BucketID: bucketID,
Bucket: bucket,
}))
}

@@ -55,8 +73,7 @@ func (s *BucketService) Create(ctx *gin.Context) {
func (s *BucketService) Delete(ctx *gin.Context) {
log := logger.WithField("HTTP", "Bucket.Delete")

var req cdssdk.BucketDeleteReq
// 尝试从HTTP请求绑定JSON请求体到结构体
var req cdssdk.BucketDelete
if err := ctx.ShouldBindJSON(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
// 绑定失败,返回错误信息
@@ -75,3 +92,25 @@ func (s *BucketService) Delete(ctx *gin.Context) {
// 删除存储桶成功,返回成功响应
ctx.JSON(http.StatusOK, OK(nil))
}

func (s *BucketService) ListUserBuckets(ctx *gin.Context) {
log := logger.WithField("HTTP", "Bucket.ListUserBuckets")

var req cdssdk.BucketListUserBucketsReq
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding query: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

buckets, err := s.svc.BucketSvc().GetUserBuckets(req.UserID)
if err != nil {
log.Warnf("getting user buckets: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get user buckets failed"))
return
}

ctx.JSON(http.StatusOK, OK(cdssdk.BucketListUserBucketsResp{
Buckets: buckets,
}))
}

+ 90
- 50
client/internal/http/object.go View File

@@ -1,16 +1,18 @@
package http

import (
"fmt"
"io"
"mime/multipart"
"net/http"
"path"
"time"

"github.com/gin-gonic/gin"
"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/pkgs/logger"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
myio "gitlink.org.cn/cloudream/common/utils/io"
myhttp "gitlink.org.cn/cloudream/common/utils/http"
)

// ObjectService 服务结构体,处理对象相关的HTTP请求
@@ -55,14 +57,28 @@ func (s *ObjectService) Upload(ctx *gin.Context) {

// 等待上传任务完成
for {
complete, _, err := s.svc.ObjectSvc().WaitUploading(taskID, time.Second*5)
complete, objs, err := s.svc.ObjectSvc().WaitUploading(taskID, time.Second*5)
if complete {
if err != nil {
log.Warnf("uploading object: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "uploading object failed"))
return
}
ctx.JSON(http.StatusOK, OK(nil))

uploadeds := make([]cdssdk.UploadedObject, len(objs.Objects))
for i, obj := range objs.Objects {
err := ""
if obj.Error != nil {
err = obj.Error.Error()
}
o := obj.Object
uploadeds[i] = cdssdk.UploadedObject{
Object: &o,
Error: err,
}
}

ctx.JSON(http.StatusOK, OK(cdssdk.ObjectUploadResp{Uploadeds: uploadeds}))
return
}

@@ -74,91 +90,115 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
}
}

// ObjectDownloadReq 定义下载对象请求的结构体
type ObjectDownloadReq struct {
UserID *cdssdk.UserID `form:"userID" binding:"required"` // 用户ID
ObjectID *cdssdk.ObjectID `form:"objectID" binding:"required"` // 对象ID
}

// Download 处理对象下载请求
func (s *ObjectService) Download(ctx *gin.Context) {
log := logger.WithField("HTTP", "Object.Download")

var req ObjectDownloadReq
var req cdssdk.ObjectDownload
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

// 下载对象
file, err := s.svc.ObjectSvc().Download(*req.UserID, *req.ObjectID)
file, err := s.svc.ObjectSvc().Download(req.UserID, req.ObjectID)
if err != nil {
log.Warnf("downloading object: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "download object failed"))
return
}

// 设置响应头,进行文件下载
mw := multipart.NewWriter(ctx.Writer)
defer mw.Close()

ctx.Writer.Header().Set("Content-Type", fmt.Sprintf("%s;boundary=%s", myhttp.ContentTypeMultiPart, mw.Boundary()))
ctx.Writer.WriteHeader(http.StatusOK)
ctx.Header("Content-Disposition", "attachment; filename=filename")
ctx.Header("Content-Type", "application/octet-stream")

// 通过流式传输返回文件内容
buf := make([]byte, 4096)
ctx.Stream(func(w io.Writer) bool {
rd, err := file.Read(buf)
if err == io.EOF {
err = myio.WriteAll(w, buf[:rd])
if err != nil {
log.Warnf("writing data to response: %s", err.Error())
}
return false
}

if err != nil {
log.Warnf("reading file data: %s", err.Error())
return false
}
fw, err := mw.CreateFormFile("file", path.Base(file.Object.Path))
if err != nil {
log.Warnf("creating form file: %s", err.Error())
return
}

err = myio.WriteAll(w, buf[:rd])
if err != nil {
log.Warnf("writing data to response: %s", err.Error())
return false
}
io.Copy(fw, file.File)
}

func (s *ObjectService) UpdateInfo(ctx *gin.Context) {
log := logger.WithField("HTTP", "Object.UpdateInfo")

var req cdssdk.ObjectUpdateInfo
if err := ctx.ShouldBindJSON(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

sucs, err := s.svc.ObjectSvc().UpdateInfo(req.UserID, req.Updatings)
if err != nil {
log.Warnf("updating objects: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "update objects failed"))
return
}

return true
})
ctx.JSON(http.StatusOK, OK(cdssdk.ObjectUpdateInfoResp{Successes: sucs}))
}

// GetPackageObjectsReq 定义获取包内对象请求的结构体
type GetPackageObjectsReq struct {
UserID *cdssdk.UserID `form:"userID" binding:"required"` // 用户ID
PackageID *cdssdk.PackageID `form:"packageID" binding:"required"` // 包ID
func (s *ObjectService) Move(ctx *gin.Context) {
log := logger.WithField("HTTP", "Object.Move")

var req cdssdk.ObjectMove
if err := ctx.ShouldBindJSON(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

sucs, err := s.svc.ObjectSvc().Move(req.UserID, req.Movings)
if err != nil {
log.Warnf("moving objects: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "move objects failed"))
return
}

ctx.JSON(http.StatusOK, OK(cdssdk.ObjectMoveResp{Successes: sucs}))
}

// GetPackageObjectsResp 定义获取包内对象响应的结构体
type GetPackageObjectsResp = cdssdk.ObjectGetPackageObjectsResp
func (s *ObjectService) Delete(ctx *gin.Context) {
log := logger.WithField("HTTP", "Object.Delete")

var req cdssdk.ObjectDelete
if err := ctx.ShouldBindJSON(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

err := s.svc.ObjectSvc().Delete(req.UserID, req.ObjectIDs)
if err != nil {
log.Warnf("deleting objects: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "delete objects failed"))
return
}

ctx.JSON(http.StatusOK, OK(nil))
}

// GetPackageObjects 处理获取包内对象的请求
func (s *ObjectService) GetPackageObjects(ctx *gin.Context) {
log := logger.WithField("HTTP", "Object.GetPackageObjects")

var req GetPackageObjectsReq
var req cdssdk.ObjectGetPackageObjects
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

// 获取包内的对象列表
objs, err := s.svc.ObjectSvc().GetPackageObjects(*req.UserID, *req.PackageID)
objs, err := s.svc.ObjectSvc().GetPackageObjects(req.UserID, req.PackageID)
if err != nil {
log.Warnf("getting package objects: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get package object failed"))
return
}

// 返回响应
ctx.JSON(http.StatusOK, OK(GetPackageObjectsResp{Objects: objs}))
ctx.JSON(http.StatusOK, OK(cdssdk.ObjectGetPackageObjectsResp{Objects: objs}))
}

+ 64
- 58
client/internal/http/package.go View File

@@ -3,6 +3,7 @@ package http
import (
"mime/multipart"
"net/http"
"net/url"

"github.com/gin-gonic/gin"
"gitlink.org.cn/cloudream/common/consts/errorcode"
@@ -10,7 +11,6 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"

"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
stgiter "gitlink.org.cn/cloudream/storage/common/pkgs/iterator"
)

@@ -26,49 +26,57 @@ func (s *Server) Package() *PackageService {
}
}

// PackageGetReq 包含获取包信息请求所需的参数。
type PackageGetReq struct {
UserID *cdssdk.UserID `form:"userID" binding:"required"`
PackageID *cdssdk.PackageID `form:"packageID" binding:"required"`
}

// PackageGetResp 包含获取包信息响应的结果。
type PackageGetResp struct {
model.Package
}

// Get 处理获取包信息的HTTP请求。
func (s *PackageService) Get(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.Get")

var req PackageGetReq
var req cdssdk.PackageGetReq
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

pkg, err := s.svc.PackageSvc().Get(*req.UserID, *req.PackageID)
pkg, err := s.svc.PackageSvc().Get(req.UserID, req.PackageID)
if err != nil {
log.Warnf("getting package: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get package failed"))
return
}

ctx.JSON(http.StatusOK, OK(PackageGetResp{Package: *pkg}))
ctx.JSON(http.StatusOK, OK(cdssdk.PackageGetResp{Package: *pkg}))
}

func (s *PackageService) GetByName(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.GetByName")

var req cdssdk.PackageGetByName
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding query: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

pkg, err := s.svc.PackageSvc().GetByName(req.UserID, req.BucketName, req.PackageName)
if err != nil {
log.Warnf("getting package by name: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get package by name failed"))
return
}

ctx.JSON(http.StatusOK, OK(cdssdk.PackageGetByNameResp{Package: *pkg}))
}

// Create 处理创建新包的HTTP请求。
func (s *PackageService) Create(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.Create")
var req cdssdk.PackageCreateReq
var req cdssdk.PackageCreate
if err := ctx.ShouldBindJSON(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

pkgID, err := s.svc.PackageSvc().Create(req.UserID, req.BucketID, req.Name)
pkg, err := s.svc.PackageSvc().Create(req.UserID, req.BucketID, req.Name)
if err != nil {
log.Warnf("creating package: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "create package failed"))
@@ -76,28 +84,21 @@ func (s *PackageService) Create(ctx *gin.Context) {
}

ctx.JSON(http.StatusOK, OK(cdssdk.PackageCreateResp{
PackageID: pkgID,
Package: pkg,
}))
}

// PackageDeleteReq 包含删除包请求所需的参数。
type PackageDeleteReq struct {
UserID *cdssdk.UserID `json:"userID" binding:"required"`
PackageID *cdssdk.PackageID `json:"packageID" binding:"required"`
}

// Delete 处理删除包的HTTP请求。
func (s *PackageService) Delete(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.Delete")

var req PackageDeleteReq
var req cdssdk.PackageDelete
if err := ctx.ShouldBindJSON(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

err := s.svc.PackageSvc().DeletePackage(*req.UserID, *req.PackageID)
err := s.svc.PackageSvc().DeletePackage(req.UserID, req.PackageID)
if err != nil {
log.Warnf("deleting package: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "delete package failed"))
@@ -107,68 +108,68 @@ func (s *PackageService) Delete(ctx *gin.Context) {
ctx.JSON(http.StatusOK, OK(nil))
}

// GetCachedNodesReq 包含获取缓存节点请求所需的参数。
type GetCachedNodesReq struct {
UserID *cdssdk.UserID `json:"userID" binding:"required"`
PackageID *cdssdk.PackageID `json:"packageID" binding:"required"`
}
func (s *PackageService) ListBucketPackages(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.ListBucketPackages")

// GetCachedNodesResp 包含获取缓存节点响应的结果。
type GetCachedNodesResp struct {
cdssdk.PackageCachingInfo
var req cdssdk.PackageListBucketPackages
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding query: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

pkgs, err := s.svc.PackageSvc().GetBucketPackages(req.UserID, req.BucketID)
if err != nil {
log.Warnf("getting bucket packages: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get bucket packages failed"))
return
}

ctx.JSON(http.StatusOK, OK(cdssdk.PackageListBucketPackagesResp{
Packages: pkgs,
}))
}

// GetCachedNodes 处理获取包的缓存节点的HTTP请求。
func (s *PackageService) GetCachedNodes(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.GetCachedNodes")

var req GetCachedNodesReq
if err := ctx.ShouldBindJSON(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
var req cdssdk.PackageGetCachedNodesReq
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding query: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

resp, err := s.svc.PackageSvc().GetCachedNodes(*req.UserID, *req.PackageID)
resp, err := s.svc.PackageSvc().GetCachedNodes(req.UserID, req.PackageID)
if err != nil {
log.Warnf("get package cached nodes failed: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get package cached nodes failed"))
return
}

ctx.JSON(http.StatusOK, OK(GetCachedNodesResp{resp}))
}

// GetLoadedNodesReq 包含获取加载节点请求所需的参数。
type GetLoadedNodesReq struct {
UserID *cdssdk.UserID `json:"userID" binding:"required"`
PackageID *cdssdk.PackageID `json:"packageID" binding:"required"`
}

// GetLoadedNodesResp 包含获取加载节点响应的结果。
type GetLoadedNodesResp struct {
NodeIDs []cdssdk.NodeID `json:"nodeIDs"`
ctx.JSON(http.StatusOK, OK(cdssdk.PackageGetCachedNodesResp{PackageCachingInfo: resp}))
}

// GetLoadedNodes 处理获取包的加载节点的HTTP请求。
func (s *PackageService) GetLoadedNodes(ctx *gin.Context) {
log := logger.WithField("HTTP", "Package.GetLoadedNodes")

var req GetLoadedNodesReq
if err := ctx.ShouldBindJSON(&req); err != nil {
log.Warnf("binding body: %s", err.Error())
var req cdssdk.PackageGetLoadedNodesReq
if err := ctx.ShouldBindQuery(&req); err != nil {
log.Warnf("binding query: %s", err.Error())
ctx.JSON(http.StatusBadRequest, Failed(errorcode.BadArgument, "missing argument or invalid argument"))
return
}

nodeIDs, err := s.svc.PackageSvc().GetLoadedNodes(*req.UserID, *req.PackageID)
nodeIDs, err := s.svc.PackageSvc().GetLoadedNodes(req.UserID, req.PackageID)
if err != nil {
log.Warnf("get package loaded nodes failed: %s", err.Error())
ctx.JSON(http.StatusOK, Failed(errorcode.OperationFailed, "get package loaded nodes failed"))
return
}

ctx.JSON(http.StatusOK, OK(GetLoadedNodesResp{
ctx.JSON(http.StatusOK, OK(cdssdk.PackageGetLoadedNodesResp{
NodeIDs: nodeIDs,
}))
}
@@ -183,8 +184,13 @@ func mapMultiPartFileToUploadingObject(files []*multipart.FileHeader) stgiter.Up
return nil, err
}

fileName, err := url.PathUnescape(file.Filename)
if err != nil {
return nil, err
}

return &stgiter.IterUploadingObject{
Path: file.Filename,
Path: fileName,
Size: file.Size,
File: stream,
}, nil


+ 17
- 13
client/internal/http/server.go View File

@@ -50,17 +50,20 @@ func (s *Server) Serve() error {
// 它主要用于配置和初始化与HTTP请求相关的所有路由,
// 包括对象存储、包管理、存储管理、缓存管理和存储桶管理等。
func (s *Server) initRouters() {
// 对象存储相关路由配置
s.engine.GET(cdssdk.ObjectDownloadPath, s.Object().Download) // 处理对象下载请求
s.engine.POST(cdssdk.ObjectUploadPath, s.Object().Upload) // 处理对象上传请求
s.engine.GET(cdssdk.ObjectGetPackageObjectsPath, s.Object().GetPackageObjects) // 处理获取包内对象请求
s.engine.GET(cdssdk.ObjectDownloadPath, s.Object().Download)
s.engine.POST(cdssdk.ObjectUploadPath, s.Object().Upload)
s.engine.GET(cdssdk.ObjectGetPackageObjectsPath, s.Object().GetPackageObjects)
s.engine.POST(cdssdk.ObjectUpdateInfoPath, s.Object().UpdateInfo)
s.engine.POST(cdssdk.ObjectMovePath, s.Object().Move)
s.engine.POST(cdssdk.ObjectDeletePath, s.Object().Delete)

// 包管理相关路由配置
s.engine.GET(cdssdk.PackageGetPath, s.Package().Get) // 处理获取包信息请求
s.engine.POST(cdssdk.PackageCreatePath, s.Package().Create) // 处理创建包请求
s.engine.POST("/package/delete", s.Package().Delete) // 处理删除包请求
s.engine.GET("/package/getCachedNodes", s.Package().GetCachedNodes) // 处理获取缓存节点请求
s.engine.GET("/package/getLoadedNodes", s.Package().GetLoadedNodes) // 处理获取已加载节点请求
s.engine.GET(cdssdk.PackageGetPath, s.Package().Get)
s.engine.GET(cdssdk.PackageGetByNamePath, s.Package().GetByName)
s.engine.POST(cdssdk.PackageCreatePath, s.Package().Create)
s.engine.POST(cdssdk.PackageDeletePath, s.Package().Delete)
s.engine.GET(cdssdk.PackageListBucketPackagesPath, s.Package().ListBucketPackages)
s.engine.GET(cdssdk.PackageGetCachedNodesPath, s.Package().GetCachedNodes)
s.engine.GET(cdssdk.PackageGetLoadedNodesPath, s.Package().GetLoadedNodes)

// 存储管理相关路由配置
s.engine.POST("/storage/loadPackage", s.Storage().LoadPackage) // 处理加载包请求
@@ -70,7 +73,8 @@ func (s *Server) initRouters() {
// 缓存管理相关路由配置
s.engine.POST(cdssdk.CacheMovePackagePath, s.Cache().MovePackage) // 处理移动包到缓存请求

// 存储桶管理相关路由配置
s.engine.POST(cdssdk.BucketCreatePath, s.Bucket().Create) // 处理创建存储桶请求
s.engine.POST(cdssdk.BucketDeletePath, s.Bucket().Delete) // 处理删除存储桶请求
s.engine.GET(cdssdk.BucketGetByNamePath, s.Bucket().GetByName)
s.engine.POST(cdssdk.BucketCreatePath, s.Bucket().Create)
s.engine.POST(cdssdk.BucketDeletePath, s.Bucket().Delete)
s.engine.GET(cdssdk.BucketListUserBucketsPath, s.Bucket().ListUserBuckets)
}

+ 19
- 14
client/internal/services/bucket.go View File

@@ -28,9 +28,21 @@ func (svc *BucketService) GetBucket(userID cdssdk.UserID, bucketID cdssdk.Bucket
panic("not implement yet")
}

// GetUserBuckets 获取指定用户的所有桶信息
// userID: 用户的唯一标识
// 返回值: 用户的所有桶信息列表和可能发生的错误
func (svc *BucketService) GetBucketByName(userID cdssdk.UserID, bucketName string) (model.Bucket, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return model.Bucket{}, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

resp, err := coorCli.GetBucketByName(coormq.ReqGetBucketByName(userID, bucketName))
if err != nil {
return model.Bucket{}, fmt.Errorf("get bucket by name failed, err: %w", err)
}

return resp.Bucket, nil
}

func (svc *BucketService) GetUserBuckets(userID cdssdk.UserID) ([]model.Bucket, error) {
// 从CoordinatorMQPool中获取Coordinator客户端
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
@@ -69,25 +81,20 @@ func (svc *BucketService) GetBucketPackages(userID cdssdk.UserID, bucketID cdssd
return resp.Packages, nil
}

// CreateBucket 创建一个新的桶
// userID: 用户的唯一标识
// bucketName: 桶的名称
// 返回值: 新创建的桶的ID和可能发生的错误
func (svc *BucketService) CreateBucket(userID cdssdk.UserID, bucketName string) (cdssdk.BucketID, error) {
// 获取Coordinator客户端
func (svc *BucketService) CreateBucket(userID cdssdk.UserID, bucketName string) (cdssdk.Bucket, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return 0, fmt.Errorf("new coordinator client: %w", err)
return cdssdk.Bucket{}, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli) // 确保客户端被释放

// 请求Coordinator创建新桶
resp, err := coorCli.CreateBucket(coormq.NewCreateBucket(userID, bucketName))
if err != nil {
return 0, fmt.Errorf("creating bucket: %w", err)
return cdssdk.Bucket{}, fmt.Errorf("creating bucket: %w", err)
}

return resp.BucketID, nil
return resp.Bucket, nil
}

// DeleteBucket 删除指定的桶
@@ -102,8 +109,6 @@ func (svc *BucketService) DeleteBucket(userID cdssdk.UserID, bucketID cdssdk.Buc
}
defer stgglb.CoordinatorMQPool.Release(coorCli) // 确保客户端被释放

// TODO: 检查用户是否有删除这个Bucket的权限。检查的时候可以只上UserBucket的Read锁

_, err = coorCli.DeleteBucket(coormq.NewDeleteBucket(userID, bucketID))
if err != nil {
return fmt.Errorf("request to coordinator failed, err: %w", err)


+ 84
- 7
client/internal/services/object.go View File

@@ -2,12 +2,12 @@ package services

import (
"fmt"
"io"
"time"

cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
mytask "gitlink.org.cn/cloudream/storage/client/internal/task"
stgglb "gitlink.org.cn/cloudream/storage/common/globals"
stgmod "gitlink.org.cn/cloudream/storage/common/models"
"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
"gitlink.org.cn/cloudream/storage/common/pkgs/iterator"
coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
@@ -47,12 +47,89 @@ func (svc *ObjectService) WaitUploading(taskID string, waitTimeout time.Duration
return false, nil, nil
}

// Download 下载对象。当前未实现。
// userID: 用户ID。
// objectID: 对象ID。
// 返回值: 读取关闭器和错误信息。
func (svc *ObjectService) Download(userID cdssdk.UserID, objectID cdssdk.ObjectID) (io.ReadCloser, error) {
panic("not implement yet!")
func (svc *ObjectService) UpdateInfo(userID cdssdk.UserID, updatings []cdssdk.UpdatingObject) ([]cdssdk.ObjectID, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

resp, err := coorCli.UpdateObjectInfos(coormq.ReqUpdateObjectInfos(userID, updatings))
if err != nil {
return nil, fmt.Errorf("requsting to coodinator: %w", err)
}

return resp.Successes, nil
}

func (svc *ObjectService) Move(userID cdssdk.UserID, movings []cdssdk.MovingObject) ([]cdssdk.ObjectID, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

resp, err := coorCli.MoveObjects(coormq.ReqMoveObjects(userID, movings))
if err != nil {
return nil, fmt.Errorf("requsting to coodinator: %w", err)
}

return resp.Successes, nil
}

// Download 用于下载指定的对象。
// userID: 表示用户的唯一标识。
// objectID: 表示要下载的对象的唯一标识。
// 返回值: 返回一个正在下载的对象的迭代器和可能遇到的错误。
func (svc *ObjectService) Download(userID cdssdk.UserID, objectID cdssdk.ObjectID) (*iterator.IterDownloadingObject, error) {
// 从协调器MQ池中获取客户端
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
// 确保在函数结束时释放客户端
defer stgglb.CoordinatorMQPool.Release(coorCli)

// 向协调器请求对象详情
resp, err := coorCli.GetObjectDetails(coormq.ReqGetObjectDetails([]cdssdk.ObjectID{objectID}))
if err != nil {
return nil, fmt.Errorf("requesting to coordinator")
}

// 检查对象是否存在
if resp.Objects[0] == nil {
return nil, fmt.Errorf("object not found")
}

// 创建下载对象的迭代器
iter := iterator.NewDownloadObjectIterator([]stgmod.ObjectDetail{*resp.Objects[0]}, &iterator.DownloadContext{
Distlock: svc.DistLock,
})
// 确保在函数结束时关闭迭代器
defer iter.Close()

// 初始化下载过程
downloading, err := iter.MoveNext()
if err != nil {
return nil, err
}

return downloading, nil
}

func (svc *ObjectService) Delete(userID cdssdk.UserID, objectIDs []cdssdk.ObjectID) error {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

_, err = coorCli.DeleteObjects(coormq.ReqDeleteObjects(userID, objectIDs))
if err != nil {
return fmt.Errorf("requsting to coodinator: %w", err)
}

return nil
}

// GetPackageObjects 获取包中的对象列表。


+ 35
- 10
client/internal/services/package.go View File

@@ -6,7 +6,6 @@ import (
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"

stgglb "gitlink.org.cn/cloudream/storage/common/globals"
"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
"gitlink.org.cn/cloudream/storage/common/pkgs/iterator"
coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
)
@@ -21,9 +20,7 @@ func (svc *Service) PackageSvc() *PackageService {
return &PackageService{Service: svc}
}

// Get 获取指定用户的指定包信息
func (svc *PackageService) Get(userID cdssdk.UserID, packageID cdssdk.PackageID) (*model.Package, error) {
// 从协调器MQ池中获取客户端
func (svc *PackageService) Get(userID cdssdk.UserID, packageID cdssdk.PackageID) (*cdssdk.Package, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
@@ -39,22 +36,50 @@ func (svc *PackageService) Get(userID cdssdk.UserID, packageID cdssdk.PackageID)
return &getResp.Package, nil
}

// Create 创建一个新的包
func (svc *PackageService) Create(userID cdssdk.UserID, bucketID cdssdk.BucketID, name string) (cdssdk.PackageID, error) {
// 从协调器MQ池中获取客户端
func (svc *PackageService) GetByName(userID cdssdk.UserID, bucketName string, packageName string) (*cdssdk.Package, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

getResp, err := coorCli.GetPackageByName(coormq.ReqGetPackageByName(userID, bucketName, packageName))
if err != nil {
return nil, fmt.Errorf("requsting to coodinator: %w", err)
}

return &getResp.Package, nil
}

func (svc *PackageService) GetBucketPackages(userID cdssdk.UserID, bucketID cdssdk.BucketID) ([]cdssdk.Package, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

getResp, err := coorCli.GetBucketPackages(coormq.NewGetBucketPackages(userID, bucketID))
if err != nil {
return nil, fmt.Errorf("requsting to coodinator: %w", err)
}

return getResp.Packages, nil
}

func (svc *PackageService) Create(userID cdssdk.UserID, bucketID cdssdk.BucketID, name string) (cdssdk.Package, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return 0, fmt.Errorf("new coordinator client: %w", err)
return cdssdk.Package{}, fmt.Errorf("new coordinator client: %w", err)
}
defer stgglb.CoordinatorMQPool.Release(coorCli)

// 向协调器发送创建包的请求
resp, err := coorCli.CreatePackage(coormq.NewCreatePackage(userID, bucketID, name))
if err != nil {
return 0, fmt.Errorf("creating package: %w", err)
return cdssdk.Package{}, fmt.Errorf("creating package: %w", err)
}

return resp.PackageID, nil
return resp.Package, nil
}

// DownloadPackage 下载指定包的内容


+ 19
- 6
common/pkgs/cmd/upload_objects.go View File

@@ -37,10 +37,9 @@ type UploadObjectsResult struct {

// ObjectUploadResult 单个对象上传结果的结构体,包含上传信息、错误和对象ID。
type ObjectUploadResult struct {
Info *iterator.IterUploadingObject
Error error
// TODO 这个字段没有被赋值
ObjectID cdssdk.ObjectID
Info *iterator.IterUploadingObject
Error error
Object cdssdk.Object
}

// UploadNodeInfo 上传节点信息的结构体,包含节点信息、延迟、是否与客户端在同一位置。
@@ -215,12 +214,26 @@ func uploadAndUpdatePackage(packageID cdssdk.PackageID, objectIter iterator.Uplo
}
}

// 更新包信息
_, err = coorCli.UpdatePackage(coormq.NewUpdatePackage(packageID, adds, nil))
updateResp, err := coorCli.UpdatePackage(coormq.NewUpdatePackage(packageID, adds, nil))
if err != nil {
return nil, fmt.Errorf("updating package: %w", err)
}

updatedObjs := make(map[string]*cdssdk.Object)
for _, obj := range updateResp.Added {
o := obj
updatedObjs[obj.Path] = &o
}

for i := range uploadRets {
obj := updatedObjs[uploadRets[i].Info.Path]
if obj == nil {
uploadRets[i].Error = fmt.Errorf("object %s not found in package", uploadRets[i].Info.Path)
continue
}
uploadRets[i].Object = *obj
}

return uploadRets, nil
}



+ 15
- 0
common/pkgs/db/bucket.go View File

@@ -18,6 +18,12 @@ func (db *DB) Bucket() *BucketDB {
return &BucketDB{DB: db}
}

func (db *BucketDB) GetByID(ctx SQLContext, bucketID cdssdk.BucketID) (cdssdk.Bucket, error) {
var ret cdssdk.Bucket
err := sqlx.Get(ctx, &ret, "select * from Bucket where BucketID = ?", bucketID)
return ret, err
}

// GetIDByName 根据BucketName查询BucketID
func (db *BucketDB) GetIDByName(bucketName string) (int64, error) {
//桶结构体
@@ -57,6 +63,15 @@ func (*BucketDB) GetUserBucket(ctx SQLContext, userID cdssdk.UserID, bucketID cd
return ret, err
}

func (*BucketDB) GetUserBucketByName(ctx SQLContext, userID cdssdk.UserID, bucketName string) (model.Bucket, error) {
var ret model.Bucket
err := sqlx.Get(ctx, &ret,
"select Bucket.* from UserBucket, Bucket where UserID = ? and"+
" UserBucket.BucketID = Bucket.BucketID and"+
" Bucket.Name = ?", userID, bucketName)
return ret, err
}

func (*BucketDB) GetUserBuckets(ctx SQLContext, userID cdssdk.UserID) ([]model.Bucket, error) {
var ret []model.Bucket
err := sqlx.Select(ctx, &ret, "select Bucket.* from UserBucket, Bucket where UserID = ? and UserBucket.BucketID = Bucket.BucketID", userID)


+ 1
- 1
common/pkgs/db/model/model.go View File

@@ -22,7 +22,7 @@ type Storage struct {

type User struct {
UserID cdssdk.UserID `db:"UserID" json:"userID"`
Password string `db:"PassWord" json:"password"`
Password string `db:"Password" json:"password"`
}

type UserBucket struct {


+ 54
- 16
common/pkgs/db/object.go View File

@@ -26,25 +26,46 @@ func (db *ObjectDB) GetByID(ctx SQLContext, objectID cdssdk.ObjectID) (model.Obj
return ret.ToObject(), err
}

func (db *ObjectDB) BatchGetPackageObjectIDs(ctx SQLContext, pkgID cdssdk.PackageID, pathes []string) ([]cdssdk.ObjectID, error) {
func (db *ObjectDB) BatchGet(ctx SQLContext, objectIDs []cdssdk.ObjectID) ([]model.Object, error) {
if len(objectIDs) == 0 {
return nil, nil
}

// TODO In语句
stmt, args, err := sqlx.In("select * from Object where ObjectID in (?) order by ObjectID asc", objectIDs)
if err != nil {
return nil, err
}
stmt = ctx.Rebind(stmt)

objs := make([]model.TempObject, 0, len(objectIDs))
err = sqlx.Select(ctx, &objs, stmt, args...)
if err != nil {
return nil, err
}

return lo.Map(objs, func(o model.TempObject, idx int) cdssdk.Object { return o.ToObject() }), nil
}

func (db *ObjectDB) BatchGetByPackagePath(ctx SQLContext, pkgID cdssdk.PackageID, pathes []string) ([]cdssdk.Object, error) {
if len(pathes) == 0 {
return nil, nil
}

// TODO In语句
stmt, args, err := sqlx.In("select ObjectID from Object force index(PackagePath) where PackageID=? and Path in (?)", pkgID, pathes)
stmt, args, err := sqlx.In("select * from Object force index(PackagePath) where PackageID=? and Path in (?)", pkgID, pathes)
if err != nil {
return nil, err
}
stmt = ctx.Rebind(stmt)

objIDs := make([]cdssdk.ObjectID, 0, len(pathes))
err = sqlx.Select(ctx, &objIDs, stmt, args...)
objs := make([]model.TempObject, 0, len(pathes))
err = sqlx.Select(ctx, &objs, stmt, args...)
if err != nil {
return nil, err
}

return objIDs, nil
return lo.Map(objs, func(o model.TempObject, idx int) cdssdk.Object { return o.ToObject() }), nil
}

func (db *ObjectDB) Create(ctx SQLContext, obj cdssdk.Object) (cdssdk.ObjectID, error) {
@@ -63,10 +84,10 @@ func (db *ObjectDB) Create(ctx SQLContext, obj cdssdk.Object) (cdssdk.ObjectID,
return cdssdk.ObjectID(objectID), nil
}

// 可以用于批量创建或者更新记录
// 用于创建时,需要额外检查PackageID+Path的唯一性
// 可以用于批量创建或者更新记录
// 用于创建时,需要额外检查PackageID+Path的唯一性
// 用于更新时,需要额外检查现存的PackageID+Path对应的ObjectID是否与待更新的ObjectID相同。不会更新CreateTime。
func (db *ObjectDB) BatchCreateOrUpdate(ctx SQLContext, objs []cdssdk.Object) error {
func (db *ObjectDB) BatchUpsertByPackagePath(ctx SQLContext, objs []cdssdk.Object) error {
if len(objs) == 0 {
return nil
}
@@ -78,6 +99,18 @@ func (db *ObjectDB) BatchCreateOrUpdate(ctx SQLContext, objs []cdssdk.Object) er
return BatchNamedExec(ctx, sql, 7, objs, nil)
}

func (db *ObjectDB) BatchUpert(ctx SQLContext, objs []cdssdk.Object) error {
if len(objs) == 0 {
return nil
}

sql := "insert into Object(ObjectID, PackageID, Path, Size, FileHash, Redundancy, CreateTime ,UpdateTime)" +
" values(:ObjectID, :PackageID,:Path,:Size,:FileHash,:Redundancy, :CreateTime, :UpdateTime) as new" +
" on duplicate key update PackageID = new.PackageID, Path = new.Path, Size = new.Size, FileHash = new.FileHash, Redundancy = new.Redundancy, UpdateTime = new.UpdateTime"

return BatchNamedExec(ctx, sql, 8, objs, nil)
}

func (*ObjectDB) GetPackageObjects(ctx SQLContext, packageID cdssdk.PackageID) ([]model.Object, error) {
var ret []model.TempObject
err := sqlx.Select(ctx, &ret, "select * from Object where PackageID = ? order by ObjectID asc", packageID)
@@ -146,7 +179,7 @@ func (db *ObjectDB) GetPackageObjectDetails(ctx SQLContext, packageID cdssdk.Pac
return rets, nil
}

func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds []coormq.AddObjectEntry) ([]cdssdk.ObjectID, error) {
func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds []coormq.AddObjectEntry) ([]cdssdk.Object, error) {
if len(adds) == 0 {
return nil, nil
}
@@ -164,7 +197,7 @@ func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds []
})
}

err := db.BatchCreateOrUpdate(ctx, objs)
err := db.BatchUpsertByPackagePath(ctx, objs)
if err != nil {
return nil, fmt.Errorf("batch create or update objects: %w", err)
}
@@ -173,17 +206,22 @@ func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds []
for _, add := range adds {
pathes = append(pathes, add.Path)
}
objIDs, err := db.BatchGetPackageObjectIDs(ctx, packageID, pathes)
// 这里可以不用检查查询结果是否与pathes的数量相同
addedObjs, err := db.BatchGetByPackagePath(ctx, packageID, pathes)
if err != nil {
return nil, fmt.Errorf("batch get object ids: %w", err)
}
addedObjIDs := make([]cdssdk.ObjectID, len(addedObjs))
for i := range addedObjs {
addedObjIDs[i] = addedObjs[i].ObjectID
}

err = db.ObjectBlock().BatchDeleteByObjectID(ctx, objIDs)
err = db.ObjectBlock().BatchDeleteByObjectID(ctx, addedObjIDs)
if err != nil {
return nil, fmt.Errorf("batch delete object blocks: %w", err)
}

err = db.PinnedObject().BatchDeleteByObjectID(ctx, objIDs)
err = db.PinnedObject().BatchDeleteByObjectID(ctx, addedObjIDs)
if err != nil {
return nil, fmt.Errorf("batch delete pinned objects: %w", err)
}
@@ -191,7 +229,7 @@ func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds []
objBlocks := make([]stgmod.ObjectBlock, 0, len(adds))
for i, add := range adds {
objBlocks = append(objBlocks, stgmod.ObjectBlock{
ObjectID: objIDs[i],
ObjectID: addedObjIDs[i],
Index: 0,
NodeID: add.NodeID,
FileHash: add.FileHash,
@@ -216,10 +254,10 @@ func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds []
return nil, fmt.Errorf("batch create caches: %w", err)
}

return objIDs, nil
return addedObjs, nil
}

func (db *ObjectDB) BatchUpdateRedundancy(ctx SQLContext, objs []coormq.ChangeObjectRedundancyEntry) error {
func (db *ObjectDB) BatchUpdateRedundancy(ctx SQLContext, objs []coormq.UpdatingObjectRedundancy) error {
if len(objs) == 0 {
return nil
}


+ 20
- 0
common/pkgs/db/object_block.go View File

@@ -24,6 +24,26 @@ func (db *ObjectBlockDB) GetByNodeID(ctx SQLContext, nodeID cdssdk.NodeID) ([]st
return rets, err
}

func (db *ObjectBlockDB) BatchGetByObjectID(ctx SQLContext, objectIDs []cdssdk.ObjectID) ([]stgmod.ObjectBlock, error) {
if len(objectIDs) == 0 {
return nil, nil
}

stmt, args, err := sqlx.In("select * from ObjectBlock where ObjectID in (?) order by ObjectID, `Index` asc", objectIDs)
if err != nil {
return nil, err
}
stmt = ctx.Rebind(stmt)

var blocks []stgmod.ObjectBlock
err = sqlx.Select(ctx, &blocks, stmt, args...)
if err != nil {
return nil, err
}

return blocks, nil
}

func (db *ObjectBlockDB) Create(ctx SQLContext, objectID cdssdk.ObjectID, index int, nodeID cdssdk.NodeID, fileHash string) error {
_, err := ctx.Exec("insert into ObjectBlock values(?,?,?,?)", objectID, index, nodeID, fileHash)
return err


+ 14
- 0
common/pkgs/db/package.go View File

@@ -77,6 +77,20 @@ func (db *PackageDB) GetUserPackage(ctx SQLContext, userID cdssdk.UserID, packag
return ret, err
}

// 在指定名称的Bucket中查找指定名称的Package
func (*PackageDB) GetUserPackageByName(ctx SQLContext, userID cdssdk.UserID, bucketName string, packageName string) (cdssdk.Package, error) {
var ret model.Package
err := sqlx.Get(ctx, &ret,
"select Package.* from Package, Bucket, UserBucket where"+
" Package.Name = ? and"+
" Package.BucketID = Bucket.BucketID and"+
" Bucket.Name = ? and"+
" UserBucket.UserID = ? and"+
" UserBucket.BucketID = Bucket.BucketID",
packageName, bucketName, userID)
return ret, err
}

func (db *PackageDB) Create(ctx SQLContext, bucketID cdssdk.BucketID, name string) (cdssdk.PackageID, error) {
// 根据packagename和bucketid查询,若不存在则插入,若存在则返回错误
var packageID int64


+ 20
- 0
common/pkgs/db/pinned_object.go View File

@@ -34,6 +34,26 @@ func (*PinnedObjectDB) Create(ctx SQLContext, nodeID cdssdk.NodeID, objectID cds
return err
}

func (*PinnedObjectDB) BatchGetByObjectID(ctx SQLContext, objectIDs []cdssdk.ObjectID) ([]cdssdk.PinnedObject, error) {
if len(objectIDs) == 0 {
return nil, nil
}

stmt, args, err := sqlx.In("select * from PinnedObject where ObjectID in (?) order by ObjectID asc", objectIDs)
if err != nil {
return nil, err
}
stmt = ctx.Rebind(stmt)

var pinneds []cdssdk.PinnedObject
err = sqlx.Select(ctx, &pinneds, stmt, args...)
if err != nil {
return nil, err
}

return pinneds, nil
}

func (*PinnedObjectDB) TryCreate(ctx SQLContext, nodeID cdssdk.NodeID, objectID cdssdk.ObjectID, createTime time.Time) error {
_, err := ctx.Exec("insert ignore into PinnedObject values(?,?,?)", nodeID, objectID, createTime)
return err


+ 1
- 2
common/pkgs/iterator/download_object_iterator.go View File

@@ -18,7 +18,6 @@ import (
stgglb "gitlink.org.cn/cloudream/storage/common/globals"
stgmod "gitlink.org.cn/cloudream/storage/common/models"
stgmodels "gitlink.org.cn/cloudream/storage/common/models"
"gitlink.org.cn/cloudream/storage/common/pkgs/db/model"
"gitlink.org.cn/cloudream/storage/common/pkgs/distlock"
"gitlink.org.cn/cloudream/storage/common/pkgs/ec"
coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
@@ -27,7 +26,7 @@ import (
type DownloadingObjectIterator = Iterator[*IterDownloadingObject]

type IterDownloadingObject struct {
Object model.Object
Object cdssdk.Object
File io.ReadCloser
}



+ 33
- 3
common/pkgs/mq/coordinator/bucket.go View File

@@ -7,6 +7,8 @@ import (
)

type BucketService interface {
GetBucketByName(msg *GetBucketByName) (*GetBucketByNameResp, *mq.CodeMessage)

GetUserBuckets(msg *GetUserBuckets) (*GetUserBucketsResp, *mq.CodeMessage)

GetBucketPackages(msg *GetBucketPackages) (*GetBucketPackagesResp, *mq.CodeMessage)
@@ -16,6 +18,34 @@ type BucketService interface {
DeleteBucket(msg *DeleteBucket) (*DeleteBucketResp, *mq.CodeMessage)
}

// 根据桶名获取桶
var _ = Register(Service.GetBucketByName)

type GetBucketByName struct {
mq.MessageBodyBase
UserID cdssdk.UserID `json:"userID"`
Name string `json:"name"`
}
type GetBucketByNameResp struct {
mq.MessageBodyBase
Bucket cdssdk.Bucket `json:"bucket"`
}

func ReqGetBucketByName(userID cdssdk.UserID, name string) *GetBucketByName {
return &GetBucketByName{
UserID: userID,
Name: name,
}
}
func RespGetBucketByName(bucket cdssdk.Bucket) *GetBucketByNameResp {
return &GetBucketByNameResp{
Bucket: bucket,
}
}
func (client *Client) GetBucketByName(msg *GetBucketByName) (*GetBucketByNameResp, error) {
return mq.Request(Service.GetBucketByName, client.rabbitCli, msg)
}

// 获取用户所有的桶
var _ = Register(Service.GetUserBuckets)

@@ -80,7 +110,7 @@ type CreateBucket struct {
}
type CreateBucketResp struct {
mq.MessageBodyBase
BucketID cdssdk.BucketID `json:"bucketID"`
Bucket cdssdk.Bucket `json:"bucket"`
}

func NewCreateBucket(userID cdssdk.UserID, bucketName string) *CreateBucket {
@@ -89,9 +119,9 @@ func NewCreateBucket(userID cdssdk.UserID, bucketName string) *CreateBucket {
BucketName: bucketName,
}
}
func NewCreateBucketResp(bucketID cdssdk.BucketID) *CreateBucketResp {
func NewCreateBucketResp(bucket cdssdk.Bucket) *CreateBucketResp {
return &CreateBucketResp{
BucketID: bucketID,
Bucket: bucket,
}
}
func (client *Client) CreateBucket(msg *CreateBucket) (*CreateBucketResp, error) {


+ 131
- 13
common/pkgs/mq/coordinator/object.go View File

@@ -13,7 +13,15 @@ type ObjectService interface {

GetPackageObjectDetails(msg *GetPackageObjectDetails) (*GetPackageObjectDetailsResp, *mq.CodeMessage)

ChangeObjectRedundancy(msg *ChangeObjectRedundancy) (*ChangeObjectRedundancyResp, *mq.CodeMessage)
GetObjectDetails(msg *GetObjectDetails) (*GetObjectDetailsResp, *mq.CodeMessage)

UpdateObjectRedundancy(msg *UpdateObjectRedundancy) (*UpdateObjectRedundancyResp, *mq.CodeMessage)

UpdateObjectInfos(msg *UpdateObjectInfos) (*UpdateObjectInfosResp, *mq.CodeMessage)

MoveObjects(msg *MoveObjects) (*MoveObjectsResp, *mq.CodeMessage)

DeleteObjects(msg *DeleteObjects) (*DeleteObjectsResp, *mq.CodeMessage)
}

// 查询Package中的所有Object,返回的Objects会按照ObjectID升序
@@ -70,31 +78,141 @@ func (client *Client) GetPackageObjectDetails(msg *GetPackageObjectDetails) (*Ge
return mq.Request(Service.GetPackageObjectDetails, client.rabbitCli, msg)
}

// 获取多个Object以及它们的分块详细信息,返回的Objects会按照ObjectID升序。
var _ = Register(Service.GetObjectDetails)

type GetObjectDetails struct {
mq.MessageBodyBase
ObjectIDs []cdssdk.ObjectID `json:"objectIDs"`
}
type GetObjectDetailsResp struct {
mq.MessageBodyBase
Objects []*stgmod.ObjectDetail `json:"objects"` // 如果没有查询到某个ID对应的信息,则此数组对应位置为nil
}

func ReqGetObjectDetails(objectIDs []cdssdk.ObjectID) *GetObjectDetails {
return &GetObjectDetails{
ObjectIDs: objectIDs,
}
}
func RespGetObjectDetails(objects []*stgmod.ObjectDetail) *GetObjectDetailsResp {
return &GetObjectDetailsResp{
Objects: objects,
}
}
func (client *Client) GetObjectDetails(msg *GetObjectDetails) (*GetObjectDetailsResp, error) {
return mq.Request(Service.GetObjectDetails, client.rabbitCli, msg)
}

// 更新Object的冗余方式
var _ = Register(Service.ChangeObjectRedundancy)
var _ = Register(Service.UpdateObjectRedundancy)

type ChangeObjectRedundancy struct {
type UpdateObjectRedundancy struct {
mq.MessageBodyBase
Entries []ChangeObjectRedundancyEntry `json:"entries"`
Updatings []UpdatingObjectRedundancy `json:"updatings"`
}
type ChangeObjectRedundancyResp struct {
type UpdateObjectRedundancyResp struct {
mq.MessageBodyBase
}
type ChangeObjectRedundancyEntry struct {
type UpdatingObjectRedundancy struct {
ObjectID cdssdk.ObjectID `json:"objectID" db:"ObjectID"`
Redundancy cdssdk.Redundancy `json:"redundancy" db:"Redundancy"`
PinnedAt []cdssdk.NodeID `json:"pinnedAt"`
Blocks []stgmod.ObjectBlock `json:"blocks"`
}

func ReqChangeObjectRedundancy(entries []ChangeObjectRedundancyEntry) *ChangeObjectRedundancy {
return &ChangeObjectRedundancy{
Entries: entries,
func ReqUpdateObjectRedundancy(updatings []UpdatingObjectRedundancy) *UpdateObjectRedundancy {
return &UpdateObjectRedundancy{
Updatings: updatings,
}
}
func RespUpdateObjectRedundancy() *UpdateObjectRedundancyResp {
return &UpdateObjectRedundancyResp{}
}
func (client *Client) UpdateObjectRedundancy(msg *UpdateObjectRedundancy) (*UpdateObjectRedundancyResp, error) {
return mq.Request(Service.UpdateObjectRedundancy, client.rabbitCli, msg)
}

// 更新Object元数据
var _ = Register(Service.UpdateObjectInfos)

type UpdateObjectInfos struct {
mq.MessageBodyBase
UserID cdssdk.UserID `json:"userID"`
Updatings []cdssdk.UpdatingObject `json:"updatings"`
}

type UpdateObjectInfosResp struct {
mq.MessageBodyBase
Successes []cdssdk.ObjectID `json:"successes"`
}

func ReqUpdateObjectInfos(userID cdssdk.UserID, updatings []cdssdk.UpdatingObject) *UpdateObjectInfos {
return &UpdateObjectInfos{
UserID: userID,
Updatings: updatings,
}
}
func RespUpdateObjectInfos(successes []cdssdk.ObjectID) *UpdateObjectInfosResp {
return &UpdateObjectInfosResp{
Successes: successes,
}
}
func (client *Client) UpdateObjectInfos(msg *UpdateObjectInfos) (*UpdateObjectInfosResp, error) {
return mq.Request(Service.UpdateObjectInfos, client.rabbitCli, msg)
}

// 移动Object
var _ = Register(Service.MoveObjects)

type MoveObjects struct {
mq.MessageBodyBase
UserID cdssdk.UserID `json:"userID"`
Movings []cdssdk.MovingObject `json:"movings"`
}

type MoveObjectsResp struct {
mq.MessageBodyBase
Successes []cdssdk.ObjectID `json:"successes"`
}

func ReqMoveObjects(userID cdssdk.UserID, movings []cdssdk.MovingObject) *MoveObjects {
return &MoveObjects{
UserID: userID,
Movings: movings,
}
}
func RespMoveObjects(successes []cdssdk.ObjectID) *MoveObjectsResp {
return &MoveObjectsResp{
Successes: successes,
}
}
func (client *Client) MoveObjects(msg *MoveObjects) (*MoveObjectsResp, error) {
return mq.Request(Service.MoveObjects, client.rabbitCli, msg)
}

// 删除Object
var _ = Register(Service.DeleteObjects)

type DeleteObjects struct {
mq.MessageBodyBase
UserID cdssdk.UserID `json:"userID"`
ObjectIDs []cdssdk.ObjectID `json:"objectIDs"`
}

type DeleteObjectsResp struct {
mq.MessageBodyBase
}

func ReqDeleteObjects(userID cdssdk.UserID, objectIDs []cdssdk.ObjectID) *DeleteObjects {
return &DeleteObjects{
UserID: userID,
ObjectIDs: objectIDs,
}
}
func RespChangeObjectRedundancy() *ChangeObjectRedundancyResp {
return &ChangeObjectRedundancyResp{}
func RespDeleteObjects() *DeleteObjectsResp {
return &DeleteObjectsResp{}
}
func (client *Client) ChangeObjectRedundancy(msg *ChangeObjectRedundancy) (*ChangeObjectRedundancyResp, error) {
return mq.Request(Service.ChangeObjectRedundancy, client.rabbitCli, msg)
func (client *Client) DeleteObjects(msg *DeleteObjects) (*DeleteObjectsResp, error) {
return mq.Request(Service.DeleteObjects, client.rabbitCli, msg)
}

+ 40
- 5
common/pkgs/mq/coordinator/package.go View File

@@ -12,6 +12,8 @@ import (
type PackageService interface {
GetPackage(msg *GetPackage) (*GetPackageResp, *mq.CodeMessage)

GetPackageByName(msg *GetPackageByName) (*GetPackageByNameResp, *mq.CodeMessage)

CreatePackage(msg *CreatePackage) (*CreatePackageResp, *mq.CodeMessage)

UpdatePackage(msg *UpdatePackage) (*UpdatePackageResp, *mq.CodeMessage)
@@ -52,6 +54,36 @@ func (client *Client) GetPackage(msg *GetPackage) (*GetPackageResp, error) {
return mq.Request(Service.GetPackage, client.rabbitCli, msg)
}

// 根据名称获取Package
var _ = Register(Service.GetPackageByName)

type GetPackageByName struct {
mq.MessageBodyBase
UserID cdssdk.UserID `json:"userID"`
BucketName string `json:"bucketName"`
PackageName string `json:"packageName"`
}
type GetPackageByNameResp struct {
mq.MessageBodyBase
Package cdssdk.Package `json:"package"`
}

func ReqGetPackageByName(userID cdssdk.UserID, bucketName string, packageName string) *GetPackageByName {
return &GetPackageByName{
UserID: userID,
BucketName: bucketName,
PackageName: packageName,
}
}
func NewGetPackageByNameResp(pkg cdssdk.Package) *GetPackageByNameResp {
return &GetPackageByNameResp{
Package: pkg,
}
}
func (client *Client) GetPackageByName(msg *GetPackageByName) (*GetPackageByNameResp, error) {
return mq.Request(Service.GetPackageByName, client.rabbitCli, msg)
}

// 创建一个Package
var _ = Register(Service.CreatePackage)

@@ -63,7 +95,7 @@ type CreatePackage struct {
}
type CreatePackageResp struct {
mq.MessageBodyBase
PackageID cdssdk.PackageID `json:"packageID"`
Package cdssdk.Package `json:"package"`
}

func NewCreatePackage(userID cdssdk.UserID, bucketID cdssdk.BucketID, name string) *CreatePackage {
@@ -73,9 +105,9 @@ func NewCreatePackage(userID cdssdk.UserID, bucketID cdssdk.BucketID, name strin
Name: name,
}
}
func NewCreatePackageResp(packageID cdssdk.PackageID) *CreatePackageResp {
func NewCreatePackageResp(pkg cdssdk.Package) *CreatePackageResp {
return &CreatePackageResp{
PackageID: packageID,
Package: pkg,
}
}

@@ -94,6 +126,7 @@ type UpdatePackage struct {
}
type UpdatePackageResp struct {
mq.MessageBodyBase
Added []cdssdk.Object `json:"added"`
}
type AddObjectEntry struct {
Path string `json:"path"`
@@ -110,8 +143,10 @@ func NewUpdatePackage(packageID cdssdk.PackageID, adds []AddObjectEntry, deletes
Deletes: deletes,
}
}
func NewUpdatePackageResp() *UpdatePackageResp {
return &UpdatePackageResp{}
func NewUpdatePackageResp(added []cdssdk.Object) *UpdatePackageResp {
return &UpdatePackageResp{
Added: added,
}
}
func NewAddObjectEntry(path string, size int64, fileHash string, uploadTime time.Time, nodeID cdssdk.NodeID) AddObjectEntry {
return AddObjectEntry{


+ 19
- 3
coordinator/internal/mq/bucket.go View File

@@ -18,6 +18,18 @@ func (svc *Service) GetBucket(userID cdssdk.UserID, bucketID cdssdk.BucketID) (m
panic("not implement yet")
}

func (svc *Service) GetBucketByName(msg *coormq.GetBucketByName) (*coormq.GetBucketByNameResp, *mq.CodeMessage) {
bucket, err := svc.db.Bucket().GetUserBucketByName(svc.db.SQLCtx(), msg.UserID, msg.Name)
if err != nil {
logger.WithField("UserID", msg.UserID).
WithField("Name", msg.Name).
Warnf("getting bucket by name: %s", err.Error())
return nil, mq.Failed(errorcode.OperationFailed, "get bucket by name failed")
}

return mq.ReplyOK(coormq.RespGetBucketByName(bucket))
}

func (svc *Service) GetUserBuckets(msg *coormq.GetUserBuckets) (*coormq.GetUserBucketsResp, *mq.CodeMessage) {
buckets, err := svc.db.Bucket().GetUserBuckets(svc.db.SQLCtx(), msg.UserID)

@@ -44,18 +56,22 @@ func (svc *Service) GetBucketPackages(msg *coormq.GetBucketPackages) (*coormq.Ge
}

func (svc *Service) CreateBucket(msg *coormq.CreateBucket) (*coormq.CreateBucketResp, *mq.CodeMessage) {
var bucketID cdssdk.BucketID
var bucket cdssdk.Bucket
err := svc.db.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error {
_, err := svc.db.User().GetByID(tx, msg.UserID)
if err != nil {
return fmt.Errorf("getting user by id: %w", err)
}

bucketID, err = svc.db.Bucket().Create(tx, msg.UserID, msg.BucketName)
bucketID, err := svc.db.Bucket().Create(tx, msg.UserID, msg.BucketName)
if err != nil {
return fmt.Errorf("creating bucket: %w", err)
}

bucket, err = svc.db.Bucket().GetByID(tx, bucketID)
if err != nil {
return fmt.Errorf("getting bucket by id: %w", err)
}
return nil
})
if err != nil {
@@ -65,7 +81,7 @@ func (svc *Service) CreateBucket(msg *coormq.CreateBucket) (*coormq.CreateBucket
return nil, mq.Failed(errorcode.OperationFailed, "create bucket failed")
}

return mq.ReplyOK(coormq.NewCreateBucketResp(bucketID))
return mq.ReplyOK(coormq.NewCreateBucketResp(bucket))
}

func (svc *Service) DeleteBucket(msg *coormq.DeleteBucket) (*coormq.DeleteBucketResp, *mq.CodeMessage) {


+ 383
- 7
coordinator/internal/mq/object.go View File

@@ -5,19 +5,34 @@ import (
"fmt"

"github.com/jmoiron/sqlx"
"github.com/samber/lo"
"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/pkgs/mq"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/sort2"
stgmod "gitlink.org.cn/cloudream/storage/common/models"
coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator"
)

func (svc *Service) GetPackageObjects(msg *coormq.GetPackageObjects) (*coormq.GetPackageObjectsResp, *mq.CodeMessage) {
// TODO 检查用户是否有权限
objs, err := svc.db.Object().GetPackageObjects(svc.db.SQLCtx(), msg.PackageID)
var objs []cdssdk.Object
err := svc.db.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error {
_, err := svc.db.Package().GetUserPackage(tx, msg.UserID, msg.PackageID)
if err != nil {
return fmt.Errorf("getting package by id: %w", err)
}

objs, err = svc.db.Object().GetPackageObjects(svc.db.SQLCtx(), msg.PackageID)
if err != nil {
return fmt.Errorf("getting package objects: %w", err)
}

return nil
})
if err != nil {
logger.WithField("PackageID", msg.PackageID).
Warnf("get package objects: %s", err.Error())
logger.WithField("UserID", msg.UserID).WithField("PackageID", msg.PackageID).
Warn(err.Error())

return nil, mq.Failed(errorcode.OperationFailed, "get package objects failed")
}
@@ -51,14 +66,375 @@ func (svc *Service) GetPackageObjectDetails(msg *coormq.GetPackageObjectDetails)
return mq.ReplyOK(coormq.NewGetPackageObjectDetailsResp(details))
}

func (svc *Service) ChangeObjectRedundancy(msg *coormq.ChangeObjectRedundancy) (*coormq.ChangeObjectRedundancyResp, *mq.CodeMessage) {
func (svc *Service) GetObjectDetails(msg *coormq.GetObjectDetails) (*coormq.GetObjectDetailsResp, *mq.CodeMessage) {
details := make([]*stgmod.ObjectDetail, len(msg.ObjectIDs))
err := svc.db.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error {
var err error

msg.ObjectIDs = sort2.SortAsc(msg.ObjectIDs)

// 根据ID依次查询Object,ObjectBlock,PinnedObject,并根据升序的特点进行合并
objs, err := svc.db.Object().BatchGet(tx, msg.ObjectIDs)
if err != nil {
return fmt.Errorf("batch get objects: %w", err)
}

objIDIdx := 0
objIdx := 0
for objIDIdx < len(msg.ObjectIDs) && objIdx < len(objs) {
if msg.ObjectIDs[objIDIdx] < objs[objIdx].ObjectID {
objIDIdx++
continue
}

// 由于是使用msg.ObjectIDs去查询Object,因此不存在msg.ObjectIDs > Object.ObjectID的情况,
// 下面同理
obj := stgmod.ObjectDetail{
Object: objs[objIDIdx],
}
details[objIDIdx] = &obj
objIdx++
}

// 查询合并
blocks, err := svc.db.ObjectBlock().BatchGetByObjectID(tx, msg.ObjectIDs)
if err != nil {
return fmt.Errorf("batch get object blocks: %w", err)
}

objIDIdx = 0
blkIdx := 0
for objIDIdx < len(msg.ObjectIDs) && blkIdx < len(blocks) {
if details[objIDIdx] == nil {
objIDIdx++
continue
}

if msg.ObjectIDs[objIDIdx] < blocks[blkIdx].ObjectID {
objIDIdx++
continue
}

details[objIDIdx].Blocks = append(details[objIDIdx].Blocks, blocks[blkIdx])
blkIdx++
}

// 查询合并
pinneds, err := svc.db.PinnedObject().BatchGetByObjectID(tx, msg.ObjectIDs)
if err != nil {
return fmt.Errorf("batch get pinned objects: %w", err)
}

objIDIdx = 0
pinIdx := 0
for objIDIdx < len(msg.ObjectIDs) && pinIdx < len(pinneds) {
if details[objIDIdx] == nil {
objIDIdx++
continue
}

if msg.ObjectIDs[objIDIdx] < pinneds[pinIdx].ObjectID {
objIDIdx++
continue
}

details[objIDIdx].PinnedAt = append(details[objIDIdx].PinnedAt, pinneds[pinIdx].NodeID)
pinIdx++
}
return nil
})

if err != nil {
logger.Warn(err.Error())
return nil, mq.Failed(errorcode.OperationFailed, "get object details failed")
}

return mq.ReplyOK(coormq.RespGetObjectDetails(details))
}

func (svc *Service) UpdateObjectRedundancy(msg *coormq.UpdateObjectRedundancy) (*coormq.UpdateObjectRedundancyResp, *mq.CodeMessage) {
err := svc.db.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error {
return svc.db.Object().BatchUpdateRedundancy(tx, msg.Entries)
return svc.db.Object().BatchUpdateRedundancy(tx, msg.Updatings)
})
if err != nil {
logger.Warnf("batch updating redundancy: %s", err.Error())
return nil, mq.Failed(errorcode.OperationFailed, "batch update redundancy failed")
}

return mq.ReplyOK(coormq.RespChangeObjectRedundancy())
return mq.ReplyOK(coormq.RespUpdateObjectRedundancy())
}

func (svc *Service) UpdateObjectInfos(msg *coormq.UpdateObjectInfos) (*coormq.UpdateObjectInfosResp, *mq.CodeMessage) {
var sucs []cdssdk.ObjectID
err := svc.db.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error {
msg.Updatings = sort2.Sort(msg.Updatings, func(o1, o2 cdssdk.UpdatingObject) int {
return sort2.Cmp(o1.ObjectID, o2.ObjectID)
})

objIDs := make([]cdssdk.ObjectID, len(msg.Updatings))
for i, obj := range msg.Updatings {
objIDs[i] = obj.ObjectID
}

oldObjs, err := svc.db.Object().BatchGet(tx, objIDs)
if err != nil {
return fmt.Errorf("batch getting objects: %w", err)
}
oldObjIDs := make([]cdssdk.ObjectID, len(oldObjs))
for i, obj := range oldObjs {
oldObjIDs[i] = obj.ObjectID
}

avaiUpdatings, notExistsObjs := pickByObjectIDs(msg.Updatings, oldObjIDs, func(obj cdssdk.UpdatingObject) cdssdk.ObjectID { return obj.ObjectID })
if len(notExistsObjs) > 0 {
// TODO 部分对象已经不存在
}

newObjs := make([]cdssdk.Object, len(avaiUpdatings))
for i := range newObjs {
newObjs[i] = oldObjs[i]
avaiUpdatings[i].ApplyTo(&newObjs[i])
}

err = svc.db.Object().BatchUpsertByPackagePath(tx, newObjs)
if err != nil {
return fmt.Errorf("batch create or update: %w", err)
}

sucs = lo.Map(newObjs, func(obj cdssdk.Object, _ int) cdssdk.ObjectID { return obj.ObjectID })
return nil
})

if err != nil {
logger.Warnf("batch updating objects: %s", err.Error())
return nil, mq.Failed(errorcode.OperationFailed, "batch update objects failed")
}

return mq.ReplyOK(coormq.RespUpdateObjectInfos(sucs))
}

// 根据objIDs从objs中挑选Object。
// len(objs) >= len(objIDs)
func pickByObjectIDs[T any](objs []T, objIDs []cdssdk.ObjectID, getID func(T) cdssdk.ObjectID) (picked []T, notFound []T) {
objIdx := 0
idIdx := 0

for idIdx < len(objIDs) && objIdx < len(objs) {
if getID(objs[objIdx]) < objIDs[idIdx] {
notFound = append(notFound, objs[objIdx])
objIdx++
continue
}

picked = append(picked, objs[objIdx])
objIdx++
idIdx++
}

return
}

func (svc *Service) MoveObjects(msg *coormq.MoveObjects) (*coormq.MoveObjectsResp, *mq.CodeMessage) {
var sucs []cdssdk.ObjectID
err := svc.db.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error {
msg.Movings = sort2.Sort(msg.Movings, func(o1, o2 cdssdk.MovingObject) int {
return sort2.Cmp(o1.ObjectID, o2.ObjectID)
})

objIDs := make([]cdssdk.ObjectID, len(msg.Movings))
for i, obj := range msg.Movings {
objIDs[i] = obj.ObjectID
}

oldObjs, err := svc.db.Object().BatchGet(tx, objIDs)
if err != nil {
return fmt.Errorf("batch getting objects: %w", err)
}
oldObjIDs := make([]cdssdk.ObjectID, len(oldObjs))
for i, obj := range oldObjs {
oldObjIDs[i] = obj.ObjectID
}

avaiMovings, notExistsObjs := pickByObjectIDs(msg.Movings, oldObjIDs, func(obj cdssdk.MovingObject) cdssdk.ObjectID { return obj.ObjectID })
if len(notExistsObjs) > 0 {
// TODO 部分对象已经不存在
}

// 筛选出PackageID变化、Path变化的对象,这两种对象要检测改变后是否有冲突
var pkgIDChangedObjs []cdssdk.Object
var pathChangedObjs []cdssdk.Object
for i := range avaiMovings {
if avaiMovings[i].PackageID != oldObjs[i].PackageID {
newObj := oldObjs[i]
avaiMovings[i].ApplyTo(&newObj)
pkgIDChangedObjs = append(pkgIDChangedObjs, newObj)
} else if avaiMovings[i].Path != oldObjs[i].Path {
newObj := oldObjs[i]
avaiMovings[i].ApplyTo(&newObj)
pathChangedObjs = append(pathChangedObjs, newObj)
}
}

var newObjs []cdssdk.Object
// 对于PackageID发生变化的对象,需要检查目标Package内是否存在同Path的对象
ensuredObjs, err := svc.ensurePackageChangedObjects(tx, msg.UserID, pkgIDChangedObjs)
if err != nil {
return err
}
newObjs = append(newObjs, ensuredObjs...)

// 对于只有Path发生变化的对象,则检查同Package内有没有同Path的对象
ensuredObjs, err = svc.ensurePathChangedObjects(tx, msg.UserID, pathChangedObjs)
if err != nil {
return err
}
newObjs = append(newObjs, ensuredObjs...)

err = svc.db.Object().BatchUpert(tx, newObjs)
if err != nil {
return fmt.Errorf("batch create or update: %w", err)
}

sucs = lo.Map(newObjs, func(obj cdssdk.Object, _ int) cdssdk.ObjectID { return obj.ObjectID })
return nil
})
if err != nil {
logger.Warn(err.Error())
return nil, mq.Failed(errorcode.OperationFailed, "move objects failed")
}

return mq.ReplyOK(coormq.RespMoveObjects(sucs))
}

func (svc *Service) ensurePackageChangedObjects(tx *sqlx.Tx, userID cdssdk.UserID, objs []cdssdk.Object) ([]cdssdk.Object, error) {
if len(objs) == 0 {
return nil, nil
}

type PackageObjects struct {
PackageID cdssdk.PackageID
ObjectByPath map[string]*cdssdk.Object
}

packages := make(map[cdssdk.PackageID]*PackageObjects)
for _, obj := range objs {
pkg, ok := packages[obj.PackageID]
if !ok {
pkg = &PackageObjects{
PackageID: obj.PackageID,
ObjectByPath: make(map[string]*cdssdk.Object),
}
packages[obj.PackageID] = pkg
}

if pkg.ObjectByPath[obj.Path] == nil {
o := obj
pkg.ObjectByPath[obj.Path] = &o
} else {
// TODO 有两个对象移动到同一个路径,有冲突
}
}

var willUpdateObjs []cdssdk.Object
for _, pkg := range packages {
_, err := svc.db.Package().GetUserPackage(tx, userID, pkg.PackageID)
if err == sql.ErrNoRows {
continue
}
if err != nil {
return nil, fmt.Errorf("getting user package by id: %w", err)
}

existsObjs, err := svc.db.Object().BatchGetByPackagePath(tx, pkg.PackageID, lo.Keys(pkg.ObjectByPath))
if err != nil {
return nil, fmt.Errorf("batch getting objects by package path: %w", err)
}

// 标记冲突的对象
for _, obj := range existsObjs {
pkg.ObjectByPath[obj.Path] = nil
// TODO 目标Package内有冲突的对象
}

for _, obj := range pkg.ObjectByPath {
if obj == nil {
continue
}
willUpdateObjs = append(willUpdateObjs, *obj)
}
}

return willUpdateObjs, nil
}

func (svc *Service) ensurePathChangedObjects(tx *sqlx.Tx, userID cdssdk.UserID, objs []cdssdk.Object) ([]cdssdk.Object, error) {
if len(objs) == 0 {
return nil, nil
}

objByPath := make(map[string]*cdssdk.Object)
for _, obj := range objs {
if objByPath[obj.Path] == nil {
o := obj
objByPath[obj.Path] = &o
} else {
// TODO 有两个对象移动到同一个路径,有冲突
}

}

_, err := svc.db.Package().GetUserPackage(tx, userID, objs[0].PackageID)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("getting user package by id: %w", err)
}

existsObjs, err := svc.db.Object().BatchGetByPackagePath(tx, objs[0].PackageID, lo.Map(objs, func(obj cdssdk.Object, idx int) string { return obj.Path }))
if err != nil {
return nil, fmt.Errorf("batch getting objects by package path: %w", err)
}

// 不支持两个对象交换位置的情况,因为数据库不支持
for _, obj := range existsObjs {
objByPath[obj.Path] = nil
}

var willMoveObjs []cdssdk.Object
for _, obj := range objByPath {
if obj == nil {
continue
}
willMoveObjs = append(willMoveObjs, *obj)
}

return willMoveObjs, nil
}

func (svc *Service) DeleteObjects(msg *coormq.DeleteObjects) (*coormq.DeleteObjectsResp, *mq.CodeMessage) {
err := svc.db.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error {
err := svc.db.Object().BatchDelete(tx, msg.ObjectIDs)
if err != nil {
return fmt.Errorf("batch deleting objects: %w", err)
}

err = svc.db.ObjectBlock().BatchDeleteByObjectID(tx, msg.ObjectIDs)
if err != nil {
return fmt.Errorf("batch deleting object blocks: %w", err)
}

err = svc.db.PinnedObject().BatchDeleteByObjectID(tx, msg.ObjectIDs)
if err != nil {
return fmt.Errorf("batch deleting pinned objects: %w", err)
}

return nil
})
if err != nil {
logger.Warnf("batch deleting objects: %s", err.Error())
return nil, mq.Failed(errorcode.OperationFailed, "batch delete objects failed")
}

return mq.ReplyOK(coormq.RespDeleteObjects())
}

+ 28
- 16
coordinator/internal/mq/package.go View File

@@ -35,15 +35,22 @@ func (svc *Service) GetPackage(msg *coormq.GetPackage) (*coormq.GetPackageResp,
return mq.ReplyOK(coormq.NewGetPackageResp(pkg))
}

// CreatePackage 创建一个新的包
// 参数:
// - msg: 包含创建包所需信息的请求消息
// 返回值:
// - *coormq.CreatePackageResp: 创建包成功的响应
// - *mq.CodeMessage: 错误时返回的错误信息
func (svc *Service) GetPackageByName(msg *coormq.GetPackageByName) (*coormq.GetPackageByNameResp, *mq.CodeMessage) {
pkg, err := svc.db.Package().GetUserPackageByName(svc.db.SQLCtx(), msg.UserID, msg.BucketName, msg.PackageName)
if err != nil {
logger.WithField("UserID", msg.UserID).
WithField("BucketName", msg.BucketName).
WithField("PackageName", msg.PackageName).
Warnf("get package by name: %s", err.Error())

return nil, mq.Failed(errorcode.OperationFailed, "get package by name failed")
}

return mq.ReplyOK(coormq.NewGetPackageByNameResp(pkg))
}

func (svc *Service) CreatePackage(msg *coormq.CreatePackage) (*coormq.CreatePackageResp, *mq.CodeMessage) {
var pkgID cdssdk.PackageID
// 在事务中执行创建包的操作
var pkg cdssdk.Package
err := svc.db.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error {
var err error

@@ -53,12 +60,16 @@ func (svc *Service) CreatePackage(msg *coormq.CreatePackage) (*coormq.CreatePack
return fmt.Errorf("bucket is not avaiable to the user")
}

// 创建包
pkgID, err = svc.db.Package().Create(tx, msg.BucketID, msg.Name)
pkgID, err := svc.db.Package().Create(tx, msg.BucketID, msg.Name)
if err != nil {
return fmt.Errorf("creating package: %w", err)
}

pkg, err = svc.db.Package().GetByID(tx, pkgID)
if err != nil {
return fmt.Errorf("getting package by id: %w", err)
}

return nil
})
if err != nil {
@@ -69,8 +80,7 @@ func (svc *Service) CreatePackage(msg *coormq.CreatePackage) (*coormq.CreatePack
return nil, mq.Failed(errorcode.OperationFailed, "creating package failed")
}

// 返回成功响应
return mq.ReplyOK(coormq.NewCreatePackageResp(pkgID))
return mq.ReplyOK(coormq.NewCreatePackageResp(pkg))
}

// UpdatePackage 更新包的信息
@@ -80,7 +90,8 @@ func (svc *Service) CreatePackage(msg *coormq.CreatePackage) (*coormq.CreatePack
// - *coormq.UpdatePackageResp: 更新包成功的响应
// - *mq.CodeMessage: 错误时返回的错误信息
func (svc *Service) UpdatePackage(msg *coormq.UpdatePackage) (*coormq.UpdatePackageResp, *mq.CodeMessage) {
// 在事务中执行更新包的操作

var added []cdssdk.Object
err := svc.db.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error {
// 验证包是否存在
_, err := svc.db.Package().GetByID(tx, msg.PackageID)
@@ -97,9 +108,11 @@ func (svc *Service) UpdatePackage(msg *coormq.UpdatePackage) (*coormq.UpdatePack

// 添加对象
if len(msg.Adds) > 0 {
if _, err := svc.db.Object().BatchAdd(tx, msg.PackageID, msg.Adds); err != nil {
ad, err := svc.db.Object().BatchAdd(tx, msg.PackageID, msg.Adds)
if err != nil {
return fmt.Errorf("adding objects: %w", err)
}
added = ad
}

return nil
@@ -110,8 +123,7 @@ func (svc *Service) UpdatePackage(msg *coormq.UpdatePackage) (*coormq.UpdatePack
return nil, mq.Failed(errorcode.OperationFailed, "update package failed")
}

// 返回成功响应
return mq.ReplyOK(coormq.NewUpdatePackageResp())
return mq.ReplyOK(coormq.NewUpdatePackageResp(added))
}

// DeletePackage 删除一个包


+ 473
- 0
log/agent.log View File

@@ -868,3 +868,476 @@
2024-04-11 12:19:52 [WARN] agent server err: deserialize error: channel is closed
2024-04-11 12:19:52 [ERRO] command server stopped with error: receive message error: channel is closed
2024-04-11 12:19:52 [INFO] command server stopped
2024-04-11 12:22:59 [DEBU] [:Collector] do testing
2024-04-11 12:27:59 [DEBU] [:Collector] do testing
2024-04-11 12:32:59 [DEBU] [:Collector] do testing
2024-04-11 12:37:59 [DEBU] [:Collector] do testing
2024-04-11 12:42:59 [DEBU] [:Collector] do testing
2024-04-11 12:47:59 [DEBU] [:Collector] do testing
2024-04-11 12:52:59 [DEBU] [:Collector] do testing
2024-04-11 12:57:59 [DEBU] [:Collector] do testing
2024-04-11 13:02:59 [DEBU] [:Collector] do testing
2024-04-11 13:07:59 [DEBU] [:Collector] do testing
2024-04-11 13:12:59 [DEBU] [:Collector] do testing
2024-04-11 13:17:59 [DEBU] [:Collector] do testing
2024-04-11 13:22:59 [DEBU] [:Collector] do testing
2024-04-11 13:27:59 [DEBU] [:Collector] do testing
2024-04-11 13:32:59 [DEBU] [:Collector] do testing
2024-04-11 13:37:59 [DEBU] [:Collector] do testing
2024-04-11 13:42:59 [DEBU] [:Collector] do testing
2024-04-11 13:47:59 [DEBU] [:Collector] do testing
2024-04-11 13:52:59 [DEBU] [:Collector] do testing
2024-04-11 13:57:59 [DEBU] [:Collector] do testing
2024-04-11 14:02:59 [DEBU] [:Collector] do testing
2024-04-11 14:07:59 [DEBU] [:Collector] do testing
2024-04-11 14:12:59 [DEBU] [:Collector] do testing
2024-04-11 14:17:59 [DEBU] [:Collector] do testing
2024-04-11 14:22:59 [DEBU] [:Collector] do testing
2024-04-11 14:27:59 [DEBU] [:Collector] do testing
2024-04-11 14:32:59 [DEBU] [:Collector] do testing
2024-04-11 14:37:59 [DEBU] [:Collector] do testing
2024-04-11 14:42:59 [DEBU] [:Collector] do testing
2024-04-11 14:47:59 [DEBU] [:Collector] do testing
2024-04-11 14:50:53 [DEBU] [:Collector] do testing
2024-04-11 14:50:53 [INFO] [:Collector] start connectivity reporter
2024-04-11 14:51:38 [INFO] start serving distlock
2024-04-11 14:51:38 [INFO] start serving command server
2024-04-11 14:51:38 [INFO] start serving grpc
2024-04-11 14:52:36 [DEBU] [:Collector] do testing
2024-04-11 14:58:21 [DEBU] [:Collector] do testing
2024-04-11 14:59:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 14:59:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:03:21 [DEBU] [:Collector] do testing
2024-04-11 15:04:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:04:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:08:21 [DEBU] [:Collector] do testing
2024-04-11 15:09:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:09:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:13:21 [DEBU] [:Collector] do testing
2024-04-11 15:14:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:14:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:18:21 [DEBU] [:Collector] do testing
2024-04-11 15:19:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:19:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:23:21 [DEBU] [:Collector] do testing
2024-04-11 15:24:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:24:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:28:21 [DEBU] [:Collector] do testing
2024-04-11 15:29:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:29:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:33:21 [DEBU] [:Collector] do testing
2024-04-11 15:34:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:34:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:38:21 [DEBU] [:Collector] do testing
2024-04-11 15:39:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:39:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:43:21 [DEBU] [:Collector] do testing
2024-04-11 15:44:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:44:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:48:21 [DEBU] [:Collector] do testing
2024-04-11 15:49:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:49:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:53:21 [DEBU] [:Collector] do testing
2024-04-11 15:54:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:54:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 15:58:21 [DEBU] [:Collector] do testing
2024-04-11 15:59:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 15:59:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:03:21 [DEBU] [:Collector] do testing
2024-04-11 16:04:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:04:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:08:21 [DEBU] [:Collector] do testing
2024-04-11 16:09:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:09:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:13:21 [DEBU] [:Collector] do testing
2024-04-11 16:14:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:14:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:18:21 [DEBU] [:Collector] do testing
2024-04-11 16:19:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:19:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:23:21 [DEBU] [:Collector] do testing
2024-04-11 16:24:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:24:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:28:21 [DEBU] [:Collector] do testing
2024-04-11 16:29:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:29:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:33:21 [DEBU] [:Collector] do testing
2024-04-11 16:34:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:34:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:38:21 [DEBU] [:Collector] do testing
2024-04-11 16:39:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:39:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:43:21 [DEBU] [:Collector] do testing
2024-04-11 16:44:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:44:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:48:21 [DEBU] [:Collector] do testing
2024-04-11 16:49:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:49:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:53:21 [DEBU] [:Collector] do testing
2024-04-11 16:54:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:54:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 16:58:21 [DEBU] [:Collector] do testing
2024-04-11 16:59:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 16:59:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 17:03:21 [DEBU] [:Collector] do testing
2024-04-11 17:04:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 17:04:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 17:08:21 [DEBU] [:Collector] do testing
2024-04-11 17:09:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 17:09:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 17:13:21 [DEBU] [:Collector] do testing
2024-04-11 17:14:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 17:14:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 17:18:21 [DEBU] [:Collector] do testing
2024-04-11 17:19:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 17:19:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 17:23:21 [DEBU] [:Collector] do testing
2024-04-11 17:24:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 17:24:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 17:28:21 [DEBU] [:Collector] do testing
2024-04-11 17:29:49 [WARN] parsing user id $RECYCLE.BIN: strconv.ParseInt: parsing "$RECYCLE.BIN": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id AppGallery: strconv.ParseInt: parsing "AppGallery": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id BrowerDownload: strconv.ParseInt: parsing "BrowerDownload": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id Config.Msi: strconv.ParseInt: parsing "Config.Msi": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id Huawei Share: strconv.ParseInt: parsing "Huawei Share": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id Others: strconv.ParseInt: parsing "Others": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id Soft: strconv.ParseInt: parsing "Soft": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id SoftPackage: strconv.ParseInt: parsing "SoftPackage": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id System Volume Information: strconv.ParseInt: parsing "System Volume Information": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id Work: strconv.ParseInt: parsing "Work": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id draw-graph: strconv.ParseInt: parsing "draw-graph": invalid syntax
2024-04-11 17:29:49 [WARN] parsing user id tmp: strconv.ParseInt: parsing "tmp": invalid syntax
2024-04-11 19:49:56 [DEBU] [:Collector] do testing
2024-04-11 19:49:56 [WARN] agent server err: deserialize error: channel is closed
2024-04-11 19:49:56 [ERRO] command server stopped with error: receive message error: channel is closed
2024-04-11 19:49:56 [INFO] command server stopped
2024-04-11 20:24:34 [DEBU] [:Collector] do testing
2024-04-11 20:28:21 [DEBU] [:Collector] do testing
2024-04-11 20:33:21 [DEBU] [:Collector] do testing
2024-04-11 20:38:21 [DEBU] [:Collector] do testing
2024-04-11 20:43:21 [DEBU] [:Collector] do testing
2024-04-11 20:48:21 [DEBU] [:Collector] do testing
2024-04-11 20:53:21 [DEBU] [:Collector] do testing
2024-04-11 20:58:21 [DEBU] [:Collector] do testing
2024-04-11 21:03:21 [DEBU] [:Collector] do testing
2024-04-11 21:08:21 [DEBU] [:Collector] do testing
2024-04-11 21:13:21 [DEBU] [:Collector] do testing
2024-04-11 21:29:30 [DEBU] [:Collector] do testing
2024-04-11 21:33:21 [DEBU] [:Collector] do testing
2024-04-11 21:38:21 [DEBU] [:Collector] do testing
2024-04-11 21:43:21 [DEBU] [:Collector] do testing
2024-04-11 21:48:21 [DEBU] [:Collector] do testing
2024-04-11 21:53:21 [DEBU] [:Collector] do testing
2024-04-11 21:58:21 [DEBU] [:Collector] do testing
2024-04-11 22:03:21 [DEBU] [:Collector] do testing
2024-04-11 22:08:21 [DEBU] [:Collector] do testing
2024-04-11 22:13:21 [DEBU] [:Collector] do testing
2024-04-11 22:18:21 [DEBU] [:Collector] do testing
2024-04-11 22:23:21 [DEBU] [:Collector] do testing
2024-04-11 22:28:21 [DEBU] [:Collector] do testing
2024-04-11 22:33:21 [DEBU] [:Collector] do testing
2024-04-11 22:38:21 [DEBU] [:Collector] do testing
2024-04-11 22:53:19 [DEBU] [:Collector] do testing
2024-04-11 22:53:21 [DEBU] [:Collector] do testing
2024-04-12 01:42:33 [DEBU] [:Collector] do testing
2024-04-12 08:50:14 [DEBU] [:Collector] do testing

+ 2291
- 0
log/scanner.log
File diff suppressed because it is too large
View File


+ 22
- 22
scanner/internal/event/check_package_redundancy.go View File

@@ -109,7 +109,7 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) {
}
}

var changedObjects []coormq.ChangeObjectRedundancyEntry
var changedObjects []coormq.UpdatingObjectRedundancy

defRep := cdssdk.DefaultRepRedundancy
defEC := cdssdk.DefaultECRedundancy
@@ -136,7 +136,7 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) {
defer mutex.Unlock()

for _, obj := range getObjs.Objects {
var entry *coormq.ChangeObjectRedundancyEntry
var updating *coormq.UpdatingObjectRedundancy
var err error

shouldUseEC := obj.Object.Size > config.Cfg().ECFileSizeThreshold
@@ -145,32 +145,32 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) {
case *cdssdk.NoneRedundancy:
if shouldUseEC {
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec")
entry, err = t.noneToEC(obj, &defEC, newECNodes)
updating, err = t.noneToEC(obj, &defEC, newECNodes)
} else {
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep")
entry, err = t.noneToRep(obj, &defRep, newRepNodes)
updating, err = t.noneToRep(obj, &defRep, newRepNodes)
}

case *cdssdk.RepRedundancy:
if shouldUseEC {
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec")
entry, err = t.repToEC(obj, &defEC, newECNodes)
updating, err = t.repToEC(obj, &defEC, newECNodes)
} else {
entry, err = t.repToRep(obj, &defRep, rechoosedRepNodes)
updating, err = t.repToRep(obj, &defRep, rechoosedRepNodes)
}

case *cdssdk.ECRedundancy:
if shouldUseEC {
uploadNodes := t.rechooseNodesForEC(obj, red, allNodes)
entry, err = t.ecToEC(obj, red, &defEC, uploadNodes)
updating, err = t.ecToEC(obj, red, &defEC, uploadNodes)
} else {
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep")
entry, err = t.ecToRep(obj, red, &defRep, newRepNodes)
updating, err = t.ecToRep(obj, red, &defRep, newRepNodes)
}
}

if entry != nil {
changedObjects = append(changedObjects, *entry)
if updating != nil {
changedObjects = append(changedObjects, *updating)
}

if err != nil {
@@ -182,7 +182,7 @@ func (t *CheckPackageRedundancy) Execute(execCtx ExecuteContext) {
return
}

_, err = coorCli.ChangeObjectRedundancy(coormq.ReqChangeObjectRedundancy(changedObjects))
_, err = coorCli.UpdateObjectRedundancy(coormq.ReqUpdateObjectRedundancy(changedObjects))
if err != nil {
log.Warnf("requesting to change object redundancy: %s", err.Error())
return
@@ -367,7 +367,7 @@ func (t *CheckPackageRedundancy) chooseSoManyNodes(count int, nodes []*NodeLoadI
return chosen
}

func (t *CheckPackageRedundancy) noneToRep(obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.ChangeObjectRedundancyEntry, error) {
func (t *CheckPackageRedundancy) noneToRep(obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
if len(obj.Blocks) == 0 {
return nil, fmt.Errorf("object is not cached on any nodes, cannot change its redundancy to rep")
}
@@ -389,14 +389,14 @@ func (t *CheckPackageRedundancy) noneToRep(obj stgmod.ObjectDetail, red *cdssdk.
})
}

return &coormq.ChangeObjectRedundancyEntry{
return &coormq.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
Redundancy: red,
Blocks: blocks,
}, nil
}

func (t *CheckPackageRedundancy) noneToEC(obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.ChangeObjectRedundancyEntry, error) {
func (t *CheckPackageRedundancy) noneToEC(obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
@@ -443,14 +443,14 @@ func (t *CheckPackageRedundancy) noneToEC(obj stgmod.ObjectDetail, red *cdssdk.E
})
}

return &coormq.ChangeObjectRedundancyEntry{
return &coormq.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
Redundancy: red,
Blocks: blocks,
}, nil
}

func (t *CheckPackageRedundancy) repToRep(obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.ChangeObjectRedundancyEntry, error) {
func (t *CheckPackageRedundancy) repToRep(obj stgmod.ObjectDetail, red *cdssdk.RepRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
if len(obj.Blocks) == 0 {
return nil, fmt.Errorf("object is not cached on any nodes, cannot change its redundancy to rep")
}
@@ -479,18 +479,18 @@ func (t *CheckPackageRedundancy) repToRep(obj stgmod.ObjectDetail, red *cdssdk.R
})
}

return &coormq.ChangeObjectRedundancyEntry{
return &coormq.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
Redundancy: red,
Blocks: blocks,
}, nil
}

func (t *CheckPackageRedundancy) repToEC(obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.ChangeObjectRedundancyEntry, error) {
func (t *CheckPackageRedundancy) repToEC(obj stgmod.ObjectDetail, red *cdssdk.ECRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
return t.noneToEC(obj, red, uploadNodes)
}

func (t *CheckPackageRedundancy) ecToRep(obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.RepRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.ChangeObjectRedundancyEntry, error) {
func (t *CheckPackageRedundancy) ecToRep(obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.RepRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
@@ -556,14 +556,14 @@ func (t *CheckPackageRedundancy) ecToRep(obj stgmod.ObjectDetail, srcRed *cdssdk
})
}

return &coormq.ChangeObjectRedundancyEntry{
return &coormq.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
Redundancy: tarRed,
Blocks: blocks,
}, nil
}

func (t *CheckPackageRedundancy) ecToEC(obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.ECRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.ChangeObjectRedundancyEntry, error) {
func (t *CheckPackageRedundancy) ecToEC(obj stgmod.ObjectDetail, srcRed *cdssdk.ECRedundancy, tarRed *cdssdk.ECRedundancy, uploadNodes []*NodeLoadInfo) (*coormq.UpdatingObjectRedundancy, error) {
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
return nil, fmt.Errorf("new coordinator client: %w", err)
@@ -654,7 +654,7 @@ func (t *CheckPackageRedundancy) ecToEC(obj stgmod.ObjectDetail, srcRed *cdssdk.
newBlocks[idx].FileHash = v.(string)
}

return &coormq.ChangeObjectRedundancyEntry{
return &coormq.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
Redundancy: tarRed,
Blocks: newBlocks,


+ 13
- 13
scanner/internal/event/clean_pinned.go View File

@@ -109,7 +109,7 @@ func (t *CleanPinned) Execute(execCtx ExecuteContext) {
pinPlans := make(map[cdssdk.NodeID]*[]string)

// 对于rep对象,统计出所有对象块分布最多的两个节点,用这两个节点代表所有rep对象块的分布,去进行退火算法
var repObjectsUpdateEntries []coormq.ChangeObjectRedundancyEntry
var repObjectsUpdating []coormq.UpdatingObjectRedundancy
repMostNodeIDs := t.summaryRepObjectBlockNodes(repObjects)
solu := t.startAnnealing(allNodeInfos, readerNodeIDs, annealingObject{
totalBlockCount: 1,
@@ -118,11 +118,11 @@ func (t *CleanPinned) Execute(execCtx ExecuteContext) {
blocks: nil,
})
for _, obj := range repObjects {
repObjectsUpdateEntries = append(repObjectsUpdateEntries, t.makePlansForRepObject(solu, obj, pinPlans))
repObjectsUpdating = append(repObjectsUpdating, t.makePlansForRepObject(solu, obj, pinPlans))
}

// 对于ec对象,则每个对象单独进行退火算法
var ecObjectsUpdateEntries []coormq.ChangeObjectRedundancyEntry
var ecObjectsUpdating []coormq.UpdatingObjectRedundancy
for _, obj := range ecObjects {
ecRed := obj.Object.Redundancy.(*cdssdk.ECRedundancy)
solu := t.startAnnealing(allNodeInfos, readerNodeIDs, annealingObject{
@@ -131,7 +131,7 @@ func (t *CleanPinned) Execute(execCtx ExecuteContext) {
pinnedAt: obj.PinnedAt,
blocks: obj.Blocks,
})
ecObjectsUpdateEntries = append(ecObjectsUpdateEntries, t.makePlansForECObject(allNodeInfos, solu, obj, &planBld))
ecObjectsUpdating = append(ecObjectsUpdating, t.makePlansForECObject(allNodeInfos, solu, obj, &planBld))
}

ioSwRets, err := t.executePlans(execCtx, pinPlans, &planBld)
@@ -141,13 +141,13 @@ func (t *CleanPinned) Execute(execCtx ExecuteContext) {
}

// 根据按照方案进行调整的结果,填充更新元数据的命令
for i := range ecObjectsUpdateEntries {
t.populateECObjectEntry(&ecObjectsUpdateEntries[i], ecObjects[i], ioSwRets)
for i := range ecObjectsUpdating {
t.populateECObjectEntry(&ecObjectsUpdating[i], ecObjects[i], ioSwRets)
}

finalEntries := append(repObjectsUpdateEntries, ecObjectsUpdateEntries...)
finalEntries := append(repObjectsUpdating, ecObjectsUpdating...)
if len(finalEntries) > 0 {
_, err = coorCli.ChangeObjectRedundancy(coormq.ReqChangeObjectRedundancy(finalEntries))
_, err = coorCli.UpdateObjectRedundancy(coormq.ReqUpdateObjectRedundancy(finalEntries))
if err != nil {
log.Warnf("changing object redundancy: %s", err.Error())
return
@@ -715,8 +715,8 @@ func (t *CleanPinned) alwaysAccept(curTemp float64, dScore float64, coolingRate
return v > rand.Float64()
}

func (t *CleanPinned) makePlansForRepObject(solu annealingSolution, obj stgmod.ObjectDetail, pinPlans map[cdssdk.NodeID]*[]string) coormq.ChangeObjectRedundancyEntry {
entry := coormq.ChangeObjectRedundancyEntry{
func (t *CleanPinned) makePlansForRepObject(solu annealingSolution, obj stgmod.ObjectDetail, pinPlans map[cdssdk.NodeID]*[]string) coormq.UpdatingObjectRedundancy {
entry := coormq.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
Redundancy: obj.Object.Redundancy,
}
@@ -748,8 +748,8 @@ func (t *CleanPinned) makePlansForRepObject(solu annealingSolution, obj stgmod.O
return entry
}

func (t *CleanPinned) makePlansForECObject(allNodeInfos map[cdssdk.NodeID]*cdssdk.Node, solu annealingSolution, obj stgmod.ObjectDetail, planBld *plans.PlanBuilder) coormq.ChangeObjectRedundancyEntry {
entry := coormq.ChangeObjectRedundancyEntry{
func (t *CleanPinned) makePlansForECObject(allNodeInfos map[cdssdk.NodeID]*cdssdk.Node, solu annealingSolution, obj stgmod.ObjectDetail, planBld *plans.PlanBuilder) coormq.UpdatingObjectRedundancy {
entry := coormq.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID,
Redundancy: obj.Object.Redundancy,
}
@@ -871,7 +871,7 @@ func (t *CleanPinned) executePlans(execCtx ExecuteContext, pinPlans map[cdssdk.N
return ioSwRets, nil
}

func (t *CleanPinned) populateECObjectEntry(entry *coormq.ChangeObjectRedundancyEntry, obj stgmod.ObjectDetail, ioRets map[string]any) {
func (t *CleanPinned) populateECObjectEntry(entry *coormq.UpdatingObjectRedundancy, obj stgmod.ObjectDetail, ioRets map[string]any) {
for i := range entry.Blocks {
if entry.Blocks[i].FileHash != "" {
continue


Loading…
Cancel
Save