diff --git a/models/error.go b/models/error.go old mode 100644 new mode 100755 index 7fd61303f..c29462e7d --- a/models/error.go +++ b/models/error.go @@ -6,9 +6,8 @@ package models import ( - "fmt" - "code.gitea.io/gitea/modules/git" + "fmt" ) // ErrNotExist represents a non-exist error. @@ -1971,3 +1970,19 @@ func IsErrOAuthApplicationNotFound(err error) bool { func (err ErrOAuthApplicationNotFound) Error() string { return fmt.Sprintf("OAuth application not found [ID: %d]", err.ID) } + +// ErrFileChunkNotExist represents a "FileChunkNotExist" kind of error. +type ErrFileChunkNotExist struct { + Md5 string + Uuid string +} + +func (err ErrFileChunkNotExist) Error() string { + return fmt.Sprintf("fileChunk does not exist [md5: %s, uuid: %s]", err.Md5, err.Uuid) +} + +// IsErrFileChunkNotExist checks if an error is a ErrFileChunkNotExist. +func IsErrFileChunkNotExist(err error) bool { + _, ok := err.(ErrFileChunkNotExist) + return ok +} diff --git a/models/file_chunk.go b/models/file_chunk.go new file mode 100755 index 000000000..5291ef7bd --- /dev/null +++ b/models/file_chunk.go @@ -0,0 +1,78 @@ +package models + +import ( + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/xorm" +) + +const ( + FileNotUploaded int = iota + FileUploaded +) + +type FileChunk struct { + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + Md5 string `xorm:"UNIQUE"` + IsUploaded int `xorm:"DEFAULT 0"` // not uploaded: 0, uploaded: 1 + HasUploaded string //chunkNumbers 1,2,3 + UploadID string `xorm:"UNIQUE"`//minio upload id + TotalChunks int64 + Size int64 + UserID int64 `xorm:"INDEX"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} + +// GetFileChunkByMD5 returns fileChunk by given id +func GetFileChunkByMD5(md5 string) (*FileChunk, error) { + return getFileChunkByMD5(x, md5) +} + +func getFileChunkByMD5(e Engine, md5 string) (*FileChunk, error) { + fileChunk := new(FileChunk) + + if has, err := e.Where("md5 = ?", md5).Get(fileChunk); err != nil { + return nil, err + } else if !has { + return nil, ErrFileChunkNotExist{md5, ""} + } + return fileChunk, nil +} + +// GetAttachmentByID returns attachment by given id +func GetFileChunkByUUID(uuid string) (*FileChunk, error) { + return getFileChunkByUUID(x, uuid) +} + +func getFileChunkByUUID(e Engine, uuid string) (*FileChunk, error) { + fileChunk := new(FileChunk) + + if has, err := e.Where("uuid = ?", uuid).Get(fileChunk); err != nil { + return nil, err + } else if !has { + return nil, ErrFileChunkNotExist{"", uuid} + } + return fileChunk, nil +} + +// InsertFileChunk insert a record into file_chunk. +func InsertFileChunk(fileChunk *FileChunk) (_ *FileChunk, err error) { + if _, err := x.Insert(fileChunk); err != nil { + return nil, err + } + + return fileChunk,nil +} + +// UpdateAttachment updates the given attachment in database +func UpdateFileChunk(fileChunk *FileChunk) error { + return updateFileChunk(x, fileChunk) +} + +func updateFileChunk(e Engine, fileChunk *FileChunk) error { + var sess *xorm.Session + sess = e.Where("uuid = ?", fileChunk.UUID) + _, err := sess.Cols("is_uploaded", "has_uploaded").Update(fileChunk) + return err +} diff --git a/models/models.go b/models/models.go old mode 100644 new mode 100755 index e3d7d934f..3009b2b3c --- a/models/models.go +++ b/models/models.go @@ -126,6 +126,7 @@ func init() { new(LanguageStat), new(EmailHash), new(Dataset), + new(FileChunk), ) gonicNames := []string{"SSL", "UID"} diff --git a/modules/storage/minio_ext.go b/modules/storage/minio_ext.go index 410ccd452..4f9d97433 100755 --- a/modules/storage/minio_ext.go +++ b/modules/storage/minio_ext.go @@ -98,13 +98,17 @@ func getClients()(*minio_ext.Client, *miniov6.Core, error){ return client, core, nil } -func GenMultiPartSignedUrl(bucketName string, objectName string, uploadId string, partNumber int, partSize int64) (string, error) { +func GenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, partSize int64) (string, error) { minioClient, _, err := getClients() if err != nil { log.Error("getClients failed:", err.Error()) return "", err } + minio := setting.Attachment.Minio + bucketName := minio.Bucket + objectName := strings.TrimPrefix(path.Join(minio.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") + return minioClient.GenUploadPartSignedUrl(uploadId, bucketName, objectName, partNumber, partSize, PresignedUploadPartUrlExpireTime) } diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 253585e7c..6803b2c5d 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -277,6 +277,28 @@ func AddAttachment(ctx *context.Context) { }) } +func GetSuccessChunks(ctx *context.Context) { + fileMD5 := ctx.Params("fileMD5") + + fileChunk, err := models.GetFileChunkByMD5(fileMD5) + if err != nil { + if models.IsErrFileChunkNotExist(err) { + ctx.Error(404) + } else { + ctx.ServerError("GetFileChunkByMD5", err) + } + return + } + + ctx.JSON(200, map[string]string{ + "uuid": fileChunk.UUID, + "uploaded": strconv.Itoa(fileChunk.IsUploaded), + "uploadID":fileChunk.UploadID, + "chunks": fileChunk.HasUploaded, + }) + +} + func NewMultipart(ctx *context.Context) { if !setting.Attachment.Enabled { ctx.Error(404, "attachment is not enabled") @@ -291,15 +313,28 @@ func NewMultipart(ctx *context.Context) { if setting.Attachment.StoreType == storage.MinioStorageType { uuid := gouuid.NewV4().String() - url, err := storage.NewMultiPartUpload(uuid) + uploadID, err := storage.NewMultiPartUpload(uuid) if err != nil { ctx.ServerError("NewMultipart", err) return } + _, err = models.InsertFileChunk(&models.FileChunk{ + UUID: uuid, + UserID: ctx.User.ID, + UploadID: uploadID, + Md5: ctx.Params("md5"), + Size: ctx.ParamsInt64("size"), + }) + + if err != nil { + ctx.Error(500, fmt.Sprintf("InsertFileChunk: %v", err)) + return + } + ctx.JSON(200, map[string]string{ "uuid": uuid, - "url": url, + "uploadID": uploadID, }) } else { ctx.Error(404, "storage type is not enabled") @@ -307,18 +342,52 @@ func NewMultipart(ctx *context.Context) { } } +func GetMultipartUploadUrl(ctx *context.Context) { + uuid := ctx.Query("uuid") + uploadID := ctx.Params("uploadID") + partNumber := ctx.ParamsInt("partNumber") + size := ctx.ParamsInt64("size") + + url,err := storage.GenMultiPartSignedUrl(uuid, uploadID, partNumber, size) + if err != nil { + ctx.Error(500, fmt.Sprintf("GenMultiPartSignedUrl failed: %v", err)) + return + } + + ctx.JSON(200, map[string]string{ + "url": url, + }) +} func CompleteMultipart(ctx *context.Context) { uuid := ctx.Query("uuid") uploadID := ctx.Query("uploadID") completedParts := ctx.Query("completedParts") - _, err := storage.CompleteMultiPartUpload(uuid, uploadID, completedParts) + fileChunk, err := models.GetFileChunkByUUID(uuid) + if err != nil { + if models.IsErrFileChunkNotExist(err) { + ctx.Error(404) + } else { + ctx.ServerError("GetFileChunkByUUID", err) + } + return + } + + _, err = storage.CompleteMultiPartUpload(uuid, uploadID, completedParts) if err != nil { ctx.Error(500, fmt.Sprintf("CompleteMultiPartUpload failed: %v", err)) return } + fileChunk.IsUploaded = models.FileUploaded + + err = models.UpdateFileChunk(fileChunk) + if err != nil { + ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err)) + return + } + _, err = models.InsertAttachment(&models.Attachment{ UUID: uuid, UploaderID: ctx.User.ID, @@ -337,3 +406,30 @@ func CompleteMultipart(ctx *context.Context) { "result_code": "0", }) } + +func UpdateMultipart(ctx *context.Context) { + uuid := ctx.Query("uuid") + partNumber := ctx.QueryInt("partNumber") + + fileChunk, err := models.GetFileChunkByUUID(uuid) + if err != nil { + if models.IsErrFileChunkNotExist(err) { + ctx.Error(404) + } else { + ctx.ServerError("GetFileChunkByUUID", err) + } + return + } + + fileChunk.HasUploaded += "," + strconv.Itoa(partNumber) + + err = models.UpdateFileChunk(fileChunk) + if err != nil { + ctx.Error(500, fmt.Sprintf("UpdateFileChunk: %v", err)) + return + } + + ctx.JSON(200, map[string]string{ + "result_code": "0", + }) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 6216261cd..ea8ea3c16 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -521,8 +521,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/get_pre_url", repo.GetPresignedPutObjectURL) m.Post("/add", repo.AddAttachment) m.Post("/private", repo.UpdatePublicAttachment) + m.Get("/get_chunks", repo.GetSuccessChunks) m.Get("/new_multipart", repo.NewMultipart) + m.Get("/get_multipart_url", repo.GetMultipartUploadUrl) m.Post("/complete_multipart", repo.CompleteMultipart) + m.Post("/update_multipart", repo.UpdateMultipart) }, reqSignIn) m.Group("/:username", func() {