From 4849a7719c1f6afd443135398923eb4b40ba8d2b Mon Sep 17 00:00:00 2001 From: yuyuanshifu <747342561@qq.com> Date: Thu, 23 Jul 2020 19:18:29 +0800 Subject: [PATCH] chunk upload --- models/file_chunk.go | 20 ++++++++++++++++++-- modules/storage/minio_ext.go | 23 ++++++++++------------- routers/repo/attachment.go | 5 ++--- web_src/js/App.vue | 32 +++++++++++++++----------------- web_src/js/index.js | 4 ++++ 5 files changed, 49 insertions(+), 35 deletions(-) diff --git a/models/file_chunk.go b/models/file_chunk.go index 9c83d54dd..d20a1cbdf 100755 --- a/models/file_chunk.go +++ b/models/file_chunk.go @@ -13,13 +13,13 @@ const ( type FileChunk struct { ID int64 `xorm:"pk autoincr"` UUID string `xorm:"uuid UNIQUE"` - Md5 string `xorm:"UNIQUE"` + Md5 string `xorm:"INDEX"` IsUploaded int `xorm:"DEFAULT 0"` // not uploaded: 0, uploaded: 1 UploadID string `xorm:"UNIQUE"`//minio upload id TotalChunks int Size int64 UserID int64 `xorm:"INDEX"` - CompletedParts []string // chunkNumber+etag eg: ,1-asqwewqe21312312.2-123hjkas + CompletedParts []string `xorm:"DEFAULT """`// chunkNumber+etag eg: ,1-asqwewqe21312312.2-123hjkas CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } @@ -40,6 +40,22 @@ func getFileChunkByMD5(e Engine, md5 string) (*FileChunk, error) { return fileChunk, nil } +// GetFileChunkByMD5 returns fileChunk by given id +func GetFileChunkByMD5AndUser(md5 string, userID int64) (*FileChunk, error) { + return getFileChunkByMD5AndUser(x, md5, userID) +} + +func getFileChunkByMD5AndUser(e Engine, md5 string, userID int64) (*FileChunk, error) { + fileChunk := new(FileChunk) + + if has, err := e.Where("md5 = ? and user_id = ?", md5, userID).Get(fileChunk); err != nil { + return nil, err + } else if !has { + return nil, ErrFileChunkNotExist{md5, ""} + } + return fileChunk, nil +} + // GetAttachmentByID returns attachment by given id func GetFileChunkByUUID(uuid string) (*FileChunk, error) { return getFileChunkByUUID(x, uuid) diff --git a/modules/storage/minio_ext.go b/modules/storage/minio_ext.go index e0d275002..251532cad 100755 --- a/modules/storage/minio_ext.go +++ b/modules/storage/minio_ext.go @@ -1,10 +1,10 @@ package storage import ( - "encoding/json" "encoding/xml" "path" "sort" + "strconv" "strings" "sync" "time" @@ -127,7 +127,7 @@ func NewMultiPartUpload(uuid string) (string, error){ return core.NewMultipartUpload(bucketName, objectName, miniov6.PutObjectOptions{}) } -func CompleteMultiPartUpload(uuid string, uploadID string, complParts string) (string, error){ +func CompleteMultiPartUpload(uuid string, uploadID string, complParts []string) (string, error){ _, core, err := getClients() if err != nil { log.Error("getClients failed:", err.Error()) @@ -138,19 +138,16 @@ func CompleteMultiPartUpload(uuid string, uploadID string, complParts string) (s bucketName := minio.Bucket objectName := strings.TrimPrefix(path.Join(minio.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") - //complParts:{"completedParts":[{"partNumber":2,"eTag":'"684929e7fe8b996d495e7b152d34ae37"'}]} - var parts CompleteParts - err = json.Unmarshal([]byte(complParts), &parts) - if err != nil { - log.Error("json.Unmarshal(%s) failed:(%s)", complParts, err.Error()) - return "", err - } - // Complete multipart upload. var complMultipartUpload completeMultipartUpload - for _,part := range parts.Data { + for _,part := range complParts { + partNumber, err := strconv.Atoi(strings.Split(part,"-")[0]) + if err != nil { + log.Error(err.Error()) + return "",err + } complMultipartUpload.Parts =append(complMultipartUpload.Parts, miniov6.CompletePart{ - PartNumber:part.PartNumber, - ETag:part.ETag, + PartNumber: partNumber, + ETag: strings.Split(part,"-")[1], }) } diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index dea78a2f0..a35eb7745 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -336,7 +336,7 @@ func UpdateAttachmentDecompressState(ctx *context.Context) { func GetSuccessChunks(ctx *context.Context) { fileMD5 := ctx.Query("md5") - fileChunk, err := models.GetFileChunkByMD5(fileMD5) + fileChunk, err := models.GetFileChunkByMD5AndUser(fileMD5, ctx.User.ID) if err != nil { if models.IsErrFileChunkNotExist(err) { ctx.JSON(200, map[string]string{ @@ -447,7 +447,6 @@ func GetMultipartUploadUrl(ctx *context.Context) { func CompleteMultipart(ctx *context.Context) { uuid := ctx.Query("uuid") uploadID := ctx.Query("uploadID") - completedParts := ctx.Query("completedParts") fileChunk, err := models.GetFileChunkByUUID(uuid) if err != nil { @@ -459,7 +458,7 @@ func CompleteMultipart(ctx *context.Context) { return } - _, err = storage.CompleteMultiPartUpload(uuid, uploadID, completedParts) + _, err = storage.CompleteMultiPartUpload(uuid, uploadID, fileChunk.CompletedParts) if err != nil { ctx.Error(500, fmt.Sprintf("CompleteMultiPartUpload failed: %v", err)) return diff --git a/web_src/js/App.vue b/web_src/js/App.vue index b70731903..76a341d03 100755 --- a/web_src/js/App.vue +++ b/web_src/js/App.vue @@ -28,7 +28,7 @@ export default { data () { return { - options: { + /*options: { target: 'http://localhost:9000/upload', testChunks: false, chunkSize: 1024*1024*64, //64MB @@ -45,9 +45,9 @@ .replace(/\sminutes?/, '分钟') .replace(/\sseconds?/, '秒') } - }, + },*/ attrs: { - accept: '*' + accept: 'img/*' }, panelShow: false, //选择文件后，展示上传panel collapse: false, @@ -83,7 +83,6 @@ file.uuid = response.data.uuid; file.uploaded = response.data.uploaded; file.chunks = response.data.chunks; - console.log(file.chunks); resolve(response); }).catch(function (error) { console.log(error); @@ -194,7 +193,6 @@ await uploadMinio(urls[currentChunk], e); if (etags[currentChunk] != "") { //更新数据库：分片上传结果 - console.log(etags[currentChunk]); await updateChunk(currentChunk); } else { return; @@ -212,7 +210,9 @@ axios.post('/attachments/complete_multipart', qs.stringify({ uuid: file.uuid, uploadID: file.uploadID, - etag: etags[currentChunk], + file_name: file.name, + size: file.size, + //dataset_id: datasetID, _csrf: csrf })).then(function (response) { resolve(response); @@ -233,8 +233,10 @@ console.log(`第${currentChunk}个分片上传完成, 开始第${currentChunk +1}/${chunks}个分片上传`); await loadNext(); } else { - completeUpload(); + //console.log(dataset_id) + await completeUpload(); console.log(`文件上传完成：${file.name} \n分片：${chunks} 大小:${file.size} 用时：${(new Date().getTime() - time)/1000} s`); + //window.location.reload(); } }; } @@ -324,22 +326,18 @@ await this.newMultiUpload(file); if (file.uploadID != "" && file.uuid != "") { file.chunks = ""; - //todo:开始分片上传:分片，获取分片上传地址，上传 this.multipartUpload(file); } else { return; } } else { if (file.uploaded == "1") { //已上传成功 - //todo:结束上传 + //秒传 + console.log("文件已上传完成"); + //window.location.reload(); } else { - //todo:查询已上传成功的分片，重新上传未成功上传的分片 - var successChunks = new Array(); - var successParts = new Array(); - successParts = file.chunks.split(","); - for (let i = 0; i < successParts.length; i++) { - successChunks[i] = successParts[i].split("-")[0].split("\"")[1]; - } + //断点续传 + this.multipartUpload(file); } } }, @@ -379,7 +377,7 @@