chunk upload

models/file_chunk.go

@@ -15,11 +15,11 @@ type FileChunk struct {
 	UUID          string `xorm:"uuid UNIQUE"`
 	Md5			  string `xorm:"UNIQUE"`
 	IsUploaded    int `xorm:"DEFAULT 0"`  // not uploaded: 0, uploaded: 1
-	HasUploaded   string 	//chunkNumbers  1,2,3
 	UploadID   	  string	`xorm:"UNIQUE"`//minio upload id
 	TotalChunks   int
 	Size		  int64
 	UserID        int64              `xorm:"INDEX"`
+	CompletedParts		  []string	// chunkNumber+etag eg: ,1-asqwewqe21312312.2-123hjkas
 	CreatedUnix   timeutil.TimeStamp `xorm:"INDEX created"`
 	UpdatedUnix   timeutil.TimeStamp `xorm:"INDEX updated"`
 }
@@ -73,6 +73,6 @@ func UpdateFileChunk(fileChunk *FileChunk) error {
 func updateFileChunk(e Engine, fileChunk *FileChunk) error {
 	var sess *xorm.Session
 	sess = e.Where("uuid = ?", fileChunk.UUID)
-	_, err := sess.Cols("is_uploaded", "has_uploaded").Update(fileChunk)
+	_, err := sess.Cols("is_uploaded", "completed_parts").Update(fileChunk)
 	return err
 }

modules/minio_ext/api.go

@@ -993,7 +993,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R
 }
 
 
-func (c Client) GenUploadPartSignedUrl(uploadID string, bucketName string, objectName string, partNumber int, size int64, expires time.Duration) (string, error){
+func (c Client) GenUploadPartSignedUrl(uploadID string, bucketName string, objectName string, partNumber int, size int64, expires time.Duration, bucketLocation string) (string, error){
 	signedUrl := ""
 
 	// Input validation.
@@ -1037,6 +1037,7 @@ func (c Client) GenUploadPartSignedUrl(uploadID string, bucketName string, objec
 		//contentMD5Base64: md5Base64,
 		//contentSHA256Hex: sha256Hex,
 		expires:		  int64(expires/time.Second),
+		bucketLocation:		bucketLocation,
 	}
 
 	req, err := c.newRequest("PUT", reqMetadata)

modules/minio_ext/constants.go

@@ -25,7 +25,7 @@ const absMinPartSize = 1024 * 1024 * 5
 
 // minPartSize - minimum part size 128MiB per object after which
 // putObject behaves internally as multipart.
-const MinPartSize = 1024 * 1024 * 128
+const MinPartSize = 1024 * 1024 * 64
 
 // maxPartsCount - maximum number of parts for a single multipart session.
 const MaxPartsCount = 10000

modules/storage/minio_ext.go

@@ -109,7 +109,7 @@ func GenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, partSiz
 	bucketName := minio.Bucket
 	objectName := strings.TrimPrefix(path.Join(minio.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")
 
-	return minioClient.GenUploadPartSignedUrl(uploadId, bucketName, objectName, partNumber, partSize, PresignedUploadPartUrlExpireTime)
+	return minioClient.GenUploadPartSignedUrl(uploadId, bucketName, objectName, partNumber, partSize, PresignedUploadPartUrlExpireTime, setting.Attachment.Minio.Location)
 
 }
 

routers/repo/attachment.go

@@ -6,6 +6,7 @@ package repo
 
 import (
 	contexExt "context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -350,11 +351,17 @@ func GetSuccessChunks(ctx *context.Context) {
 		return
 	}
 
+	chunks,err :=json.Marshal(fileChunk.CompletedParts)
+	if err != nil {
+		ctx.ServerError("json.Marshal failed", err)
+		return
+	}
+
 	ctx.JSON(200, map[string]string{
 		"uuid": fileChunk.UUID,
 		"uploaded": strconv.Itoa(fileChunk.IsUploaded),
 		"uploadID":fileChunk.UploadID,
-		"chunks": fileChunk.HasUploaded,
+		"chunks": string(chunks),
 	})
 
 }
@@ -487,7 +494,8 @@ func CompleteMultipart(ctx *context.Context) {
 
 func UpdateMultipart(ctx *context.Context) {
 	uuid := ctx.Query("uuid")
-	partNumber := ctx.QueryInt("partNumber")
+	partNumber := ctx.QueryInt("chunkNumber")
+	etag := ctx.Query("etag")
 
 	fileChunk, err := models.GetFileChunkByUUID(uuid)
 	if err != nil {
@@ -499,9 +507,7 @@ func UpdateMultipart(ctx *context.Context) {
 		return
 	}
 
-	fileChunk.HasUploaded += "," + strconv.Itoa(partNumber)
-
-	//todo:chunk parts
+	fileChunk.CompletedParts = append(fileChunk.CompletedParts, strconv.Itoa(partNumber) + "-" + strings.Replace(etag, "\"","", -1))
 
 	err = models.UpdateFileChunk(fileChunk)
 	if err != nil {

routers/routes/routes.go

@@ -525,7 +525,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Get("/new_multipart", repo.NewMultipart)
 		m.Get("/get_multipart_url", repo.GetMultipartUploadUrl)
 		m.Post("/complete_multipart", repo.CompleteMultipart)
-		m.Post("/update_multipart", repo.UpdateMultipart)
+		m.Post("/update_chunk", repo.UpdateMultipart)
 	}, reqSignIn)
 
 	m.Group("/attachments", func() {

web_src/js/App.vue

@@ -31,7 +31,7 @@
         options: {
           target: 'http://localhost:9000/upload',
           testChunks: false,
-          chunkSize: 1024*1024*128,  //128MB
+          chunkSize: 1024*1024*64,  //64MB
           simultaneousUploads: 1, //并发上传数
           headers: {
                 'access-token': 'abcd1234'
@@ -79,11 +79,11 @@
               md5: file.uniqueIdentifier,
               _csrf: csrf
             }}).then(function (response) {
-              console.log(response.data.uploadID, response.data.uuid, response.data.uploaded, response.data.chunks);
               file.uploadID = response.data.uploadID;
               file.uuid = response.data.uuid;
               file.uploaded = response.data.uploaded;
               file.chunks = response.data.chunks;
+              console.log(file.chunks);
               resolve(response);
             }).catch(function (error) {
               console.log(error);
@@ -101,7 +101,6 @@
               fileType: file.fileType,
               _csrf: csrf
             }}).then(function (response) {
-              console.log(response.data.uploadID, response.data.uuid);
               file.uploadID = response.data.uploadID;
               file.uuid = response.data.uuid;
               resolve(response);
@@ -113,7 +112,7 @@
         },
         multipartUpload(file) {
           let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
-            chunkSize = 1024*1024*128,
+            chunkSize = 1024*1024*64,
             chunks = Math.ceil(file.size / chunkSize),
             currentChunk = 0,
             fileReader = new FileReader(),
@@ -135,54 +134,124 @@
             return true;
           }
 
-          function uploadChunk(e) {
-            //判断是否已上传
-            if (!checkSuccessChunks()) {
-              let start = currentChunk * chunkSize;
-              let partSize = ((start + chunkSize) >= file.size) ? file.size -start : chunkSize;
-              //获取分片url
-              axios.get('/attachments/get_multipart_url', {params :{
-                uuid: file.uuid,
-                uploadID: file.uploadID,
-                size: partSize,
-                chunkNumber: currentChunk+1,
-                _csrf: csrf
-              }}).then(function (response) {
-                
-                //todo:分片上传
-                axios.put(response.data.url, e.target.result
+          function getUploadChunkUrl(currentChunk, partSize) {
+            return new Promise((resolve, reject) => {
+                axios.get('/attachments/get_multipart_url', {params :{
+                  uuid: file.uuid,
+                  uploadID: file.uploadID,
+                  size: partSize,
+                  chunkNumber: currentChunk+1,
+                  _csrf: csrf
+                }}).then(function (response) {
+                  urls[currentChunk] = response.data.url
+                  resolve(response);
+                }).catch(function (error) {
+                  console.log(error);
+                  reject(error);
+                });
+              })
+          }
+
+          function uploadMinio(url, e) {
+            return new Promise((resolve, reject) => {
+              
+              axios.put(url, e.target.result
                 ).then(function (res) {
-                  console.log(res.headers.etag);
+                  etags[currentChunk] = res.headers.etag;
+                  resolve(res);
+                }).catch(function (err) {
+                  console.log(err);
+                  reject(err);
+                });
+            });
+          }
+
+          function updateChunk(currentChunk) {
+            return new Promise((resolve, reject) => {
+                axios.post('/attachments/update_chunk', qs.stringify({
+                  uuid: file.uuid,
+                  chunkNumber: currentChunk+1,
+                  etag: etags[currentChunk],
+                  _csrf: csrf
+                })).then(function (response) {
+                  resolve(response);
                 }).catch(function (error) {
                   console.log(error);
+                  reject(error);
                 });
+              })
+          }
+
+          async function uploadChunk(e) {
+            if (!checkSuccessChunks()) {
+              let start = currentChunk * chunkSize;
+              let partSize = ((start + chunkSize) >= file.size) ? file.size -start : chunkSize;
 
-              }).catch(function (error) {
-                console.log(error);
-              });
+              //获取分片上传url
+              await getUploadChunkUrl(currentChunk, partSize);
+              if (urls[currentChunk] != "") {
+                //上传到minio
+                await uploadMinio(urls[currentChunk], e);
+                if (etags[currentChunk] != "") {
+                  //更新数据库：分片上传结果
+                  console.log(etags[currentChunk]);
+                  await updateChunk(currentChunk);
+                } else {
+                  return;
+                }
+              } else {
+                return;
+              }
+              
             }
             
+          };
+
+          function completeUpload(){
+            return new Promise((resolve, reject) => {
+                axios.post('/attachments/complete_multipart', qs.stringify({
+                  uuid: file.uuid,
+                  uploadID: file.uploadID,
+                  etag: etags[currentChunk],
+                  _csrf: csrf
+                })).then(function (response) {
+                  resolve(response);
+                }).catch(function (error) {
+                  console.log(error);
+                  reject(error);
+                });
+              })
+          }
+
+          function upload() {
+            loadNext();
+            fileReader.onload = async (e) => {
+              await uploadChunk(e);
+              currentChunk++;
+        
+              if (currentChunk < chunks) {
+                  console.log(`第${currentChunk}个分片上传完成, 开始第${currentChunk +1}/${chunks}个分片上传`);
+                  await loadNext();
+              } else {
+                  completeUpload();
+                  console.log(`文件上传完成：${file.name} \n分片：${chunks} 大小:${file.size} 用时：${(new Date().getTime() - time)/1000} s`);
+              }
+            };
           }
 
           var successChunks = new Array();
-          successChunks = file.chunks.split(",");
+          var successParts = new Array();
+          successParts = file.chunks.split(",");
+          for (let i = 0; i < successParts.length; i++) {
+            successChunks[i] = successParts[i].split("-")[0].split("\"")[1];
+          }
+          
+          var urls = new Array();
+          var etags = new Array();
 
           console.log('上传分片...');
-          loadNext();
-          fileReader.onload = (e) => {
-            uploadChunk(e);
-            currentChunk++;
-      
-            if (currentChunk < chunks && !checkSuccessChunks()) {
-                console.log(`第${currentChunk}个分片上传完成, 开始第${currentChunk +1} / ${chunks}个分片上传`);
-                loadNext();
-            } else {
-                console.log(`文件上传完成：${file.name} \n分片：${chunks} 大小:${file.size} 用时：${(new Date().getTime() - time)/1000} s`);
-                
-
-                //this.computeMD5Success(file);
-            }
-          };
+          
+          upload();
 
         },
         chkMd5(file) {
@@ -202,7 +271,7 @@
         //计算MD5
         computeMD5(file) {
             let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
-                chunkSize = 1024*1024*128,
+                chunkSize = 1024*1024*64,
                 chunks = Math.ceil(file.size / chunkSize),
                 currentChunk = 0,
                 spark = new SparkMD5.ArrayBuffer(),
@@ -220,7 +289,7 @@
                 currentChunk++;
          
                 if (currentChunk < chunks) {
-                    console.log(`第${currentChunk}分片解析完成, 开始第${currentChunk +1} / ${chunks}分片解析`);
+                    console.log(`第${currentChunk}分片解析完成, 开始第${currentChunk +1}/${chunks}分片解析`);
                     // let percent = Math.floor(currentChunk / chunks * 100);
                     // console.log(percent);
                     // file.cmd5progress = percent;
@@ -252,7 +321,6 @@
             await this.getSuccessChunks(file);
             
             if (file.uploadID == "" || file.uuid == "") { //未上传过
-              console.log("test1");
               await this.newMultiUpload(file);
               if (file.uploadID != "" && file.uuid != "") {
                 file.chunks = "";
@@ -262,11 +330,16 @@
                 return;
               }
             } else {
-              console.log("test2");
               if (file.uploaded == "1") {  //已上传成功
                 //todo:结束上传
               } else {
                 //todo:查询已上传成功的分片，重新上传未成功上传的分片
+                var successChunks = new Array();
+                var successParts = new Array();
+                successParts = file.chunks.split(",");
+                for (let i = 0; i < successParts.length; i++) {
+                  successChunks[i] = successParts[i].split("-")[0].split("\"")[1];
+                }
               }
             }
         },