From df77d0cb57962345aa07f9773fe9f00f52fa7e77 Mon Sep 17 00:00:00 2001
From: yuyuanshifu <747342561@qq.com>
Date: Thu, 4 Mar 2021 16:35:31 +0800
Subject: [PATCH] mod obs upload

---
 modules/storage/obs.go                        |  50 +-
 routers/repo/attachment.go                    |  32 --
 routers/routes/routes.go                      |   1 -
 .../js/components/MinioUploader.vue.opendata  | 487 ++++++++++++++++++
 web_src/js/components/ObsUploader.vue         |   1 -
 5 files changed, 497 insertions(+), 74 deletions(-)
 create mode 100755 web_src/js/components/MinioUploader.vue.opendata

diff --git a/modules/storage/obs.go b/modules/storage/obs.go
index d174b8fba..4e364fe9c 100755
--- a/modules/storage/obs.go
+++ b/modules/storage/obs.go
@@ -5,7 +5,7 @@
 package storage
 
 import (
-	"io"
+	"github.com/unknwon/com"
 	"path"
 	"strconv"
 	"strings"
@@ -102,57 +102,27 @@ func CompleteObsMultiPartUpload(uuid string, uploadID string) error {
 	return nil
 }
 
-func ObsUploadPart(uuid string, uploadId string, partNumber int, partSize int64, body io.Reader) (string, error) {
-	input := &obs.UploadPartInput{}
-	input.PartNumber = partNumber
-	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/")
-	input.UploadId = uploadId
-	input.Bucket = setting.Bucket
-	input.PartSize = partSize
-	input.Body = body
-	output, err := ObsCli.UploadPart(input)
-	if err != nil {
-		log.Error("UploadPart failed:", err.Error())
-		return "", err
-	}
-
-	return output.ETag, nil
-}
-
 func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, partSize int64) (string, error) {
-	/*
+
 	input := &obs.CreateSignedUrlInput{}
 	input.Bucket = setting.Bucket
-	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")
-	input.Expires = int(PresignedUploadPartUrlExpireTime)
-	input.Method = obs.HTTP_PUT
+	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/")
+	input.Expires = 60 * 60
+	input.Method = obs.HttpMethodPut
 
 	input.QueryParams = map[string]string{
-		"Bucket":	input.Bucket,
-		"Key":		input.Key,
-		"PartNumber": com.ToStr(partNumber,10),
-		"UploadId":	uploadId,
-		"PartSize":	com.ToStr(partSize,10),
+		"partNumber": com.ToStr(partNumber,10),
+		"uploadId":	uploadId,
+		//"partSize":	com.ToStr(partSize,10),
 	}
 
-	input.Headers = map[string]string{
-
-	}
-
-	 */
-
-	Key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/")
-	req, err := ObsCli.CreateUploadPartSignedUrl(setting.Bucket, Key, uploadId, partNumber, partSize)
+	output, err := ObsCli.CreateSignedUrl(input)
 	if err != nil {
 		log.Error("CreateSignedUrl failed:", err.Error())
 		return "", err
 	}
 
-	log.Info(req.URL.String())
-	log.Info("", req.Header)
-
-	return req.URL.String(), nil
-
+	return output.SignedUrl, nil
 }
 
 func ObsGetPreSignedUrl(uuid, fileName string) (string, error) {
diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go
index 6874a1eda..62bdc2ea7 100755
--- a/routers/repo/attachment.go
+++ b/routers/repo/attachment.go
@@ -633,38 +633,6 @@ func GetObsKey(ctx *context.Context) {
 	})
 }
 
-func UploadPart(ctx *context.Context) {
-	tmp, err := ctx.Req.Body().String()
-	log.Info(tmp)
-
-	err = ctx.Req.ParseMultipartForm(100*1024*1024)
-	if err != nil {
-		ctx.Error(http.StatusBadRequest, fmt.Sprintf("ParseMultipartForm failed: %v", err))
-		return
-	}
-
-	file, fileHeader, err := ctx.Req.FormFile("file")
-	log.Info(ctx.Req.Form.Get("file"))
-	if err != nil {
-		ctx.Error(http.StatusBadRequest, fmt.Sprintf("FormFile failed: %v", err))
-		return
-	}
-
-
-
-	log.Info(fileHeader.Filename)
-
-	etag, err := storage.ObsUploadPart("", "", 1, 1, file)
-	if err != nil {
-		ctx.Error(500, fmt.Sprintf("ObsUploadPart failed: %v", err))
-		return
-	}
-
-	ctx.JSON(200, map[string]string{
-		"etag": etag,
-	})
-}
-
 func CompleteMultipart(ctx *context.Context) {
 	uuid := ctx.Query("uuid")
 	uploadID := ctx.Query("uploadID")
diff --git a/routers/routes/routes.go b/routers/routes/routes.go
index 312707314..adc8938a9 100755
--- a/routers/routes/routes.go
+++ b/routers/routes/routes.go
@@ -529,7 +529,6 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Get("/get_multipart_url", repo.GetMultipartUploadUrl)
 		m.Post("/complete_multipart", repo.CompleteMultipart)
 		m.Post("/update_chunk", repo.UpdateMultipart)
-		m.Post("/upload_part", repo.UploadPart)
 		m.Get("/get_obs_key", repo.GetObsKey)
 	}, reqSignIn)
 
diff --git a/web_src/js/components/MinioUploader.vue.opendata b/web_src/js/components/MinioUploader.vue.opendata
new file mode 100755
index 000000000..1775fd02b
--- /dev/null
+++ b/web_src/js/components/MinioUploader.vue.opendata
@@ -0,0 +1,487 @@
+<template>
+  <div class="dropzone-wrapper dataset-files">
+    <div
+      id="dataset"
+      class="dropzone"
+    />
+    <p class="upload-info">
+      {{ file_status_text }}
+      <span class="success">{{ status }}</span>
+    </p>
+  </div>
+</template>
+
+<script>
+/* eslint-disable eqeqeq */
+// import Dropzone from 'dropzone/dist/dropzone.js';
+// import 'dropzone/dist/dropzone.css'
+import SparkMD5 from 'spark-md5';
+import axios from 'axios';
+import qs from 'qs';
+import createDropzone from '../features/dropzone.js';
+
+const {_AppSubUrl, _StaticUrlPrefix, csrf} = window.config;
+const CloudBrainType = 1
+
+export default {
+  data() {
+    return {
+      dropzoneUploader: null,
+      maxFiles: 1,
+      maxFilesize: 1 * 1024 * 1024 * 1024 * 1024,
+      acceptedFiles: '*/*',
+      progress: 0,
+      status: '',
+      dropzoneParams: {},
+      file_status_text: ''
+    };
+  },
+
+  async mounted() {
+    this.dropzoneParams = $('div#minioUploader-params');
+    this.file_status_text = this.dropzoneParams.data('file-status');
+    this.status = this.dropzoneParams.data('file-init-status');
+
+    let previewTemplate = '';
+    previewTemplate += '<div class="dz-preview dz-file-preview">\n ';
+    previewTemplate += '    <div class="dz-details">\n ';
+    previewTemplate += '        <div class="dz-filename">';
+    previewTemplate +=
+      '            <span data-dz-name data-dz-thumbnail></span>';
+    previewTemplate += '        </div>\n    ';
+    previewTemplate += '        <div class="dz-size" data-dz-size></div>\n  ';
+    previewTemplate += '    </div>\n  ';
+    previewTemplate += '    <div class="dz-progress ui active progress">';
+    previewTemplate +=
+      '        <div class="dz-upload bar" data-dz-uploadprogress><div class="progress"></div></div>\n  ';
+    previewTemplate += '    </div>\n  ';
+    previewTemplate += '    <div class="dz-success-mark">';
+    previewTemplate += '        <span>上传成功</span>';
+    previewTemplate += '    </div>\n  ';
+    previewTemplate += '    <div class="dz-error-mark">';
+    previewTemplate += '        <span>上传失败</span>';
+    previewTemplate += '    </div>\n  ';
+    previewTemplate += '    <div class="dz-error-message">';
+    previewTemplate += '        <span data-dz-errormessage></span>';
+    previewTemplate += '    </div>\n';
+    previewTemplate += '</div>';
+
+    const $dropzone = $('div#dataset');
+    console.log('createDropzone');
+    const dropzoneUploader = await createDropzone($dropzone[0], {
+      url: '/todouploader',
+      maxFiles: this.maxFiles,
+      maxFilesize: this.maxFileSize,
+      timeout: 0,
+      autoQueue: false,
+      dictDefaultMessage: this.dropzoneParams.data('default-message'),
+      dictInvalidFileType: this.dropzoneParams.data('invalid-input-type'),
+      dictFileTooBig: this.dropzoneParams.data('file-too-big'),
+      dictRemoveFile: this.dropzoneParams.data('remove-file'),
+      previewTemplate
+    });
+    dropzoneUploader.on('addedfile', (file) => {
+      setTimeout(() => {
+        // eslint-disable-next-line no-unused-expressions
+        file.accepted && this.onFileAdded(file);
+      }, 200);
+    });
+    dropzoneUploader.on('maxfilesexceeded', function (file) {
+      if (this.files[0].status !== 'success') {
+        alert(this.dropzoneParams.data('waitting-uploading'));
+        this.removeFile(file);
+        return;
+      }
+      this.removeAllFiles();
+      this.addFile(file);
+    });
+
+    this.dropzoneUploader = dropzoneUploader;
+  },
+  methods: {
+    resetStatus() {
+      this.progress = 0;
+      this.status = '';
+    },
+    updateProgress(file, progress) {
+      file.previewTemplate.querySelector(
+        '.dz-upload'
+      ).style.width = `${progress}%`;
+    },
+    emitDropzoneSuccess(file) {
+      file.status = 'success';
+      this.dropzoneUploader.emit('success', file);
+      this.dropzoneUploader.emit('complete', file);
+    },
+    emitDropzoneFailed(file) {
+      this.status = this.dropzoneParams.data('falied');
+      file.status = 'error';
+      this.dropzoneUploader.emit('error', file);
+      // this.dropzoneUploader.emit('complete', file);
+    },
+    onFileAdded(file) {
+      file.datasetId = document
+        .getElementById('datasetId')
+        .getAttribute('datasetId');
+      this.resetStatus();
+      this.computeMD5(file);
+    },
+
+    finishUpload(file) {
+      this.emitDropzoneSuccess(file);
+      setTimeout(() => {
+        //window.location.reload();
+      }, 1000);
+    },
+
+    computeMD5(file) {
+      this.resetStatus();
+      const blobSlice =
+          File.prototype.slice ||
+          File.prototype.mozSlice ||
+          File.prototype.webkitSlice,
+        chunkSize = 1024 * 1024 * 64,
+        chunks = Math.ceil(file.size / chunkSize),
+        spark = new SparkMD5.ArrayBuffer(),
+        fileReader = new FileReader();
+      let currentChunk = 0;
+
+      const time = new Date().getTime();
+      // console.log('计算MD5...')
+      this.status = this.dropzoneParams.data('md5-computing');
+      file.totalChunkCounts = chunks;
+      loadNext();
+
+      fileReader.onload = (e) => {
+        fileLoaded.call(this, e);
+      };
+      fileReader.onerror = (err) => {
+        console.warn('oops, something went wrong.', err);
+        file.cancel();
+      };
+
+      function fileLoaded(e) {
+        spark.append(e.target.result); // Append array buffer
+        currentChunk++;
+        if (currentChunk < chunks) {
+          // console.log(`第${currentChunk}分片解析完成, 开始第${currentChunk +1}/${chunks}分片解析`);
+          this.status = `${this.dropzoneParams.data('loading-file')} ${(
+            (currentChunk / chunks) *
+            100
+          ).toFixed(2)}% (${currentChunk}/${chunks})`;
+          this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2));
+          loadNext();
+          return;
+        }
+
+        const md5 = spark.end();
+        console.log(
+          `MD5计算完成：${file.name} \nMD5：${md5} \n分片：${chunks} 大小:${
+            file.size
+          } 用时：${(new Date().getTime() - time) / 1000} s`
+        );
+        spark.destroy(); // 释放缓存
+        file.uniqueIdentifier = md5; // 将文件md5赋值给文件唯一标识
+        file.cmd5 = false; // 取消计算md5状态
+        this.computeMD5Success(file);
+      }
+
+      function loadNext() {
+        const start = currentChunk * chunkSize;
+        const end =
+          start + chunkSize >= file.size ? file.size : start + chunkSize;
+        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
+      }
+    },
+
+    async computeMD5Success(md5edFile) {
+      const file = await this.getSuccessChunks(md5edFile);
+      try {
+        if (file.uploadID == '' || file.uuid == '') {
+        // 未上传过
+          await this.newMultiUpload(file);
+          if (file.uploadID != '' && file.uuid != '') {
+            file.chunks = '';
+            this.multipartUpload(file);
+          } else {
+          // 失败如何处理
+            return;
+          }
+          return;
+        }
+
+        if (file.uploaded == '1') {
+        // 已上传成功
+        // 秒传
+          if (file.attachID == '0') {
+          // 删除数据集记录，未删除文件
+            await addAttachment(file);
+          }
+          //不同数据集上传同一个文件
+          if (file.datasetID != '') {
+            if (Number(file.datasetID) != file.datasetId) {
+              var info = "该文件已上传，对应数据集（" + file.datasetName + "）-文件（" + file.realName + ")";
+              window.alert(info);
+              window.location.reload();
+            }
+          }
+          console.log('文件已上传完成');
+          this.progress = 100;
+          this.status = this.dropzoneParams.data('upload-complete');
+          this.finishUpload(file);
+        } else {
+        // 断点续传
+          this.multipartUpload(file);
+        }
+      } catch (error) {
+        this.emitDropzoneFailed(file);
+        console.log(error);
+      }
+
+      async function addAttachment(file) {
+        return await axios.post(
+          '/attachments/add',
+          qs.stringify({
+            uuid: file.uuid,
+            file_name: file.name,
+            size: file.size,
+            dataset_id: file.datasetId,
+            type: CloudBrainType,
+            _csrf: csrf,
+          })
+        );
+      }
+    },
+
+    async getSuccessChunks(file) {
+      const params = {
+        params: {
+          md5: file.uniqueIdentifier,
+          type: CloudBrainType,
+          _csrf: csrf
+        }
+      };
+      try {
+        const response = await axios.get('/attachments/get_chunks', params);
+        file.uploadID = response.data.uploadID;
+        file.uuid = response.data.uuid;
+        file.uploaded = response.data.uploaded;
+        file.chunks = response.data.chunks;
+        file.attachID = response.data.attachID;
+        file.datasetID = response.data.datasetID;
+        file.datasetName = response.data.datasetName;
+        file.realName = response.data.fileName;
+        return file;
+      } catch (error) {
+        this.emitDropzoneFailed(file);
+        console.log('getSuccessChunks catch: ', error);
+        return null;
+      }
+    },
+
+    async newMultiUpload(file) {
+      const res = await axios.get('/attachments/new_multipart', {
+        params: {
+          totalChunkCounts: file.totalChunkCounts,
+          md5: file.uniqueIdentifier,
+          size: file.size,
+          fileType: file.type,
+          type: CloudBrainType,
+          _csrf: csrf
+        }
+      });
+      file.uploadID = res.data.uploadID;
+      file.uuid = res.data.uuid;
+    },
+
+    multipartUpload(file) {
+      const blobSlice =
+          File.prototype.slice ||
+          File.prototype.mozSlice ||
+          File.prototype.webkitSlice,
+        chunkSize = 1024 * 1024 * 64,
+        chunks = Math.ceil(file.size / chunkSize),
+        fileReader = new FileReader(),
+        time = new Date().getTime();
+      let currentChunk = 0;
+
+      function loadNext() {
+        const start = currentChunk * chunkSize;
+        const end =
+          start + chunkSize >= file.size ? file.size : start + chunkSize;
+        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
+      }
+
+      function checkSuccessChunks() {
+        const index = successChunks.indexOf((currentChunk + 1).toString());
+        if (index == -1) {
+          return false;
+        }
+        return true;
+      }
+
+      async function getUploadChunkUrl(currentChunk, partSize) {
+        const res = await axios.get('/attachments/get_multipart_url', {
+          params: {
+            uuid: file.uuid,
+            uploadID: file.uploadID,
+            size: partSize,
+            chunkNumber: currentChunk + 1,
+            type: CloudBrainType,
+            _csrf: csrf
+          }
+        });
+        urls[currentChunk] = res.data.url;
+      }
+
+      async function uploadMinio(url, e) {
+        let urls = [];
+        console.log(url);
+        const res = await axios.put(url, e.target.result, {
+            headers: {
+              'Content-Type': ''
+            }});
+        //const res = await axios.put(url);
+//         $.ajax({
+//           url: url,//服务器地址
+//           //data:e.target.result,//读到arraybuffer中的待上传数据
+//           type: 'PUT', //设置post上传数据类型
+// //          dataType: 'text',//here
+//           processData: false,//数据不进行预处理
+//           //headers: {'Content-Type': 'application/octet-stream'},
+//           xhrFields: {//跨越请求，必须加上这个字段
+//               withCredentials: true
+//               },
+//           success: function (data) {
+//             alert('请求成功');
+//             // console.log(res);
+//             // etags[currentChunk] = res.headers.etag;
+
+//           },
+//           error:function(msg){
+//             alert('请求发生错误');
+//           }
+//       });     
+        console.log(res);
+        etags[currentChunk] = res.headers.etag;
+      }
+
+      async function updateChunk(currentChunk) {
+        await axios.post(
+          '/attachments/update_chunk',
+          qs.stringify({
+            uuid: file.uuid,
+            chunkNumber: currentChunk + 1,
+            etag: etags[currentChunk],
+            type: CloudBrainType,
+            _csrf: csrf
+          })
+        );
+      }
+      async function uploadChunk(e) {
+        try {
+          if (!checkSuccessChunks()) {
+            const start = currentChunk * chunkSize;
+            const partSize =
+            start + chunkSize >= file.size ? file.size - start : chunkSize;
+            // 获取分片上传url
+            await getUploadChunkUrl(currentChunk, partSize);
+            if (urls[currentChunk] != '') {
+            // 上传到minio
+              await uploadMinio(urls[currentChunk], e);
+              if (etags[currentChunk] != '') {
+              // 更新数据库：分片上传结果
+                //await updateChunk(currentChunk);
+              } else {
+                console.log("上传到minio uploadChunk etags[currentChunk] == ''");// TODO
+              }
+            } else {
+              console.log("uploadChunk urls[currentChunk] != ''");// TODO
+            }
+          }
+        } catch (error) {
+          this.emitDropzoneFailed(file);
+          console.log(error);
+        }
+      }
+
+      async function completeUpload() {
+        return await axios.post(
+          '/attachments/complete_multipart',
+          qs.stringify({
+            uuid: file.uuid,
+            uploadID: file.uploadID,
+            file_name: file.name,
+            size: file.size,
+            dataset_id: file.datasetId,
+            type: CloudBrainType,
+            _csrf: csrf
+          })
+        );
+      }
+
+      const successChunks = [];
+      let successParts = [];
+      successParts = file.chunks.split(',');
+      for (let i = 0; i < successParts.length; i++) {
+        successChunks[i] = successParts[i].split('-')[0];
+      }
+      const urls = []; // TODO const ?
+      const etags = [];
+      console.log('上传分片...');
+      this.status = this.dropzoneParams.data('uploading');
+      loadNext();
+      fileReader.onload = async (e) => {
+        await uploadChunk(e);
+        fileReader.abort();
+        currentChunk++;
+        if (currentChunk < chunks) {
+          console.log(
+            `第${currentChunk}个分片上传完成, 开始第${currentChunk +
+              1}/${chunks}个分片上传`
+          );
+          this.progress = Math.ceil((currentChunk / chunks) * 100);
+          this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2));
+          this.status = `${this.dropzoneParams.data('uploading')} ${(
+            (currentChunk / chunks) *
+            100
+          ).toFixed(2)}%`;
+          await loadNext();
+        } else {
+          await completeUpload();
+          console.log(
+            `文件上传完成：${file.name} \n分片：${chunks} 大小:${
+              file.size
+            } 用时：${(new Date().getTime() - time) / 1000} s`
+          );
+          this.progress = 100;
+          this.status = this.dropzoneParams.data('upload-complete');
+          this.finishUpload(file);
+        }
+      };
+    }
+  }
+};
+</script>
+
+<style>
+.dropzone-wrapper {
+  margin: 2em auto;
+}
+.ui .dropzone {
+  border: 2px dashed #0087f5;
+  box-shadow: none !important;
+  padding: 0;
+  min-height: 5rem;
+  border-radius: 4px;
+}
+.dataset .dataset-files #dataset .dz-preview.dz-file-preview,
+.dataset .dataset-files #dataset .dz-preview.dz-processing {
+  display: flex;
+  align-items: center;
+}
+.dataset .dataset-files #dataset .dz-preview {
+  border-bottom: 1px solid #dadce0;
+  min-height: 0;
+}
+</style>
\ No newline at end of file
diff --git a/web_src/js/components/ObsUploader.vue b/web_src/js/components/ObsUploader.vue
index 1ff118d94..0e9e7d2eb 100755
--- a/web_src/js/components/ObsUploader.vue
+++ b/web_src/js/components/ObsUploader.vue
@@ -152,7 +152,6 @@ export default {
 
     // 断点续传
     do_multi_uploader(file){
-      //var file = evt.target.files[0]
       const result = this.get_result()
       const upload_datasetId = document
         .getElementById('datasetId')