diff --git a/modules/storage/obs.go b/modules/storage/obs.go index f733eef6c..08a354359 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -30,6 +30,8 @@ type FileInfo struct { } type FileInfoList []FileInfo +const MAX_LIST_PARTS = 1000 + func (ulist FileInfoList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } func (ulist FileInfoList) Len() int { return len(ulist) } func (ulist FileInfoList) Less(i, j int) bool { @@ -97,29 +99,48 @@ func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error { input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.UploadId = uploadID - output, err := ObsCli.ListParts(&obs.ListPartsInput{ - Bucket: setting.Bucket, - Key: input.Key, - UploadId: uploadID, - }) - if err != nil { - log.Error("ListParts failed:", err.Error()) - return err - } - for _, partInfo := range output.Parts { - input.Parts = append(input.Parts, obs.Part{ - PartNumber: partInfo.PartNumber, - ETag: partInfo.ETag, + partNumberMarker := 0 + for { + output, err := ObsCli.ListParts(&obs.ListPartsInput{ + Bucket: setting.Bucket, + Key: input.Key, + UploadId: uploadID, + MaxParts: MAX_LIST_PARTS, + PartNumberMarker: partNumberMarker, }) + if err != nil { + log.Error("ListParts failed:", err.Error()) + return err + } + + partNumberMarker = output.NextPartNumberMarker + log.Info("uuid:%s, MaxParts:%d, PartNumberMarker:%d, NextPartNumberMarker:%d, len:%d", uuid, output.MaxParts, output.PartNumberMarker, output.NextPartNumberMarker, len(output.Parts)) + + for _, partInfo := range output.Parts { + input.Parts = append(input.Parts, obs.Part{ + PartNumber: partInfo.PartNumber, + ETag: partInfo.ETag, + }) + } + + if len(output.Parts) < output.MaxParts { + break + } else { + continue + } + + break } - _, err = ObsCli.CompleteMultipartUpload(input) + output, err := ObsCli.CompleteMultipartUpload(input) if err != nil { log.Error("CompleteMultipartUpload failed:", err.Error()) return err } + log.Info("uuid:%s, RequestId:%s", uuid, output.RequestId) + return nil } diff --git a/web_src/js/components/MinioUploader.vue b/web_src/js/components/MinioUploader.vue index 12edccb7c..908535ad8 100755 --- a/web_src/js/components/MinioUploader.vue +++ b/web_src/js/components/MinioUploader.vue @@ -26,7 +26,7 @@ import qs from 'qs'; import createDropzone from '../features/dropzone.js'; const {_AppSubUrl, _StaticUrlPrefix, csrf} = window.config; -// const uploadtype = 0; +const chunkSize = 1024 * 1024 * 64; export default { props:{ @@ -137,7 +137,6 @@ export default { resetStatus() { this.progress = 0; this.status = ''; - console.log(this.uploadtype) }, updateProgress(file, progress) { console.log("progress---",progress) @@ -165,7 +164,6 @@ export default { .getElementById('datasetId') .getAttribute('datasetId'); this.resetStatus(); - console.log(this.file,!this.file?.upload) if(!this.file?.upload){ this.btnFlag = false return @@ -186,7 +184,7 @@ export default { File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice, - chunkSize = 1024 * 1024 * 64, + //chunkSize = 1024 * 1024 * 64, chunks = Math.ceil(file.size / chunkSize), spark = new SparkMD5.ArrayBuffer(), fileReader = new FileReader(); @@ -327,7 +325,6 @@ export default { }, async newMultiUpload(file) { - console.log(this.uploadtype,this) const res = await axios.get('/attachments/new_multipart', { params: { totalChunkCounts: file.totalChunkCounts, @@ -348,7 +345,7 @@ export default { File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice, - chunkSize = 1024 * 1024 * 32, + //chunkSize = 1024 * 1024 * 32, chunks = Math.ceil(file.size / chunkSize), fileReader = new FileReader(), time = new Date().getTime(); @@ -457,7 +454,6 @@ export default { } async function completeUpload() { - console.log(_this.uploadtype) return await axios.post( '/attachments/complete_multipart', qs.stringify({ @@ -494,7 +490,6 @@ export default { 1}/${chunks}个分片上传` ); this.progress = Math.ceil((currentChunk / chunks) * 100); - console.log("((currentChunk / chunks) * 100).toFixed(2)",((currentChunk / chunks) * 100).toFixed(2)) this.updateProgress(file, ((currentChunk / chunks) * 100).toFixed(2)); this.status = `${this.dropzoneParams.data('uploading')} ${( (currentChunk / chunks) *