Merge pull request 'upload to minio directly' (#18) from presignedUrl into develop

Reviewed-by: berry <senluowanxiangt@gmail.com>

models/attachment.go

@@ -318,3 +318,13 @@ func (a *Attachment) LinkedDataSet() (*Dataset, error) {
 	}
 	return nil, nil
 }
+
+// InsertAttachment insert a record into attachment.
+func InsertAttachment(attach *Attachment) (_ *Attachment, err error) {
+
+	if _, err := x.Insert(attach); err != nil {
+		return nil, err
+	}
+
+	return attach, nil
+}

modules/storage/local.go

@@ -65,6 +65,14 @@ func (l *LocalStorage) Delete(path string) error {
 	return os.Remove(p)
 }
 
-func (l *LocalStorage) PresignedGetURL(path string, fileName string) (string,error) {
-	return "",nil
+func (l *LocalStorage) PresignedGetURL(path string, fileName string) (string, error) {
+	return "", nil
+}
+
+func (l *LocalStorage) PresignedPutURL(path string) (string, error) {
+	return "", nil
+}
+
+func (l *LocalStorage) HasObject(path string) (bool, error) {
+	return false, nil
 }

modules/storage/minio.go

@@ -18,7 +18,10 @@ var (
 	_ ObjectStorage = &MinioStorage{}
 )
 
-const PRESIGNED_URL_EXPIRE_TIME = time.Hour * 24 * 7
+const (
+	PresignedGetUrlExpireTime = time.Hour * 24 * 7
+	PresignedPutUrlExpireTime = time.Hour * 24 * 7
+)
 
 // MinioStorage returns a minio bucket storage
 type MinioStorage struct {
@@ -73,17 +76,49 @@ func (m *MinioStorage) Delete(path string) error {
 	return m.client.RemoveObject(m.bucket, m.buildMinioPath(path))
 }
 
-//Get Presigned URL
-func (m *MinioStorage) PresignedGetURL(path string, fileName string) (string,error) {
+//Get Presigned URL for get object
+func (m *MinioStorage) PresignedGetURL(path string, fileName string) (string, error) {
 	// Set request parameters for content-disposition.
 	reqParams := make(url.Values)
-	reqParams.Set("response-content-disposition", "attachment; filename=\"" + fileName + "\"")
+	reqParams.Set("response-content-disposition", "attachment; filename=\""+fileName+"\"")
+
+	var preURL *url.URL
+	preURL, err := m.client.PresignedGetObject(m.bucket, m.buildMinioPath(path), PresignedGetUrlExpireTime, reqParams)
+	if err != nil {
+		return "", err
+	}
+
+	return preURL.String(), nil
+}
 
+//Get Presigned URL for put object
+func (m *MinioStorage) PresignedPutURL(path string) (string, error) {
 	var preURL *url.URL
-	preURL,err := m.client.PresignedGetObject(m.bucket, m.buildMinioPath(path), PRESIGNED_URL_EXPIRE_TIME, reqParams)
+	preURL, err := m.client.PresignedPutObject(m.bucket, m.buildMinioPath(path), PresignedPutUrlExpireTime)
 	if err != nil {
-		return "",err
+		return "", err
+	}
+
+	return preURL.String(), nil
+}
+
+//check if has the object
+func (m *MinioStorage) HasObject(path string) (bool, error) {
+	hasObject := false
+	// Create a done channel to control 'ListObjects' go routine.
+	doneCh := make(chan struct{})
+
+	// Indicate to our routine to exit cleanly upon return.
+	defer close(doneCh)
+
+	objectCh := m.client.ListObjects(m.bucket, m.buildMinioPath(path), false, doneCh)
+	for object := range objectCh {
+		if object.Err != nil {
+			return hasObject, object.Err
+		}
+
+		hasObject = true
 	}
 
-	return preURL.String(),nil
+	return hasObject, nil
 }

modules/storage/storage.go

@@ -22,6 +22,8 @@ type ObjectStorage interface {
 	Open(path string) (io.ReadCloser, error)
 	Delete(path string) error
 	PresignedGetURL(path string, fileName string) (string, error)
+	PresignedPutURL(path string) (string, error)
+	HasObject(path string) (bool, error)
 }
 
 // Copy copys a file from source ObjectStorage to dest ObjectStorage

modules/upload/filetype.go

@@ -31,6 +31,10 @@ func (err ErrFileTypeForbidden) Error() string {
 func VerifyAllowedContentType(buf []byte, allowedTypes []string) error {
 	fileType := http.DetectContentType(buf)
 
+	return VerifyFileType(fileType, allowedTypes)
+}
+
+func VerifyFileType(fileType string, allowedTypes []string) error {
 	for _, t := range allowedTypes {
 		t := strings.Trim(t, " ")
 

routers/repo/attachment.go

@@ -16,6 +16,8 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/upload"
+
+	gouuid "github.com/satori/go.uuid"
 )
 
 func RenderAttachmentSettings(ctx *context.Context) {
@@ -210,3 +212,66 @@ func increaseDownloadCount(attach *models.Attachment, dataSet *models.Dataset) e
 
 	return nil
 }
+
+// Get a presigned url for put object
+func GetPresignedPutObjectURL(ctx *context.Context) {
+	if !setting.Attachment.Enabled {
+		ctx.Error(404, "attachment is not enabled")
+		return
+	}
+
+	err := upload.VerifyFileType(ctx.Params("file_type"), strings.Split(setting.Attachment.AllowedTypes, ","))
+	if err != nil {
+		ctx.Error(400, err.Error())
+		return
+	}
+
+	if setting.Attachment.StoreType == storage.MinioStorageType {
+		uuid := gouuid.NewV4().String()
+		url, err := storage.Attachments.PresignedPutURL(models.AttachmentRelativePath(uuid))
+		if err != nil {
+			ctx.ServerError("PresignedPutURL", err)
+			return
+		}
+
+		ctx.JSON(200, map[string]string{
+			"uuid": uuid,
+			"url":  url,
+		})
+	} else {
+		ctx.Error(404, "storage type is not enabled")
+		return
+	}
+}
+
+// AddAttachment response for add attachment record
+func AddAttachment(ctx *context.Context) {
+	uuid := ctx.Query("uuid")
+	has, err := storage.Attachments.HasObject(models.AttachmentRelativePath(uuid))
+	if err != nil {
+		ctx.ServerError("HasObject", err)
+		return
+	}
+
+	if !has {
+		ctx.Error(404, "attachment has not been uploaded")
+		return
+	}
+
+	_, err = models.InsertAttachment(&models.Attachment{
+		UUID:       uuid,
+		UploaderID: ctx.User.ID,
+		Name:       ctx.Query("file_name"),
+		Size:       ctx.QueryInt64("size"),
+		DatasetID:  ctx.QueryInt64("dataset_id"),
+	})
+
+	if err != nil {
+		ctx.Error(500, fmt.Sprintf("InsertAttachment: %v", err))
+		return
+	}
+
+	ctx.JSON(200, map[string]string{
+		"result_code": "0",
+	})
+}

routers/repo/dataset.go

@@ -1,13 +1,18 @@
 package repo
 
 import (
+	"net/url"
 	"sort"
 
+	"code.gitea.io/gitea/modules/storage"
+
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
+
+	gouuid "github.com/satori/go.uuid"
 )
 
 const (
@@ -77,6 +82,18 @@ func DatasetIndex(ctx *context.Context) {
 	ctx.Data["dataset"] = dataset
 	ctx.Data["Attachments"] = attachments
 	ctx.Data["IsOwner"] = true
+	uuid := gouuid.NewV4().String()
+	tmpUrl, err := storage.Attachments.PresignedPutURL(models.AttachmentRelativePath(uuid))
+	if err != nil {
+		ctx.ServerError("PresignedPutURL", err)
+	}
+	preUrl, err := url.QueryUnescape(tmpUrl)
+	if err != nil {
+		ctx.ServerError("QueryUnescape", err)
+	}
+
+	ctx.Data["uuid"] = uuid
+	ctx.Data["url"] = preUrl
 	renderAttachmentSettings(ctx)
 
 	ctx.HTML(200, tplIndex)

routers/routes/routes.go

@@ -518,6 +518,8 @@ func RegisterRoutes(m *macaron.Macaron) {
 	m.Group("/attachments", func() {
 		m.Post("", repo.UploadAttachment)
 		m.Post("/delete", repo.DeleteAttachment)
+		m.Get("/get_pre_url", repo.GetPresignedPutObjectURL)
+		m.Post("/add", repo.AddAttachment)
 		m.Post("/private", repo.UpdatePublicAttachment)
 	}, reqSignIn)
 

templates/repo/datasets/dataset.tmpl

@@ -2,7 +2,7 @@
   <div class="field required dataset-files">
     <label>{{.i18n.Tr "dataset.file"}}</label>
     <div class="files"></div>
-    <div class="ui dropzone" id="dataset" data-upload-url="{{AppSubUrl}}/attachments" data-accepts="{{.AttachmentAllowedTypes}}" data-remove-url="{{AppSubUrl}}/attachments/delete" data-csrf="{{.CsrfToken}}" dataset-id={{.dataset.ID}} data-max-file="100"  data-dataset-id="{{.dataset.ID}}" data-max-size="{{.AttachmentMaxSize}}" data-default-message="{{.i18n.Tr "dropzone.default_message"}}" data-invalid-input-type="{{.i18n.Tr "dropzone.invalid_input_type"}}" data-file-too-big="{{.i18n.Tr "dropzone.file_too_big"}}" data-remove-file="{{.i18n.Tr "dropzone.remove_file"}}">
+    <div class="ui dropzone" id="dataset" data-upload-url="{{.url}}" data-uuid="{{.uuid}}" data-add-url="{{AppSubUrl}}/attachments/add" data-accepts="{{.AttachmentAllowedTypes}}" data-remove-url="{{AppSubUrl}}/attachments/delete" data-csrf="{{.CsrfToken}}" dataset-id={{.dataset.ID}} data-max-file="100"  data-dataset-id="{{.dataset.ID}}" data-max-size="{{.AttachmentMaxSize}}" data-default-message="{{.i18n.Tr "dropzone.default_message"}}" data-invalid-input-type="{{.i18n.Tr "dropzone.invalid_input_type"}}" data-file-too-big="{{.i18n.Tr "dropzone.file_too_big"}}" data-remove-file="{{.i18n.Tr "dropzone.remove_file"}}">
     </div>
   </div>
 </div>

web_src/js/index.js

@@ -2396,6 +2396,7 @@ $(document).ready(async () => {
 
     await createDropzone('#dataset', {
       url: $dataset.data('upload-url'),
+      method: 'put',
       headers: {'X-Csrf-Token': csrf},
       maxFiles: $dataset.data('max-file'),
       maxFilesize: $dataset.data('max-size'),
@@ -2411,13 +2412,19 @@ $(document).ready(async () => {
         this.on('sending', (_file, _xhr, formData) => {
           formData.append('dataset_id', $dataset.data('dataset-id'));
         });
-        this.on('success', (file, data) => {
-          filenameDict[file.name] = data.uuid;
-          const input = $(`<input id="${data.uuid}" name="files" type="hidden">`).val(data.uuid);
-          $('.files').append(input);
-        });
-        this.on('queuecomplete', () => {
-          window.location.realod();
+        this.on('success', (file, _data) => {
+          const uuid = $dataset.data('uuid');
+          if ($dataset.data('add-url') && $dataset.data('csrf')) {
+            $.post($dataset.data('add-url'), {
+              uuid,
+              file_name: file.name,
+              size: file.size,
+              dataset_id: $dataset.data('dataset-id'),
+              _csrf: $dataset.data('csrf')
+            }).done(() => {
+              window.location.reload();
+            });
+          }
         });
         this.on('removedfile', (file) => {
           if (file.name in filenameDict) {