Add setting for minio and flags for migrate-storage

cmd/migrate_storage.go

@@ -6,6 +6,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/models/migrations"
@@ -22,6 +23,53 @@ var CmdMigrateStorage = cli.Command{
 	Usage:       "Migrate the storage",
 	Description: "This is a command for migrating storage.",
 	Action:      runMigrateStorage,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "type, t",
+			Value: "",
+			Usage: "Files type to migrate, currently should be attachments",
+		},
+		cli.StringFlag{
+			Name:  "store, s",
+			Value: "local",
+			Usage: "New storage type, local or minio",
+		},
+		cli.StringFlag{
+			Name:  "path, p",
+			Value: "",
+			Usage: "New storage placement if store is local",
+		},
+		cli.StringFlag{
+			Name:  "minio-endpoint",
+			Value: "",
+			Usage: "New storage placement if store is local",
+		},
+		cli.StringFlag{
+			Name:  "minio-access-key-id",
+			Value: "",
+			Usage: "New storage placement if store is local",
+		},
+		cli.StringFlag{
+			Name:  "minio-scret-access-key",
+			Value: "",
+			Usage: "New storage placement if store is local",
+		},
+		cli.StringFlag{
+			Name:  "minio-bucket",
+			Value: "",
+			Usage: "New storage placement if store is local",
+		},
+		cli.StringFlag{
+			Name:  "minio-location",
+			Value: "",
+			Usage: "New storage placement if store is local",
+		},
+		cli.StringFlag{
+			Name:  "minio-use-ssl",
+			Value: "",
+			Usage: "New storage placement if store is local",
+		},
+	},
 }
 
 func migrateAttachments(dstStorage storage.ObjectStorage) error {
@@ -47,17 +95,32 @@ func runMigrateStorage(ctx *cli.Context) error {
 		return err
 	}
 
-	tp := ctx.String("type")
-
-	// TODO: init setting
-
 	if err := storage.Init(); err != nil {
 		return err
 	}
 
+	tp := ctx.String("type")
 	switch tp {
 	case "attachments":
-		dstStorage, err := storage.NewLocalStorage(ctx.String("dst"))
+		var dstStorage storage.ObjectStorage
+		var err error
+		switch ctx.String("store") {
+		case "local":
+			dstStorage, err = storage.NewLocalStorage(ctx.String("dst"))
+		case "minio":
+			dstStorage, err = storage.NewMinioStorage(
+				ctx.String("minio-endpoint"),
+				ctx.String("minio-access-key-id"),
+				ctx.String("minio-secret-access-key"),
+				ctx.String("minio-bucket"),
+				ctx.String("minio-location"),
+				ctx.String("minio-basePath"),
+				ctx.Bool("minio-useSSL"),
+			)
+		default:
+			return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
+		}
+
 		if err != nil {
 			return err
 		}

modules/context/context.go

@@ -317,7 +317,7 @@ func Contexter() macaron.Handler {
 
 		// If request sends files, parse them here otherwise the Query() can't be parsed and the CsrfToken will be invalid.
 		if ctx.Req.Method == "POST" && strings.Contains(ctx.Req.Header.Get("Content-Type"), "multipart/form-data") {
-			if err := ctx.Req.ParseMultipartForm(setting.AttachmentMaxSize << 20); err != nil && !strings.Contains(err.Error(), "EOF") { // 32MB max size
+			if err := ctx.Req.ParseMultipartForm(setting.Attachment.MaxSize << 20); err != nil && !strings.Contains(err.Error(), "EOF") { // 32MB max size
 				ctx.ServerError("ParseMultipartForm", err)
 				return
 			}

modules/setting/setting.go

@@ -298,11 +298,38 @@ var (
 	EnableXORMLog      bool
 
 	// Attachment settings
-	AttachmentPath         string
-	AttachmentAllowedTypes string
-	AttachmentMaxSize      int64
-	AttachmentMaxFiles     int
-	AttachmentEnabled      bool
+	Attachment = struct {
+		StoreType string
+		Path      string
+		Minio     struct {
+			Endpoint        string
+			AccessKeyID     string
+			SecretAccessKey string
+			UseSSL          bool
+			Bucket          string
+			Location        string
+			BasePath        string
+		}
+		AllowedTypes string
+		MaxSize      int64
+		MaxFiles     int
+		Enabled      bool
+	}{
+		StoreType: "local",
+		Minio: struct {
+			Endpoint        string
+			AccessKeyID     string
+			SecretAccessKey string
+			UseSSL          bool
+			Bucket          string
+			Location        string
+			BasePath        string
+		}{},
+		AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
+		MaxSize:      4,
+		MaxFiles:     5,
+		Enabled:      true,
+	}
 
 	// Time settings
 	TimeFormat string
@@ -845,14 +872,27 @@ func NewContext() {
 	}
 
 	sec = Cfg.Section("attachment")
-	AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
-	if !filepath.IsAbs(AttachmentPath) {
-		AttachmentPath = path.Join(AppWorkPath, AttachmentPath)
-	}
-	AttachmentAllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
-	AttachmentMaxSize = sec.Key("MAX_SIZE").MustInt64(4)
-	AttachmentMaxFiles = sec.Key("MAX_FILES").MustInt(5)
-	AttachmentEnabled = sec.Key("ENABLED").MustBool(true)
+	Attachment.StoreType = sec.Key("STORE_TYPE").MustString("local")
+	switch Attachment.StoreType {
+	case "local":
+		Attachment.Path = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
+		if !filepath.IsAbs(Attachment.Path) {
+			Attachment.Path = path.Join(AppWorkPath, Attachment.Path)
+		}
+	case "minio":
+		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
+		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
+		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
+		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
+		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
+		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
+		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
+	}
+
+	Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
+	Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4)
+	Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5)
+	Attachment.Enabled = sec.Key("ENABLED").MustBool(true)
 
 	timeFormatKey := Cfg.Section("time").Key("FORMAT").MustString("")
 	if timeFormatKey != "" {

modules/storage/minio.go

@@ -6,6 +6,7 @@ package storage
 
 import (
 	"io"
+	"path"
 	"strings"
 
 	"github.com/minio/minio-go"
@@ -18,34 +19,41 @@ var (
 // MinioStorage returns a minio bucket storage
 type MinioStorage struct {
 	client   *minio.Client
-	location string
 	bucket   string
+	location string
 	basePath string
 }
 
 // NewMinioStorage returns a minio storage
-func NewMinioStorage(endpoint, accessKeyID, secretAccessKey, location, bucket, basePath string, useSSL bool) (*MinioStorage, error) {
+func NewMinioStorage(endpoint, accessKeyID, secretAccessKey, bucket, location, basePath string, useSSL bool) (*MinioStorage, error) {
 	minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL)
 	if err != nil {
 		return nil, err
 	}
 
+	if err := minioClient.MakeBucket(bucket, location); err != nil {
+		// Check to see if we already own this bucket (which happens if you run this twice)
+		exists, errBucketExists := minioClient.BucketExists(bucket)
+		if !exists || errBucketExists != nil {
+			return nil, err
+		}
+	}
+
 	return &MinioStorage{
-		location: location,
 		client:   minioClient,
 		bucket:   bucket,
 		basePath: basePath,
 	}, nil
 }
 
-func buildMinioPath(p string) string {
-	return strings.TrimPrefix(p, "/")
+func (m *MinioStorage) buildMinioPath(p string) string {
+	return strings.TrimPrefix(path.Join(m.basePath, p), "/")
 }
 
 // Open open a file
 func (m *MinioStorage) Open(path string) (io.ReadCloser, error) {
 	var opts = minio.GetObjectOptions{}
-	object, err := m.client.GetObject(m.bucket, buildMinioPath(path), opts)
+	object, err := m.client.GetObject(m.bucket, m.buildMinioPath(path), opts)
 	if err != nil {
 		return nil, err
 	}
@@ -54,10 +62,10 @@ func (m *MinioStorage) Open(path string) (io.ReadCloser, error) {
 
 // Save save a file to minio
 func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) {
-	return m.client.PutObject(m.bucket, buildMinioPath(path), r, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"})
+	return m.client.PutObject(m.bucket, m.buildMinioPath(path), r, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"})
 }
 
 // Delete delete a file
 func (m *MinioStorage) Delete(path string) error {
-	return m.client.RemoveObject(m.bucket, buildMinioPath(path))
+	return m.client.RemoveObject(m.bucket, m.buildMinioPath(path))
 }

modules/storage/storage.go

@@ -5,6 +5,7 @@
 package storage
 
 import (
+	"fmt"
 	"io"
 
 	"code.gitea.io/gitea/modules/setting"
@@ -36,7 +37,24 @@ var (
 // Init init the stoarge
 func Init() error {
 	var err error
-	Attachments, err = NewLocalStorage(setting.AttachmentPath)
+	switch setting.Attachment.StoreType {
+	case "local":
+		Attachments, err = NewLocalStorage(setting.Attachment.Path)
+	case "minio":
+		minio := setting.Attachment.Minio
+		Attachments, err = NewMinioStorage(
+			minio.Endpoint,
+			minio.AccessKeyID,
+			minio.SecretAccessKey,
+			minio.Bucket,
+			minio.Location,
+			minio.BasePath,
+			minio.UseSSL,
+		)
+	default:
+		return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType)
+	}
+
 	if err != nil {
 		return err
 	}

routers/api/v1/repo/release_attachment.go

@@ -154,7 +154,7 @@ func CreateReleaseAttachment(ctx *context.APIContext) {
 	//     "$ref": "#/responses/error"
 
 	// Check if attachments are enabled
-	if !setting.AttachmentEnabled {
+	if !setting.Attachment.Enabled {
 		ctx.NotFound("Attachment is not enabled")
 		return
 	}
@@ -182,7 +182,7 @@ func CreateReleaseAttachment(ctx *context.APIContext) {
 	}
 
 	// Check if the filetype is allowed by the settings
-	err = upload.VerifyAllowedContentType(buf, strings.Split(setting.AttachmentAllowedTypes, ","))
+	err = upload.VerifyAllowedContentType(buf, strings.Split(setting.Attachment.AllowedTypes, ","))
 	if err != nil {
 		ctx.Error(http.StatusBadRequest, "DetectContentType", err)
 		return

routers/repo/attachment.go

@@ -18,15 +18,16 @@ import (
 )
 
 func renderAttachmentSettings(ctx *context.Context) {
-	ctx.Data["IsAttachmentEnabled"] = setting.AttachmentEnabled
-	ctx.Data["AttachmentAllowedTypes"] = setting.AttachmentAllowedTypes
-	ctx.Data["AttachmentMaxSize"] = setting.AttachmentMaxSize
-	ctx.Data["AttachmentMaxFiles"] = setting.AttachmentMaxFiles
+	ctx.Data["IsAttachmentEnabled"] = setting.Attachment.Enabled
+	ctx.Data["AttachmentStoreType"] = setting.Attachment.StoreType
+	ctx.Data["AttachmentAllowedTypes"] = setting.Attachment.AllowedTypes
+	ctx.Data["AttachmentMaxSize"] = setting.Attachment.MaxSize
+	ctx.Data["AttachmentMaxFiles"] = setting.Attachment.MaxFiles
 }
 
 // UploadAttachment response for uploading issue's attachment
 func UploadAttachment(ctx *context.Context) {
-	if !setting.AttachmentEnabled {
+	if !setting.Attachment.Enabled {
 		ctx.Error(404, "attachment is not enabled")
 		return
 	}
@@ -44,7 +45,7 @@ func UploadAttachment(ctx *context.Context) {
 		buf = buf[:n]
 	}
 
-	err = upload.VerifyAllowedContentType(buf, strings.Split(setting.AttachmentAllowedTypes, ","))
+	err = upload.VerifyAllowedContentType(buf, strings.Split(setting.Attachment.AllowedTypes, ","))
 	if err != nil {
 		ctx.Error(400, err.Error())
 		return

routers/repo/issue.go

@@ -613,7 +613,7 @@ func NewIssuePost(ctx *context.Context, form auth.CreateIssueForm) {
 		return
 	}
 
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachments = form.Files
 	}
 
@@ -1516,7 +1516,7 @@ func NewComment(ctx *context.Context, form auth.CreateCommentForm) {
 	}
 
 	var attachments []string
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachments = form.Files
 	}
 

routers/repo/pull.go

@@ -896,7 +896,7 @@ func CompareAndPullRequestPost(ctx *context.Context, form auth.CreateIssueForm)
 		return
 	}
 
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachments = form.Files
 	}
 

routers/repo/release.go

@@ -215,7 +215,7 @@ func NewReleasePost(ctx *context.Context, form auth.NewReleaseForm) {
 	}
 
 	var attachmentUUIDs []string
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachmentUUIDs = form.Files
 	}
 
@@ -336,7 +336,7 @@ func EditReleasePost(ctx *context.Context, form auth.EditReleaseForm) {
 	}
 
 	var attachmentUUIDs []string
-	if setting.AttachmentEnabled {
+	if setting.Attachment.Enabled {
 		attachmentUUIDs = form.Files
 	}