diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index b92100135..496f18e11 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -6,6 +6,7 @@ package cmd import ( "context" + "fmt" "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/migrations" @@ -22,6 +23,53 @@ var CmdMigrateStorage = cli.Command{ Usage: "Migrate the storage", Description: "This is a command for migrating storage.", Action: runMigrateStorage, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "type, t", + Value: "", + Usage: "Files type to migrate, currently should be attachments", + }, + cli.StringFlag{ + Name: "store, s", + Value: "local", + Usage: "New storage type, local or minio", + }, + cli.StringFlag{ + Name: "path, p", + Value: "", + Usage: "New storage placement if store is local", + }, + cli.StringFlag{ + Name: "minio-endpoint", + Value: "", + Usage: "New storage placement if store is local", + }, + cli.StringFlag{ + Name: "minio-access-key-id", + Value: "", + Usage: "New storage placement if store is local", + }, + cli.StringFlag{ + Name: "minio-scret-access-key", + Value: "", + Usage: "New storage placement if store is local", + }, + cli.StringFlag{ + Name: "minio-bucket", + Value: "", + Usage: "New storage placement if store is local", + }, + cli.StringFlag{ + Name: "minio-location", + Value: "", + Usage: "New storage placement if store is local", + }, + cli.StringFlag{ + Name: "minio-use-ssl", + Value: "", + Usage: "New storage placement if store is local", + }, + }, } func migrateAttachments(dstStorage storage.ObjectStorage) error { @@ -47,17 +95,32 @@ func runMigrateStorage(ctx *cli.Context) error { return err } - tp := ctx.String("type") - - // TODO: init setting - if err := storage.Init(); err != nil { return err } + tp := ctx.String("type") switch tp { case "attachments": - dstStorage, err := storage.NewLocalStorage(ctx.String("dst")) + var dstStorage storage.ObjectStorage + var err error + switch ctx.String("store") { + case "local": + dstStorage, err = storage.NewLocalStorage(ctx.String("dst")) + case "minio": + dstStorage, err = storage.NewMinioStorage( + ctx.String("minio-endpoint"), + ctx.String("minio-access-key-id"), + ctx.String("minio-secret-access-key"), + ctx.String("minio-bucket"), + ctx.String("minio-location"), + ctx.String("minio-basePath"), + ctx.Bool("minio-useSSL"), + ) + default: + return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store")) + } + if err != nil { return err } diff --git a/modules/context/context.go b/modules/context/context.go index f8663b9c0..71c8986fb 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -317,7 +317,7 @@ func Contexter() macaron.Handler { // If request sends files, parse them here otherwise the Query() can't be parsed and the CsrfToken will be invalid. if ctx.Req.Method == "POST" && strings.Contains(ctx.Req.Header.Get("Content-Type"), "multipart/form-data") { - if err := ctx.Req.ParseMultipartForm(setting.AttachmentMaxSize << 20); err != nil && !strings.Contains(err.Error(), "EOF") { // 32MB max size + if err := ctx.Req.ParseMultipartForm(setting.Attachment.MaxSize << 20); err != nil && !strings.Contains(err.Error(), "EOF") { // 32MB max size ctx.ServerError("ParseMultipartForm", err) return } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index ede4687c8..65d4d250c 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -298,11 +298,38 @@ var ( EnableXORMLog bool // Attachment settings - AttachmentPath string - AttachmentAllowedTypes string - AttachmentMaxSize int64 - AttachmentMaxFiles int - AttachmentEnabled bool + Attachment = struct { + StoreType string + Path string + Minio struct { + Endpoint string + AccessKeyID string + SecretAccessKey string + UseSSL bool + Bucket string + Location string + BasePath string + } + AllowedTypes string + MaxSize int64 + MaxFiles int + Enabled bool + }{ + StoreType: "local", + Minio: struct { + Endpoint string + AccessKeyID string + SecretAccessKey string + UseSSL bool + Bucket string + Location string + BasePath string + }{}, + AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip", + MaxSize: 4, + MaxFiles: 5, + Enabled: true, + } // Time settings TimeFormat string @@ -845,14 +872,27 @@ func NewContext() { } sec = Cfg.Section("attachment") - AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments")) - if !filepath.IsAbs(AttachmentPath) { - AttachmentPath = path.Join(AppWorkPath, AttachmentPath) - } - AttachmentAllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1) - AttachmentMaxSize = sec.Key("MAX_SIZE").MustInt64(4) - AttachmentMaxFiles = sec.Key("MAX_FILES").MustInt(5) - AttachmentEnabled = sec.Key("ENABLED").MustBool(true) + Attachment.StoreType = sec.Key("STORE_TYPE").MustString("local") + switch Attachment.StoreType { + case "local": + Attachment.Path = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments")) + if !filepath.IsAbs(Attachment.Path) { + Attachment.Path = path.Join(AppWorkPath, Attachment.Path) + } + case "minio": + Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000") + Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("") + Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("") + Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea") + Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1") + Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/") + Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false) + } + + Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1) + Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4) + Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5) + Attachment.Enabled = sec.Key("ENABLED").MustBool(true) timeFormatKey := Cfg.Section("time").Key("FORMAT").MustString("") if timeFormatKey != "" { diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 445ac546c..46bac876c 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -6,6 +6,7 @@ package storage import ( "io" + "path" "strings" "github.com/minio/minio-go" @@ -18,34 +19,41 @@ var ( // MinioStorage returns a minio bucket storage type MinioStorage struct { client *minio.Client - location string bucket string + location string basePath string } // NewMinioStorage returns a minio storage -func NewMinioStorage(endpoint, accessKeyID, secretAccessKey, location, bucket, basePath string, useSSL bool) (*MinioStorage, error) { +func NewMinioStorage(endpoint, accessKeyID, secretAccessKey, bucket, location, basePath string, useSSL bool) (*MinioStorage, error) { minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) if err != nil { return nil, err } + if err := minioClient.MakeBucket(bucket, location); err != nil { + // Check to see if we already own this bucket (which happens if you run this twice) + exists, errBucketExists := minioClient.BucketExists(bucket) + if !exists || errBucketExists != nil { + return nil, err + } + } + return &MinioStorage{ - location: location, client: minioClient, bucket: bucket, basePath: basePath, }, nil } -func buildMinioPath(p string) string { - return strings.TrimPrefix(p, "/") +func (m *MinioStorage) buildMinioPath(p string) string { + return strings.TrimPrefix(path.Join(m.basePath, p), "/") } // Open open a file func (m *MinioStorage) Open(path string) (io.ReadCloser, error) { var opts = minio.GetObjectOptions{} - object, err := m.client.GetObject(m.bucket, buildMinioPath(path), opts) + object, err := m.client.GetObject(m.bucket, m.buildMinioPath(path), opts) if err != nil { return nil, err } @@ -54,10 +62,10 @@ func (m *MinioStorage) Open(path string) (io.ReadCloser, error) { // Save save a file to minio func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) { - return m.client.PutObject(m.bucket, buildMinioPath(path), r, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + return m.client.PutObject(m.bucket, m.buildMinioPath(path), r, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) } // Delete delete a file func (m *MinioStorage) Delete(path string) error { - return m.client.RemoveObject(m.bucket, buildMinioPath(path)) + return m.client.RemoveObject(m.bucket, m.buildMinioPath(path)) } diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 88e84137a..963b38d0f 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -5,6 +5,7 @@ package storage import ( + "fmt" "io" "code.gitea.io/gitea/modules/setting" @@ -36,7 +37,24 @@ var ( // Init init the stoarge func Init() error { var err error - Attachments, err = NewLocalStorage(setting.AttachmentPath) + switch setting.Attachment.StoreType { + case "local": + Attachments, err = NewLocalStorage(setting.Attachment.Path) + case "minio": + minio := setting.Attachment.Minio + Attachments, err = NewMinioStorage( + minio.Endpoint, + minio.AccessKeyID, + minio.SecretAccessKey, + minio.Bucket, + minio.Location, + minio.BasePath, + minio.UseSSL, + ) + default: + return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType) + } + if err != nil { return err } diff --git a/routers/api/v1/repo/release_attachment.go b/routers/api/v1/repo/release_attachment.go index 6ba6489e2..3d1084f21 100644 --- a/routers/api/v1/repo/release_attachment.go +++ b/routers/api/v1/repo/release_attachment.go @@ -154,7 +154,7 @@ func CreateReleaseAttachment(ctx *context.APIContext) { // "$ref": "#/responses/error" // Check if attachments are enabled - if !setting.AttachmentEnabled { + if !setting.Attachment.Enabled { ctx.NotFound("Attachment is not enabled") return } @@ -182,7 +182,7 @@ func CreateReleaseAttachment(ctx *context.APIContext) { } // Check if the filetype is allowed by the settings - err = upload.VerifyAllowedContentType(buf, strings.Split(setting.AttachmentAllowedTypes, ",")) + err = upload.VerifyAllowedContentType(buf, strings.Split(setting.Attachment.AllowedTypes, ",")) if err != nil { ctx.Error(http.StatusBadRequest, "DetectContentType", err) return diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 505468fee..479e8b395 100644 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -18,15 +18,16 @@ import ( ) func renderAttachmentSettings(ctx *context.Context) { - ctx.Data["IsAttachmentEnabled"] = setting.AttachmentEnabled - ctx.Data["AttachmentAllowedTypes"] = setting.AttachmentAllowedTypes - ctx.Data["AttachmentMaxSize"] = setting.AttachmentMaxSize - ctx.Data["AttachmentMaxFiles"] = setting.AttachmentMaxFiles + ctx.Data["IsAttachmentEnabled"] = setting.Attachment.Enabled + ctx.Data["AttachmentStoreType"] = setting.Attachment.StoreType + ctx.Data["AttachmentAllowedTypes"] = setting.Attachment.AllowedTypes + ctx.Data["AttachmentMaxSize"] = setting.Attachment.MaxSize + ctx.Data["AttachmentMaxFiles"] = setting.Attachment.MaxFiles } // UploadAttachment response for uploading issue's attachment func UploadAttachment(ctx *context.Context) { - if !setting.AttachmentEnabled { + if !setting.Attachment.Enabled { ctx.Error(404, "attachment is not enabled") return } @@ -44,7 +45,7 @@ func UploadAttachment(ctx *context.Context) { buf = buf[:n] } - err = upload.VerifyAllowedContentType(buf, strings.Split(setting.AttachmentAllowedTypes, ",")) + err = upload.VerifyAllowedContentType(buf, strings.Split(setting.Attachment.AllowedTypes, ",")) if err != nil { ctx.Error(400, err.Error()) return diff --git a/routers/repo/issue.go b/routers/repo/issue.go index afe64c731..1e8f1d81a 100644 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -613,7 +613,7 @@ func NewIssuePost(ctx *context.Context, form auth.CreateIssueForm) { return } - if setting.AttachmentEnabled { + if setting.Attachment.Enabled { attachments = form.Files } @@ -1516,7 +1516,7 @@ func NewComment(ctx *context.Context, form auth.CreateCommentForm) { } var attachments []string - if setting.AttachmentEnabled { + if setting.Attachment.Enabled { attachments = form.Files } diff --git a/routers/repo/pull.go b/routers/repo/pull.go index d23c93d0b..a7af5bdb3 100644 --- a/routers/repo/pull.go +++ b/routers/repo/pull.go @@ -896,7 +896,7 @@ func CompareAndPullRequestPost(ctx *context.Context, form auth.CreateIssueForm) return } - if setting.AttachmentEnabled { + if setting.Attachment.Enabled { attachments = form.Files } diff --git a/routers/repo/release.go b/routers/repo/release.go index 1eac3dce9..5ded48f61 100644 --- a/routers/repo/release.go +++ b/routers/repo/release.go @@ -215,7 +215,7 @@ func NewReleasePost(ctx *context.Context, form auth.NewReleaseForm) { } var attachmentUUIDs []string - if setting.AttachmentEnabled { + if setting.Attachment.Enabled { attachmentUUIDs = form.Files } @@ -336,7 +336,7 @@ func EditReleasePost(ctx *context.Context, form auth.EditReleaseForm) { } var attachmentUUIDs []string - if setting.AttachmentEnabled { + if setting.Attachment.Enabled { attachmentUUIDs = form.Files }