Browse Source

fmt code, improve some issue

tags/vopendata0.1.2
palytoxin 5 years ago
parent
commit
eba1670c6f
12 changed files with 76 additions and 71 deletions
  1. +12
    -12
      models/file_chunk.go
  2. +3
    -0
      modules/minio_ext/api-error-response.go
  3. +10
    -12
      modules/minio_ext/api.go
  4. +3
    -0
      modules/minio_ext/object.go
  5. +2
    -2
      modules/minio_ext/util.go
  6. +2
    -2
      modules/setting/setting.go
  7. +21
    -20
      modules/storage/minio_ext.go
  8. +0
    -4
      routers/repo/attachment.go
  9. +18
    -17
      routers/repo/dir.go
  10. +1
    -1
      templates/base/footer.tmpl
  11. +3
    -1
      templates/home.tmpl
  12. +1
    -0
      web_src/js/components/MinioUploader.vue

+ 12
- 12
models/file_chunk.go View File

@@ -11,17 +11,17 @@ const (
)

type FileChunk struct {
ID int64 `xorm:"pk autoincr"`
UUID string `xorm:"uuid UNIQUE"`
Md5 string `xorm:"INDEX"`
IsUploaded int `xorm:"DEFAULT 0"` // not uploaded: 0, uploaded: 1
UploadID string `xorm:"UNIQUE"`//minio upload id
TotalChunks int
Size int64
UserID int64 `xorm:"INDEX"`
CompletedParts []string `xorm:"DEFAULT """`// chunkNumber+etag eg: ,1-asqwewqe21312312.2-123hjkas
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
ID int64 `xorm:"pk autoincr"`
UUID string `xorm:"uuid UNIQUE"`
Md5 string `xorm:"INDEX"`
IsUploaded int `xorm:"DEFAULT 0"` // not uploaded: 0, uploaded: 1
UploadID string `xorm:"UNIQUE"` //minio upload id
TotalChunks int
Size int64
UserID int64 `xorm:"INDEX"`
CompletedParts []string `xorm:"DEFAULT """` // chunkNumber+etag eg: ,1-asqwewqe21312312.2-123hjkas
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}

// GetFileChunkByMD5 returns fileChunk by given id
@@ -78,7 +78,7 @@ func InsertFileChunk(fileChunk *FileChunk) (_ *FileChunk, err error) {
return nil, err
}

return fileChunk,nil
return fileChunk, nil
}

// UpdateAttachment updates the given attachment in database


+ 3
- 0
modules/minio_ext/api-error-response.go View File

@@ -27,9 +27,11 @@ type ErrorResponse struct {
func (e ErrorResponse) Error() string {
return e.Message
}

const (
reportIssue = "Please report this issue at https://github.com/minio/minio/issues."
)

// httpRespToErrorResponse returns a new encoded ErrorResponse
// structure as error.
func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) error {
@@ -122,6 +124,7 @@ func ToErrorResponse(err error) ErrorResponse {
return ErrorResponse{}
}
}

// ErrInvalidArgument - Invalid argument response.
func ErrInvalidArgument(message string) error {
return ErrorResponse{


+ 10
- 12
modules/minio_ext/api.go View File

@@ -156,7 +156,6 @@ func (r *lockedRandSource) Seed(seed int64) {
r.lk.Unlock()
}


// Different types of url lookup supported by the server.Initialized to BucketLookupAuto
const (
BucketLookupAuto BucketLookupType = iota
@@ -992,8 +991,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R
return req, nil
}


func (c Client) GenUploadPartSignedUrl(uploadID string, bucketName string, objectName string, partNumber int, size int64, expires time.Duration, bucketLocation string) (string, error){
func (c Client) GenUploadPartSignedUrl(uploadID string, bucketName string, objectName string, partNumber int, size int64, expires time.Duration, bucketLocation string) (string, error) {
signedUrl := ""

// Input validation.
@@ -1027,17 +1025,17 @@ func (c Client) GenUploadPartSignedUrl(uploadID string, bucketName string, objec
customHeader := make(http.Header)

reqMetadata := requestMetadata{
presignURL: true,
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
customHeader: customHeader,
presignURL: true,
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
customHeader: customHeader,
//contentBody: reader,
contentLength: size,
contentLength: size,
//contentMD5Base64: md5Base64,
//contentSHA256Hex: sha256Hex,
expires: int64(expires/time.Second),
bucketLocation: bucketLocation,
expires: int64(expires / time.Second),
bucketLocation: bucketLocation,
}

req, err := c.newRequest("PUT", reqMetadata)
@@ -1047,5 +1045,5 @@ func (c Client) GenUploadPartSignedUrl(uploadID string, bucketName string, objec
}

signedUrl = req.URL.String()
return signedUrl,nil
return signedUrl, nil
}

+ 3
- 0
modules/minio_ext/object.go View File

@@ -8,10 +8,12 @@ import (

// StringMap represents map with custom UnmarshalXML
type StringMap map[string]string

// CommonPrefix container for prefix response.
type CommonPrefix struct {
Prefix string
}

// ObjectInfo container for object metadata.
type ObjectInfo struct {
// An ETag is optionally set to md5sum of an object. In case of multipart objects,
@@ -44,6 +46,7 @@ type ObjectInfo struct {
// Error
Err error `json:"-"`
}

// ListBucketResult container for listObjects response.
type ListBucketResult struct {
// A response can contain CommonPrefixes only if you have


+ 2
- 2
modules/minio_ext/util.go View File

@@ -1,6 +1,7 @@
package minio_ext

import (
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"io"
@@ -10,7 +11,6 @@ import (
"net/url"
"regexp"
"strings"
"crypto/sha256"

"github.com/minio/minio-go/v6/pkg/s3utils"
)
@@ -21,7 +21,6 @@ var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/")
// regCred matches signature string in HTTP header
var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)")


// xmlDecoder provide decoded value in xml.
func xmlDecoder(body io.Reader, v interface{}) error {
d := xml.NewDecoder(body)
@@ -44,6 +43,7 @@ func redactSignature(origAuth string) string {
// Strip out 256-bit signature from: Signature=<256-bit signature>
return regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**")
}

// closeResponse close non nil response with any response Body.
// convenient wrapper to drain any remaining data on response body.
//


+ 2
- 2
modules/setting/setting.go View File

@@ -309,7 +309,7 @@ var (
Bucket string
Location string
BasePath string
RealPath string
RealPath string
}
AllowedTypes string
MaxSize int64
@@ -325,7 +325,7 @@ var (
Bucket string
Location string
BasePath string
RealPath string
RealPath string
}{},
AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
MaxSize: 4,


+ 21
- 20
modules/storage/minio_ext.go View File

@@ -21,39 +21,41 @@ const (
)

type ComplPart struct {
PartNumber int `json:"partNumber"`
ETag string `json:"eTag"`
PartNumber int `json:"partNumber"`
ETag string `json:"eTag"`
}

type CompleteParts struct {
Data []ComplPart `json:"completedParts"`
Data []ComplPart `json:"completedParts"`
}

// completedParts is a collection of parts sortable by their part numbers.
// used for sorting the uploaded parts before completing the multipart request.
type completedParts []miniov6.CompletePart

func (a completedParts) Len() int { return len(a) }
func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }

// completeMultipartUpload container for completing multipart upload.
type completeMultipartUpload struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"`
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"`
Parts []miniov6.CompletePart `xml:"Part"`
}

var (
adminClient * minio_ext.Client = nil
coreClient *miniov6.Core = nil
adminClient *minio_ext.Client = nil
coreClient *miniov6.Core = nil
)

var mutex *sync.Mutex

func init(){
func init() {
mutex = new(sync.Mutex)
}

func getClients()(*minio_ext.Client, *miniov6.Core, error){
var client * minio_ext.Client
func getClients() (*minio_ext.Client, *miniov6.Core, error) {
var client *minio_ext.Client
var core *miniov6.Core
mutex.Lock()

@@ -73,8 +75,8 @@ func getClients()(*minio_ext.Client, *miniov6.Core, error){
minio.AccessKeyID,
minio.SecretAccessKey,
minio.UseSSL,
)
if nil != err{
)
if nil != err {
return nil, nil, err
}
}
@@ -88,7 +90,7 @@ func getClients()(*minio_ext.Client, *miniov6.Core, error){
minio.SecretAccessKey,
minio.UseSSL,
)
if nil != err{
if nil != err {
return nil, nil, err
}
}
@@ -113,7 +115,7 @@ func GenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, partSiz

}

func NewMultiPartUpload(uuid string) (string, error){
func NewMultiPartUpload(uuid string) (string, error) {
_, core, err := getClients()
if err != nil {
log.Error("getClients failed:", err.Error())
@@ -127,7 +129,7 @@ func NewMultiPartUpload(uuid string) (string, error){
return core.NewMultipartUpload(bucketName, objectName, miniov6.PutObjectOptions{})
}

func CompleteMultiPartUpload(uuid string, uploadID string, complParts []string) (string, error){
func CompleteMultiPartUpload(uuid string, uploadID string, complParts []string) (string, error) {
_, core, err := getClients()
if err != nil {
log.Error("getClients failed:", err.Error())
@@ -139,15 +141,15 @@ func CompleteMultiPartUpload(uuid string, uploadID string, complParts []string)
objectName := strings.TrimPrefix(path.Join(minio.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")

var complMultipartUpload completeMultipartUpload
for _,part := range complParts {
partNumber, err := strconv.Atoi(strings.Split(part,"-")[0])
for _, part := range complParts {
partNumber, err := strconv.Atoi(strings.Split(part, "-")[0])
if err != nil {
log.Error(err.Error())
return "",err
return "", err
}
complMultipartUpload.Parts =append(complMultipartUpload.Parts, miniov6.CompletePart{
complMultipartUpload.Parts = append(complMultipartUpload.Parts, miniov6.CompletePart{
PartNumber: partNumber,
ETag: strings.Split(part,"-")[1],
ETag: strings.Split(part, "-")[1],
})
}

@@ -156,4 +158,3 @@ func CompleteMultiPartUpload(uuid string, uploadID string, complParts []string)

return core.CompleteMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload.Parts)
}


+ 0
- 4
routers/repo/attachment.go View File

@@ -21,10 +21,6 @@ import (
"code.gitea.io/gitea/modules/upload"
"code.gitea.io/gitea/modules/worker"

"net/http"
"strconv"
"strings"

gouuid "github.com/satori/go.uuid"
)



+ 18
- 17
routers/repo/dir.go View File

@@ -1,15 +1,16 @@
package repo

import (
"errors"
"io/ioutil"
"path"
"strings"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"errors"
"io/ioutil"
"path"
"strings"
)

const (
@@ -17,12 +18,12 @@ const (
)

type FileInfo struct {
FileName string
ModTime string
IsDir bool
Size int64
ParenDir string
UUID string
FileName string
ModTime string
IsDir bool
Size int64
ParenDir string
UUID string
}

func DirIndex(ctx *context.Context) {
@@ -52,7 +53,7 @@ func DirIndex(ctx *context.Context) {
}

files, err := ioutil.ReadDir(setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.Attachment.Minio.BasePath +
path.Join(uuid[0:1], uuid[1:2], uuid + uuid) + "/" + parentDir)
path.Join(uuid[0:1], uuid[1:2], uuid+uuid) + "/" + parentDir)
if err != nil {
log.Error("ReadDir failed:", err.Error())
ctx.ServerError("ReadDir failed:", err)
@@ -76,12 +77,12 @@ func DirIndex(ctx *context.Context) {
}

fileInfos = append(fileInfos, FileInfo{
FileName:file.Name(),
ModTime:file.ModTime().Format("2006-01-02 15:04:05"),
IsDir:file.IsDir(),
Size:file.Size(),
ParenDir:tmp,
UUID:uuid,
FileName: file.Name(),
ModTime: file.ModTime().Format("2006-01-02 15:04:05"),
IsDir: file.IsDir(),
Size: file.Size(),
ParenDir: tmp,
UUID: uuid,
})
i++
}


+ 1
- 1
templates/base/footer.tmpl View File

@@ -38,7 +38,7 @@
<script src="{{StaticUrlPrefix}}/vendor/plugins/tribute/tribute.min.js"></script>
{{end}}
{{if .PageIsHome}}
<link rel="stylesheet" href="{{StaticUrlPrefix}}/vendor/plugins/jquery.particleground/jquery.particleground.min.js">
<script rel="stylesheet" src="{{StaticUrlPrefix}}/vendor/plugins/jquery.particleground/jquery.particleground.min.js"></script>
{{end}}
<script src="{{StaticUrlPrefix}}/fomantic/semantic.min.js?v={{MD5 AppVer}}"></script>
<script src="{{StaticUrlPrefix}}/js/index.js?v={{MD5 AppVer}}"></script>


+ 3
- 1
templates/home.tmpl View File

@@ -7,7 +7,9 @@
lineColor: 'rgba(68,114,196,0.1)'
});
var intro = document.getElementById('intro');
intro.style.marginTop = - intro.offsetHeight / 2 + 'px';
if (intro) {
intro.style.marginTop = - intro.offsetHeight / 2 + 'px';
}
}, false);
</script>
<div id="index-project">


+ 1
- 0
web_src/js/components/MinioUploader.vue View File

@@ -65,6 +65,7 @@ export default {
this.file_status_text = this.dropzoneParams.data('file-status');
this.status = this.dropzoneParams.data('file-init-status');
const $dropzone = $('div#dataset');
console.log('createDropzone');
const dropzoneUploader = await createDropzone($dropzone[0], {
url: '/todouploader',
maxFiles: this.maxFiles,


Loading…
Cancel
Save