diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 63ea95e65..9249946c1 100755 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -753,8 +753,6 @@ MINIO_LOCATION = us-east-1 MINIO_BASE_PATH = attachments/ ; Minio enabled ssl only available when STORE_TYPE is `minio` MINIO_USE_SSL = false -; real Minio storage path -MINIO_REAL_PATH = /mnt/test/minio/data/ [time] ; Specifies the format for fully outputted dates. Defaults to RFC1123 @@ -1041,11 +1039,19 @@ RETRY_BACKOFF = 3 [machinery] ; redis conf for decompress dataset zip file. -BROKER = redis://localhost:6379 +BROKER = amqp://admin:pcladmin@127.0.0.1:5672/ DEFAULT_QUEUE = DecompressTasksQueue -RESULT_BACKEND = redis://localhost:6379 +RESULT_BACKEND = amqp://admin:pcladmin@127.0.0.1:5672/ +EXCHANGE = decompress_exchange +EXCHANGE_TYPE = direct +BINDING_KEY = decompress_task [cloudbrain] HOST = http://192.168.204.24 USERNAME = PASSWORD = + +[decompress] +HOST = http://192.168.207.34:39987 +USER = cW4cMtH24eoWPE7X +PASSWORD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2085b0873..60f4b41c1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -309,7 +309,6 @@ var ( Bucket string Location string BasePath string - RealPath string } AllowedTypes string MaxSize int64 @@ -325,7 +324,6 @@ var ( Bucket string Location string BasePath string - RealPath string }{}, AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip", MaxSize: 4, @@ -425,6 +423,14 @@ var ( Broker string DefaultQueue string ResultBackend string + Exchange string + ExchangeType string + BindingKey string + + //decompress config + DecompressAddress string + AuthUser string + AuthPassword string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -894,7 +900,6 @@ func NewContext() { Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1") Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/") Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false) - Attachment.Minio.RealPath = sec.Key("MINIO_REAL_PATH").MustString("/mnt/test/minio/data/") } Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1) @@ -1084,9 +1089,17 @@ func NewContext() { } sec = Cfg.Section("machinery") - Broker = sec.Key("BROKER").MustString("redis://localhost:6379") + Broker = sec.Key("BROKER").MustString("amqp://admin:pcladmin@127.0.0.1:5672/") DefaultQueue = sec.Key("DEFAULT_QUEUE").MustString("DecompressTasksQueue") - ResultBackend = sec.Key("RESULT_BACKEND").MustString("redis://localhost:6379") + ResultBackend = sec.Key("RESULT_BACKEND").MustString("amqp://admin:pcladmin@127.0.0.1:5672/") + Exchange = sec.Key("EXCHANGE").MustString("decompress_exchange") + ExchangeType = sec.Key("EXCHANGE_TYPE").MustString("direct") + BindingKey = sec.Key("BINDING_KEY").MustString("decompress_task") + + sec = Cfg.Section("decompress") + DecompressAddress = sec.Key("HOST").MustString("http://192.168.207.34:39987") + AuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") + AuthPassword = sec.Key("PASSWORD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") } func loadInternalToken(sec *ini.Section) string { diff --git a/modules/worker/worker.go b/modules/worker/worker.go index 37445c978..1dbfff3b3 100755 --- a/modules/worker/worker.go +++ b/modules/worker/worker.go @@ -15,6 +15,11 @@ func NewTaskCenter() { Broker: setting.Broker, DefaultQueue: setting.DefaultQueue, ResultBackend: setting.ResultBackend, + AMQP: &mchConf.AMQPConfig{ + Exchange: "machinery_exchange", + ExchangeType: "direct", + BindingKey: "machinery_task", + }, } tc, err := machinery.NewServer(cnf) if err != nil { diff --git a/routers/repo/dir.go b/routers/repo/dir.go index 4d7cdafe8..7665d9ff7 100755 --- a/routers/repo/dir.go +++ b/routers/repo/dir.go @@ -1,10 +1,10 @@ package repo import ( + "encoding/json" "errors" - "os" - "path" - "sort" + "io/ioutil" + "net/http" "strings" "code.gitea.io/gitea/models" @@ -19,12 +19,17 @@ const ( ) type FileInfo struct { - FileName string - ModTime string - IsDir bool - Size int64 - ParenDir string - UUID string + FileName string `json:"FileName"` + ModTime string `json:"ModTime"` + IsDir bool `json:"IsDir"` + Size int64 `json:"Size"` + ParenDir string `json:"ParenDir"` + UUID string `json:"UUID"` +} + +type RespGetDirs struct { + ResultCode string `json:"resultCode"` + FileInfos string `json:"fileInfos"` } func DirIndex(ctx *context.Context) { @@ -53,39 +58,19 @@ func DirIndex(ctx *context.Context) { dirArray = []string{attachment.Name} } - files, err := readDir(setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.Attachment.Minio.BasePath + - path.Join(uuid[0:1], uuid[1:2], uuid+uuid) + "/" + parentDir) + dirs, err := getDirs(uuid, parentDir) if err != nil { - log.Error("ReadDir failed:", err.Error()) - ctx.ServerError("ReadDir failed:", err) + log.Error("getDirs failed:", err.Error()) + ctx.ServerError("getDirs failed:", err) return } - i := 1 var fileInfos []FileInfo - for _, file := range files { - if i > 100 { - break - } - - log.Info(file.Name()) - - var tmp string - if parentDir == "" { - tmp = file.Name() - } else { - tmp = parentDir + "/" + file.Name() - } - - fileInfos = append(fileInfos, FileInfo{ - FileName: file.Name(), - ModTime: file.ModTime().Format("2006-01-02 15:04:05"), - IsDir: file.IsDir(), - Size: file.Size(), - ParenDir: tmp, - UUID: uuid, - }) - i++ + err = json.Unmarshal([]byte(dirs), &fileInfos) + if err != nil { + log.Error("json.Unmarshal failed:", err.Error()) + ctx.ServerError("json.Unmarshal failed:", err) + return } ctx.Data["Path"] = dirArray @@ -95,18 +80,53 @@ func DirIndex(ctx *context.Context) { ctx.HTML(200, tplDirIndex) } -// readDir reads the directory named by dirname and returns -// a list of directory entries sorted by filename. -func readDir(dirname string) ([]os.FileInfo, error) { - f, err := os.Open(dirname) +func getDirs(uuid string, parentDir string) (string,error) { + var dirs string + var req string + if parentDir == "" { + req = "uuid=" + uuid + } else { + req = "uuid=" + uuid + "&parentDir=" + parentDir + } + + url := setting.DecompressAddress + "/dirs?" + req + reqHttp, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + log.Error("http.NewRequest failed:", err.Error()) + return dirs, err + } + + reqHttp.SetBasicAuth(setting.AuthUser, setting.AuthPassword) + res, err := http.DefaultClient.Do(reqHttp) if err != nil { - return nil, err + log.Error("send http to decompress failed:", err.Error()) + return dirs, err } - list, err := f.Readdir(100) - f.Close() + + if res.StatusCode != http.StatusOK { + log.Error("the response from decompress is failed") + return dirs, errors.New("the response from decompress is failed") + } + + body,err := ioutil.ReadAll(res.Body) + if err != nil { + log.Error("read resp body failed:", err.Error()) + return dirs, err + } + + var resp RespGetDirs + err = json.Unmarshal(body, &resp) if err != nil { - return nil, err + log.Error("unmarshal resp failed:", err.Error()) + return dirs, err + } + + if resp.ResultCode != "0" { + log.Error("GetDirs failed:", resp.ResultCode) + return dirs, errors.New("GetDirs failed") } - sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() }) - return list, nil + + dirs = resp.FileInfos + + return dirs, nil }