Merge pull request 'move-getdir' (#68) from move-getdir into develop

Reviewed-by: palytoxin <sxty32@126.com>

custom/conf/app.ini.sample

@@ -753,8 +753,6 @@ MINIO_LOCATION = us-east-1
 MINIO_BASE_PATH = attachments/
 ; Minio enabled ssl only available when STORE_TYPE is `minio`
 MINIO_USE_SSL = false
-; real Minio storage path
-MINIO_REAL_PATH = /mnt/test/minio/data/
 
 [time]
 ; Specifies the format for fully outputted dates. Defaults to RFC1123
@@ -1041,11 +1039,19 @@ RETRY_BACKOFF = 3
 
 [machinery]
 ; redis conf for decompress dataset zip file.
-BROKER = redis://localhost:6379
+BROKER = amqp://admin:pcladmin@127.0.0.1:5672/
 DEFAULT_QUEUE = DecompressTasksQueue
-RESULT_BACKEND = redis://localhost:6379
+RESULT_BACKEND = amqp://admin:pcladmin@127.0.0.1:5672/
+EXCHANGE = decompress_exchange
+EXCHANGE_TYPE = direct
+BINDING_KEY = decompress_task
 
 [cloudbrain]
 HOST     = http://192.168.204.24
 USERNAME =
 PASSWORD =
+
+[decompress]
+HOST     = http://192.168.207.34:39987
+USER     = cW4cMtH24eoWPE7X
+PASSWORD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC

modules/setting/setting.go

@@ -309,7 +309,6 @@ var (
 			Bucket          string
 			Location        string
 			BasePath        string
-			RealPath        string
 		}
 		AllowedTypes string
 		MaxSize      int64
@@ -325,7 +324,6 @@ var (
 			Bucket          string
 			Location        string
 			BasePath        string
-			RealPath        string
 		}{},
 		AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
 		MaxSize:      4,
@@ -425,6 +423,14 @@ var (
 	Broker        string
 	DefaultQueue  string
 	ResultBackend string
+	Exchange      string
+	ExchangeType  string
+	BindingKey    string
+
+	//decompress config
+	DecompressAddress        string
+	AuthUser				 string
+	AuthPassword			 string
 )
 
 // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -894,7 +900,6 @@ func NewContext() {
 		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
 		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
 		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
-		Attachment.Minio.RealPath = sec.Key("MINIO_REAL_PATH").MustString("/mnt/test/minio/data/")
 	}
 
 	Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
@@ -1084,9 +1089,17 @@ func NewContext() {
 	}
 
 	sec = Cfg.Section("machinery")
-	Broker = sec.Key("BROKER").MustString("redis://localhost:6379")
+	Broker = sec.Key("BROKER").MustString("amqp://admin:pcladmin@127.0.0.1:5672/")
 	DefaultQueue = sec.Key("DEFAULT_QUEUE").MustString("DecompressTasksQueue")
-	ResultBackend = sec.Key("RESULT_BACKEND").MustString("redis://localhost:6379")
+	ResultBackend = sec.Key("RESULT_BACKEND").MustString("amqp://admin:pcladmin@127.0.0.1:5672/")
+	Exchange = sec.Key("EXCHANGE").MustString("decompress_exchange")
+	ExchangeType = sec.Key("EXCHANGE_TYPE").MustString("direct")
+	BindingKey = sec.Key("BINDING_KEY").MustString("decompress_task")
+
+	sec = Cfg.Section("decompress")
+	DecompressAddress = sec.Key("HOST").MustString("http://192.168.207.34:39987")
+	AuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X")
+	AuthPassword = sec.Key("PASSWORD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC")
 }
 
 func loadInternalToken(sec *ini.Section) string {

modules/worker/worker.go

@@ -15,6 +15,11 @@ func NewTaskCenter() {
 		Broker:        setting.Broker,
 		DefaultQueue:  setting.DefaultQueue,
 		ResultBackend: setting.ResultBackend,
+		AMQP:          &mchConf.AMQPConfig{
+			Exchange:     "machinery_exchange",
+			ExchangeType: "direct",
+			BindingKey:   "machinery_task",
+		},
 	}
 	tc, err := machinery.NewServer(cnf)
 	if err != nil {

routers/repo/dir.go

@@ -1,10 +1,10 @@
 package repo
 
 import (
+	"encoding/json"
 	"errors"
-	"os"
-	"path"
-	"sort"
+	"io/ioutil"
+	"net/http"
 	"strings"
 
 	"code.gitea.io/gitea/models"
@@ -19,12 +19,17 @@ const (
 )
 
 type FileInfo struct {
-	FileName string
-	ModTime  string
-	IsDir    bool
-	Size     int64
-	ParenDir string
-	UUID     string
+	FileName string `json:"FileName"`
+	ModTime  string `json:"ModTime"`
+	IsDir    bool `json:"IsDir"`
+	Size     int64 `json:"Size"`
+	ParenDir string `json:"ParenDir"`
+	UUID     string `json:"UUID"`
+}
+
+type RespGetDirs struct {
+	ResultCode string `json:"resultCode"`
+	FileInfos string `json:"fileInfos"`
 }
 
 func DirIndex(ctx *context.Context) {
@@ -53,39 +58,19 @@ func DirIndex(ctx *context.Context) {
 		dirArray = []string{attachment.Name}
 	}
 
-	files, err := readDir(setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.Attachment.Minio.BasePath +
-		path.Join(uuid[0:1], uuid[1:2], uuid+uuid) + "/" + parentDir)
+	dirs, err := getDirs(uuid, parentDir)
 	if err != nil {
-		log.Error("ReadDir failed:", err.Error())
-		ctx.ServerError("ReadDir failed:", err)
+		log.Error("getDirs failed:", err.Error())
+		ctx.ServerError("getDirs failed:", err)
 		return
 	}
 
-	i := 1
 	var fileInfos []FileInfo
-	for _, file := range files {
-		if i > 100 {
-			break
-		}
-
-		log.Info(file.Name())
-
-		var tmp string
-		if parentDir == "" {
-			tmp = file.Name()
-		} else {
-			tmp = parentDir + "/" + file.Name()
-		}
-
-		fileInfos = append(fileInfos, FileInfo{
-			FileName: file.Name(),
-			ModTime:  file.ModTime().Format("2006-01-02 15:04:05"),
-			IsDir:    file.IsDir(),
-			Size:     file.Size(),
-			ParenDir: tmp,
-			UUID:     uuid,
-		})
-		i++
+	err = json.Unmarshal([]byte(dirs), &fileInfos)
+	if err != nil {
+		log.Error("json.Unmarshal failed:", err.Error())
+		ctx.ServerError("json.Unmarshal failed:", err)
+		return
 	}
 
 	ctx.Data["Path"] = dirArray
@@ -95,18 +80,53 @@ func DirIndex(ctx *context.Context) {
 	ctx.HTML(200, tplDirIndex)
 }
 
-// readDir reads the directory named by dirname and returns
-// a list of directory entries sorted by filename.
-func readDir(dirname string) ([]os.FileInfo, error) {
-	f, err := os.Open(dirname)
+func getDirs(uuid string, parentDir string) (string,error) {
+	var dirs string
+	var req string
+	if parentDir == "" {
+		req = "uuid=" + uuid
+	} else {
+		req = "uuid=" + uuid + "&parentDir=" + parentDir
+	}
+
+	url := setting.DecompressAddress + "/dirs?" + req
+	reqHttp, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		log.Error("http.NewRequest failed:", err.Error())
+		return dirs, err
+	}
+
+	reqHttp.SetBasicAuth(setting.AuthUser, setting.AuthPassword)
+	res, err := http.DefaultClient.Do(reqHttp)
 	if err != nil {
-		return nil, err
+		log.Error("send http to decompress failed:", err.Error())
+		return dirs, err
 	}
-	list, err := f.Readdir(100)
-	f.Close()
+
+	if res.StatusCode != http.StatusOK {
+		log.Error("the response from decompress is failed")
+		return dirs, errors.New("the response from decompress is failed")
+	}
+
+	body,err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		log.Error("read resp body failed:", err.Error())
+		return dirs, err
+	}
+
+	var resp RespGetDirs
+	err = json.Unmarshal(body, &resp)
 	if err != nil {
-		return nil, err
+		log.Error("unmarshal resp failed:", err.Error())
+		return dirs, err
+	}
+
+	if resp.ResultCode != "0" {
+		log.Error("GetDirs failed:", resp.ResultCode)
+		return dirs, errors.New("GetDirs failed")
 	}
-	sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() })
-	return list, nil
+
+	dirs = resp.FileInfos
+
+	return dirs, nil
 }