JointCloud
/
JCS-pub
Not watched
1
0
Fork 0
Code
Releases 0 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

代码优化

feature_gxh
Sydonian 4 months ago
parent
367faa3251
commit
89ebc0e01f
7 changed files with 17 additions and 422 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -323
      client/internal/http/v1/aws_auth.go
  2. +0
    -31
      client/sdk/api/v1/cache.go
  3. +0
    -24
      client/sdk/api/v1/package.go
  4. +9
    -35
      client/sdk/api/v1/user_space.go
  5. +4
    -5
      common/assets/confs/client.config.json
  6. +2
    -2
      common/pkgs/ioswitch2/parser/opt/pin.go
  7. +2
    -2
      common/pkgs/ioswitchlrc/parser/passes.go

+ 0
- 323
client/internal/http/v1/aws_auth.go View File

@@ -1,323 +0,0 @@
package http

/*
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"

"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/gin-gonic/gin"
"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
)

const (
AuthRegion = "any"
AuthService = "jcs"
AuthorizationHeader = "Authorization"
)

type AWSAuth struct {
cfg *types.ConfigJSON
cred aws.Credentials
signer *v4.Signer
}

func NewAWSAuth(cfg *types.ConfigJSON) *AWSAuth {
auth := &AWSAuth{
cfg: cfg,
}

if cfg.AuthAccessKey != "" && cfg.AuthSecretKey != "" {
prod := credentials.NewStaticCredentialsProvider(cfg.AuthAccessKey, cfg.AuthSecretKey, "")
cred, _ := prod.Retrieve(context.TODO())
auth.cred = cred
auth.signer = v4.NewSigner()
}

return auth
}

func (a *AWSAuth) Auth(c *gin.Context) {
if a.signer == nil {
c.Next()
return
}

authorizationHeader := c.GetHeader(AuthorizationHeader)
if authorizationHeader == "" {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.Unauthorized, "authorization header is missing"))
return
}

_, headers, reqSig, err := parseAuthorizationHeader(authorizationHeader)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.Unauthorized, "invalid Authorization header format"))
return
}

// 限制请求体大小
rd := io.LimitReader(c.Request.Body, a.cfg.MaxBodySize)
body, err := io.ReadAll(rd)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "read request body failed"))
return
}

timestamp, err := time.Parse("20060102T150405Z", c.GetHeader("X-Amz-Date"))
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "invalid X-Amz-Date header format"))
return
}

if time.Now().After(timestamp.Add(5 * time.Minute)) {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "X-Amz-Date is expired"))
return
}

payloadHash := sha256.Sum256(body)
hexPayloadHash := hex.EncodeToString(payloadHash[:])

// 构造验签用的请求
verifyReq, err := http.NewRequest(c.Request.Method, c.Request.URL.String(), nil)
if err != nil {
c.AbortWithStatusJSON(http.StatusOK, types.Failed(ecode.OperationFailed, err.Error()))
return
}
for _, h := range headers {
if strings.EqualFold(h, "content-length") {
verifyReq.ContentLength = c.Request.ContentLength
} else if strings.EqualFold(h, "host") {
verifyReq.Host = c.Request.Host
} else {
verifyReq.Header.Add(h, c.Request.Header.Get(h))
}
}

signer := v4.NewSigner()
err = signer.SignHTTP(context.TODO(), a.cred, verifyReq, hexPayloadHash, AuthService, AuthRegion, timestamp)
if err != nil {
logger.Warnf("sign request: %v", err)
c.AbortWithStatusJSON(http.StatusOK, types.Failed(ecode.OperationFailed, "sign request failed"))
return
}

verifySig := getSignatureFromAWSHeader(verifyReq)
if !strings.EqualFold(verifySig, reqSig) {
logger.Warnf("signature mismatch, input header: %s, verify: %s", authorizationHeader, verifyReq.Header.Get(AuthorizationHeader))
c.AbortWithStatusJSON(http.StatusOK, types.Failed(ecode.Unauthorized, "signature mismatch"))
return
}

c.Request.Body = io.NopCloser(bytes.NewReader(body))

c.Next()
}

func (a *AWSAuth) AuthWithoutBody(c *gin.Context) {
if a.signer == nil {
c.Next()
return
}

authorizationHeader := c.GetHeader(AuthorizationHeader)
if authorizationHeader == "" {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.Unauthorized, "authorization header is missing"))
return
}

_, headers, reqSig, err := parseAuthorizationHeader(authorizationHeader)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.Unauthorized, "invalid Authorization header format"))
return
}

timestamp, err := time.Parse("20060102T150405Z", c.GetHeader("X-Amz-Date"))
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "invalid X-Amz-Date header format"))
return
}

if time.Now().After(timestamp.Add(5 * time.Minute)) {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "X-Amz-Date is expired"))
return
}

// 构造验签用的请求
verifyReq, err := http.NewRequest(c.Request.Method, c.Request.URL.String(), nil)
if err != nil {
c.AbortWithStatusJSON(http.StatusOK, types.Failed(ecode.OperationFailed, err.Error()))
return
}
for _, h := range headers {
if strings.EqualFold(h, "content-length") {
verifyReq.ContentLength = c.Request.ContentLength
} else if strings.EqualFold(h, "host") {
verifyReq.Host = c.Request.Host
} else {
verifyReq.Header.Add(h, c.Request.Header.Get(h))
}
}

err = a.signer.SignHTTP(context.TODO(), a.cred, verifyReq, "", AuthService, AuthRegion, timestamp)

if err != nil {
logger.Warnf("sign request: %v", err)
c.AbortWithStatusJSON(http.StatusOK, types.Failed(ecode.OperationFailed, "sign request failed"))
return
}

verifySig := getSignatureFromAWSHeader(verifyReq)
if !strings.EqualFold(verifySig, reqSig) {
logger.Warnf("signature mismatch, input header: %s, verify: %s", authorizationHeader, verifySig)
c.AbortWithStatusJSON(http.StatusOK, types.Failed(ecode.Unauthorized, "signature mismatch"))
return
}

c.Next()
}

func (a *AWSAuth) PresignedAuth(c *gin.Context) {
if a.signer == nil {
c.Next()
return
}

query := c.Request.URL.Query()

signature := query.Get("X-Amz-Signature")
query.Del("X-Amz-Signature")
if signature == "" {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "missing X-Amz-Signature query parameter"))
return
}

// alg := c.Request.URL.Query().Get("X-Amz-Algorithm")
// cred := c.Request.URL.Query().Get("X-Amz-Credential")

date := query.Get("X-Amz-Date")
expiresStr := query.Get("X-Expires")
expires, err := strconv.ParseInt(expiresStr, 10, 64)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "invalid X-Expires format"))
return
}

signedHeaders := strings.Split(query.Get("X-Amz-SignedHeaders"), ";")

c.Request.URL.RawQuery = query.Encode()

verifyReq, err := http.NewRequest(c.Request.Method, c.Request.URL.String(), nil)
if err != nil {
c.AbortWithStatusJSON(http.StatusOK, types.Failed(ecode.OperationFailed, err.Error()))
return
}
for _, h := range signedHeaders {
if strings.EqualFold(h, "content-length") {
verifyReq.ContentLength = c.Request.ContentLength
} else if strings.EqualFold(h, "host") {
verifyReq.Host = c.Request.Host
} else {
verifyReq.Header.Add(h, c.Request.Header.Get(h))
}
}

timestamp, err := time.Parse("20060102T150405Z", date)
if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, types.Failed(ecode.BadArgument, "invalid X-Amz-Date format"))
return
}

if time.Now().After(timestamp.Add(time.Duration(expires) * time.Second)) {
c.AbortWithStatusJSON(http.StatusUnauthorized, types.Failed(ecode.Unauthorized, "request expired"))
return
}

signer := v4.NewSigner()
uri, _, err := signer.PresignHTTP(context.TODO(), a.cred, verifyReq, "", AuthService, AuthRegion, timestamp)
if err != nil {
c.AbortWithStatusJSON(http.StatusOK, types.Failed(ecode.OperationFailed, "sign request failed"))
return
}

verifySig := getSignatureFromAWSQuery(uri)
if !strings.EqualFold(verifySig, signature) {
logger.Warnf("signature mismatch, input: %s, verify: %s", signature, verifySig)
c.AbortWithStatusJSON(http.StatusOK, types.Failed(ecode.Unauthorized, "signature mismatch"))
return
}

c.Next()
}

// 解析 Authorization 头部
func parseAuthorizationHeader(authorizationHeader string) (string, []string, string, error) {
if !strings.HasPrefix(authorizationHeader, "AWS4-HMAC-SHA256 ") {
return "", nil, "", fmt.Errorf("invalid Authorization header format")
}

authorizationHeader = strings.TrimPrefix(authorizationHeader, "AWS4-HMAC-SHA256")

parts := strings.Split(authorizationHeader, ",")
if len(parts) != 3 {
return "", nil, "", fmt.Errorf("invalid Authorization header format")
}

var credential, signedHeaders, signature string
for _, part := range parts {
part = strings.TrimSpace(part)

if strings.HasPrefix(part, "Credential=") {
credential = strings.TrimPrefix(part, "Credential=")
}
if strings.HasPrefix(part, "SignedHeaders=") {
signedHeaders = strings.TrimPrefix(part, "SignedHeaders=")
}
if strings.HasPrefix(part, "Signature=") {
signature = strings.TrimPrefix(part, "Signature=")
}
}

if credential == "" || signedHeaders == "" || signature == "" {
return "", nil, "", fmt.Errorf("missing necessary parts in Authorization header")
}

headers := strings.Split(signedHeaders, ";")
return credential, headers, signature, nil
}

func getSignatureFromAWSHeader(req *http.Request) string {
auth := req.Header.Get(AuthorizationHeader)
idx := strings.Index(auth, "Signature=")
if idx == -1 {
return ""
}

return auth[idx+len("Signature="):]
}

func getSignatureFromAWSQuery(uri string) string {
idx := strings.Index(uri, "X-Amz-Signature=")
if idx == -1 {
return ""
}

andIdx := strings.Index(uri[idx:], "&")
if andIdx == -1 {
return uri[idx+len("X-Amz-Signature="):]
}

return uri[idx+len("X-Amz-Signature=") : andIdx]
}
*/

+ 0
- 31
client/sdk/api/v1/cache.go View File

@@ -1,31 +0,0 @@
package api

/*
import (
"net/http"

"gitlink.org.cn/cloudream/common/sdks"
cdssdk "gitlink.org.cn/cloudream/jcs-pub/client/types"
)

const CacheMovePackagePath = "/cache/movePackage"

type CacheMovePackageReq struct {
PackageID cdssdk.PackageID `json:"packageID"`
StorageID cdssdk.StorageID `json:"storageID"`
}

func (r *CacheMovePackageReq) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, CacheMovePackagePath, r)
}

type CacheMovePackageResp struct{}

func (r *CacheMovePackageResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

func (c *Client) CacheMovePackage(req CacheMovePackageReq) (*CacheMovePackageResp, error) {
return JSONAPI(&c.cfg, c.httpCli, &req, &CacheMovePackageResp{})
}
*/

+ 0
- 24
client/sdk/api/v1/package.go View File

@@ -207,27 +207,3 @@ func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error
func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) { func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) {
return JSONAPI(&c.cfg, c.httpCli, &req, &PackageListBucketPackagesResp{}) return JSONAPI(&c.cfg, c.httpCli, &req, &PackageListBucketPackagesResp{})
} }

const PackageGetCachedStoragesPath = "/package/getCachedStorages"

type PackageGetCachedStoragesReq struct {
PackageID clitypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
}

func (r *PackageGetCachedStoragesReq) MakeParam() *sdks.RequestParam {
return sdks.MakeQueryParam(http.MethodGet, PackageGetCachedStoragesPath, r)
}

/*
type PackageGetCachedStoragesResp struct {
cdssdk.PackageCachingInfo
}

func (r *PackageGetCachedStoragesResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) {
return JSONAPI(&c.cfg, c.httpCli, &req, &PackageGetCachedStoragesResp{})
}
*/

+ 9
- 35
client/sdk/api/v1/user_space.go View File

@@ -8,7 +8,7 @@ import (
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


const UserSpaceDownloadPackagePath = "/userspace/downloadPackage"
const UserSpaceDownloadPackagePath = "/userSpace/downloadPackage"


type UserSpaceDownloadPackageReq struct { type UserSpaceDownloadPackageReq struct {
PackageID clitypes.PackageID `json:"packageID" binding:"required"` PackageID clitypes.PackageID `json:"packageID" binding:"required"`
@@ -30,7 +30,7 @@ func (c *Client) UserSpaceDownloadPackage(req UserSpaceDownloadPackageReq) (*Use
return JSONAPI(&c.cfg, c.httpCli, &req, &UserSpaceDownloadPackageResp{}) return JSONAPI(&c.cfg, c.httpCli, &req, &UserSpaceDownloadPackageResp{})
} }


const UserSpaceCreatePackagePath = "/userspace/createPackage"
const UserSpaceCreatePackagePath = "/userSpace/createPackage"


type UserSpaceCreatePackageReq struct { type UserSpaceCreatePackageReq struct {
UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"` UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"`
@@ -56,7 +56,7 @@ func (c *Client) UserSpaceCreatePackage(req UserSpaceCreatePackageReq) (*UserSpa
return JSONAPI(&c.cfg, c.httpCli, &req, &UserSpaceCreatePackageResp{}) return JSONAPI(&c.cfg, c.httpCli, &req, &UserSpaceCreatePackageResp{})
} }


const UserSpaceGetPath = "/userspace/get"
const UserSpaceGetPath = "/userSpace/get"


type UserSpaceGet struct { type UserSpaceGet struct {
UserSpaceID clitypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"` UserSpaceID clitypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"`
@@ -67,7 +67,7 @@ func (r *UserSpaceGet) MakeParam() *sdks.RequestParam {
} }


type UserSpaceGetResp struct { type UserSpaceGetResp struct {
clitypes.UserSpace
UserSpace clitypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceGetResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceGetResp) ParseResponse(resp *http.Response) error {
@@ -79,7 +79,7 @@ func (c *Client) UserSpaceGet(req UserSpaceGet) (*UserSpaceGetResp, error) {
} }


// 创建用户空间 // 创建用户空间
const UserSpaceCreatePath = "/userspace/create"
const UserSpaceCreatePath = "/userSpace/create"


type UserSpaceCreate struct { type UserSpaceCreate struct {
Name string `json:"name" binding:"required"` Name string `json:"name" binding:"required"`
@@ -95,7 +95,7 @@ func (r *UserSpaceCreate) MakeParam() *sdks.RequestParam {
} }


type UserSpaceCreateResp struct { type UserSpaceCreateResp struct {
clitypes.UserSpace
UserSpace clitypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceCreateResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceCreateResp) ParseResponse(resp *http.Response) error {
@@ -107,7 +107,7 @@ func (c *Client) UserSpaceCreate(req UserSpaceCreate) (*UserSpaceCreateResp, err
} }


// 更新用户空间。一些重要的配置不可再二次修改 // 更新用户空间。一些重要的配置不可再二次修改
const UserSpaceUpdatePath = "/userspace/update"
const UserSpaceUpdatePath = "/userSpace/update"


type UserSpaceUpdate struct { type UserSpaceUpdate struct {
UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"` UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"`
@@ -133,7 +133,7 @@ func (c *Client) UserSpaceUpdate(req UserSpaceUpdate) (*UserSpaceUpdateResp, err
} }


// 删除用户空间 // 删除用户空间
const UserSpaceDeletePath = "/userspace/delete"
const UserSpaceDeletePath = "/userSpace/delete"


type UserSpaceDelete struct { type UserSpaceDelete struct {
UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"` UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"`
@@ -154,7 +154,7 @@ func (c *Client) UserSpaceDelete(req UserSpaceDelete) (*UserSpaceDeleteResp, err
} }


// 测试给定用户空间的配置是否有效 // 测试给定用户空间的配置是否有效
const UserSpaceTestPath = "/userspace/test"
const UserSpaceTestPath = "/userSpace/test"


type UserSpaceTest struct { type UserSpaceTest struct {
Storage cortypes.StorageType `json:"storage" binding:"required"` Storage cortypes.StorageType `json:"storage" binding:"required"`
@@ -177,29 +177,3 @@ func (r *UserSpaceTestResp) ParseResponse(resp *http.Response) error {
func (c *Client) UserSpaceTest(req UserSpaceTest) (*UserSpaceTestResp, error) { func (c *Client) UserSpaceTest(req UserSpaceTest) (*UserSpaceTestResp, error) {
return JSONAPI(&c.cfg, c.httpCli, &req, &UserSpaceTestResp{}) return JSONAPI(&c.cfg, c.httpCli, &req, &UserSpaceTestResp{})
} }

// 存储服务间直传
const UserSpaceSpaceToSpacePath = "/userspace/spaceToSpace"

type UserSpaceSpaceToSpace struct {
SrcUserSpaceID clitypes.UserSpaceID `json:"srcUserSpaceID" binding:"required"`
DstUserSpaceID clitypes.UserSpaceID `json:"dstUserSpaceID" binding:"required"`
SrcPath string `json:"srcPath" binding:"required"`
DstPath string `json:"dstPath" binding:"required"`
}

func (r *UserSpaceSpaceToSpace) MakeParam() *sdks.RequestParam {
return sdks.MakeJSONParam(http.MethodPost, UserSpaceSpaceToSpacePath, r)
}

type UserSpaceSpaceToSpaceResp struct {
clitypes.SpaceToSpaceResult
}

func (r *UserSpaceSpaceToSpaceResp) ParseResponse(resp *http.Response) error {
return sdks.ParseCodeDataJSONResponse(resp, r)
}

func (c *Client) UserSpaceSpaceToSpace(req UserSpaceSpaceToSpace) (*UserSpaceSpaceToSpaceResp, error) {
return JSONAPI(&c.cfg, c.httpCli, &req, &UserSpaceSpaceToSpaceResp{})
}

+ 4
- 5
common/assets/confs/client.config.json View File

@@ -1,7 +1,4 @@
{ {
"local": {
"locationID": 1
},
"hubRPC": { "hubRPC": {
"rootCA": "" "rootCA": ""
}, },
@@ -45,9 +42,11 @@
"http": { "http": {
"enabled": true, "enabled": true,
"listen": "127.0.0.1:7890", "listen": "127.0.0.1:7890",
"rootCA": "",
"serverCert": "",
"serverKey": "",
"clientCerts": [],
"userSpaceID": 0, "userSpaceID": 0,
"authAccessKey": "",
"authSecretKey": "",
"maxBodySize": 5242880 "maxBodySize": 5242880
}, },
"mount": { "mount": {


+ 2
- 2
common/pkgs/ioswitch2/parser/opt/pin.go View File

@@ -18,7 +18,7 @@ func Pin(ctx *state.GenerateState) bool {
var toEnv *dag.NodeEnv var toEnv *dag.NodeEnv
for _, out := range node.OutputStreams().Slots.RawArray() { for _, out := range node.OutputStreams().Slots.RawArray() {
for _, to := range out.Dst.RawArray() { for _, to := range out.Dst.RawArray() {
if to.Env().Type == dag.EnvUnknown {
if to.Env().Type == dag.EnvAny {
continue continue
} }


@@ -43,7 +43,7 @@ func Pin(ctx *state.GenerateState) bool {
// 否则根据输入流的始发地来固定 // 否则根据输入流的始发地来固定
var fromEnv *dag.NodeEnv var fromEnv *dag.NodeEnv
for _, in := range node.InputStreams().Slots.RawArray() { for _, in := range node.InputStreams().Slots.RawArray() {
if in.Src.Env().Type == dag.EnvUnknown {
if in.Src.Env().Type == dag.EnvAny {
continue continue
} }




+ 2
- 2
common/pkgs/ioswitchlrc/parser/passes.go View File

@@ -142,7 +142,7 @@ func pin(ctx *GenerateContext) bool {
var toEnv *dag.NodeEnv var toEnv *dag.NodeEnv
for _, out := range node.OutputStreams().Slots.RawArray() { for _, out := range node.OutputStreams().Slots.RawArray() {
for _, to := range out.Dst.RawArray() { for _, to := range out.Dst.RawArray() {
if to.Env().Type == dag.EnvUnknown {
if to.Env().Type == dag.EnvAny {
continue continue
} }


@@ -167,7 +167,7 @@ func pin(ctx *GenerateContext) bool {
// 否则根据输入流的始发地来固定 // 否则根据输入流的始发地来固定
var fromEnv *dag.NodeEnv var fromEnv *dag.NodeEnv
for _, in := range node.InputStreams().Slots.RawArray() { for _, in := range node.InputStreams().Slots.RawArray() {
if in.Src.Env().Type == dag.EnvUnknown {
if in.Src.Env().Type == dag.EnvAny {
continue continue
} }




Write Preview
Loading…
Cancel
Save