From 63cf68a413212a582c1be1f07184bef756d77b22 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 26 Feb 2025 15:19:55 +0800 Subject: [PATCH 01/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=91=E5=AE=9A=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 12 +++- sdks/pcmscheduler/models.go | 21 ++++--- sdks/scheduler/models.go | 4 +- sdks/uploader/models.go | 31 ++++++++--- sdks/uploader/uploader.go | 107 ++++++++++++++++++------------------ 5 files changed, 100 insertions(+), 75 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index dca430c..ace7fac 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -64,6 +64,11 @@ type CreateJobReq struct { DataDistribute DataDistribute `json:"dataDistributes"` } +type CommonJsonData struct { + ID string `json:"id"` + Name string `json:"name"` +} + type DataDistribute struct { Dataset []DatasetDistribute `json:"dataset"` Code []CodeDistribute `json:"code"` @@ -90,9 +95,10 @@ type CodeDistribute struct { } type ImageDistribute struct { - DataName string `json:"dataName"` - PackageID cdssdk.PackageID `json:"packageID"` - Clusters []DataDetail `json:"clusters"` + DataName string `json:"dataName"` + //PackageID cdssdk.PackageID `json:"packageID"` + ImageID schsdk.ImageID `json:"imageID"` + Clusters []DataDetail `json:"clusters"` } type ModelDistribute struct { diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index d0ab2ff..4007f4a 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -58,9 +58,13 @@ const ( PlatformSugon = "sugon" PlatformOpenI = "openi" PlatformModelArts = "modelarts" + + URL = "url" + ID = "id" ) type TaskID int64 +type DataID int64 type ClusterDetail struct { // 集群ID @@ -395,14 +399,15 @@ type ModelBinding struct { type CodeBinding struct { serder.Metadata `union:"code"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - ImageID int64 `json:"imageID"` - ObjectID cdssdk.ObjectID `json:"objectID"` - FilePath string `json:"filePath"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + //ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Description string `json:"description"` + ImageID schsdk.ImageID `json:"imageID"` + BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` + //FilePath string `json:"filePath"` + PackageID cdssdk.PackageID `json:"packageID"` } //type ImageBinding struct { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 906796d..8992f5b 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -115,7 +115,7 @@ type PCMJobRuntimeInfo struct { //} type JobResource interface { - Noop2() + Noop() } var JobResourceTypeUnion = types.NewTypeUnion[JobResource]( @@ -132,7 +132,7 @@ var _ = serder.UseTypeUnionInternallyTagged(&JobResourceTypeUnion, "type") type JobResourceBase struct{} -func (d *JobResourceBase) Noop2() {} +func (d *JobResourceBase) Noop() {} type CPU struct { serder.Metadata `union:"CPU"` diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 1fe59d1..4d28dc7 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -25,14 +25,24 @@ func (BlockChain) TableName() string { } type Binding struct { - ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - Name string `gorm:"column:name" json:"Name"` - DataType string `gorm:"column:data_type" json:"dataType"` - //JsonData string `gorm:"column:json_data" json:"jsonData"` - Content string `gorm:"column:content" json:"Content"` - AccessLevel string `gorm:"column:access_level" json:"accessLevel"` - CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` +} + +type BindingDAO struct { + ID DataID `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name" json:"Name"` + DataType string `gorm:"column:data_type" json:"dataType"` + Content string `gorm:"column:content" json:"Content"` + AccessLevel string `gorm:"column:access_level" json:"accessLevel"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + BindingCluster []BindingCluster `gorm:"foreignKey:binding_id;references:id" json:"bindingCluster"` } type BindingAccessData struct { @@ -73,6 +83,10 @@ type BindingCluster struct { JsonData string `gorm:"column:json_data" json:"jsonData"` } +func (BindingCluster) TableName() string { + return "bindingCluster" // 确保和数据库中的表名一致 +} + type Folder struct { PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` Path string `gorm:"column:path_name" json:"path"` @@ -136,6 +150,7 @@ type PackageCloneDAO struct { ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` ImageID string `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:created_at" json:"createTime"` } diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go index 7a99b2c..54f17df 100644 --- a/sdks/uploader/uploader.go +++ b/sdks/uploader/uploader.go @@ -3,7 +3,6 @@ package uploadersdk import ( "fmt" "gitlink.org.cn/cloudream/common/pkgs/types" - sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" "gitlink.org.cn/cloudream/common/utils/http2" @@ -12,65 +11,65 @@ import ( "strings" ) -type DataScheduleReq struct { - PackageID cdssdk.PackageID `json:"packageID"` - DataType string `json:"dataType"` - ScheduleTarget ScheduleTarget `json:"scheduleTarget"` -} +//type DataScheduleReq struct { +// PackageID cdssdk.PackageID `json:"packageID"` +// DataType string `json:"dataType"` +// ScheduleTarget ScheduleTarget `json:"scheduleTarget"` +//} //type DataScheduleResp struct { // Results []sch.DataScheduleResult `json:"data"` //} -type TmpDataScheduleResult struct { - Cluster sch.DataDetail `json:"cluster"` - PackageID cdssdk.PackageID `json:"packageID"` - Status bool `json:"status"` - Msg string `json:"msg"` -} - -func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule") - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - println(resp.Body) - - contType := resp.Header.Get("Content-Type") - if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[[]TmpDataScheduleResult] - if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return nil, fmt.Errorf("parsing response: %w", err) - } - - if codeResp.Code == ResponseCodeOK { - var results []sch.DataScheduleResult - for _, tmpResult := range codeResp.Data { - result := sch.DataScheduleResult{ - PackageID: tmpResult.PackageID, - Status: tmpResult.Status, - Msg: tmpResult.Msg, - Clusters: []sch.DataDetail{ - tmpResult.Cluster, - }, - } - results = append(results, result) - } - return results, nil - } - - return nil, codeResp.ToError() - } +//type TmpDataScheduleResult struct { +// Cluster sch.DataDetail `json:"cluster"` +// PackageID cdssdk.PackageID `json:"packageID"` +// Status bool `json:"status"` +// Msg string `json:"msg"` +//} - return nil, fmt.Errorf("unknow response content type: %s", contType) -} +//func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) { +// targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule") +// if err != nil { +// return nil, err +// } +// +// resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ +// Body: req, +// }) +// if err != nil { +// return nil, err +// } +// println(resp.Body) +// +// contType := resp.Header.Get("Content-Type") +// if strings.Contains(contType, http2.ContentTypeJSON) { +// var codeResp response[[]TmpDataScheduleResult] +// if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { +// return nil, fmt.Errorf("parsing response: %w", err) +// } +// +// if codeResp.Code == ResponseCodeOK { +// var results []sch.DataScheduleResult +// for _, tmpResult := range codeResp.Data { +// result := sch.DataScheduleResult{ +// PackageID: tmpResult.PackageID, +// Status: tmpResult.Status, +// Msg: tmpResult.Msg, +// Clusters: []sch.DataDetail{ +// tmpResult.Cluster, +// }, +// } +// results = append(results, result) +// } +// return results, nil +// } +// +// return nil, codeResp.ToError() +// } +// +// return nil, fmt.Errorf("unknow response content type: %s", contType) +//} type UploadReq struct { DataType string `json:"dataType"` From bea84c8eb4027b1d021c684bd070501b899c2d9a Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 26 Feb 2025 16:04:37 +0800 Subject: [PATCH 02/29] =?UTF-8?q?=E5=AD=90=E7=AE=97=E6=B3=95=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 4d28dc7..68c5d8c 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -149,7 +149,7 @@ type PackageCloneDAO struct { BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` - ImageID string `gorm:"column:image_id" json:"imageID"` + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:created_at" json:"createTime"` } @@ -167,7 +167,7 @@ type PackageCloneParam struct { BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `json:"clusterID"` ParentImageID schsdk.ImageID `json:"parentImageID"` - ImageID string `json:"imageID"` + ImageID schsdk.ImageID `json:"imageID"` } type PackageCloneVO struct { @@ -180,6 +180,7 @@ type PackageCloneVO struct { ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` ImageID string `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:created_at" json:"createTime"` ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"` } From 13658da85d310f824746037ffcb0f34a4a6526fb Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 27 Feb 2025 10:35:42 +0800 Subject: [PATCH 03/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=8F=90=E4=BA=A4=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 7 ++++--- sdks/scheduler/models.go | 8 +++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index ace7fac..8bbdd4e 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -78,8 +78,9 @@ type DataDistribute struct { type DataDetail struct { ClusterID schsdk.ClusterID `json:"clusterID"` - StorageID cdssdk.StorageID `json:"storageID"` - JsonData string `json:"jsonData"` + //StorageID cdssdk.StorageID `json:"storageID"` + StorageID cdssdk.StorageID + JsonData string `json:"jsonData"` } type DatasetDistribute struct { @@ -97,7 +98,7 @@ type CodeDistribute struct { type ImageDistribute struct { DataName string `json:"dataName"` //PackageID cdssdk.PackageID `json:"packageID"` - ImageID schsdk.ImageID `json:"imageID"` + ImageID schsdk.ImageID `json:"packageID"` Clusters []DataDetail `json:"clusters"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 8992f5b..cbf0ae7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -99,9 +99,11 @@ type JobResources struct { } type ClusterInfo struct { - ClusterID ClusterID `json:"clusterID"` - Resources []JobResource `json:"resources"` - Runtime PCMJobRuntimeInfo `json:"runtime"` + ClusterID ClusterID `json:"clusterID"` + Resources []JobResource `json:"resources"` + //Files JobFilesInfo `json:"files"` + Code JobFileInfo `json:"code"` + Runtime PCMJobRuntimeInfo `json:"runtime"` } type PCMJobRuntimeInfo struct { From fb1cc453b0054726d41c24f84eea2f47aef68c09 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 27 Feb 2025 15:08:12 +0800 Subject: [PATCH 04/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 47 +++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 68c5d8c..2a92a89 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -127,12 +127,12 @@ type Package struct { } type PackageDAO struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + //JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:create_time" json:"createTime"` UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 @@ -148,10 +148,10 @@ type PackageCloneDAO struct { Description string `gorm:"column:description" json:"description"` BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` - ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` - ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + //ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` } func (PackageCloneDAO) TableName() string { @@ -159,14 +159,12 @@ func (PackageCloneDAO) TableName() string { } type PackageCloneParam struct { - PackageID cdssdk.PackageID `json:"packageID" binding:"required"` - PackageName string `json:"packageName" binding:"required"` - //BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + PackageName string `json:"packageName" binding:"required"` Name string `json:"name"` Description string `json:"description"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `json:"clusterID"` - ParentImageID schsdk.ImageID `json:"parentImageID"` ImageID schsdk.ImageID `json:"imageID"` } @@ -178,11 +176,11 @@ type PackageCloneVO struct { Description string `gorm:"column:description" json:"description"` BootstrapObjectID cdssdk.ObjectID `gorm:"column:bootstrap_object_id" json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` - ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` - ImageID string `gorm:"column:image_id" json:"imageID"` - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:created_at" json:"createTime"` - ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"` + //ParentImageID schsdk.ImageID `gorm:"column:parent_image_id" json:"parentImageID"` + ImageID string `gorm:"column:image_id" json:"imageID"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:created_at" json:"createTime"` + ClusterMapping ClusterMapping `gorm:"foreignKey:cluster_id;references:cluster_id" json:"cluster"` } type ClusterMapping struct { @@ -197,17 +195,6 @@ func (ClusterMapping) TableName() string { return "ClusterMapping" } -//type PackageCloneClusterDAO struct { -// ID DataID `gorm:"column:id" json:"ID"` -// ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` -// ClusterName string `gorm:"column:cluster_name" json:"clusterName"` -//} -// -//type PackageCloneCluster struct { -// ClusterID schsdk.ClusterID `json:"clusterID"` -// ClusterName string `json:"clusterName"` -//} - type ScheduleTarget interface { Noop() } From 91563ca94108529e6323f03fef11acaeb30a30eb Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 28 Feb 2025 13:50:51 +0800 Subject: [PATCH 05/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 4007f4a..32c55f7 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -61,6 +61,8 @@ const ( URL = "url" ID = "id" + + Startup = "startup" ) type TaskID int64 From 1016cdffd5c44a581f6349be306f11ab9826359e Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 28 Feb 2025 14:50:47 +0800 Subject: [PATCH 06/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=AD=90=E7=AE=97=E6=B3=95=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 2a92a89..c4126bc 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -66,7 +66,7 @@ type BindingDetail struct { SSOId string `json:"ssoID"` Name string `json:"Name"` Info sch.DataBinding `json:"info"` - Packages []Package `json:"packages"` + Package Package `json:"package"` Status string `json:"status"` AccessLevel string `json:"accessLevel"` CreateTime time.Time `json:"createTime"` From e888412c58a6d363397cd6e17ee4c3ceb56ef9b0 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 4 Mar 2025 11:16:30 +0800 Subject: [PATCH 07/29] =?UTF-8?q?=E6=96=B0=E5=A2=9Etoken?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 10 +++++++-- sdks/pcmscheduler/models.go | 42 +++++++++++++++++++------------------ sdks/uploader/models.go | 1 + 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 8bbdd4e..6f6a366 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -120,7 +120,7 @@ type ScheduleData struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } -func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { +func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "/createTask") if err != nil { return nil, err @@ -135,6 +135,9 @@ func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, + Header: map[string]string{ + "Authorization": token, + }, }) if err != nil { return nil, err @@ -176,7 +179,7 @@ type DataScheduleResults struct { Results []DataScheduleResult `json:"results"` } -func (c *Client) RunJob(req RunJobReq) error { +func (c *Client) RunJob(req RunJobReq, token string) error { targetUrl, err := url.JoinPath(c.baseURL, "runTask") if err != nil { return err @@ -184,6 +187,9 @@ func (c *Client) RunJob(req RunJobReq) error { resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, + Header: map[string]string{ + "Authorization": token, + }, }) if err != nil { return err diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 32c55f7..4c4a490 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -376,40 +376,42 @@ func (d *DataBindingBase) Noop() {} type DatasetBinding struct { serder.Metadata `union:"dataset"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Category string `json:"category"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` } type ModelBinding struct { serder.Metadata `union:"model"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Category string `json:"category"` - ModelType string `json:"modelType"` - Env string `json:"env"` - Version string `json:"version"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` } type CodeBinding struct { serder.Metadata `union:"code"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - //ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Type string `json:"type"` + Name string `json:"name"` ClusterID schsdk.ClusterID `json:"clusterID"` Description string `json:"description"` ImageID schsdk.ImageID `json:"imageID"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` - //FilePath string `json:"filePath"` - PackageID cdssdk.PackageID `json:"packageID"` + PackageID cdssdk.PackageID `json:"packageID"` + // 当集群为openi的时候，需要传入分支 + Branch string `json:"branch"` } //type ImageBinding struct { diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index c4126bc..7a9879a 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -80,6 +80,7 @@ type BindingCluster struct { BindingID DataID `gorm:"column:binding_id" json:"bindingID"` ClusterID ClusterID `gorm:"column:cluster_id" json:"clusterID"` Status string `gorm:"column:status" json:"status"` + Param string `gorm:"column:param" json:"Param"` JsonData string `gorm:"column:json_data" json:"jsonData"` } From e85760ff14c9be62ceb3a86a108fb4c5347d42d8 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 4 Mar 2025 16:52:01 +0800 Subject: [PATCH 08/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 4c4a490..dcb1719 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -56,8 +56,8 @@ const ( ParentType = "parent" PlatformSugon = "sugon" - PlatformOpenI = "openi" - PlatformModelArts = "modelarts" + PlatformOpenI = "OpenI" + PlatformModelArts = "ModelArts" URL = "url" ID = "id" From 8c629b18b4aef4c75ad1da8792e016286f422424 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 6 Mar 2025 08:58:33 +0800 Subject: [PATCH 09/29] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=8A=B6=E6=80=81=E5=9B=9E=E6=98=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index cbf0ae7..999d8b7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -140,6 +140,7 @@ type CPU struct { serder.Metadata `union:"CPU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -147,6 +148,7 @@ type GPU struct { serder.Metadata `union:"GPU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -154,6 +156,7 @@ type NPU struct { serder.Metadata `union:"NPU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -161,6 +164,7 @@ type Memory struct { serder.Metadata `union:"Memory"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -168,6 +172,7 @@ type DCU struct { serder.Metadata `union:"DCU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -175,6 +180,7 @@ type MLU struct { serder.Metadata `union:"MLU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -182,6 +188,7 @@ type PRICE struct { serder.Metadata `union:"PRICE"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } From dad73eba53d24a7d8fc2ff98ed7a51b0a32b8d0c Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 7 Mar 2025 15:03:38 +0800 Subject: [PATCH 10/29] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 2 ++ sdks/pcmscheduler/models.go | 11 ++++++++++- sdks/scheduler/models.go | 1 + sdks/uploader/models.go | 12 ++++++------ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 6f6a366..269151a 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -163,6 +163,8 @@ func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, erro type RunJobReq struct { TaskID TaskID `json:"taskID"` + JobSetID schsdk.JobSetID `json:"jobSetID"` + LocalJobID string `json:"localJobID"` ScheduledDatas []DataScheduleResults `json:"scheduledDatas"` } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index dcb1719..daca965 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -24,6 +24,7 @@ const ( DATASET = "dataset" IMAGE = "image" MODEL = "model" + RESULT = "result" OrderByName = "name" OrderBySize = "size" @@ -278,10 +279,18 @@ type TaskMessage struct { Message string `json:"message"` } +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` +} + type UploadParams struct { DataType string `json:"dataType"` UploadInfo UploadInfo `json:"uploadInfo"` - //UploadPriority UploadPriority `json:"uploadPriority"` } type UploadInfo interface { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 999d8b7..6ba148c 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -220,6 +220,7 @@ type DataReturnJobInfo struct { JobInfoBase Type string `json:"type"` BucketID cdssdk.BucketID `json:"bucketID"` + BindingType string `json:"bindingType"` TargetLocalJobID string `json:"targetLocalJobID"` } diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 7a9879a..0637b8c 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -112,12 +112,12 @@ func (Cluster) TableName() string { } type Package struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + //JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:create_time" json:"createTime"` Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` From 41767cd308f1d7aafad37194ae79253c3cad3ad9 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 10 Mar 2025 15:42:38 +0800 Subject: [PATCH 11/29] =?UTF-8?q?=E5=AD=98=E5=82=A8=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=A2=9E=E5=8A=A0=E9=89=B4=E6=9D=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 54 ++-- go.sum | 133 ++++----- sdks/sdks.go | 183 ++++++++++++- sdks/storage/cdsapi/bucket.go | 136 +++------- sdks/storage/cdsapi/cache.go | 37 +-- sdks/storage/cdsapi/client.go | 4 +- sdks/storage/cdsapi/config.go | 4 +- sdks/storage/cdsapi/hub.go | 38 +-- sdks/storage/cdsapi/hub_io.go | 10 +- sdks/storage/cdsapi/object.go | 371 ++++++++------------------ sdks/storage/cdsapi/package.go | 245 ++++++----------- sdks/storage/cdsapi/presigned.go | 12 +- sdks/storage/cdsapi/presigned_test.go | 18 +- sdks/storage/cdsapi/storage.go | 112 +++----- sdks/storage/cdsapi/user.go | 66 ++--- sdks/storage/cdsapi/utils.go | 68 +++++ utils/io2/io.go | 14 + utils/serder/serder.go | 10 + 18 files changed, 691 insertions(+), 824 deletions(-) diff --git a/go.mod b/go.mod index fc87255..00dc24a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module gitlink.org.cn/cloudream/common -go 1.22 +go 1.23.0 toolchain go1.23.2 @@ -8,7 +8,7 @@ require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/credentials v1.17.62 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.15 github.com/json-iterator/go v1.1.12 @@ -16,41 +16,41 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/modern-go/reflect2 v1.0.2 github.com/otiai10/copy v1.12.0 - github.com/samber/lo v1.36.0 - github.com/sirupsen/logrus v1.9.2 - github.com/smartystreets/goconvey v1.8.0 + github.com/samber/lo v1.49.1 + github.com/sirupsen/logrus v1.9.3 + github.com/smartystreets/goconvey v1.8.1 github.com/streadway/amqp v1.1.0 github.com/zyedidia/generic v1.2.1 - go.etcd.io/etcd/client/v3 v3.5.9 - golang.org/x/exp v0.0.0-20230519143937-03e91628a987 + go.etcd.io/etcd/client/v3 v3.5.12 + golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa ) -require github.com/aws/smithy-go v1.22.2 // indirect +require ( + github.com/aws/smithy-go v1.22.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/smarty/assertions v1.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect +) require ( - github.com/benbjohnson/clock v1.3.0 // indirect - github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/smartystreets/assertions v1.13.1 // indirect - github.com/stretchr/testify v1.8.2 // indirect - go.etcd.io/etcd/api/v3 v3.5.9 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/goleak v1.1.12 // indirect - go.uber.org/multierr v1.9.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect - google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect - google.golang.org/grpc v1.54.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + go.etcd.io/etcd/api/v3 v3.5.12 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect ) diff --git a/go.sum b/go.sum index f099e44..58cfdfc 100644 --- a/go.sum +++ b/go.sum @@ -6,30 +6,27 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBa github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8= github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -45,122 +42,96 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw= -github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= -github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= -github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= -github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= -github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w= -github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew= +github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM= github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= -github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= -go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= -go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= -go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= -go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= -go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= -go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= +go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= +go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= +go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= +go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= +go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20230519143937-03e91628a987 h1:3xJIFvzUFbu4ls0BTBYcgbCGhA63eAOEMxIHugyXJqA= -golang.org/x/exp v0.0.0-20230519143937-03e91628a987/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/sdks/sdks.go b/sdks/sdks.go index c556cd5..1fedfad 100644 --- a/sdks/sdks.go +++ b/sdks/sdks.go @@ -1,6 +1,19 @@ package sdks -import "fmt" +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/google/go-querystring/query" + "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/io2" + "gitlink.org.cn/cloudream/common/utils/serder" +) type CodeMessageError struct { Code string @@ -10,3 +23,171 @@ type CodeMessageError struct { func (e *CodeMessageError) Error() string { return fmt.Sprintf("code: %s, message: %s", e.Code, e.Message) } + +type RequestParam struct { + Method string + Path string + Query url.Values + Header http.Header + Body RequestBody +} + +func (p *RequestParam) MakeRequest(baseURL string) (*http.Request, error) { + var body io.ReadCloser + bodyLen := int64(-1) + if p.Body != nil { + body = p.Body.IntoStream() + bodyLen = p.Body.Length() + } + + req, err := http.NewRequest(p.Method, joinUrlUnsafe(baseURL, p.Path), body) + if err != nil { + return nil, err + } + req.ContentLength = bodyLen + req.URL.RawQuery = p.Query.Encode() + req.Header = p.Header + + return req, nil +} + +func (p *RequestParam) Do(baseURL string, cli *http.Client) (*http.Response, error) { + req, err := p.MakeRequest(baseURL) + if err != nil { + return nil, err + } + + return cli.Do(req) +} + +func joinUrlUnsafe(base string, path string) string { + if strings.HasSuffix(base, "/") { + if strings.HasPrefix(path, "/") { + return base + path[1:] + } + + return base + path + } + + if strings.HasPrefix(path, "/") { + return base + path + } + + return base + "/" + path +} + +type RequestBody interface { + // 请求体长度，如果长度未知，返回-1 + Length() int64 + // 将内部值变成一个流，用于发送请求 + IntoStream() io.ReadCloser +} + +type StringBody struct { + Value string +} + +func (s StringBody) Length() int64 { + return int64(len(s.Value)) +} + +func (s StringBody) IntoStream() io.ReadCloser { + return io.NopCloser(bytes.NewReader([]byte(s.Value))) +} + +type BytesBody struct { + Value []byte +} + +func (b BytesBody) Length() int64 { + return int64(len(b.Value)) +} + +func (b BytesBody) IntoStream() io.ReadCloser { + return io.NopCloser(bytes.NewReader(b.Value)) +} + +type StreamBody struct { + Stream io.ReadCloser + LengthHint int64 // 长度提示，如果长度未知，可以设置为-1 +} + +func (s StreamBody) Length() int64 { + return s.LengthHint +} + +func (s StreamBody) IntoStream() io.ReadCloser { + return s.Stream +} + +type APIRequest interface { + MakeParam() *RequestParam +} + +func MakeJSONParam(method string, path string, body any) *RequestParam { + data, err := serder.ObjectToJSONEx(body) + if err != nil { + // 开发人员应该保证param是可序列化的 + panic(err) + } + + return &RequestParam{ + Method: method, + Path: path, + Body: BytesBody{Value: data}, + } +} + +func MakeQueryParam(method string, path string, q any) *RequestParam { + values, err := query.Values(q) + if err != nil { + // 开发人员应该保证param是可序列化的 + panic(err) + } + + return &RequestParam{ + Method: method, + Path: path, + Query: values, + } +} + +type APIResponse interface { + ParseResponse(resp *http.Response) error +} + +type CodeDataResponse[T any] struct { + Code string `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +func (r *CodeDataResponse[T]) Unwarp() (T, error) { + if r.Code == errorcode.OK { + return r.Data, nil + } + var def T + return def, &CodeMessageError{Code: r.Code, Message: r.Message} +} + +func ParseCodeDataJSONResponse[T any](resp *http.Response, ret T) error { + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var err error + var r CodeDataResponse[T] + + if err = serder.JSONToObjectStreamExRaw(resp.Body, &r); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + return nil + } + + cont, err := io2.ReadMost(resp.Body, 200) + if err != nil { + return fmt.Errorf("unknow response content type: %s, status: %d", contType, resp.StatusCode) + } + strCont := string(cont) + + return fmt.Errorf("unknow response content type: %s, status: %d, body[:200]: %s", contType, resp.StatusCode, strCont) +} diff --git a/sdks/storage/cdsapi/bucket.go b/sdks/storage/cdsapi/bucket.go index 2cb2e9c..0299949 100644 --- a/sdks/storage/cdsapi/bucket.go +++ b/sdks/storage/cdsapi/bucket.go @@ -1,11 +1,10 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) type BucketService struct { @@ -19,36 +18,24 @@ func (c *Client) Bucket() *BucketService { const BucketGetByNamePath = "/bucket/getByName" type BucketGetByName struct { - UserID cdssdk.UserID `json:"userID" form:"userID" binding:"required"` - Name string `json:"name" form:"name" binding:"required"` + UserID cdssdk.UserID `url:"userID" form:"userID" binding:"required"` + Name string `url:"name" form:"name" binding:"required"` } + +func (r *BucketGetByName) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, BucketGetByNamePath, r) +} + type BucketGetByNameResp struct { Bucket cdssdk.Bucket `json:"bucket"` } -func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) { - url, err := url.JoinPath(c.baseURL, BucketGetByNamePath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[BucketGetByNameResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) { + return JSONAPI[*BucketGetByNameResp](c.cfg, http.DefaultClient, &req) } const BucketCreatePath = "/bucket/create" @@ -58,33 +45,20 @@ type BucketCreate struct { Name string `json:"name" binding:"required"` } +func (r *BucketCreate) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, BucketCreatePath, r) +} + type BucketCreateResp struct { Bucket cdssdk.Bucket `json:"bucket"` } -func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) { - url, err := url.JoinPath(c.baseURL, BucketCreatePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[BucketCreateResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *BucketCreateResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) { + return JSONAPI[*BucketCreateResp](c.cfg, http.DefaultClient, &req) } const BucketDeletePath = "/bucket/delete" @@ -94,64 +68,38 @@ type BucketDelete struct { BucketID cdssdk.BucketID `json:"bucketID" binding:"required"` } -type BucketDeleteResp struct{} - -func (c *BucketService) Delete(req BucketDelete) error { - url, err := url.JoinPath(c.baseURL, BucketDeletePath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } +func (r *BucketDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, BucketDeletePath, r) +} - codeResp, err := ParseJSONResponse[response[BucketDeleteResp]](resp) - if err != nil { - return err - } +type BucketDeleteResp struct{} - if codeResp.Code == errorcode.OK { - return nil - } +func (r *BucketDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return codeResp.ToError() +func (c *BucketService) Delete(req BucketDelete) error { + return JSONAPINoData[*BucketDeleteResp](c.cfg, http.DefaultClient, &req) } const BucketListUserBucketsPath = "/bucket/listUserBuckets" type BucketListUserBucketsReq struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` +} + +func (r *BucketListUserBucketsReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, BucketListUserBucketsPath, r) } type BucketListUserBucketsResp struct { Buckets []cdssdk.Bucket `json:"buckets"` } +func (r *BucketListUserBucketsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *BucketService) ListUserBuckets(req BucketListUserBucketsReq) (*BucketListUserBucketsResp, error) { - url, err := url.JoinPath(c.baseURL, BucketListUserBucketsPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[BucketListUserBucketsResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() + return JSONAPI[*BucketListUserBucketsResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/cache.go b/sdks/storage/cdsapi/cache.go index 609cdd0..bb28e58 100644 --- a/sdks/storage/cdsapi/cache.go +++ b/sdks/storage/cdsapi/cache.go @@ -1,11 +1,10 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) const CacheMovePackagePath = "/cache/movePackage" @@ -15,29 +14,17 @@ type CacheMovePackageReq struct { PackageID cdssdk.PackageID `json:"packageID"` StorageID cdssdk.StorageID `json:"storageID"` } + +func (r *CacheMovePackageReq) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, CacheMovePackagePath, r) +} + type CacheMovePackageResp struct{} +func (r *CacheMovePackageResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) CacheMovePackage(req CacheMovePackageReq) (*CacheMovePackageResp, error) { - url, err := url.JoinPath(c.baseURL, CacheMovePackagePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[CacheMovePackageResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } - - return nil, jsonResp.ToError() + return JSONAPI[*CacheMovePackageResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/client.go b/sdks/storage/cdsapi/client.go index 40af4dd..8e12be3 100644 --- a/sdks/storage/cdsapi/client.go +++ b/sdks/storage/cdsapi/client.go @@ -18,12 +18,12 @@ func (r *response[T]) ToError() *sdks.CodeMessageError { } type Client struct { - baseURL string + cfg *Config } func NewClient(cfg *Config) *Client { return &Client{ - baseURL: cfg.URL, + cfg: cfg, } } diff --git a/sdks/storage/cdsapi/config.go b/sdks/storage/cdsapi/config.go index 484808c..448039e 100644 --- a/sdks/storage/cdsapi/config.go +++ b/sdks/storage/cdsapi/config.go @@ -1,5 +1,7 @@ package cdsapi type Config struct { - URL string `json:"url"` + URL string `json:"url"` + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` } diff --git a/sdks/storage/cdsapi/hub.go b/sdks/storage/cdsapi/hub.go index d1ed7b7..fb360a4 100644 --- a/sdks/storage/cdsapi/hub.go +++ b/sdks/storage/cdsapi/hub.go @@ -1,44 +1,30 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) var HubGetHubsPath = "/hub/getHubs" type HubGetHubsReq struct { - HubIDs []cdssdk.HubID `json:"hubIDs"` + HubIDs []cdssdk.HubID `form:"hubIDs" url:"hubIDs"` +} + +func (r *HubGetHubsReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, HubGetHubsPath, r) } type HubGetHubsResp struct { Hubs []*cdssdk.Hub `json:"hubs"` } +func (r *HubGetHubsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) HubGetHubs(req HubGetHubsReq) (*HubGetHubsResp, error) { - url, err := url.JoinPath(c.baseURL, HubGetHubsPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[HubGetHubsResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } - - return nil, jsonResp.ToError() + return JSONAPI[*HubGetHubsResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/hub_io.go b/sdks/storage/cdsapi/hub_io.go index bd5f1a3..826c425 100644 --- a/sdks/storage/cdsapi/hub_io.go +++ b/sdks/storage/cdsapi/hub_io.go @@ -24,7 +24,7 @@ type GetStreamReq struct { } func (c *Client) GetStream(req GetStreamReq) (io.ReadCloser, error) { - targetUrl, err := url.JoinPath(c.baseURL, GetStreamPath) + targetUrl, err := url.JoinPath(c.cfg.URL, GetStreamPath) if err != nil { return nil, err } @@ -65,7 +65,7 @@ type SendStreamInfo struct { } func (c *Client) SendStream(req SendStreamReq) error { - targetUrl, err := url.JoinPath(c.baseURL, SendStreamPath) + targetUrl, err := url.JoinPath(c.cfg.URL, SendStreamPath) if err != nil { return err } @@ -133,7 +133,7 @@ type ExecuteIOPlanReq struct { } func (c *Client) ExecuteIOPlan(req ExecuteIOPlanReq) error { - targetUrl, err := url.JoinPath(c.baseURL, ExecuteIOPlanPath) + targetUrl, err := url.JoinPath(c.cfg.URL, ExecuteIOPlanPath) if err != nil { return err } @@ -171,7 +171,7 @@ type SendVarReq struct { } func (c *Client) SendVar(req SendVarReq) error { - targetUrl, err := url.JoinPath(c.baseURL, SendVarPath) + targetUrl, err := url.JoinPath(c.cfg.URL, SendVarPath) if err != nil { return err } @@ -214,7 +214,7 @@ type GetVarResp struct { } func (c *Client) GetVar(req GetVarReq) (*GetVarResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, GetVarPath) + targetUrl, err := url.JoinPath(c.cfg.URL, GetVarPath) if err != nil { return nil, err } diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 5ceda5f..80fc18f 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -4,12 +4,14 @@ import ( "fmt" "io" "mime" + "net/http" "net/url" "strings" "time" "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/pkgs/iterator" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" @@ -28,76 +30,52 @@ func (c *Client) Object() *ObjectService { const ObjectListPathByPath = "/object/listByPath" type ObjectListByPath struct { - UserID cdssdk.UserID `form:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" binding:"required"` - Path string `form:"path"` // 允许为空字符串 - IsPrefix bool `form:"isPrefix"` - NoRecursive bool `form:"noRecursive"` // 仅当isPrefix为true时有效，表示仅查询直接属于Prefix下的对象，对于更深的对象，返回它们的公共前缀 + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + PackageID cdssdk.PackageID `form:"packageID" binding:"required" url:"packageID"` + Path string `form:"path" url:"path"` // 允许为空字符串 + IsPrefix bool `form:"isPrefix" url:"isPrefix"` + NoRecursive bool `form:"noRecursive" url:"noRecursive"` // 仅当isPrefix为true时有效，表示仅查询直接属于Prefix下的对象，对于更深的对象，返回它们的公共前缀 } + +func (r *ObjectListByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectListPathByPath, r) +} + type ObjectListByPathResp struct { CommonPrefixes []string `json:"commonPrefixes"` // 仅在IsPrefix为true且NoRecursive为true时有效，包含更深层对象的shared prefix Objects []cdssdk.Object `json:"objects"` // 如果IsPrefix为true且NoRecursive为false，则返回所有匹配的对象，否则只返回直接属于Prefix下的对象 } -func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectListPathByPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectListByPathResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectListByPathResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) { + return JSONAPI[*ObjectListByPathResp](c.cfg, http.DefaultClient, &req) } const ObjectListByIDsPath = "/object/listByIDs" type ObjectListByIDs struct { - UserID cdssdk.UserID `json:"userID" binding:"required"` - ObjectIDs []cdssdk.ObjectID `json:"objectIDs" binding:"required"` + UserID cdssdk.UserID `form:"userID" binding:"required" url:"userID"` + ObjectIDs []cdssdk.ObjectID `form:"objectIDs" binding:"required" url:"objectIDs"` +} + +func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r) } + type ObjectListByIDsResp struct { Objects []*cdssdk.Object `json:"object"` // 与ObjectIDs一一对应，如果某个ID不存在，则对应位置为nil } -func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectListByIDsPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectListByIDsResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectListByIDsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) { + return JSONAPI[*ObjectListByIDsResp](c.cfg, http.DefaultClient, &req) } const ObjectUploadPath = "/object/upload" @@ -127,7 +105,7 @@ type ObjectUploadResp struct { } func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUploadPath) + url, err := url.JoinPath(c.cfg.URL, ObjectUploadPath) if err != nil { return nil, err } @@ -184,7 +162,7 @@ type DownloadingObject struct { } func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) { - url, err := url.JoinPath(c.baseURL, ObjectDownloadPath) + url, err := url.JoinPath(c.cfg.URL, ObjectDownloadPath) if err != nil { return nil, err } @@ -229,7 +207,7 @@ type ObjectDownloadByPath struct { } func (c *ObjectService) DownloadByPath(req ObjectDownloadByPath) (*DownloadingObject, error) { - url, err := url.JoinPath(c.baseURL, ObjectDownloadByPathPath) + url, err := url.JoinPath(c.cfg.URL, ObjectDownloadByPathPath) if err != nil { return nil, err } @@ -279,33 +257,20 @@ type ObjectUpdateInfo struct { Updatings []UpdatingObject `json:"updatings" binding:"required"` } +func (r *ObjectUpdateInfo) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectUpdateInfoPath, r) +} + type ObjectUpdateInfoResp struct { Successes []cdssdk.ObjectID `json:"successes"` } -func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUpdateInfoPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectUpdateInfoResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectUpdateInfoResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) { + return JSONAPI[*ObjectUpdateInfoResp](c.cfg, http.DefaultClient, &req) } const ObjectUpdateInfoByPathPath = "/object/updateInfoByPath" @@ -317,31 +282,18 @@ type ObjectUpdateInfoByPath struct { UpdateTime time.Time `json:"updateTime" binding:"required"` } -type ObjectUpdateInfoByPathResp struct{} - -func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUpdateInfoByPathPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } +func (r *ObjectUpdateInfoByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectUpdateInfoByPathPath, r) +} - jsonResp, err := ParseJSONResponse[response[ObjectUpdateInfoByPathResp]](resp) - if err != nil { - return nil, err - } +type ObjectUpdateInfoByPathResp struct{} - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectUpdateInfoByPathResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) { + return JSONAPI[*ObjectUpdateInfoByPathResp](c.cfg, http.DefaultClient, &req) } const ObjectMovePath = "/object/move" @@ -362,33 +314,20 @@ type ObjectMove struct { Movings []MovingObject `json:"movings" binding:"required"` } +func (r *ObjectMove) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectMovePath, r) +} + type ObjectMoveResp struct { Successes []cdssdk.ObjectID `json:"successes"` } -func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectMovePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectMoveResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectMoveResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) { + return JSONAPI[*ObjectMoveResp](c.cfg, http.DefaultClient, &req) } const ObjectDeletePath = "/object/delete" @@ -398,31 +337,18 @@ type ObjectDelete struct { ObjectIDs []cdssdk.ObjectID `json:"objectIDs" binding:"required"` } -type ObjectDeleteResp struct{} - -func (c *ObjectService) Delete(req ObjectDelete) error { - url, err := url.JoinPath(c.baseURL, ObjectDeletePath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } +func (r *ObjectDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectDeletePath, r) +} - jsonResp, err := ParseJSONResponse[response[ObjectDeleteResp]](resp) - if err != nil { - return err - } +type ObjectDeleteResp struct{} - if jsonResp.Code == errorcode.OK { - return nil - } +func (r *ObjectDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return jsonResp.ToError() +func (c *ObjectService) Delete(req ObjectDelete) error { + return JSONAPINoData[*ObjectDeleteResp](c.cfg, http.DefaultClient, &req) } const ObjectDeleteByPathPath = "/object/deleteByPath" @@ -432,31 +358,19 @@ type ObjectDeleteByPath struct { PackageID cdssdk.PackageID `json:"packageID" binding:"required"` Path string `json:"path" binding:"required"` } -type ObjectDeleteByPathResp struct{} -func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error { - url, err := url.JoinPath(c.baseURL, ObjectDeleteByPathPath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } +func (r *ObjectDeleteByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectDeleteByPathPath, r) +} - jsonResp, err := ParseJSONResponse[response[ObjectDeleteByPathResp]](resp) - if err != nil { - return err - } +type ObjectDeleteByPathResp struct{} - if jsonResp.Code == errorcode.OK { - return nil - } +func (r *ObjectDeleteByPathResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return jsonResp.ToError() +func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error { + return JSONAPINoData[*ObjectDeleteByPathResp](c.cfg, http.DefaultClient, &req) } const ObjectClonePath = "/object/clone" @@ -466,6 +380,10 @@ type ObjectClone struct { Clonings []CloningObject `json:"clonings" binding:"required"` } +func (r *ObjectClone) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectClonePath, r) +} + type CloningObject struct { ObjectID cdssdk.ObjectID `json:"objectID" binding:"required"` NewPath string `json:"newPath" binding:"required"` @@ -476,64 +394,35 @@ type ObjectCloneResp struct { Objects []*cdssdk.Object `json:"objects"` } -func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectClonePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectCloneResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectCloneResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) { + return JSONAPI[*ObjectCloneResp](c.cfg, http.DefaultClient, &req) } const ObjectGetPackageObjectsPath = "/object/getPackageObjects" type ObjectGetPackageObjects struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` } + +func (r *ObjectGetPackageObjects) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectGetPackageObjectsPath, r) +} + type ObjectGetPackageObjectsResp struct { Objects []cdssdk.Object `json:"objects"` } -func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectGetPackageObjectsPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectGetPackageObjectsResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectGetPackageObjectsResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) { + return JSONAPI[*ObjectGetPackageObjectsResp](c.cfg, http.DefaultClient, &req) } const ObjectNewMultipartUploadPath = "/v1/object/newMultipartUpload" @@ -544,33 +433,20 @@ type ObjectNewMultipartUpload struct { Path string `json:"path" binding:"required"` } +func (r *ObjectNewMultipartUpload) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectNewMultipartUploadPath, r) +} + type ObjectNewMultipartUploadResp struct { Object cdssdk.Object `json:"object"` } -func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectNewMultipartUploadPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectNewMultipartUploadResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectNewMultipartUploadResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) { + return JSONAPI[*ObjectNewMultipartUploadResp](c.cfg, http.DefaultClient, &req) } const ObjectUploadPartPath = "/v1/object/uploadPart" @@ -589,7 +465,7 @@ type ObjectUploadPartInfo struct { type ObjectUploadPartResp struct{} func (c *ObjectService) UploadPart(req ObjectUploadPart) (*ObjectUploadPartResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectUploadPartPath) + url, err := url.JoinPath(c.cfg.URL, ObjectUploadPartPath) if err != nil { return nil, err } @@ -636,31 +512,18 @@ type ObjectCompleteMultipartUpload struct { Indexes []int `json:"indexes" binding:"required"` } +func (r *ObjectCompleteMultipartUpload) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, ObjectCompleteMultipartUploadPath, r) +} + type ObjectCompleteMultipartUploadResp struct { Object cdssdk.Object `json:"object"` } -func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) { - url, err := url.JoinPath(c.baseURL, ObjectCompleteMultipartUploadPath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - jsonResp, err := ParseJSONResponse[response[ObjectCompleteMultipartUploadResp]](resp) - if err != nil { - return nil, err - } - - if jsonResp.Code == errorcode.OK { - return &jsonResp.Data, nil - } +func (r *ObjectCompleteMultipartUploadResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, jsonResp.ToError() +func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) { + return JSONAPI[*ObjectCompleteMultipartUploadResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index e72ae3e..085871f 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -2,11 +2,12 @@ package cdsapi import ( "fmt" + "net/http" "net/url" - "strings" "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/pkgs/iterator" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" @@ -23,72 +24,48 @@ func (c *Client) Package() *PackageService { const PackageGetPath = "/package/get" type PackageGetReq struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` } + +func (r *PackageGetReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageGetPath, r) +} + type PackageGetResp struct { cdssdk.Package } -func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageGetResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { + return JSONAPI[*PackageGetResp](c.cfg, http.DefaultClient, &req) } const PackageGetByFullNamePath = "/package/getByFullName" type PackageGetByFullName struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - BucketName string `form:"bucketName" json:"bucketName" binding:"required"` - PackageName string `form:"packageName" json:"packageName" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + BucketName string `form:"bucketName" url:"bucketName" binding:"required"` + PackageName string `form:"packageName" url:"packageName" binding:"required"` } + +func (r *PackageGetByFullName) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageGetByFullNamePath, r) +} + type PackageGetByFullNameResp struct { Package cdssdk.Package `json:"package"` } -func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetByFullNamePath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetByFullNameResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { + return JSONAPI[*PackageGetByFullNameResp](c.cfg, http.DefaultClient, &req) } const PackageCreatePath = "/package/create" @@ -99,33 +76,20 @@ type PackageCreate struct { Name string `json:"name"` } +func (r *PackageCreate) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageCreatePath, r) +} + type PackageCreateResp struct { Package cdssdk.Package `json:"package"` } -func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) { - url, err := url.JoinPath(s.baseURL, PackageCreatePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageCreateResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageCreateResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) { + return JSONAPI[*PackageCreateResp](s.cfg, http.DefaultClient, &req) } const PackageCreateLoadPath = "/package/createLoad" @@ -147,7 +111,7 @@ type PackageCreateLoadResp struct { } func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadResp, error) { - url, err := url.JoinPath(c.baseURL, PackageCreateLoadPath) + url, err := url.JoinPath(c.cfg.URL, PackageCreateLoadPath) if err != nil { return nil, err } @@ -190,35 +154,18 @@ type PackageDelete struct { PackageID cdssdk.PackageID `json:"packageID" binding:"required"` } -func (c *PackageService) Delete(req PackageDelete) error { - url, err := url.JoinPath(c.baseURL, PackageDeletePath) - if err != nil { - return err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } - - contType := resp.Header.Get("Content-Type") - - if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[any] - if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return fmt.Errorf("parsing response: %w", err) - } +func (r *PackageDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageDeletePath, r) +} - if codeResp.Code == errorcode.OK { - return nil - } +type PackageDeleteResp struct{} - return codeResp.ToError() - } +func (r *PackageDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return fmt.Errorf("unknow response content type: %s", contType) +func (c *PackageService) Delete(req PackageDelete) error { + return JSONAPINoData[*PackageDeleteResp](c.cfg, http.DefaultClient, &req) } const PackageClonePath = "/package/clone" @@ -230,102 +177,64 @@ type PackageClone struct { Name string `json:"name" binding:"required"` } +func (r *PackageClone) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, PackageClonePath, r) +} + type PackageCloneResp struct { Package cdssdk.Package `json:"package"` } -func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) { - url, err := url.JoinPath(c.baseURL, PackageClonePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageCloneResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageCloneResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) { + return JSONAPI[*PackageCloneResp](c.cfg, http.DefaultClient, &req) } const PackageListBucketPackagesPath = "/package/listBucketPackages" type PackageListBucketPackages struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - BucketID cdssdk.BucketID `form:"bucketID" json:"bucketID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + BucketID cdssdk.BucketID `form:"bucketID" url:"bucketID" binding:"required"` +} + +func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageListBucketPackagesPath, r) } type PackageListBucketPackagesResp struct { Packages []cdssdk.Package `json:"packages"` } -func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) { - url, err := url.JoinPath(c.baseURL, PackageListBucketPackagesPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageListBucketPackagesResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) { + return JSONAPI[*PackageListBucketPackagesResp](c.cfg, http.DefaultClient, &req) } const PackageGetCachedStoragesPath = "/package/getCachedStorages" type PackageGetCachedStoragesReq struct { - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` +} + +func (r *PackageGetCachedStoragesReq) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, PackageGetCachedStoragesPath, r) } type PackageGetCachedStoragesResp struct { cdssdk.PackageCachingInfo } -func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) { - url, err := url.JoinPath(c.baseURL, PackageGetCachedStoragesPath) - if err != nil { - return nil, err - } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[PackageGetCachedStoragesResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *PackageGetCachedStoragesResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) { + return JSONAPI[*PackageGetCachedStoragesResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/presigned.go b/sdks/storage/cdsapi/presigned.go index ec7f357..ae83642 100644 --- a/sdks/storage/cdsapi/presigned.go +++ b/sdks/storage/cdsapi/presigned.go @@ -20,15 +20,11 @@ const ( type PresignedService struct { *Client - accessKey string - secretKey string } -func (c *Client) Presigned(accessKey string, secretKey string) *PresignedService { +func (c *Client) Presigned() *PresignedService { return &PresignedService{ - Client: c, - accessKey: accessKey, - secretKey: secretKey, + Client: c, } } @@ -112,7 +108,7 @@ func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectComp } func (c *PresignedService) presign(req any, path string, method string, expireIn int) (string, error) { - u, err := url.Parse(c.baseURL) + u, err := url.Parse(c.cfg.URL) if err != nil { return "", err } @@ -126,7 +122,7 @@ func (c *PresignedService) presign(req any, path string, method string, expireIn u.RawQuery = us.Encode() - prod := credentials.NewStaticCredentialsProvider(c.accessKey, c.secretKey, "") + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") cred, err := prod.Retrieve(context.TODO()) if err != nil { return "", err diff --git a/sdks/storage/cdsapi/presigned_test.go b/sdks/storage/cdsapi/presigned_test.go index ae34e3d..37c5b87 100644 --- a/sdks/storage/cdsapi/presigned_test.go +++ b/sdks/storage/cdsapi/presigned_test.go @@ -9,11 +9,13 @@ import ( func Test_Presigned(t *testing.T) { cli := NewClient(&Config{ - URL: "http://localhost:7890", + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", }) Convey("下载文件", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ UserID: 1, PackageID: 3, @@ -26,7 +28,7 @@ func Test_Presigned(t *testing.T) { }) Convey("上传文件", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectUpload(PresignedObjectUpload{ UserID: 1, PackageID: 3, @@ -43,7 +45,7 @@ func Test_PresignedObjectDownload(t *testing.T) { }) Convey("下载文件", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ UserID: 1, PackageID: 3, @@ -62,7 +64,7 @@ func Test_PresignedObjectUpload(t *testing.T) { }) Convey("上传文件", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectUpload(PresignedObjectUpload{ UserID: 1, PackageID: 3, @@ -79,7 +81,7 @@ func Test_PresignedNewMultipartUpload(t *testing.T) { }) Convey("启动分片上传", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectNewMultipartUpload(PresignedObjectNewMultipartUpload{ UserID: 1, PackageID: 3, @@ -96,7 +98,7 @@ func Test_PresignedObjectUploadPart(t *testing.T) { }) Convey("上传分片", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectUploadPart(PresignedObjectUploadPart{ UserID: 1, ObjectID: 7, @@ -113,7 +115,7 @@ func Test_PresignedCompleteMultipartUpload(t *testing.T) { }) Convey("合并分片", t, func() { - pre := cli.Presigned("123456", "123456") + pre := cli.Presigned() url, err := pre.ObjectCompleteMultipartUpload(PresignedObjectCompleteMultipartUpload{ UserID: 1, ObjectID: 7, diff --git a/sdks/storage/cdsapi/storage.go b/sdks/storage/cdsapi/storage.go index f45b234..b1986b8 100644 --- a/sdks/storage/cdsapi/storage.go +++ b/sdks/storage/cdsapi/storage.go @@ -1,14 +1,10 @@ package cdsapi import ( - "fmt" - "net/url" - "strings" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" - "gitlink.org.cn/cloudream/common/utils/serder" ) const StorageLoadPackagePath = "/storage/loadPackage" @@ -19,31 +15,19 @@ type StorageLoadPackageReq struct { StorageID cdssdk.StorageID `json:"storageID" binding:"required"` RootPath string `json:"rootPath"` } + +func (r *StorageLoadPackageReq) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, StorageLoadPackagePath, r) +} + type StorageLoadPackageResp struct{} +func (r *StorageLoadPackageResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) StorageLoadPackage(req StorageLoadPackageReq) (*StorageLoadPackageResp, error) { - url, err := url.JoinPath(c.baseURL, StorageLoadPackagePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[StorageLoadPackageResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() + return JSONAPI[*StorageLoadPackageResp](c.cfg, http.DefaultClient, &req) } const StorageCreatePackagePath = "/storage/createPackage" @@ -57,71 +41,41 @@ type StorageCreatePackageReq struct { StorageAffinity cdssdk.StorageID `json:"storageAffinity"` } +func (r *StorageCreatePackageReq) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, StorageCreatePackagePath, r) +} + type StorageCreatePackageResp struct { PackageID cdssdk.PackageID `json:"packageID"` } +func (r *StorageCreatePackageResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) StorageCreatePackage(req StorageCreatePackageReq) (*StorageCreatePackageResp, error) { - url, err := url.JoinPath(c.baseURL, StorageCreatePackagePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - - contType := resp.Header.Get("Content-Type") - if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[StorageCreatePackageResp] - if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { - return nil, fmt.Errorf("parsing response: %w", err) - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() - } - - return nil, fmt.Errorf("unknow response content type: %s", contType) + return JSONAPI[*StorageCreatePackageResp](c.cfg, http.DefaultClient, &req) } const StorageGetPath = "/storage/get" type StorageGet struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - StorageID cdssdk.StorageID `form:"storageID" json:"storageID" binding:"required"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + StorageID cdssdk.StorageID `form:"storageID" url:"storageID" binding:"required"` +} + +func (r *StorageGet) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, StorageGetPath, r) } + type StorageGetResp struct { cdssdk.Storage } +func (r *StorageGetResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} + func (c *Client) StorageGet(req StorageGet) (*StorageGetResp, error) { - url, err := url.JoinPath(c.baseURL, StorageGetPath) - if err != nil { - return nil, err - } - - resp, err := http2.GetForm(url, http2.RequestParam{ - Query: req, - }) - if err != nil { - return nil, err - } - - codeResp, err := ParseJSONResponse[response[StorageGetResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } - - return nil, codeResp.ToError() + return JSONAPI[*StorageGetResp](c.cfg, http.DefaultClient, &req) } diff --git a/sdks/storage/cdsapi/user.go b/sdks/storage/cdsapi/user.go index 7397f89..295e322 100644 --- a/sdks/storage/cdsapi/user.go +++ b/sdks/storage/cdsapi/user.go @@ -1,11 +1,10 @@ package cdsapi import ( - "net/url" + "net/http" - "gitlink.org.cn/cloudream/common/consts/errorcode" + "gitlink.org.cn/cloudream/common/sdks" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gitlink.org.cn/cloudream/common/utils/http2" ) const UserCreatePath = "/v1/user/create" @@ -13,32 +12,21 @@ const UserCreatePath = "/v1/user/create" type UserCreate struct { Name string `json:"name" binding:"required"` } + +func (r *UserCreate) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, UserCreatePath, r) +} + type UserCreateResp struct { User cdssdk.User `json:"user"` } -func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) { - url, err := url.JoinPath(c.baseURL, UserCreatePath) - if err != nil { - return nil, err - } - - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return nil, err - } - codeResp, err := ParseJSONResponse[response[UserCreateResp]](resp) - if err != nil { - return nil, err - } - - if codeResp.Code == errorcode.OK { - return &codeResp.Data, nil - } +func (r *UserCreateResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return nil, codeResp.ToError() +func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) { + return JSONAPI[*UserCreateResp](c.cfg, http.DefaultClient, req) } const UserDeletePath = "/v1/user/delete" @@ -47,28 +35,16 @@ type UserDelete struct { UserID cdssdk.UserID `json:"userID" binding:"required"` } -type UserDeleteResp struct{} - -func (c *Client) UserDelete(req *UserDelete) error { - url, err := url.JoinPath(c.baseURL, UserDeletePath) - if err != nil { - return err - } +func (r *UserDelete) MakeParam() *sdks.RequestParam { + return sdks.MakeJSONParam(http.MethodPost, UserDeletePath, r) +} - resp, err := http2.PostJSON(url, http2.RequestParam{ - Body: req, - }) - if err != nil { - return err - } - codeResp, err := ParseJSONResponse[response[UserDeleteResp]](resp) - if err != nil { - return err - } +type UserDeleteResp struct{} - if codeResp.Code == errorcode.OK { - return nil - } +func (r *UserDeleteResp) ParseResponse(resp *http.Response) error { + return sdks.ParseCodeDataJSONResponse(resp, r) +} - return codeResp.ToError() +func (c *Client) UserDelete(req *UserDelete) error { + return JSONAPINoData[*UserDeleteResp](c.cfg, http.DefaultClient, req) } diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 2ef1bc8..079d50e 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -1,12 +1,17 @@ package cdsapi import ( + "context" "fmt" "io" "net/http" "path/filepath" "strings" + "time" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" + "gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/serder" @@ -36,3 +41,66 @@ func ParseJSONResponse[TBody any](resp *http.Response) (TBody, error) { return ret, fmt.Errorf("unknow response content type: %s, status: %d, body(prefix): %s", contType, resp.StatusCode, strCont[:math2.Min(len(strCont), 200)]) } + +func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) (Resp, error) { + var re Resp + + param := req.MakeParam() + + httpReq, err := param.MakeRequest(cfg.URL) + if err != nil { + return re, err + } + + if cfg.AccessKey != "" && cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return re, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return re, err + } + } + + resp, err := cli.Do(httpReq) + if err != nil { + return re, err + } + + err = re.ParseResponse(resp) + return re, err +} + +func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) error { + param := req.MakeParam() + + httpReq, err := param.MakeRequest(cfg.URL) + if err != nil { + return err + } + + if cfg.AccessKey != "" && cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + } + + resp, err := cli.Do(httpReq) + if err != nil { + return err + } + + return sdks.ParseCodeDataJSONResponse(resp, any(nil)) +} diff --git a/utils/io2/io.go b/utils/io2/io.go index 8a0d113..5107ae0 100644 --- a/utils/io2/io.go +++ b/utils/io2/io.go @@ -196,3 +196,17 @@ func DropWithBuf(str io.Reader, buf []byte) { } } } + +func ReadMost(str io.Reader, n int) ([]byte, error) { + buf := make([]byte, n) + n, err := io.ReadFull(str, buf) + if err == nil { + return buf, nil + } + + if err == io.EOF || err == io.ErrUnexpectedEOF { + return buf[:n], nil + } + + return buf[:n], err +} diff --git a/utils/serder/serder.go b/utils/serder/serder.go index 9967ae5..cbc7fbb 100644 --- a/utils/serder/serder.go +++ b/utils/serder/serder.go @@ -80,6 +80,16 @@ func JSONToObjectStreamEx[T any](stream io.Reader) (T, error) { return ret, nil } +func JSONToObjectStreamExRaw(stream io.Reader, ret any) error { + dec := defaultAPI.NewDecoder(stream) + err := dec.Decode(ret) + if err != nil { + return err + } + + return nil +} + // 将对象转为JSON字符串。如果需要支持解析TypeUnion类型，则使用"Ex"结尾的同名函数。 // // 注：[]byte会被base64编码，如果要JSON内容要给外部解析，那么应该避免使用[]byte。 From 66da854b0bd75e6dd40945a4dfd1237c3dc9ee8d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 10 Mar 2025 16:40:23 +0800 Subject: [PATCH 12/29] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/job.go | 16 ++++++++++++++++ sdks/pcmscheduler/jobset.go | 3 +-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 sdks/pcmscheduler/job.go diff --git a/sdks/pcmscheduler/job.go b/sdks/pcmscheduler/job.go new file mode 100644 index 0000000..0470ec8 --- /dev/null +++ b/sdks/pcmscheduler/job.go @@ -0,0 +1,16 @@ +package sch + +import ( + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) + +type PCMJob struct { + ID string `gorm:"column:id" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + JobSetID schsdk.JobSetID `gorm:"column:jobset_id" json:"jobSetID"` + LocalJobID string `gorm:"column:local_job_id" json:"localJobID"` + Param string `gorm:"column:param" json:"param"` + CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` +} diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 269151a..e4d7cde 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -92,6 +92,7 @@ type DatasetDistribute struct { type CodeDistribute struct { DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` Clusters []DataDetail `json:"clusters"` } @@ -163,8 +164,6 @@ func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, erro type RunJobReq struct { TaskID TaskID `json:"taskID"` - JobSetID schsdk.JobSetID `json:"jobSetID"` - LocalJobID string `json:"localJobID"` ScheduledDatas []DataScheduleResults `json:"scheduledDatas"` } From 510fe885e4c75bfa06ac59e6d8a7d4522ffd6aff Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 10 Mar 2025 17:15:48 +0800 Subject: [PATCH 13/29] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=92=8C=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E6=8E=A5=E5=8F=A3=E6=94=AF=E6=8C=81=E9=89=B4=E6=9D=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/object.go | 79 +++++++++++++++------- sdks/storage/cdsapi/package.go | 9 ++- sdks/storage/cdsapi/utils.go | 115 ++++++++++++++++++++++++++++++++- 3 files changed, 174 insertions(+), 29 deletions(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 80fc18f..9b4b8ed 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -1,6 +1,7 @@ package cdsapi import ( + "context" "fmt" "io" "mime" @@ -9,6 +10,8 @@ import ( "strings" "time" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/sdks" @@ -115,16 +118,15 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { return nil, fmt.Errorf("upload info to json: %w", err) } - resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{ - Form: map[string]string{"info": string(infoJSON)}, - Files: iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { + resp, err := PostMultiPart(c.cfg, url, + map[string]string{"info": string(infoJSON)}, + iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { return &http2.IterMultiPartFile{ FieldName: "files", FileName: src.Path, File: src.File, }, nil - }), - }) + })) if err != nil { return nil, err } @@ -151,25 +153,42 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { const ObjectDownloadPath = "/object/download" type ObjectDownload struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - ObjectID cdssdk.ObjectID `form:"objectID" json:"objectID" binding:"required"` - Offset int64 `form:"offset" json:"offset,omitempty"` - Length *int64 `form:"length" json:"length,omitempty"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `form:"objectID" url:"objectID" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (r *ObjectDownload) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectDownloadPath, r) } + type DownloadingObject struct { Path string File io.ReadCloser } func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) { - url, err := url.JoinPath(c.cfg.URL, ObjectDownloadPath) + httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL) if err != nil { return nil, err } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) + if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + resp, err := http.DefaultClient.Do(httpReq) if err != nil { return nil, err } @@ -199,22 +218,38 @@ func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) const ObjectDownloadByPathPath = "/object/downloadByPath" type ObjectDownloadByPath struct { - UserID cdssdk.UserID `form:"userID" json:"userID" binding:"required"` - PackageID cdssdk.PackageID `form:"packageID" json:"packageID" binding:"required"` - Path string `form:"path" json:"path" binding:"required"` - Offset int64 `form:"offset" json:"offset,omitempty"` - Length *int64 `form:"length" json:"length,omitempty"` + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + PackageID cdssdk.PackageID `form:"packageID" url:"packageID" binding:"required"` + Path string `form:"path" url:"path" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (r *ObjectDownloadByPath) MakeParam() *sdks.RequestParam { + return sdks.MakeQueryParam(http.MethodGet, ObjectDownloadByPathPath, r) } func (c *ObjectService) DownloadByPath(req ObjectDownloadByPath) (*DownloadingObject, error) { - url, err := url.JoinPath(c.cfg.URL, ObjectDownloadByPathPath) + httpReq, err := req.MakeParam().MakeRequest(c.cfg.URL) if err != nil { return nil, err } - resp, err := http2.GetJSON(url, http2.RequestParam{ - Query: req, - }) + if c.cfg.AccessKey != "" && c.cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(c.cfg.AccessKey, c.cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + resp, err := http.DefaultClient.Do(httpReq) if err != nil { return nil, err } diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index 085871f..621f64b 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -121,16 +121,15 @@ func (c *PackageService) CreateLoad(req PackageCreateLoad) (*PackageCreateLoadRe return nil, fmt.Errorf("upload info to json: %w", err) } - resp, err := http2.PostMultiPart(url, http2.MultiPartRequestParam{ - Form: map[string]string{"info": string(infoJSON)}, - Files: iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { + resp, err := PostMultiPart(c.cfg, url, + map[string]string{"info": string(infoJSON)}, + iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { return &http2.IterMultiPartFile{ FieldName: "files", FileName: src.Path, File: src.File, }, nil - }), - }) + })) if err != nil { return nil, err } diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 079d50e..5c573ae 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -2,15 +2,21 @@ package cdsapi import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "io" + "mime/multipart" "net/http" + ul "net/url" "path/filepath" "strings" "time" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/google/go-querystring/query" + "gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/math2" @@ -60,7 +66,7 @@ func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http. } signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + err = signer.SignHTTP(context.Background(), cred, httpReq, calcSha256(param.Body), AuthService, AuthRegion, time.Now()) if err != nil { return re, err } @@ -91,7 +97,7 @@ func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli } signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, httpReq, "", AuthService, AuthRegion, time.Now()) + err = signer.SignHTTP(context.Background(), cred, httpReq, calcSha256(param.Body), AuthService, AuthRegion, time.Now()) if err != nil { return err } @@ -104,3 +110,108 @@ func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli return sdks.ParseCodeDataJSONResponse(resp, any(nil)) } + +func calcSha256(body sdks.RequestBody) string { + hasher := sha256.New() + switch body := body.(type) { + case *sdks.StringBody: + return hex.EncodeToString(hasher.Sum([]byte(body.Value))) + + case *sdks.BytesBody: + return hex.EncodeToString(hasher.Sum(body.Value)) + + default: + return "" + } +} + +func PostMultiPart(cfg *Config, url string, info any, files http2.MultiPartFileIterator) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return nil, err + } + + pr, pw := io.Pipe() + muWriter := multipart.NewWriter(pw) + + req.Header.Set("Content-Type", fmt.Sprintf("%s;boundary=%s", http2.ContentTypeMultiPart, muWriter.Boundary())) + + writeResult := make(chan error, 1) + go func() { + writeResult <- func() error { + defer pw.Close() + defer muWriter.Close() + + if info != nil { + mp, err := query.Values(info) + if err != nil { + return fmt.Errorf("formValues object to map failed, err: %w", err) + } + + for k, v := range mp { + err := muWriter.WriteField(k, v[0]) + if err != nil { + return fmt.Errorf("write form field failed, err: %w", err) + } + } + } + + for { + file, err := files.MoveNext() + if err == iterator.ErrNoMoreItem { + break + } + if err != nil { + return fmt.Errorf("opening file: %w", err) + } + + err = sendFileOnePart(muWriter, file.FieldName, file.FileName, file.File) + file.File.Close() + if err != nil { + return err + } + } + + return nil + }() + }() + + req.Body = pr + + if cfg.AccessKey != "" && cfg.SecretKey != "" { + prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return nil, err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return nil, err + } + } + + cli := http.Client{} + resp, err := cli.Do(req) + if err != nil { + return nil, err + } + + writeErr := <-writeResult + if writeErr != nil { + return nil, writeErr + } + + return resp, nil +} + +func sendFileOnePart(muWriter *multipart.Writer, fieldName, fileName string, file io.ReadCloser) error { + w, err := muWriter.CreateFormFile(fieldName, ul.PathEscape(fileName)) + if err != nil { + return fmt.Errorf("create form file failed, err: %w", err) + } + + _, err = io.Copy(w, file) + return err +} From 794e9708ad56364c119ef61788e2d8db7e819742 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 09:43:10 +0800 Subject: [PATCH 14/29] =?UTF-8?q?=E6=8B=86=E5=88=86=E7=AD=BE=E5=90=8D?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/presigned.go | 5 -- sdks/storage/cdsapi/signer.go | 113 +++++++++++++++++++++++++++++++ sdks/storage/cdsapi/utils.go | 37 ++-------- 3 files changed, 120 insertions(+), 35 deletions(-) create mode 100644 sdks/storage/cdsapi/signer.go diff --git a/sdks/storage/cdsapi/presigned.go b/sdks/storage/cdsapi/presigned.go index ae83642..5e755fb 100644 --- a/sdks/storage/cdsapi/presigned.go +++ b/sdks/storage/cdsapi/presigned.go @@ -13,11 +13,6 @@ import ( cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" ) -const ( - AuthService = "jcs" - AuthRegion = "any" -) - type PresignedService struct { *Client } diff --git a/sdks/storage/cdsapi/signer.go b/sdks/storage/cdsapi/signer.go new file mode 100644 index 0000000..b4f8975 --- /dev/null +++ b/sdks/storage/cdsapi/signer.go @@ -0,0 +1,113 @@ +package cdsapi + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "time" + + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/credentials" +) + +const ( + AuthService = "jcs" + AuthRegion = "any" +) + +// 对一个请求进行签名，并将签名信息添加到请求头中。 +// +// 会读取请求体计算sha256哈希值。如果hash值已知，可以使用SignWithPayloadHash方法。 +func Sign(req *http.Request, accessKey, secretKey string) error { + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + payloadHash := "" + if req.Body != nil { + data, err := io.ReadAll(req.Body) + if err != nil { + return err + } + req.Body.Close() + req.Body = io.NopCloser(bytes.NewReader(data)) + + hasher := sha256.New() + hasher.Write(data) + payloadHash = hex.EncodeToString(hasher.Sum(nil)) + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, payloadHash, AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + + return nil +} + +// 对一个请求进行签名，并将签名信息添加到请求头中。 +// +// 不计算请求体的哈希，适合上传文件接口。 +func SignWithoutBody(req *http.Request, accessKey, secretKey string) error { + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + + return nil +} + +// 对一个请求进行签名，签名时使用指定的哈希值作为请求体的哈希值。 +// +// 参数payloadHash必须为sha256哈希值的16进制字符串，全小写。 +func SignWithPayloadHash(req *http.Request, payloadHash string, accessKey, secretKey string) error { + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return err + } + + signer := v4.NewSigner() + err = signer.SignHTTP(context.Background(), cred, req, payloadHash, AuthService, AuthRegion, time.Now()) + if err != nil { + return err + } + + return nil +} + +// 生成一个带签名的URL。 +// +// expiration为签名过期时间，单位为秒。 +// +// 签名时不会包含请求体的哈希值。 +func Presign(req *http.Request, accessKey, secretKey string, expiration int) (string, error) { + query := req.URL.Query() + query.Add("X-Expires", fmt.Sprintf("%v", expiration)) + + req.URL.RawQuery = query.Encode() + + prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + cred, err := prod.Retrieve(context.TODO()) + if err != nil { + return "", err + } + + signer := v4.NewSigner() + signedURL, _, err := signer.PresignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + return signedURL, err +} diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 5c573ae..4ba07df 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -1,7 +1,6 @@ package cdsapi import ( - "context" "crypto/sha256" "encoding/hex" "fmt" @@ -11,10 +10,7 @@ import ( ul "net/url" "path/filepath" "strings" - "time" - v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" - "github.com/aws/aws-sdk-go-v2/credentials" "github.com/google/go-querystring/query" "gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/sdks" @@ -59,14 +55,7 @@ func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http. } if cfg.AccessKey != "" && cfg.SecretKey != "" { - prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") - cred, err := prod.Retrieve(context.TODO()) - if err != nil { - return re, err - } - - signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, httpReq, calcSha256(param.Body), AuthService, AuthRegion, time.Now()) + err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) if err != nil { return re, err } @@ -90,14 +79,7 @@ func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli } if cfg.AccessKey != "" && cfg.SecretKey != "" { - prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") - cred, err := prod.Retrieve(context.TODO()) - if err != nil { - return err - } - - signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, httpReq, calcSha256(param.Body), AuthService, AuthRegion, time.Now()) + err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) if err != nil { return err } @@ -115,10 +97,12 @@ func calcSha256(body sdks.RequestBody) string { hasher := sha256.New() switch body := body.(type) { case *sdks.StringBody: - return hex.EncodeToString(hasher.Sum([]byte(body.Value))) + hasher.Write([]byte(body.Value)) + return hex.EncodeToString(hasher.Sum(nil)) case *sdks.BytesBody: - return hex.EncodeToString(hasher.Sum(body.Value)) + hasher.Write(body.Value) + return hex.EncodeToString(hasher.Sum(nil)) default: return "" @@ -179,14 +163,7 @@ func PostMultiPart(cfg *Config, url string, info any, files http2.MultiPartFileI req.Body = pr if cfg.AccessKey != "" && cfg.SecretKey != "" { - prod := credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretKey, "") - cred, err := prod.Retrieve(context.TODO()) - if err != nil { - return nil, err - } - - signer := v4.NewSigner() - err = signer.SignHTTP(context.Background(), cred, req, "", AuthService, AuthRegion, time.Now()) + err = SignWithoutBody(req, cfg.AccessKey, cfg.SecretKey) if err != nil { return nil, err } From 8761088ec2d720992fce77323a2b53c3e02f9efc Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 10:32:04 +0800 Subject: [PATCH 15/29] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/sdks.go | 7 ++++++- sdks/storage/cdsapi/bucket.go | 8 ++++---- sdks/storage/cdsapi/cache.go | 2 +- sdks/storage/cdsapi/hub.go | 2 +- sdks/storage/cdsapi/object.go | 28 ++++++++++++++++------------ sdks/storage/cdsapi/package.go | 14 +++++++------- sdks/storage/cdsapi/storage.go | 6 +++--- sdks/storage/cdsapi/user.go | 4 ++-- sdks/storage/cdsapi/utils.go | 17 ++++++++--------- 9 files changed, 48 insertions(+), 40 deletions(-) diff --git a/sdks/sdks.go b/sdks/sdks.go index 1fedfad..857c6b7 100644 --- a/sdks/sdks.go +++ b/sdks/sdks.go @@ -34,7 +34,7 @@ type RequestParam struct { func (p *RequestParam) MakeRequest(baseURL string) (*http.Request, error) { var body io.ReadCloser - bodyLen := int64(-1) + bodyLen := int64(0) if p.Body != nil { body = p.Body.IntoStream() bodyLen = p.Body.Length() @@ -175,11 +175,16 @@ func ParseCodeDataJSONResponse[T any](resp *http.Response, ret T) error { if strings.Contains(contType, http2.ContentTypeJSON) { var err error var r CodeDataResponse[T] + r.Data = ret if err = serder.JSONToObjectStreamExRaw(resp.Body, &r); err != nil { return fmt.Errorf("parsing response: %w", err) } + if r.Code != errorcode.OK { + return &CodeMessageError{Code: r.Code, Message: r.Message} + } + return nil } diff --git a/sdks/storage/cdsapi/bucket.go b/sdks/storage/cdsapi/bucket.go index 0299949..f0c1ad1 100644 --- a/sdks/storage/cdsapi/bucket.go +++ b/sdks/storage/cdsapi/bucket.go @@ -35,7 +35,7 @@ func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error { } func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) { - return JSONAPI[*BucketGetByNameResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketGetByNameResp{}) } const BucketCreatePath = "/bucket/create" @@ -58,7 +58,7 @@ func (r *BucketCreateResp) ParseResponse(resp *http.Response) error { } func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) { - return JSONAPI[*BucketCreateResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketCreateResp{}) } const BucketDeletePath = "/bucket/delete" @@ -79,7 +79,7 @@ func (r *BucketDeleteResp) ParseResponse(resp *http.Response) error { } func (c *BucketService) Delete(req BucketDelete) error { - return JSONAPINoData[*BucketDeleteResp](c.cfg, http.DefaultClient, &req) + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const BucketListUserBucketsPath = "/bucket/listUserBuckets" @@ -101,5 +101,5 @@ func (r *BucketListUserBucketsResp) ParseResponse(resp *http.Response) error { } func (c *BucketService) ListUserBuckets(req BucketListUserBucketsReq) (*BucketListUserBucketsResp, error) { - return JSONAPI[*BucketListUserBucketsResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &BucketListUserBucketsResp{}) } diff --git a/sdks/storage/cdsapi/cache.go b/sdks/storage/cdsapi/cache.go index bb28e58..b71725d 100644 --- a/sdks/storage/cdsapi/cache.go +++ b/sdks/storage/cdsapi/cache.go @@ -26,5 +26,5 @@ func (r *CacheMovePackageResp) ParseResponse(resp *http.Response) error { } func (c *Client) CacheMovePackage(req CacheMovePackageReq) (*CacheMovePackageResp, error) { - return JSONAPI[*CacheMovePackageResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &CacheMovePackageResp{}) } diff --git a/sdks/storage/cdsapi/hub.go b/sdks/storage/cdsapi/hub.go index fb360a4..20a9d18 100644 --- a/sdks/storage/cdsapi/hub.go +++ b/sdks/storage/cdsapi/hub.go @@ -26,5 +26,5 @@ func (r *HubGetHubsResp) ParseResponse(resp *http.Response) error { } func (c *Client) HubGetHubs(req HubGetHubsReq) (*HubGetHubsResp, error) { - return JSONAPI[*HubGetHubsResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &HubGetHubsResp{}) } diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 9b4b8ed..aca0aa9 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -55,7 +55,7 @@ func (r *ObjectListByPathResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) ListByPath(req ObjectListByPath) (*ObjectListByPathResp, error) { - return JSONAPI[*ObjectListByPathResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectListByPathResp{}) } const ObjectListByIDsPath = "/object/listByIDs" @@ -78,7 +78,7 @@ func (r *ObjectListByIDsResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) ListByIDs(req ObjectListByIDs) (*ObjectListByIDsResp, error) { - return JSONAPI[*ObjectListByIDsResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectListByIDsResp{}) } const ObjectUploadPath = "/object/upload" @@ -108,6 +108,10 @@ type ObjectUploadResp struct { } func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { + type uploadInfo struct { + Info string `url:"info"` + } + url, err := url.JoinPath(c.cfg.URL, ObjectUploadPath) if err != nil { return nil, err @@ -119,7 +123,7 @@ func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) { } resp, err := PostMultiPart(c.cfg, url, - map[string]string{"info": string(infoJSON)}, + uploadInfo{Info: string(infoJSON)}, iterator.Map(req.Files, func(src *UploadingObject) (*http2.IterMultiPartFile, error) { return &http2.IterMultiPartFile{ FieldName: "files", @@ -305,7 +309,7 @@ func (r *ObjectUpdateInfoResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) { - return JSONAPI[*ObjectUpdateInfoResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectUpdateInfoResp{}) } const ObjectUpdateInfoByPathPath = "/object/updateInfoByPath" @@ -328,7 +332,7 @@ func (r *ObjectUpdateInfoByPathResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) UpdateInfoByPath(req ObjectUpdateInfoByPath) (*ObjectUpdateInfoByPathResp, error) { - return JSONAPI[*ObjectUpdateInfoByPathResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectUpdateInfoByPathResp{}) } const ObjectMovePath = "/object/move" @@ -362,7 +366,7 @@ func (r *ObjectMoveResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) { - return JSONAPI[*ObjectMoveResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectMoveResp{}) } const ObjectDeletePath = "/object/delete" @@ -383,7 +387,7 @@ func (r *ObjectDeleteResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) Delete(req ObjectDelete) error { - return JSONAPINoData[*ObjectDeleteResp](c.cfg, http.DefaultClient, &req) + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const ObjectDeleteByPathPath = "/object/deleteByPath" @@ -405,7 +409,7 @@ func (r *ObjectDeleteByPathResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) DeleteByPath(req ObjectDeleteByPath) error { - return JSONAPINoData[*ObjectDeleteByPathResp](c.cfg, http.DefaultClient, &req) + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const ObjectClonePath = "/object/clone" @@ -434,7 +438,7 @@ func (r *ObjectCloneResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) Clone(req ObjectClone) (*ObjectCloneResp, error) { - return JSONAPI[*ObjectCloneResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectCloneResp{}) } const ObjectGetPackageObjectsPath = "/object/getPackageObjects" @@ -457,7 +461,7 @@ func (r *ObjectGetPackageObjectsResp) ParseResponse(resp *http.Response) error { } func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) { - return JSONAPI[*ObjectGetPackageObjectsResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectGetPackageObjectsResp{}) } const ObjectNewMultipartUploadPath = "/v1/object/newMultipartUpload" @@ -481,7 +485,7 @@ func (r *ObjectNewMultipartUploadResp) ParseResponse(resp *http.Response) error } func (c *ObjectService) NewMultipartUpload(req ObjectNewMultipartUpload) (*ObjectNewMultipartUploadResp, error) { - return JSONAPI[*ObjectNewMultipartUploadResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectNewMultipartUploadResp{}) } const ObjectUploadPartPath = "/v1/object/uploadPart" @@ -560,5 +564,5 @@ func (r *ObjectCompleteMultipartUploadResp) ParseResponse(resp *http.Response) e } func (c *ObjectService) CompleteMultipartUpload(req ObjectCompleteMultipartUpload) (*ObjectCompleteMultipartUploadResp, error) { - return JSONAPI[*ObjectCompleteMultipartUploadResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &ObjectCompleteMultipartUploadResp{}) } diff --git a/sdks/storage/cdsapi/package.go b/sdks/storage/cdsapi/package.go index 621f64b..6c1d2c3 100644 --- a/sdks/storage/cdsapi/package.go +++ b/sdks/storage/cdsapi/package.go @@ -41,7 +41,7 @@ func (r *PackageGetResp) ParseResponse(resp *http.Response) error { } func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) { - return JSONAPI[*PackageGetResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetResp{}) } const PackageGetByFullNamePath = "/package/getByFullName" @@ -65,7 +65,7 @@ func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error { } func (c *PackageService) GetByName(req PackageGetByFullName) (*PackageGetByFullNameResp, error) { - return JSONAPI[*PackageGetByFullNameResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetByFullNameResp{}) } const PackageCreatePath = "/package/create" @@ -89,7 +89,7 @@ func (r *PackageCreateResp) ParseResponse(resp *http.Response) error { } func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) { - return JSONAPI[*PackageCreateResp](s.cfg, http.DefaultClient, &req) + return JSONAPI(s.cfg, http.DefaultClient, &req, &PackageCreateResp{}) } const PackageCreateLoadPath = "/package/createLoad" @@ -164,7 +164,7 @@ func (r *PackageDeleteResp) ParseResponse(resp *http.Response) error { } func (c *PackageService) Delete(req PackageDelete) error { - return JSONAPINoData[*PackageDeleteResp](c.cfg, http.DefaultClient, &req) + return JSONAPINoData(c.cfg, http.DefaultClient, &req) } const PackageClonePath = "/package/clone" @@ -189,7 +189,7 @@ func (r *PackageCloneResp) ParseResponse(resp *http.Response) error { } func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) { - return JSONAPI[*PackageCloneResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageCloneResp{}) } const PackageListBucketPackagesPath = "/package/listBucketPackages" @@ -212,7 +212,7 @@ func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error } func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) { - return JSONAPI[*PackageListBucketPackagesResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageListBucketPackagesResp{}) } const PackageGetCachedStoragesPath = "/package/getCachedStorages" @@ -235,5 +235,5 @@ func (r *PackageGetCachedStoragesResp) ParseResponse(resp *http.Response) error } func (c *PackageService) GetCachedStorages(req PackageGetCachedStoragesReq) (*PackageGetCachedStoragesResp, error) { - return JSONAPI[*PackageGetCachedStoragesResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &PackageGetCachedStoragesResp{}) } diff --git a/sdks/storage/cdsapi/storage.go b/sdks/storage/cdsapi/storage.go index b1986b8..22ad944 100644 --- a/sdks/storage/cdsapi/storage.go +++ b/sdks/storage/cdsapi/storage.go @@ -27,7 +27,7 @@ func (r *StorageLoadPackageResp) ParseResponse(resp *http.Response) error { } func (c *Client) StorageLoadPackage(req StorageLoadPackageReq) (*StorageLoadPackageResp, error) { - return JSONAPI[*StorageLoadPackageResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageLoadPackageResp{}) } const StorageCreatePackagePath = "/storage/createPackage" @@ -54,7 +54,7 @@ func (r *StorageCreatePackageResp) ParseResponse(resp *http.Response) error { } func (c *Client) StorageCreatePackage(req StorageCreatePackageReq) (*StorageCreatePackageResp, error) { - return JSONAPI[*StorageCreatePackageResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageCreatePackageResp{}) } const StorageGetPath = "/storage/get" @@ -77,5 +77,5 @@ func (r *StorageGetResp) ParseResponse(resp *http.Response) error { } func (c *Client) StorageGet(req StorageGet) (*StorageGetResp, error) { - return JSONAPI[*StorageGetResp](c.cfg, http.DefaultClient, &req) + return JSONAPI(c.cfg, http.DefaultClient, &req, &StorageGetResp{}) } diff --git a/sdks/storage/cdsapi/user.go b/sdks/storage/cdsapi/user.go index 295e322..2230451 100644 --- a/sdks/storage/cdsapi/user.go +++ b/sdks/storage/cdsapi/user.go @@ -26,7 +26,7 @@ func (r *UserCreateResp) ParseResponse(resp *http.Response) error { } func (c *Client) UserCreate(req *UserCreate) (*UserCreateResp, error) { - return JSONAPI[*UserCreateResp](c.cfg, http.DefaultClient, req) + return JSONAPI(c.cfg, http.DefaultClient, req, &UserCreateResp{}) } const UserDeletePath = "/v1/user/delete" @@ -46,5 +46,5 @@ func (r *UserDeleteResp) ParseResponse(resp *http.Response) error { } func (c *Client) UserDelete(req *UserDelete) error { - return JSONAPINoData[*UserDeleteResp](c.cfg, http.DefaultClient, req) + return JSONAPINoData(c.cfg, http.DefaultClient, req) } diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 4ba07df..988bb67 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -44,33 +44,32 @@ func ParseJSONResponse[TBody any](resp *http.Response) (TBody, error) { return ret, fmt.Errorf("unknow response content type: %s, status: %d, body(prefix): %s", contType, resp.StatusCode, strCont[:math2.Min(len(strCont), 200)]) } -func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) (Resp, error) { - var re Resp +func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req, resp Resp) (Resp, error) { param := req.MakeParam() httpReq, err := param.MakeRequest(cfg.URL) if err != nil { - return re, err + return resp, err } if cfg.AccessKey != "" && cfg.SecretKey != "" { err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) if err != nil { - return re, err + return resp, err } } - resp, err := cli.Do(httpReq) + httpResp, err := cli.Do(httpReq) if err != nil { - return re, err + return resp, err } - err = re.ParseResponse(resp) - return re, err + err = resp.ParseResponse(httpResp) + return resp, err } -func JSONAPINoData[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) error { +func JSONAPINoData[Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) error { param := req.MakeParam() httpReq, err := param.MakeRequest(cfg.URL) From eacc4f6bd344c01199a70cb45c15e9bdf447ee42 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 14:35:02 +0800 Subject: [PATCH 16/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/sdks.go | 19 +++++----- sdks/storage/cdsapi/signer.go | 7 ++-- sdks/storage/cdsapi/storage_test.go | 56 +++++++++++++++++++++++++++++ sdks/storage/cdsapi/utils.go | 10 ++++-- 4 files changed, 78 insertions(+), 14 deletions(-) diff --git a/sdks/sdks.go b/sdks/sdks.go index 857c6b7..6f303c9 100644 --- a/sdks/sdks.go +++ b/sdks/sdks.go @@ -46,7 +46,10 @@ func (p *RequestParam) MakeRequest(baseURL string) (*http.Request, error) { } req.ContentLength = bodyLen req.URL.RawQuery = p.Query.Encode() - req.Header = p.Header + + if p.Header != nil { + req.Header = p.Header + } return req, nil } @@ -87,11 +90,11 @@ type StringBody struct { Value string } -func (s StringBody) Length() int64 { +func (s *StringBody) Length() int64 { return int64(len(s.Value)) } -func (s StringBody) IntoStream() io.ReadCloser { +func (s *StringBody) IntoStream() io.ReadCloser { return io.NopCloser(bytes.NewReader([]byte(s.Value))) } @@ -99,11 +102,11 @@ type BytesBody struct { Value []byte } -func (b BytesBody) Length() int64 { +func (b *BytesBody) Length() int64 { return int64(len(b.Value)) } -func (b BytesBody) IntoStream() io.ReadCloser { +func (b *BytesBody) IntoStream() io.ReadCloser { return io.NopCloser(bytes.NewReader(b.Value)) } @@ -112,11 +115,11 @@ type StreamBody struct { LengthHint int64 // 长度提示，如果长度未知，可以设置为-1 } -func (s StreamBody) Length() int64 { +func (s *StreamBody) Length() int64 { return s.LengthHint } -func (s StreamBody) IntoStream() io.ReadCloser { +func (s *StreamBody) IntoStream() io.ReadCloser { return s.Stream } @@ -134,7 +137,7 @@ func MakeJSONParam(method string, path string, body any) *RequestParam { return &RequestParam{ Method: method, Path: path, - Body: BytesBody{Value: data}, + Body: &BytesBody{Value: data}, } } diff --git a/sdks/storage/cdsapi/signer.go b/sdks/storage/cdsapi/signer.go index b4f8975..d0bea77 100644 --- a/sdks/storage/cdsapi/signer.go +++ b/sdks/storage/cdsapi/signer.go @@ -41,6 +41,9 @@ func Sign(req *http.Request, accessKey, secretKey string) error { hasher := sha256.New() hasher.Write(data) payloadHash = hex.EncodeToString(hasher.Sum(nil)) + } else { + hash := sha256.Sum256([]byte("")) + payloadHash = hex.EncodeToString(hash[:]) } signer := v4.NewSigner() @@ -52,9 +55,7 @@ func Sign(req *http.Request, accessKey, secretKey string) error { return nil } -// 对一个请求进行签名，并将签名信息添加到请求头中。 -// -// 不计算请求体的哈希，适合上传文件接口。 +// 对一个请求进行签名，不计算请求体的哈希，适合上传文件接口。 func SignWithoutBody(req *http.Request, accessKey, secretKey string) error { prod := credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") cred, err := prod.Retrieve(context.TODO()) diff --git a/sdks/storage/cdsapi/storage_test.go b/sdks/storage/cdsapi/storage_test.go index edc1994..c61ad46 100644 --- a/sdks/storage/cdsapi/storage_test.go +++ b/sdks/storage/cdsapi/storage_test.go @@ -245,3 +245,59 @@ func Test_Cache(t *testing.T) { So(err, ShouldBeNil) }) } + +func Test_Sign(t *testing.T) { + Convey("签名接口", t, func() { + cli := NewClient(&Config{ + URL: "http://localhost:7890/v1", + AccessKey: "123456", + SecretKey: "123456", + }) + + fileData := make([]byte, 4096) + for i := 0; i < len(fileData); i++ { + fileData[i] = byte(i) + } + + pkgName := uuid.NewString() + createResp, err := cli.Package().Create(PackageCreate{ + UserID: 1, + BucketID: 1, + Name: pkgName, + }) + So(err, ShouldBeNil) + + _, err = cli.Object().Upload(ObjectUpload{ + ObjectUploadInfo: ObjectUploadInfo{ + UserID: 1, + PackageID: createResp.Package.PackageID, + }, + Files: iterator.Array( + &UploadingObject{ + Path: "abc/test", + File: io.NopCloser(bytes.NewBuffer(fileData)), + }, + &UploadingObject{ + Path: "test4", + File: io.NopCloser(bytes.NewBuffer(fileData)), + }, + ), + }) + So(err, ShouldBeNil) + + getResp, err := cli.Package().Get(PackageGetReq{ + UserID: 1, + PackageID: createResp.Package.PackageID, + }) + So(err, ShouldBeNil) + + So(getResp.PackageID, ShouldEqual, createResp.Package.PackageID) + So(getResp.Package.Name, ShouldEqual, pkgName) + + err = cli.Package().Delete(PackageDelete{ + UserID: 1, + PackageID: createResp.Package.PackageID, + }) + So(err, ShouldBeNil) + }) +} diff --git a/sdks/storage/cdsapi/utils.go b/sdks/storage/cdsapi/utils.go index 988bb67..811c225 100644 --- a/sdks/storage/cdsapi/utils.go +++ b/sdks/storage/cdsapi/utils.go @@ -54,7 +54,7 @@ func JSONAPI[Resp sdks.APIResponse, Req sdks.APIRequest](cfg *Config, cli *http. } if cfg.AccessKey != "" && cfg.SecretKey != "" { - err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) + err = SignWithPayloadHash(httpReq, calcSha256(param.Body), cfg.AccessKey, cfg.SecretKey) if err != nil { return resp, err } @@ -78,7 +78,7 @@ func JSONAPINoData[Req sdks.APIRequest](cfg *Config, cli *http.Client, req Req) } if cfg.AccessKey != "" && cfg.SecretKey != "" { - err = SignWithPayloadHash(httpReq, cfg.AccessKey, cfg.SecretKey, calcSha256(param.Body)) + err = SignWithPayloadHash(httpReq, calcSha256(param.Body), cfg.AccessKey, cfg.SecretKey) if err != nil { return err } @@ -103,8 +103,12 @@ func calcSha256(body sdks.RequestBody) string { hasher.Write(body.Value) return hex.EncodeToString(hasher.Sum(nil)) - default: + case *sdks.StreamBody: return "" + + default: + hash := sha256.Sum256([]byte("")) + return hex.EncodeToString(hash[:]) } } From 1589ed240695c5dcc4d56f7f470a0f906e6490bb Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 11 Mar 2025 16:19:02 +0800 Subject: [PATCH 17/29] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=80=9A=E8=BF=87ID?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E6=96=87=E4=BB=B6=E7=9A=84=E9=A2=84=E7=AD=BE?= =?UTF-8?q?=E5=90=8D=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/presigned.go | 15 ++++++++++- sdks/storage/cdsapi/presigned_test.go | 38 ++++++++++++++++++++++----- sdks/storage/cdsapi/signer.go | 2 +- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/sdks/storage/cdsapi/presigned.go b/sdks/storage/cdsapi/presigned.go index 5e755fb..b357e71 100644 --- a/sdks/storage/cdsapi/presigned.go +++ b/sdks/storage/cdsapi/presigned.go @@ -33,10 +33,23 @@ type PresignedObjectDownloadByPath struct { Length *int64 `form:"length" url:"length,omitempty"` } -func (c *PresignedService) ObjectDownload(req PresignedObjectDownloadByPath, expireIn int) (string, error) { +func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) { return c.presign(req, PresignedObjectDownloadByPathPath, http.MethodGet, expireIn) } +const PresignedObjectDownloadPath = "/v1/presigned/object/download" + +type PresignedObjectDownload struct { + UserID cdssdk.UserID `form:"userID" url:"userID" binding:"required"` + ObjectID cdssdk.ObjectID `form:"objectID" url:"objectID" binding:"required"` + Offset int64 `form:"offset" url:"offset,omitempty"` + Length *int64 `form:"length" url:"length,omitempty"` +} + +func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) { + return c.presign(req, PresignedObjectDownloadPath, http.MethodGet, expireIn) +} + const PresignedObjectUploadPath = "/v1/presigned/object/upload" type PresignedObjectUpload struct { diff --git a/sdks/storage/cdsapi/presigned_test.go b/sdks/storage/cdsapi/presigned_test.go index 37c5b87..f4071c9 100644 --- a/sdks/storage/cdsapi/presigned_test.go +++ b/sdks/storage/cdsapi/presigned_test.go @@ -16,7 +16,7 @@ func Test_Presigned(t *testing.T) { Convey("下载文件", t, func() { pre := cli.Presigned() - url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ + url, err := pre.ObjectDownloadByPath(PresignedObjectDownloadByPath{ UserID: 1, PackageID: 3, Path: "example.java", @@ -39,14 +39,16 @@ func Test_Presigned(t *testing.T) { }) } -func Test_PresignedObjectDownload(t *testing.T) { +func Test_PresignedObjectDownloadByPath(t *testing.T) { cli := NewClient(&Config{ - URL: "http://localhost:7890", + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", }) Convey("下载文件", t, func() { pre := cli.Presigned() - url, err := pre.ObjectDownload(PresignedObjectDownloadByPath{ + url, err := pre.ObjectDownloadByPath(PresignedObjectDownloadByPath{ UserID: 1, PackageID: 3, Path: "example.java", @@ -58,6 +60,26 @@ func Test_PresignedObjectDownload(t *testing.T) { }) } +func Test_PresignedObjectDownload(t *testing.T) { + cli := NewClient(&Config{ + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", + }) + + Convey("下载文件", t, func() { + pre := cli.Presigned() + url, err := pre.ObjectDownload(PresignedObjectDownload{ + UserID: 1, + ObjectID: 1039, + // Offset: 1, + // Length: types.Ref(int64(100)), + }, 100) + So(err, ShouldEqual, nil) + t.Logf("url: %s", url) + }) +} + func Test_PresignedObjectUpload(t *testing.T) { cli := NewClient(&Config{ URL: "http://localhost:7890", @@ -94,7 +116,9 @@ func Test_PresignedNewMultipartUpload(t *testing.T) { func Test_PresignedObjectUploadPart(t *testing.T) { cli := NewClient(&Config{ - URL: "http://localhost:7890", + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", }) Convey("上传分片", t, func() { @@ -111,7 +135,9 @@ func Test_PresignedObjectUploadPart(t *testing.T) { func Test_PresignedCompleteMultipartUpload(t *testing.T) { cli := NewClient(&Config{ - URL: "http://localhost:7890", + URL: "http://localhost:7890", + AccessKey: "123456", + SecretKey: "123456", }) Convey("合并分片", t, func() { diff --git a/sdks/storage/cdsapi/signer.go b/sdks/storage/cdsapi/signer.go index d0bea77..67e39e5 100644 --- a/sdks/storage/cdsapi/signer.go +++ b/sdks/storage/cdsapi/signer.go @@ -95,7 +95,7 @@ func SignWithPayloadHash(req *http.Request, payloadHash string, accessKey, secre // // expiration为签名过期时间，单位为秒。 // -// 签名时不会包含请求体的哈希值。 +// 签名时不会包含请求体的哈希值。注：不要设置任何额外的Header（除了自动添加的Host），以免签名校验不通过 func Presign(req *http.Request, accessKey, secretKey string, expiration int) (string, error) { query := req.URL.Query() query.Add("X-Expires", fmt.Sprintf("%v", expiration)) From fe198e76bfb40bdbff88e7bf1f38239fa5322161 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 11 Mar 2025 16:27:40 +0800 Subject: [PATCH 18/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 8 ++++---- sdks/pcmscheduler/models.go | 15 ++++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index e4d7cde..be1e3ae 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -58,10 +58,10 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) } type CreateJobReq struct { - Name string `json:"name"` - Description string `json:"description"` - JobResources schsdk.JobResources `json:"jobResources"` - DataDistribute DataDistribute `json:"dataDistributes"` + Name string `json:"name"` + Description string `json:"description"` + JobResources schsdk.JobResources `json:"jobResources"` + DataDistributes DataDistribute `json:"dataDistributes"` } type CommonJsonData struct { diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index daca965..a993342 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -385,13 +385,14 @@ func (d *DataBindingBase) Noop() {} type DatasetBinding struct { serder.Metadata `union:"dataset"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Category string `json:"category"` - PackageID cdssdk.PackageID `json:"packageID"` - RepositoryName string `json:"repositoryName"` + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints float64 `json:"points"` } type ModelBinding struct { From fdace7b42b8c9866ad1966a7c2f6dc441cc96e34 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 13 Mar 2025 09:17:24 +0800 Subject: [PATCH 19/29] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E8=BE=93=E5=87=BA=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index a993342..1b3eaff 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -420,6 +420,7 @@ type CodeBinding struct { ImageID schsdk.ImageID `json:"imageID"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` // 当集群为openi的时候，需要传入分支 Branch string `json:"branch"` } From a936cffa134eece2e32229f4ff2e39f592777d1b Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 13 Mar 2025 16:59:03 +0800 Subject: [PATCH 20/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=9B=E5=BB=BApacka?= =?UTF-8?q?ge=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 6ba148c..7393c28 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -126,8 +126,9 @@ var JobResourceTypeUnion = types.NewTypeUnion[JobResource]( (*NPU)(nil), (*MLU)(nil), (*DCU)(nil), - (*Memory)(nil), + (*MEMORY)(nil), (*PRICE)(nil), + (*STORAGE)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobResourceTypeUnion, "type") @@ -144,6 +145,14 @@ type CPU struct { Number int64 `json:"number"` } +type STORAGE struct { + serder.Metadata `union:"STORAGE"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + type GPU struct { serder.Metadata `union:"GPU"` JobResourceBase @@ -160,8 +169,8 @@ type NPU struct { Number int64 `json:"number"` } -type Memory struct { - serder.Metadata `union:"Memory"` +type MEMORY struct { + serder.Metadata `union:"MEMORY"` JobResourceBase Type string `json:"type"` Name string `json:"name"` From 4374600529fdec8617b4ef92fc065ed47ee0f1bd Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 14:23:47 +0800 Subject: [PATCH 21/29] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B6=85=E7=AE=97?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/client.go | 63 +++++ sdks/hpc/config.go | 5 + sdks/hpc/job.go | 71 ++++++ sdks/hpc/models.go | 525 +++++++++++++++++++++++++++++++++++++++ sdks/scheduler/models.go | 21 ++ 5 files changed, 685 insertions(+) create mode 100644 sdks/hpc/client.go create mode 100644 sdks/hpc/config.go create mode 100644 sdks/hpc/job.go create mode 100644 sdks/hpc/models.go diff --git a/sdks/hpc/client.go b/sdks/hpc/client.go new file mode 100644 index 0000000..07b4393 --- /dev/null +++ b/sdks/hpc/client.go @@ -0,0 +1,63 @@ +package hpc + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type respons2[T any] struct { + Code int `json:"code"` + Message string `json:"msg"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/hpc/config.go b/sdks/hpc/config.go new file mode 100644 index 0000000..d168c41 --- /dev/null +++ b/sdks/hpc/config.go @@ -0,0 +1,5 @@ +package hpc + +type Config struct { + URL string `json:"url"` +} diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go new file mode 100644 index 0000000..ddc5205 --- /dev/null +++ b/sdks/hpc/job.go @@ -0,0 +1,71 @@ +package hpc + +import ( + "fmt" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "net/url" + "strings" +) + +type CreateJobReq struct { + Name string `json:"name"` + ClusterID schsdk.ClusterID `json:"clusterId"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + Parameters HPCParameter `json:"parameters"` +} + +type HPCParameter struct { + JobName string `json:"jobName"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + InputFile string `json:"inputFile"` +} + +type CreateJobResp struct { + Backend string `json:"backend"` + JobInfo HPCJobInfo `json:"jobInfo"` +} + +type HPCJobInfo struct { + JobDir string `json:"jobDir"` + JobID string `json:"jobId"` +} + +func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/hpc/commitHpcTask") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, fmt.Errorf("error: %s", codeResp.Message) + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) + +} diff --git a/sdks/hpc/models.go b/sdks/hpc/models.go new file mode 100644 index 0000000..6511bb9 --- /dev/null +++ b/sdks/hpc/models.go @@ -0,0 +1,525 @@ +package hpc + +import ( + "gitlink.org.cn/cloudream/common/pkgs/types" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/serder" + "time" +) + +type ResourceType string + +const ( + ResourceTypeCPU ResourceType = "CPU" + ResourceTypeNPU ResourceType = "NPU" + ResourceTypeGPU ResourceType = "GPU" + ResourceTypeMLU ResourceType = "MLU" + ResourceTypeStorage ResourceType = "STORAGE" + ResourceTypeMemory ResourceType = "MEMORY" + + Split = "/" + + CODE = "code" + DATASET = "dataset" + IMAGE = "image" + MODEL = "model" + RESULT = "result" + + OrderByName = "name" + OrderBySize = "size" + OrderByTime = "time" + + StorageTypeURL = "url" + StorageTypeJCS = "jcs" + + RejectedStatus = "rejected" + PendingStatus = "pending" + ApprovedStatus = "approved" + RevokedStatus = "revoked" + CancelStatus = "cancel" + ExpiredStatus = "expired" + + ApplyAccess = "apply" + PrivateAccess = "private" + PublicAccess = "public" + + PreferencePriority = "preference" + SpecifyClusterPriority = "specify" + + FailedStatus = "failed" + SuccessStatus = "success" + + Query = "query" + Delete = "delete" + + ChildrenType = "children" + ParentType = "parent" + + PlatformSugon = "sugon" + PlatformOpenI = "OpenI" + PlatformModelArts = "ModelArts" + + URL = "url" + ID = "id" + + Startup = "startup" +) + +type TaskID int64 +type DataID int64 + +type ClusterDetail struct { + // 集群ID + ClusterId schsdk.ClusterID `json:"clusterID"` + // 集群功能类型：云算，智算，超算 + ClusterType string `json:"clusterType"` + // 集群地区：华东地区、华南地区、华北地区、华中地区、西南地区、西北地区、东北地区 + Region string `json:"region"` + // 资源类型 + Resources2 []ResourceData `json:"resources1,omitempty"` + //Resources2 []ResourceData `json:"resources"` + Resources []ClusterResource `json:"resources"` +} + +type ClusterResource struct { + Resource TmpResourceData `json:"resource"` + BaseResources []TmpResourceData `json:"baseResources"` +} + +type TmpResourceData struct { + Type ResourceType `json:"type"` + Name string `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceData interface { + Noop() +} + +var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData]( + (*CPUResourceData)(nil), + (*NPUResourceData)(nil), + (*GPUResourceData)(nil), + (*MLUResourceData)(nil), + (*DCUResourceData)(nil), + (*GCUResourceData)(nil), + (*GPGPUResourceData)(nil), + (*StorageResourceData)(nil), + (*MemoryResourceData)(nil), + (*BalanceResourceData)(nil), + (*RateResourceData)(nil), +) +var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "type") + +type ResourceDataBase struct{} + +func (d *ResourceDataBase) Noop() {} + +type UnitValue[T any] struct { + Unit string `json:"unit"` + Value T `json:"value"` +} + +type CPUResourceData struct { + serder.Metadata `union:"CPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type NPUResourceData struct { + serder.Metadata `union:"NPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPUResourceData struct { + serder.Metadata `union:"GPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type MLUResourceData struct { + serder.Metadata `union:"MLU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type DCUResourceData struct { + serder.Metadata `union:"DCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GCUResourceData struct { + serder.Metadata `union:"GCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPGPUResourceData struct { + serder.Metadata `union:"ILUVATAR-GPGPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type StorageResourceData struct { + serder.Metadata `union:"STORAGE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type MemoryResourceData struct { + serder.Metadata `union:"MEMORY"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type BalanceResourceData struct { + serder.Metadata `union:"BALANCE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type RateResourceData struct { + serder.Metadata `union:"RATE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceRange struct { + UserID cdssdk.UserID `json:"userID"` + Type ResourceType `json:"type"` + GPU Range `json:"gpu"` + GPUNumber int `json:"gpuNumber"` + CPU Range `json:"cpu"` + Memory Range `json:"memory"` + Storage Range `json:"storage"` +} + +type Range struct { + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +type ResourcePriority interface { + Noop() +} + +type ResourcePriorityBase struct { +} + +var ResourcePriorityTypeUnion = types.NewTypeUnion[ResourcePriority]( + (*RegionPriority)(nil), + (*ChipPriority)(nil), + (*BiasPriority)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&ResourcePriorityTypeUnion, "type") + +func (d *ResourcePriorityBase) Noop() {} + +type RegionPriority struct { + serder.Metadata `union:"region"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type ChipPriority struct { + serder.Metadata `union:"chip"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type BiasPriority struct { + serder.Metadata `union:"bias"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type TaskMessage struct { + Status string `json:"status"` + Message string `json:"message"` +} + +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` +} + +type UploadParams struct { + DataType string `json:"dataType"` + UploadInfo UploadInfo `json:"uploadInfo"` +} + +type UploadInfo interface { + Noop() +} + +var UploadInfoTypeUnion = types.NewTypeUnion[UploadInfo]( + (*LocalUploadInfo)(nil), + (*RemoteUploadInfo)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadInfoTypeUnion, "type") + +type LocalUploadInfo struct { + serder.Metadata `union:"local"` + UploadInfoBase + Type string `json:"type"` + LocalPath string `json:"localPath"` + ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` +} + +type RemoteUploadInfo struct { + serder.Metadata `union:"url"` + UploadInfoBase + Type string `json:"type"` + Url string `json:"url"` + Branch string `json:"branch"` + DataName string `json:"dataName"` + Cluster schsdk.ClusterID `json:"clusterID"` +} + +type UploadInfoBase struct{} + +func (d *UploadInfoBase) Noop() {} + +type UploadPriority interface { + Noop() +} + +var UploadPriorityTypeUnion = types.NewTypeUnion[UploadPriority]( + (*Preferences)(nil), + (*SpecifyCluster)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadPriorityTypeUnion, "type") + +type Preferences struct { + serder.Metadata `union:"preference"` + UploadPriorityBase + Type string `json:"type"` + ResourcePriorities []ResourcePriority `json:"priorities"` +} + +type SpecifyCluster struct { + serder.Metadata `union:"specify"` + UploadPriorityBase + Type string `json:"type"` + Clusters []schsdk.ClusterID `json:"clusters"` +} + +type UploadPriorityBase struct{} + +func (d *UploadPriorityBase) Noop() {} + +type QueryData struct { + DataType string `json:"dataType" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Path string `json:"path"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + CurrentPage int `json:"currentPage" binding:"required"` + PageSize int `json:"pageSize" binding:"required"` + OrderBy string `json:"orderBy" binding:"required"` +} + +type DataBinding interface { + Noop() +} + +var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( + (*DatasetBinding)(nil), + (*ModelBinding)(nil), + (*CodeBinding)(nil), + (*ImageBinding)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") + +type DataBindingBase struct{} + +func (d *DataBindingBase) Noop() {} + +type DatasetBinding struct { + serder.Metadata `union:"dataset"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints float64 `json:"points"` +} + +type ModelBinding struct { + serder.Metadata `union:"model"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` +} + +type CodeBinding struct { + serder.Metadata `union:"code"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Description string `json:"description"` + ImageID schsdk.ImageID `json:"imageID"` + BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` + PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` + // 当集群为openi的时候，需要传入分支 + Branch string `json:"branch"` +} + +//type ImageBinding struct { +// serder.Metadata `union:"image"` +// DataBindingBase +// Type string `json:"type"` +// Name string `json:"name"` +// ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` +// Description string `json:"description"` +// Architecture string `json:"architecture"` +// ResourceType string `json:"resourceType"` +// Tags []string `json:"tags"` +// PackageID cdssdk.PackageID `json:"packageID"` +//} + +type ImageBinding struct { + serder.Metadata `union:"image"` + DataBindingBase + Type string `json:"type"` + ID int64 `json:"id"` + Name string `json:"name"` + IDType string `json:"idType"` + ImageID string `json:"imageID"` + ClusterID schsdk.ClusterID `json:"clusterID"` +} + +type Image struct { + ImageID schsdk.ImageID `json:"imageID" gorm:"column:ImageID;primaryKey"` + Name string `json:"name" gorm:"column:Name"` + CreateTime time.Time `json:"createTime" gorm:"column:CreateTime"` + ClusterImage []ClusterImage `gorm:"foreignKey:image_id;references:ImageID" json:"clusterImages"` +} + +type ClusterImage struct { + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + OriginImageType string `gorm:"column:origin_image_type" json:"originImageType"` + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + OriginImageName string `gorm:"column:origin_image_name" json:"originImageName"` + ClusterImageCard []ClusterImageCard `gorm:"foreignKey:origin_image_id;references:origin_image_id" json:"cards"` +} + +func (ClusterImage) TableName() string { + return "clusterImage" +} + +type ClusterImageCard struct { + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + Card string `gorm:"column:card" json:"card"` +} + +func (ClusterImageCard) TableName() string { + return "clusterImageCard" +} + +type QueryBindingFilters struct { + Status string `json:"status"` + Name string `json:"name"` +} + +type QueryBindingDataParam interface { + Noop() +} + +var QueryBindingDataParamTypeUnion = types.NewTypeUnion[QueryBindingDataParam]( + (*PrivateLevel)(nil), + (*ApplyLevel)(nil), + (*PublicLevel)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&QueryBindingDataParamTypeUnion, "type") + +type QueryBindingDataParamBase struct{} + +func (d *QueryBindingDataParamBase) Noop() {} + +type PrivateLevel struct { + serder.Metadata `union:"private"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + BindingID int64 `json:"bindingID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type ApplyLevel struct { + serder.Metadata `union:"apply"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type PublicLevel struct { + serder.Metadata `union:"public"` + QueryBindingDataParamBase + UserID cdssdk.UserID `json:"userID" binding:"required"` + Type string `json:"type" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 7393c28..45026f2 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -60,6 +60,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*FinetuningJobInfo)(nil), (*DataPreprocessJobInfo)(nil), (*PCMJobInfo)(nil), + (*HPCJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -92,6 +93,26 @@ type PCMJobInfo struct { JobResources JobResources `json:"jobResources"` } +type HPCJobInfo struct { + serder.Metadata `union:"HPC"` + JobInfoBase + Type string `json:"type"` + Name string `json:"name"` + ClusterID ClusterID `json:"clusterID"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + Parameters HPCParameter `json:"parameters"` +} + +type HPCParameter struct { + JobName string `json:"jobName"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + InputFile string `json:"inputFile"` +} + type JobResources struct { //任务分配策略：负载均衡、积分优先、随机分配等，dataLocality, leastLoadFirst ScheduleStrategy string `json:"scheduleStrategy"` From a460daa8614e99f9102c40f810257d4b502bd17a Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 15:30:07 +0800 Subject: [PATCH 22/29] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E5=AD=90=E7=AE=97=E6=B3=95=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 0637b8c..a20cf30 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -166,6 +166,7 @@ type PackageCloneParam struct { Description string `json:"description"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` ImageID schsdk.ImageID `json:"imageID"` } From 48681fa14707fde4818e780ce876ebd5ec584e09 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 14 Mar 2025 16:59:12 +0800 Subject: [PATCH 23/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BB=8EStorage?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0Package=E7=9A=84=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/storage/cdsapi/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/storage/cdsapi/storage.go b/sdks/storage/cdsapi/storage.go index 22ad944..d275432 100644 --- a/sdks/storage/cdsapi/storage.go +++ b/sdks/storage/cdsapi/storage.go @@ -46,7 +46,7 @@ func (r *StorageCreatePackageReq) MakeParam() *sdks.RequestParam { } type StorageCreatePackageResp struct { - PackageID cdssdk.PackageID `json:"packageID"` + Package cdssdk.Package `json:"package"` } func (r *StorageCreatePackageResp) ParseResponse(resp *http.Response) error { From e4a9171da2241341899ea6df0c46defe2198b826 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 17:47:11 +0800 Subject: [PATCH 24/29] =?UTF-8?q?=E8=B6=85=E7=AE=97=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 3 ++- sdks/scheduler/models.go | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index ddc5205..363be1a 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -15,7 +15,8 @@ type CreateJobReq struct { Backend string `json:"backend"` App string `json:"app"` OperateType string `json:"operateType"` - Parameters HPCParameter `json:"parameters"` + //Parameters HPCParameter `json:"parameters"` + Parameters map[string]string `json:"parameters"` } type HPCParameter struct { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 45026f2..9051489 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -106,11 +106,17 @@ type HPCJobInfo struct { } type HPCParameter struct { - JobName string `json:"jobName"` - Partition string `json:"partition"` - Ntasks string `json:"ntasks"` - Nodes string `json:"nodes"` - InputFile string `json:"inputFile"` + JobName string `json:"jobName"` + JobDir string `json:"jobDir"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"` +} + +type HPCBindingFile struct { + ParamName string `json:"paramName"` + BindingID int64 `json:"bindingID"` } type JobResources struct { From 1da627d0baf98717ef71c6066a4a5c5d9941a43f Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 18:47:48 +0800 Subject: [PATCH 25/29] fix --- sdks/storage/cdsapi/object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index aca0aa9..49c0b3a 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -66,7 +66,7 @@ type ObjectListByIDs struct { } func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam { - return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r) + return sdks.MakeQueryParam(http.MethodPost, ObjectListByIDsPath, r) } type ObjectListByIDsResp struct { From f5cad317d16abeb085dafaceb3db4a5fb93b376a Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 20:01:28 +0800 Subject: [PATCH 26/29] fix --- sdks/storage/cdsapi/object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 49c0b3a..aca0aa9 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -66,7 +66,7 @@ type ObjectListByIDs struct { } func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam { - return sdks.MakeQueryParam(http.MethodPost, ObjectListByIDsPath, r) + return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r) } type ObjectListByIDsResp struct { From 058904bff5240be8b0345b2a4ccde0c400ef0845 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 20:51:02 +0800 Subject: [PATCH 27/29] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=B6=85=E7=AE=97?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 1 + sdks/scheduler/models.go | 1 + 2 files changed, 2 insertions(+) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index 363be1a..f860bc1 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -24,6 +24,7 @@ type HPCParameter struct { Partition string `json:"partition"` Ntasks string `json:"ntasks"` Nodes string `json:"nodes"` + BamFile string `json:"bamFile"` InputFile string `json:"inputFile"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9051489..4890e53 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -111,6 +111,7 @@ type HPCParameter struct { Partition string `json:"partition"` Ntasks string `json:"ntasks"` Nodes string `json:"nodes"` + BamFile string `json:"bamFile"` HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"` } From 733345a5f9857da27239b9044056267ad2becbf1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 18 Mar 2025 11:23:34 +0800 Subject: [PATCH 28/29] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=95=B0=E9=87=8F=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 4890e53..ae87c13 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -65,13 +65,18 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` + LocalJobID string `json:"localJobID"` + TargetLocalJobID string `json:"targetLocalJobID"` } func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } +func (i *JobInfoBase) GetTargetLocalJobID() string { + return i.TargetLocalJobID +} + type NormalJobInfo struct { serder.Metadata `union:"Normal"` JobInfoBase From ce4cb96abe16f9a899c1a9657b13e8e4859d286c Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 18 Mar 2025 17:03:37 +0800 Subject: [PATCH 29/29] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 5 +++-- sdks/scheduler/models.go | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 9439b9c..bf5cd7b 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -49,8 +49,9 @@ const ( PreferencePriority = "preference" SpecifyClusterPriority = "specify" - FailedStatus = "failed" - SuccessStatus = "success" + FailedStatus = "failed" + SuccessStatus = "success" + UploadingStatus = "uploading" Query = "query" Delete = "delete" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index ae87c13..9b94602 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -49,6 +49,7 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string + GetTargetLocalJobID() []string } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -65,16 +66,16 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` - TargetLocalJobID string `json:"targetLocalJobID"` + LocalJobID string `json:"localJobID"` + TargetLocalJobIDs []string `json:"targetLocalJobIDs"` } func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } -func (i *JobInfoBase) GetTargetLocalJobID() string { - return i.TargetLocalJobID +func (i *JobInfoBase) GetTargetLocalJobID() []string { + return i.TargetLocalJobIDs } type NormalJobInfo struct {