From 91563ca94108529e6323f03fef11acaeb30a30eb Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 28 Feb 2025 13:50:51 +0800 Subject: [PATCH 01/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 4007f4a..32c55f7 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -61,6 +61,8 @@ const ( URL = "url" ID = "id" + + Startup = "startup" ) type TaskID int64 From 1016cdffd5c44a581f6349be306f11ab9826359e Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 28 Feb 2025 14:50:47 +0800 Subject: [PATCH 02/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=AD=90=E7=AE=97=E6=B3=95=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 2a92a89..c4126bc 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -66,7 +66,7 @@ type BindingDetail struct { SSOId string `json:"ssoID"` Name string `json:"Name"` Info sch.DataBinding `json:"info"` - Packages []Package `json:"packages"` + Package Package `json:"package"` Status string `json:"status"` AccessLevel string `json:"accessLevel"` CreateTime time.Time `json:"createTime"` From e888412c58a6d363397cd6e17ee4c3ceb56ef9b0 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 4 Mar 2025 11:16:30 +0800 Subject: [PATCH 03/73] =?UTF-8?q?=E6=96=B0=E5=A2=9Etoken?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 10 +++++++-- sdks/pcmscheduler/models.go | 42 +++++++++++++++++++------------------ sdks/uploader/models.go | 1 + 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 8bbdd4e..6f6a366 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -120,7 +120,7 @@ type ScheduleData struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } -func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { +func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "/createTask") if err != nil { return nil, err @@ -135,6 +135,9 @@ func (c *Client) CreateJob(req CreateJobReq) (*CreateJobResp, error) { resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, + Header: map[string]string{ + "Authorization": token, + }, }) if err != nil { return nil, err @@ -176,7 +179,7 @@ type DataScheduleResults struct { Results []DataScheduleResult `json:"results"` } -func (c *Client) RunJob(req RunJobReq) error { +func (c *Client) RunJob(req RunJobReq, token string) error { targetUrl, err := url.JoinPath(c.baseURL, "runTask") if err != nil { return err @@ -184,6 +187,9 @@ func (c *Client) RunJob(req RunJobReq) error { resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, + Header: map[string]string{ + "Authorization": token, + }, }) if err != nil { return err diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 32c55f7..4c4a490 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -376,40 +376,42 @@ func (d *DataBindingBase) Noop() {} type DatasetBinding struct { serder.Metadata `union:"dataset"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Category string `json:"category"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` } type ModelBinding struct { serder.Metadata `union:"model"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Category string `json:"category"` - ModelType string `json:"modelType"` - Env string `json:"env"` - Version string `json:"version"` - PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` } type CodeBinding struct { serder.Metadata `union:"code"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - //ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Type string `json:"type"` + Name string `json:"name"` ClusterID schsdk.ClusterID `json:"clusterID"` Description string `json:"description"` ImageID schsdk.ImageID `json:"imageID"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` - //FilePath string `json:"filePath"` - PackageID cdssdk.PackageID `json:"packageID"` + PackageID cdssdk.PackageID `json:"packageID"` + // 当集群为openi的时候，需要传入分支 + Branch string `json:"branch"` } //type ImageBinding struct { diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index c4126bc..7a9879a 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -80,6 +80,7 @@ type BindingCluster struct { BindingID DataID `gorm:"column:binding_id" json:"bindingID"` ClusterID ClusterID `gorm:"column:cluster_id" json:"clusterID"` Status string `gorm:"column:status" json:"status"` + Param string `gorm:"column:param" json:"Param"` JsonData string `gorm:"column:json_data" json:"jsonData"` } From e85760ff14c9be62ceb3a86a108fb4c5347d42d8 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 4 Mar 2025 16:52:01 +0800 Subject: [PATCH 04/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 4c4a490..dcb1719 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -56,8 +56,8 @@ const ( ParentType = "parent" PlatformSugon = "sugon" - PlatformOpenI = "openi" - PlatformModelArts = "modelarts" + PlatformOpenI = "OpenI" + PlatformModelArts = "ModelArts" URL = "url" ID = "id" From 8c629b18b4aef4c75ad1da8792e016286f422424 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 6 Mar 2025 08:58:33 +0800 Subject: [PATCH 05/73] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=8A=B6=E6=80=81=E5=9B=9E=E6=98=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index cbf0ae7..999d8b7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -140,6 +140,7 @@ type CPU struct { serder.Metadata `union:"CPU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -147,6 +148,7 @@ type GPU struct { serder.Metadata `union:"GPU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -154,6 +156,7 @@ type NPU struct { serder.Metadata `union:"NPU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -161,6 +164,7 @@ type Memory struct { serder.Metadata `union:"Memory"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -168,6 +172,7 @@ type DCU struct { serder.Metadata `union:"DCU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -175,6 +180,7 @@ type MLU struct { serder.Metadata `union:"MLU"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } @@ -182,6 +188,7 @@ type PRICE struct { serder.Metadata `union:"PRICE"` JobResourceBase Type string `json:"type"` + Name string `json:"name"` Number int64 `json:"number"` } From dad73eba53d24a7d8fc2ff98ed7a51b0a32b8d0c Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 7 Mar 2025 15:03:38 +0800 Subject: [PATCH 06/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 2 ++ sdks/pcmscheduler/models.go | 11 ++++++++++- sdks/scheduler/models.go | 1 + sdks/uploader/models.go | 12 ++++++------ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 6f6a366..269151a 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -163,6 +163,8 @@ func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, erro type RunJobReq struct { TaskID TaskID `json:"taskID"` + JobSetID schsdk.JobSetID `json:"jobSetID"` + LocalJobID string `json:"localJobID"` ScheduledDatas []DataScheduleResults `json:"scheduledDatas"` } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index dcb1719..daca965 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -24,6 +24,7 @@ const ( DATASET = "dataset" IMAGE = "image" MODEL = "model" + RESULT = "result" OrderByName = "name" OrderBySize = "size" @@ -278,10 +279,18 @@ type TaskMessage struct { Message string `json:"message"` } +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` +} + type UploadParams struct { DataType string `json:"dataType"` UploadInfo UploadInfo `json:"uploadInfo"` - //UploadPriority UploadPriority `json:"uploadPriority"` } type UploadInfo interface { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 999d8b7..6ba148c 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -220,6 +220,7 @@ type DataReturnJobInfo struct { JobInfoBase Type string `json:"type"` BucketID cdssdk.BucketID `json:"bucketID"` + BindingType string `json:"bindingType"` TargetLocalJobID string `json:"targetLocalJobID"` } diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 7a9879a..0637b8c 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -112,12 +112,12 @@ func (Cluster) TableName() string { } type Package struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + //JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:create_time" json:"createTime"` Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` From 66da854b0bd75e6dd40945a4dfd1237c3dc9ee8d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 10 Mar 2025 16:40:23 +0800 Subject: [PATCH 07/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/job.go | 16 ++++++++++++++++ sdks/pcmscheduler/jobset.go | 3 +-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 sdks/pcmscheduler/job.go diff --git a/sdks/pcmscheduler/job.go b/sdks/pcmscheduler/job.go new file mode 100644 index 0000000..0470ec8 --- /dev/null +++ b/sdks/pcmscheduler/job.go @@ -0,0 +1,16 @@ +package sch + +import ( + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) + +type PCMJob struct { + ID string `gorm:"column:id" json:"ID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + JobSetID schsdk.JobSetID `gorm:"column:jobset_id" json:"jobSetID"` + LocalJobID string `gorm:"column:local_job_id" json:"localJobID"` + Param string `gorm:"column:param" json:"param"` + CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` +} diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 269151a..e4d7cde 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -92,6 +92,7 @@ type DatasetDistribute struct { type CodeDistribute struct { DataName string `json:"dataName"` PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` Clusters []DataDetail `json:"clusters"` } @@ -163,8 +164,6 @@ func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, erro type RunJobReq struct { TaskID TaskID `json:"taskID"` - JobSetID schsdk.JobSetID `json:"jobSetID"` - LocalJobID string `json:"localJobID"` ScheduledDatas []DataScheduleResults `json:"scheduledDatas"` } From fe198e76bfb40bdbff88e7bf1f38239fa5322161 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 11 Mar 2025 16:27:40 +0800 Subject: [PATCH 08/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 8 ++++---- sdks/pcmscheduler/models.go | 15 ++++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index e4d7cde..be1e3ae 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -58,10 +58,10 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) } type CreateJobReq struct { - Name string `json:"name"` - Description string `json:"description"` - JobResources schsdk.JobResources `json:"jobResources"` - DataDistribute DataDistribute `json:"dataDistributes"` + Name string `json:"name"` + Description string `json:"description"` + JobResources schsdk.JobResources `json:"jobResources"` + DataDistributes DataDistribute `json:"dataDistributes"` } type CommonJsonData struct { diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index daca965..a993342 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -385,13 +385,14 @@ func (d *DataBindingBase) Noop() {} type DatasetBinding struct { serder.Metadata `union:"dataset"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` - Description string `json:"description"` - Category string `json:"category"` - PackageID cdssdk.PackageID `json:"packageID"` - RepositoryName string `json:"repositoryName"` + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints float64 `json:"points"` } type ModelBinding struct { From fdace7b42b8c9866ad1966a7c2f6dc441cc96e34 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 13 Mar 2025 09:17:24 +0800 Subject: [PATCH 09/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E8=BE=93=E5=87=BA=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index a993342..1b3eaff 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -420,6 +420,7 @@ type CodeBinding struct { ImageID schsdk.ImageID `json:"imageID"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` // 当集群为openi的时候，需要传入分支 Branch string `json:"branch"` } From a936cffa134eece2e32229f4ff2e39f592777d1b Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 13 Mar 2025 16:59:03 +0800 Subject: [PATCH 10/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=9B=E5=BB=BApacka?= =?UTF-8?q?ge=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 6ba148c..7393c28 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -126,8 +126,9 @@ var JobResourceTypeUnion = types.NewTypeUnion[JobResource]( (*NPU)(nil), (*MLU)(nil), (*DCU)(nil), - (*Memory)(nil), + (*MEMORY)(nil), (*PRICE)(nil), + (*STORAGE)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobResourceTypeUnion, "type") @@ -144,6 +145,14 @@ type CPU struct { Number int64 `json:"number"` } +type STORAGE struct { + serder.Metadata `union:"STORAGE"` + JobResourceBase + Type string `json:"type"` + Name string `json:"name"` + Number int64 `json:"number"` +} + type GPU struct { serder.Metadata `union:"GPU"` JobResourceBase @@ -160,8 +169,8 @@ type NPU struct { Number int64 `json:"number"` } -type Memory struct { - serder.Metadata `union:"Memory"` +type MEMORY struct { + serder.Metadata `union:"MEMORY"` JobResourceBase Type string `json:"type"` Name string `json:"name"` From 4374600529fdec8617b4ef92fc065ed47ee0f1bd Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 14:23:47 +0800 Subject: [PATCH 11/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B6=85=E7=AE=97?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/client.go | 63 +++++ sdks/hpc/config.go | 5 + sdks/hpc/job.go | 71 ++++++ sdks/hpc/models.go | 525 +++++++++++++++++++++++++++++++++++++++ sdks/scheduler/models.go | 21 ++ 5 files changed, 685 insertions(+) create mode 100644 sdks/hpc/client.go create mode 100644 sdks/hpc/config.go create mode 100644 sdks/hpc/job.go create mode 100644 sdks/hpc/models.go diff --git a/sdks/hpc/client.go b/sdks/hpc/client.go new file mode 100644 index 0000000..07b4393 --- /dev/null +++ b/sdks/hpc/client.go @@ -0,0 +1,63 @@ +package hpc + +import ( + "fmt" + + "gitlink.org.cn/cloudream/common/sdks" +) + +type response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type respons2[T any] struct { + Code int `json:"code"` + Message string `json:"msg"` + Data T `json:"data"` +} + +const ( + ResponseCodeOK int = 200 +) + +func (r *response[T]) ToError() *sdks.CodeMessageError { + return &sdks.CodeMessageError{ + Code: fmt.Sprintf("%d", r.Code), + Message: r.Message, + } +} + +type Client struct { + baseURL string +} + +func NewClient(cfg *Config) *Client { + return &Client{ + baseURL: cfg.URL, + } +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + cfg *Config +} + +func NewPool(cfg *Config) Pool { + return &pool{ + cfg: cfg, + } +} +func (p *pool) Acquire() (*Client, error) { + cli := NewClient(p.cfg) + return cli, nil +} + +func (p *pool) Release(cli *Client) { + +} diff --git a/sdks/hpc/config.go b/sdks/hpc/config.go new file mode 100644 index 0000000..d168c41 --- /dev/null +++ b/sdks/hpc/config.go @@ -0,0 +1,5 @@ +package hpc + +type Config struct { + URL string `json:"url"` +} diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go new file mode 100644 index 0000000..ddc5205 --- /dev/null +++ b/sdks/hpc/job.go @@ -0,0 +1,71 @@ +package hpc + +import ( + "fmt" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + "gitlink.org.cn/cloudream/common/utils/http2" + "gitlink.org.cn/cloudream/common/utils/serder" + "net/url" + "strings" +) + +type CreateJobReq struct { + Name string `json:"name"` + ClusterID schsdk.ClusterID `json:"clusterId"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + Parameters HPCParameter `json:"parameters"` +} + +type HPCParameter struct { + JobName string `json:"jobName"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + InputFile string `json:"inputFile"` +} + +type CreateJobResp struct { + Backend string `json:"backend"` + JobInfo HPCJobInfo `json:"jobInfo"` +} + +type HPCJobInfo struct { + JobDir string `json:"jobDir"` + JobID string `json:"jobId"` +} + +func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/hpc/commitHpcTask") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, fmt.Errorf("error: %s", codeResp.Message) + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) + +} diff --git a/sdks/hpc/models.go b/sdks/hpc/models.go new file mode 100644 index 0000000..6511bb9 --- /dev/null +++ b/sdks/hpc/models.go @@ -0,0 +1,525 @@ +package hpc + +import ( + "gitlink.org.cn/cloudream/common/pkgs/types" + schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/serder" + "time" +) + +type ResourceType string + +const ( + ResourceTypeCPU ResourceType = "CPU" + ResourceTypeNPU ResourceType = "NPU" + ResourceTypeGPU ResourceType = "GPU" + ResourceTypeMLU ResourceType = "MLU" + ResourceTypeStorage ResourceType = "STORAGE" + ResourceTypeMemory ResourceType = "MEMORY" + + Split = "/" + + CODE = "code" + DATASET = "dataset" + IMAGE = "image" + MODEL = "model" + RESULT = "result" + + OrderByName = "name" + OrderBySize = "size" + OrderByTime = "time" + + StorageTypeURL = "url" + StorageTypeJCS = "jcs" + + RejectedStatus = "rejected" + PendingStatus = "pending" + ApprovedStatus = "approved" + RevokedStatus = "revoked" + CancelStatus = "cancel" + ExpiredStatus = "expired" + + ApplyAccess = "apply" + PrivateAccess = "private" + PublicAccess = "public" + + PreferencePriority = "preference" + SpecifyClusterPriority = "specify" + + FailedStatus = "failed" + SuccessStatus = "success" + + Query = "query" + Delete = "delete" + + ChildrenType = "children" + ParentType = "parent" + + PlatformSugon = "sugon" + PlatformOpenI = "OpenI" + PlatformModelArts = "ModelArts" + + URL = "url" + ID = "id" + + Startup = "startup" +) + +type TaskID int64 +type DataID int64 + +type ClusterDetail struct { + // 集群ID + ClusterId schsdk.ClusterID `json:"clusterID"` + // 集群功能类型：云算，智算，超算 + ClusterType string `json:"clusterType"` + // 集群地区：华东地区、华南地区、华北地区、华中地区、西南地区、西北地区、东北地区 + Region string `json:"region"` + // 资源类型 + Resources2 []ResourceData `json:"resources1,omitempty"` + //Resources2 []ResourceData `json:"resources"` + Resources []ClusterResource `json:"resources"` +} + +type ClusterResource struct { + Resource TmpResourceData `json:"resource"` + BaseResources []TmpResourceData `json:"baseResources"` +} + +type TmpResourceData struct { + Type ResourceType `json:"type"` + Name string `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceData interface { + Noop() +} + +var ResourceDataTypeUnion = types.NewTypeUnion[ResourceData]( + (*CPUResourceData)(nil), + (*NPUResourceData)(nil), + (*GPUResourceData)(nil), + (*MLUResourceData)(nil), + (*DCUResourceData)(nil), + (*GCUResourceData)(nil), + (*GPGPUResourceData)(nil), + (*StorageResourceData)(nil), + (*MemoryResourceData)(nil), + (*BalanceResourceData)(nil), + (*RateResourceData)(nil), +) +var _ = serder.UseTypeUnionInternallyTagged(&ResourceDataTypeUnion, "type") + +type ResourceDataBase struct{} + +func (d *ResourceDataBase) Noop() {} + +type UnitValue[T any] struct { + Unit string `json:"unit"` + Value T `json:"value"` +} + +type CPUResourceData struct { + serder.Metadata `union:"CPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type NPUResourceData struct { + serder.Metadata `union:"NPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPUResourceData struct { + serder.Metadata `union:"GPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type MLUResourceData struct { + serder.Metadata `union:"MLU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type DCUResourceData struct { + serder.Metadata `union:"DCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GCUResourceData struct { + serder.Metadata `union:"GCU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type GPGPUResourceData struct { + serder.Metadata `union:"ILUVATAR-GPGPU"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[int64] `json:"total"` + Available UnitValue[int64] `json:"available"` +} + +type StorageResourceData struct { + serder.Metadata `union:"STORAGE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type MemoryResourceData struct { + serder.Metadata `union:"MEMORY"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type BalanceResourceData struct { + serder.Metadata `union:"BALANCE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type RateResourceData struct { + serder.Metadata `union:"RATE"` + ResourceDataBase + Type string `json:"type"` + Name ResourceType `json:"name"` + Total UnitValue[float64] `json:"total"` + Available UnitValue[float64] `json:"available"` +} + +type ResourceRange struct { + UserID cdssdk.UserID `json:"userID"` + Type ResourceType `json:"type"` + GPU Range `json:"gpu"` + GPUNumber int `json:"gpuNumber"` + CPU Range `json:"cpu"` + Memory Range `json:"memory"` + Storage Range `json:"storage"` +} + +type Range struct { + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +type ResourcePriority interface { + Noop() +} + +type ResourcePriorityBase struct { +} + +var ResourcePriorityTypeUnion = types.NewTypeUnion[ResourcePriority]( + (*RegionPriority)(nil), + (*ChipPriority)(nil), + (*BiasPriority)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&ResourcePriorityTypeUnion, "type") + +func (d *ResourcePriorityBase) Noop() {} + +type RegionPriority struct { + serder.Metadata `union:"region"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type ChipPriority struct { + serder.Metadata `union:"chip"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type BiasPriority struct { + serder.Metadata `union:"bias"` + ResourcePriorityBase + Type string `json:"type"` + Options []string `json:"options"` +} + +type TaskMessage struct { + Status string `json:"status"` + Message string `json:"message"` +} + +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` +} + +type UploadParams struct { + DataType string `json:"dataType"` + UploadInfo UploadInfo `json:"uploadInfo"` +} + +type UploadInfo interface { + Noop() +} + +var UploadInfoTypeUnion = types.NewTypeUnion[UploadInfo]( + (*LocalUploadInfo)(nil), + (*RemoteUploadInfo)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadInfoTypeUnion, "type") + +type LocalUploadInfo struct { + serder.Metadata `union:"local"` + UploadInfoBase + Type string `json:"type"` + LocalPath string `json:"localPath"` + ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` +} + +type RemoteUploadInfo struct { + serder.Metadata `union:"url"` + UploadInfoBase + Type string `json:"type"` + Url string `json:"url"` + Branch string `json:"branch"` + DataName string `json:"dataName"` + Cluster schsdk.ClusterID `json:"clusterID"` +} + +type UploadInfoBase struct{} + +func (d *UploadInfoBase) Noop() {} + +type UploadPriority interface { + Noop() +} + +var UploadPriorityTypeUnion = types.NewTypeUnion[UploadPriority]( + (*Preferences)(nil), + (*SpecifyCluster)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&UploadPriorityTypeUnion, "type") + +type Preferences struct { + serder.Metadata `union:"preference"` + UploadPriorityBase + Type string `json:"type"` + ResourcePriorities []ResourcePriority `json:"priorities"` +} + +type SpecifyCluster struct { + serder.Metadata `union:"specify"` + UploadPriorityBase + Type string `json:"type"` + Clusters []schsdk.ClusterID `json:"clusters"` +} + +type UploadPriorityBase struct{} + +func (d *UploadPriorityBase) Noop() {} + +type QueryData struct { + DataType string `json:"dataType" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Path string `json:"path"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` + CurrentPage int `json:"currentPage" binding:"required"` + PageSize int `json:"pageSize" binding:"required"` + OrderBy string `json:"orderBy" binding:"required"` +} + +type DataBinding interface { + Noop() +} + +var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( + (*DatasetBinding)(nil), + (*ModelBinding)(nil), + (*CodeBinding)(nil), + (*ImageBinding)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") + +type DataBindingBase struct{} + +func (d *DataBindingBase) Noop() {} + +type DatasetBinding struct { + serder.Metadata `union:"dataset"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints float64 `json:"points"` +} + +type ModelBinding struct { + serder.Metadata `union:"model"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` +} + +type CodeBinding struct { + serder.Metadata `union:"code"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Description string `json:"description"` + ImageID schsdk.ImageID `json:"imageID"` + BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` + PackageID cdssdk.PackageID `json:"packageID"` + Output string `json:"output"` + // 当集群为openi的时候，需要传入分支 + Branch string `json:"branch"` +} + +//type ImageBinding struct { +// serder.Metadata `union:"image"` +// DataBindingBase +// Type string `json:"type"` +// Name string `json:"name"` +// ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` +// Description string `json:"description"` +// Architecture string `json:"architecture"` +// ResourceType string `json:"resourceType"` +// Tags []string `json:"tags"` +// PackageID cdssdk.PackageID `json:"packageID"` +//} + +type ImageBinding struct { + serder.Metadata `union:"image"` + DataBindingBase + Type string `json:"type"` + ID int64 `json:"id"` + Name string `json:"name"` + IDType string `json:"idType"` + ImageID string `json:"imageID"` + ClusterID schsdk.ClusterID `json:"clusterID"` +} + +type Image struct { + ImageID schsdk.ImageID `json:"imageID" gorm:"column:ImageID;primaryKey"` + Name string `json:"name" gorm:"column:Name"` + CreateTime time.Time `json:"createTime" gorm:"column:CreateTime"` + ClusterImage []ClusterImage `gorm:"foreignKey:image_id;references:ImageID" json:"clusterImages"` +} + +type ClusterImage struct { + ImageID schsdk.ImageID `gorm:"column:image_id" json:"imageID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + OriginImageType string `gorm:"column:origin_image_type" json:"originImageType"` + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + OriginImageName string `gorm:"column:origin_image_name" json:"originImageName"` + ClusterImageCard []ClusterImageCard `gorm:"foreignKey:origin_image_id;references:origin_image_id" json:"cards"` +} + +func (ClusterImage) TableName() string { + return "clusterImage" +} + +type ClusterImageCard struct { + OriginImageID string `gorm:"column:origin_image_id" json:"originImageID"` + Card string `gorm:"column:card" json:"card"` +} + +func (ClusterImageCard) TableName() string { + return "clusterImageCard" +} + +type QueryBindingFilters struct { + Status string `json:"status"` + Name string `json:"name"` +} + +type QueryBindingDataParam interface { + Noop() +} + +var QueryBindingDataParamTypeUnion = types.NewTypeUnion[QueryBindingDataParam]( + (*PrivateLevel)(nil), + (*ApplyLevel)(nil), + (*PublicLevel)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&QueryBindingDataParamTypeUnion, "type") + +type QueryBindingDataParamBase struct{} + +func (d *QueryBindingDataParamBase) Noop() {} + +type PrivateLevel struct { + serder.Metadata `union:"private"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + BindingID int64 `json:"bindingID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type ApplyLevel struct { + serder.Metadata `union:"apply"` + QueryBindingDataParamBase + Type string `json:"type" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} + +type PublicLevel struct { + serder.Metadata `union:"public"` + QueryBindingDataParamBase + UserID cdssdk.UserID `json:"userID" binding:"required"` + Type string `json:"type" binding:"required"` + Info DataBinding `json:"info"` // 可选，用于精细筛选，功能暂未实现 +} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 7393c28..45026f2 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -60,6 +60,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*FinetuningJobInfo)(nil), (*DataPreprocessJobInfo)(nil), (*PCMJobInfo)(nil), + (*HPCJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -92,6 +93,26 @@ type PCMJobInfo struct { JobResources JobResources `json:"jobResources"` } +type HPCJobInfo struct { + serder.Metadata `union:"HPC"` + JobInfoBase + Type string `json:"type"` + Name string `json:"name"` + ClusterID ClusterID `json:"clusterID"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + Parameters HPCParameter `json:"parameters"` +} + +type HPCParameter struct { + JobName string `json:"jobName"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + InputFile string `json:"inputFile"` +} + type JobResources struct { //任务分配策略：负载均衡、积分优先、随机分配等，dataLocality, leastLoadFirst ScheduleStrategy string `json:"scheduleStrategy"` From a460daa8614e99f9102c40f810257d4b502bd17a Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 15:30:07 +0800 Subject: [PATCH 12/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E5=AD=90=E7=AE=97=E6=B3=95=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 0637b8c..a20cf30 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -166,6 +166,7 @@ type PackageCloneParam struct { Description string `json:"description"` BootstrapObjectID cdssdk.ObjectID `json:"bootstrapObjectID"` ClusterID schsdk.ClusterID `json:"clusterID"` + Output string `json:"output"` ImageID schsdk.ImageID `json:"imageID"` } From e4a9171da2241341899ea6df0c46defe2198b826 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 17:47:11 +0800 Subject: [PATCH 13/73] =?UTF-8?q?=E8=B6=85=E7=AE=97=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 3 ++- sdks/scheduler/models.go | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index ddc5205..363be1a 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -15,7 +15,8 @@ type CreateJobReq struct { Backend string `json:"backend"` App string `json:"app"` OperateType string `json:"operateType"` - Parameters HPCParameter `json:"parameters"` + //Parameters HPCParameter `json:"parameters"` + Parameters map[string]string `json:"parameters"` } type HPCParameter struct { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 45026f2..9051489 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -106,11 +106,17 @@ type HPCJobInfo struct { } type HPCParameter struct { - JobName string `json:"jobName"` - Partition string `json:"partition"` - Ntasks string `json:"ntasks"` - Nodes string `json:"nodes"` - InputFile string `json:"inputFile"` + JobName string `json:"jobName"` + JobDir string `json:"jobDir"` + Partition string `json:"partition"` + Ntasks string `json:"ntasks"` + Nodes string `json:"nodes"` + HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"` +} + +type HPCBindingFile struct { + ParamName string `json:"paramName"` + BindingID int64 `json:"bindingID"` } type JobResources struct { From 1da627d0baf98717ef71c6066a4a5c5d9941a43f Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 18:47:48 +0800 Subject: [PATCH 14/73] fix --- sdks/storage/cdsapi/object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index aca0aa9..49c0b3a 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -66,7 +66,7 @@ type ObjectListByIDs struct { } func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam { - return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r) + return sdks.MakeQueryParam(http.MethodPost, ObjectListByIDsPath, r) } type ObjectListByIDsResp struct { From f5cad317d16abeb085dafaceb3db4a5fb93b376a Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 20:01:28 +0800 Subject: [PATCH 15/73] fix --- sdks/storage/cdsapi/object.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/storage/cdsapi/object.go b/sdks/storage/cdsapi/object.go index 49c0b3a..aca0aa9 100644 --- a/sdks/storage/cdsapi/object.go +++ b/sdks/storage/cdsapi/object.go @@ -66,7 +66,7 @@ type ObjectListByIDs struct { } func (r *ObjectListByIDs) MakeParam() *sdks.RequestParam { - return sdks.MakeQueryParam(http.MethodPost, ObjectListByIDsPath, r) + return sdks.MakeQueryParam(http.MethodGet, ObjectListByIDsPath, r) } type ObjectListByIDsResp struct { From 058904bff5240be8b0345b2a4ccde0c400ef0845 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 14 Mar 2025 20:51:02 +0800 Subject: [PATCH 16/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=B6=85=E7=AE=97?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 1 + sdks/scheduler/models.go | 1 + 2 files changed, 2 insertions(+) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index 363be1a..f860bc1 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -24,6 +24,7 @@ type HPCParameter struct { Partition string `json:"partition"` Ntasks string `json:"ntasks"` Nodes string `json:"nodes"` + BamFile string `json:"bamFile"` InputFile string `json:"inputFile"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9051489..4890e53 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -111,6 +111,7 @@ type HPCParameter struct { Partition string `json:"partition"` Ntasks string `json:"ntasks"` Nodes string `json:"nodes"` + BamFile string `json:"bamFile"` HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"` } From 733345a5f9857da27239b9044056267ad2becbf1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 18 Mar 2025 11:23:34 +0800 Subject: [PATCH 17/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=95=B0=E9=87=8F=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 4890e53..ae87c13 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -65,13 +65,18 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` + LocalJobID string `json:"localJobID"` + TargetLocalJobID string `json:"targetLocalJobID"` } func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } +func (i *JobInfoBase) GetTargetLocalJobID() string { + return i.TargetLocalJobID +} + type NormalJobInfo struct { serder.Metadata `union:"Normal"` JobInfoBase From ce4cb96abe16f9a899c1a9657b13e8e4859d286c Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 18 Mar 2025 17:03:37 +0800 Subject: [PATCH 18/73] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 5 +++-- sdks/scheduler/models.go | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 9439b9c..bf5cd7b 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -49,8 +49,9 @@ const ( PreferencePriority = "preference" SpecifyClusterPriority = "specify" - FailedStatus = "failed" - SuccessStatus = "success" + FailedStatus = "failed" + SuccessStatus = "success" + UploadingStatus = "uploading" Query = "query" Delete = "delete" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index ae87c13..9b94602 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -49,6 +49,7 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string + GetTargetLocalJobID() []string } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -65,16 +66,16 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` - TargetLocalJobID string `json:"targetLocalJobID"` + LocalJobID string `json:"localJobID"` + TargetLocalJobIDs []string `json:"targetLocalJobIDs"` } func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } -func (i *JobInfoBase) GetTargetLocalJobID() string { - return i.TargetLocalJobID +func (i *JobInfoBase) GetTargetLocalJobID() []string { + return i.TargetLocalJobIDs } type NormalJobInfo struct { From f5a4b7176dd5d475532c42e9c7bbadc6d5dae7d3 Mon Sep 17 00:00:00 2001 From: qiwang <1364512070@qq.com> Date: Thu, 20 Mar 2025 16:48:51 +0800 Subject: [PATCH 19/73] feat:query taskResult --- sdks/pcmscheduler/models.go | 93 +++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index af991f5..bea1599 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -227,6 +227,99 @@ type ResourceRange struct { Ids []string `json:"ids"` } +type PCMJobData struct { + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + Param string `json:"param"` +} + +type JobInfo struct { + Name string `json:"name"` + Description string `json:"description,optional"` + JobResources JobResources `json:"jobResources"` + DataDistributeData DataDistributeData `json:"dataDistributes"` + ResultFile ResultFile `json:"resultFile"` +} + +type ResultFile struct { + Path string `json:"path"` + UpdateTime string `json:"updateTime"` + Size int `json:"size"` +} + +type DataDistributeData struct { + Dataset []DataItem `json:"dataset"` + Code []DataItem `json:"code"` + Image []DataItem `json:"image"` + Model []DataItem `json:"model"` +} + +// DataItem 定义 dataDistributes 中的数据项结构体 +type DataItem struct { + DataName string `json:"dataName"` + PackageID int `json:"packageID"` + PackageName string `json:"packageName"` + ResourceFile string `json:"resourceFile"` + Clusters []DataDetail `json:"clusters"` +} + +type JobResources struct { + ScheduleStrategy string `json:"scheduleStrategy"` + Clusters []*JobClusterInfo `json:"clusters"` +} + +type JobClusterInfo struct { + ClusterID string `json:"clusterID"` + Resources []map[string]interface{} `json:"resources,optional"` + Runtime JobRuntimeInfo `json:"runtime,optional"` +} + +type JobRuntimeInfo struct { + Command string `json:"command,optional"` + Envs map[string]interface{} `json:"envs,optional"` + Params map[string]interface{} `json:"params,optional"` +} + +// Engine 定义 engine 结构体 +type Engine struct { + EngineName string `json:"EngineName"` + EngineVersion string `json:"EngineVersion"` + ImageUrl string `json:"ImageUrl"` + InstallSysPackages bool `json:"InstallSysPackages"` +} + +// bindingCluster 定义根结构体 +type BindingParam struct { + Name string `json:"name"` + Description string `json:"description"` + Engine Engine `json:"engine"` + CodeDir string `json:"codeDir"` + BootFile string `json:"bootFile"` + Branch string `json:"branch"` + Platform string `json:"platform"` + UserID int `json:"userID"` + PackageId int `json:"packageId"` + BucketID int `json:"bucketID"` +} + +// bindingCluster 定义根结构体 +type BindingCluster struct { + BindingId int `json:"binding_id"` + ClusterId string `json:"cluster_id"` + Status string `json:"status"` + Param string `json:"param"` +} + +/*type TaskResult struct { + Name string `json:"name"` + Description string `json:"description"` + DataDistribute DataDistribute `json:"dataDistributes"` +} + +type ClusterResources struct { + ClusterId string `json:"clusterId"` +}*/ + type Range struct { Min float64 `json:"min"` Max float64 `json:"max"` From a7c1a55bf026d1e7f4395f6a86b1772679f9e5b1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 21 Mar 2025 11:14:11 +0800 Subject: [PATCH 20/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E5=92=8C=E5=9B=9E=E6=BA=90=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/job.go | 7 +++++ sdks/pcmscheduler/models.go | 3 ++ sdks/scheduler/models.go | 55 +++++++++++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/job.go b/sdks/pcmscheduler/job.go index 0470ec8..fd1997b 100644 --- a/sdks/pcmscheduler/job.go +++ b/sdks/pcmscheduler/job.go @@ -8,9 +8,16 @@ import ( type PCMJob struct { ID string `gorm:"column:id" json:"ID"` + JobType string `gorm:"column:job_type" json:"jobType"` UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` JobSetID schsdk.JobSetID `gorm:"column:jobset_id" json:"jobSetID"` LocalJobID string `gorm:"column:local_job_id" json:"localJobID"` Param string `gorm:"column:param" json:"param"` CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` } + +type PCMJobDataReturn struct { + JobID string `gorm:"column:job_id" json:"jobID"` + ClusterID schsdk.ClusterID `gorm:"column:cluster_id" json:"clusterID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` +} diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index bf5cd7b..2eb6f3f 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -63,6 +63,9 @@ const ( PlatformOpenI = "OpenI" PlatformModelArts = "ModelArts" + PlatformPCM = "PCM" + PlatformHPC = "HPC" + URL = "url" ID = "id" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9b94602..3faf9e2 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -8,6 +8,7 @@ import ( const ( JobTypeNormal = "Normal" + JobTypePCM = "PCM" JobTypeResource = "Resource" JobTypeInstance = "Instance" JobTypeFinetuning = "Finetuning" @@ -49,7 +50,8 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string - GetTargetLocalJobID() []string + GetTargetLocalJobIDs() []string + RemoveTargetLocalJobID(targetID string) } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -62,6 +64,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*DataPreprocessJobInfo)(nil), (*PCMJobInfo)(nil), (*HPCJobInfo)(nil), + (*BindingJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -74,10 +77,23 @@ func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } -func (i *JobInfoBase) GetTargetLocalJobID() []string { +func (i *JobInfoBase) GetTargetLocalJobIDs() []string { return i.TargetLocalJobIDs } +func (i *JobInfoBase) GetTargetLocalJobIDs2() []string { + return i.TargetLocalJobIDs +} + +func (i *JobInfoBase) RemoveTargetLocalJobID(targetID string) { + // 从i.TargetLocalJobIDs中删除id + for j, id := range i.TargetLocalJobIDs { + if targetID == id { + i.TargetLocalJobIDs = append(i.TargetLocalJobIDs[:j], i.TargetLocalJobIDs[j+1:]...) + } + } +} + type NormalJobInfo struct { serder.Metadata `union:"Normal"` JobInfoBase @@ -99,6 +115,41 @@ type PCMJobInfo struct { JobResources JobResources `json:"jobResources"` } +type BindingJobInfo struct { + serder.Metadata `union:"Binding"` + JobInfoBase + Type string `json:"type"` + Info DataBinding `json:"info"` +} + +type DataBinding interface { + Noop() +} + +var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( + (*ModelBinding)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") + +type DataBindingBase struct{} + +func (d *DataBindingBase) Noop() {} + +type ModelBinding struct { + serder.Metadata `union:"model"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + //Description string `json:"description"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + //PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` +} + type HPCJobInfo struct { serder.Metadata `union:"HPC"` JobInfoBase From 9682b7422b9962d0e31bebdabc366d35fe789dbb Mon Sep 17 00:00:00 2001 From: tzwang Date: Fri, 21 Mar 2025 17:23:04 +0800 Subject: [PATCH 21/73] added pcm inference job creation --- sdks/pcmscheduler/jobset.go | 44 ++++++++++++++++++++++++++++++++++--- sdks/pcmscheduler/models.go | 5 +++-- sdks/scheduler/models.go | 11 ++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index be1e3ae..0d9ea55 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -57,6 +57,10 @@ func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) return nil, fmt.Errorf("unknow response content type: %s", contType) } +type CreateInferenceJobResp struct { + TaskId string `json:"taskId"` +} + type CreateJobReq struct { Name string `json:"name"` Description string `json:"description"` @@ -121,8 +125,42 @@ type ScheduleData struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } +func (c *Client) CreateInferenceJob(req CreateJobReq, token string) (*CreateInferenceJobResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") + if err != nil { + return nil, err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return nil, err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateInferenceJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return &codeResp.Data, nil + } + + return nil, fmt.Errorf("error: %s", codeResp.Message) + } + + return nil, fmt.Errorf("unknow response content type: %s", contType) + +} + func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/createTask") + targetUrl, err := url.JoinPath(c.baseURL, "schedule/createTask") if err != nil { return nil, err } @@ -181,7 +219,7 @@ type DataScheduleResults struct { } func (c *Client) RunJob(req RunJobReq, token string) error { - targetUrl, err := url.JoinPath(c.baseURL, "runTask") + targetUrl, err := url.JoinPath(c.baseURL, "schedule/runTask") if err != nil { return err } @@ -220,7 +258,7 @@ type CancelJobReq struct { } func (c *Client) CancelJob(req CancelJobReq) error { - targetUrl, err := url.JoinPath(c.baseURL, "/queryResources") + targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") if err != nil { return err } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 2eb6f3f..d8642cf 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -63,8 +63,9 @@ const ( PlatformOpenI = "OpenI" PlatformModelArts = "ModelArts" - PlatformPCM = "PCM" - PlatformHPC = "HPC" + PlatformPCM = "PCM" + PlatformPCMInference = "PCM_Inference" + PlatformHPC = "HPC" URL = "url" ID = "id" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 3faf9e2..e676645 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -65,6 +65,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*PCMJobInfo)(nil), (*HPCJobInfo)(nil), (*BindingJobInfo)(nil), + (*PCMInferenceJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -105,6 +106,16 @@ type NormalJobInfo struct { ModelJobInfo ModelJobInfo `json:"modelJobInfo"` } +type PCMInferenceJobInfo struct { + serder.Metadata `union:"PCM_Inference"` + JobInfoBase + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Files JobFilesInfo `json:"files"` + JobResources JobResources `json:"jobResources"` +} + type PCMJobInfo struct { serder.Metadata `union:"PCM"` JobInfoBase From f9b6e073764c21ad644ae3881665bc0b4626b5b1 Mon Sep 17 00:00:00 2001 From: qiwang <1364512070@qq.com> Date: Fri, 21 Mar 2025 17:35:05 +0800 Subject: [PATCH 22/73] feat:query taskResult --- go.mod | 2 +- sdks/pcmscheduler/models.go | 29 ++++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index fc87255..33c40ed 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/credentials v1.17.62 + github.com/google/go-querystring v1.1.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.15 @@ -33,7 +34,6 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-querystring v1.1.0 github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index bea1599..b898051 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -227,6 +227,12 @@ type ResourceRange struct { Ids []string `json:"ids"` } +type QueryTaskResultReq struct { + TaskID string `json:"taskID" binding:"required"` + UserID cdssdk.UserID `json:"userID" binding:"required"` + PackageID cdssdk.PackageID `json:"packageID" binding:"required"` +} + type PCMJobData struct { ID string `json:"id"` CreatedAt time.Time `json:"created_at"` @@ -242,9 +248,26 @@ type JobInfo struct { } type ResultFile struct { - Path string `json:"path"` - UpdateTime string `json:"updateTime"` - Size int `json:"size"` + CommonPrefixes []interface{} `json:"commonPrefixes"` + Objects []Object `json:"objects"` +} + +// Object 表示对象结构体 +type Object struct { + ObjectID int `json:"objectID"` + PackageID int `json:"packageID"` + Path string `json:"path"` + Size string `json:"size"` + FileHash string `json:"fileHash"` + Redundancy Redundancy `json:"redundancy"` + CreateTime time.Time `json:"createTime"` + UpdateTime time.Time `json:"updateTime"` +} + +// Redundancy 表示冗余信息结构体 +type Redundancy struct { + Type string `json:"type"` + RepCount int `json:"repCount"` } type DataDistributeData struct { From 54238a337c651177092bb133635fffe2f7689691 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 24 Mar 2025 14:52:52 +0800 Subject: [PATCH 23/73] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E8=B5=84=E6=BA=90URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 0d9ea55..e6b8b41 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -20,7 +20,7 @@ type GetClusterInfoReq struct { //} func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/queryResources") + targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") if err != nil { return nil, err } From 2ce33f2288149d0a4b21aa85f6b3cc9016ca109d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 27 Mar 2025 10:31:35 +0800 Subject: [PATCH 24/73] =?UTF-8?q?=E4=BC=98=E5=8C=96hpc=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index f860bc1..df7f714 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -36,6 +36,7 @@ type CreateJobResp struct { type HPCJobInfo struct { JobDir string `json:"jobDir"` JobID string `json:"jobId"` + TaskID string `json:"taskId"` } func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { From b936b4b0f1d4fe306ceaf8a6b20bbce475a66892 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 27 Mar 2025 15:08:25 +0800 Subject: [PATCH 25/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E8=AF=A6=E6=83=85=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 20 +++--- sdks/pcmscheduler/jobset.go | 6 +- sdks/pcmscheduler/models.go | 124 ++++++++---------------------------- 3 files changed, 38 insertions(+), 112 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index df7f714..2bc4c23 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -9,7 +9,7 @@ import ( "strings" ) -type CreateJobReq struct { +type CreateHPCJobReq struct { Name string `json:"name"` ClusterID schsdk.ClusterID `json:"clusterId"` Backend string `json:"backend"` @@ -19,14 +19,14 @@ type CreateJobReq struct { Parameters map[string]string `json:"parameters"` } -type HPCParameter struct { - JobName string `json:"jobName"` - Partition string `json:"partition"` - Ntasks string `json:"ntasks"` - Nodes string `json:"nodes"` - BamFile string `json:"bamFile"` - InputFile string `json:"inputFile"` -} +//type HPCParameter struct { +// JobName string `json:"jobName"` +// Partition string `json:"partition"` +// Ntasks string `json:"ntasks"` +// Nodes string `json:"nodes"` +// BamFile string `json:"bamFile"` +// InputFile string `json:"inputFile"` +//} type CreateJobResp struct { Backend string `json:"backend"` @@ -39,7 +39,7 @@ type HPCJobInfo struct { TaskID string `json:"taskId"` } -func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { +func (c *Client) CreateJob(req CreateHPCJobReq, token string) (*CreateJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "/hpc/commitHpcTask") if err != nil { return nil, err diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index e6b8b41..87a6cfe 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -61,7 +61,7 @@ type CreateInferenceJobResp struct { TaskId string `json:"taskId"` } -type CreateJobReq struct { +type CreatePCMJobReq struct { Name string `json:"name"` Description string `json:"description"` JobResources schsdk.JobResources `json:"jobResources"` @@ -125,7 +125,7 @@ type ScheduleData struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } -func (c *Client) CreateInferenceJob(req CreateJobReq, token string) (*CreateInferenceJobResp, error) { +func (c *Client) CreateInferenceJob(req CreatePCMJobReq, token string) (*CreateInferenceJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") if err != nil { return nil, err @@ -159,7 +159,7 @@ func (c *Client) CreateInferenceJob(req CreateJobReq, token string) (*CreateInfe } -func (c *Client) CreateJob(req CreateJobReq, token string) (*CreateJobResp, error) { +func (c *Client) CreateJob(req CreatePCMJobReq, token string) (*CreateJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "schedule/createTask") if err != nil { return nil, err diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index b6895cd..c7b22bd 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -2,6 +2,7 @@ package sch import ( "gitlink.org.cn/cloudream/common/pkgs/types" + "gitlink.org.cn/cloudream/common/sdks/hpc" schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/serder" @@ -239,122 +240,47 @@ type ResourceRange struct { Ids []string `json:"ids"` } -type QueryTaskResultReq struct { - TaskID string `json:"taskID" binding:"required"` - UserID cdssdk.UserID `json:"userID" binding:"required"` - PackageID cdssdk.PackageID `json:"packageID" binding:"required"` -} - -type PCMJobData struct { - ID string `json:"id"` - CreatedAt time.Time `json:"created_at"` - Param string `json:"param"` -} - type JobInfo struct { - Name string `json:"name"` - Description string `json:"description,optional"` - JobResources JobResources `json:"jobResources"` - DataDistributeData DataDistributeData `json:"dataDistributes"` - ResultFile ResultFile `json:"resultFile"` + TaskID string `json:"taskID"` + JobSubmitInfo JobSubmitInfo `json:"jobSubmitInfo"` + ResultFiles []ResultFile `json:"resultFiles"` } type ResultFile struct { - CommonPrefixes []interface{} `json:"commonPrefixes"` - Objects []Object `json:"objects"` -} - -// Object 表示对象结构体 -type Object struct { - ObjectID int `json:"objectID"` - PackageID int `json:"packageID"` - Path string `json:"path"` - Size string `json:"size"` - FileHash string `json:"fileHash"` - Redundancy Redundancy `json:"redundancy"` - CreateTime time.Time `json:"createTime"` - UpdateTime time.Time `json:"updateTime"` -} - -// Redundancy 表示冗余信息结构体 -type Redundancy struct { - Type string `json:"type"` - RepCount int `json:"repCount"` -} - -type DataDistributeData struct { - Dataset []DataItem `json:"dataset"` - Code []DataItem `json:"code"` - Image []DataItem `json:"image"` - Model []DataItem `json:"model"` -} - -// DataItem 定义 dataDistributes 中的数据项结构体 -type DataItem struct { - DataName string `json:"dataName"` - PackageID int `json:"packageID"` - PackageName string `json:"packageName"` - ResourceFile string `json:"resourceFile"` - Clusters []DataDetail `json:"clusters"` + ClusterID schsdk.ClusterID `json:"clusterID"` + Objects []cdssdk.Object `json:"objects"` } -type JobResources struct { - ScheduleStrategy string `json:"scheduleStrategy"` - Clusters []*JobClusterInfo `json:"clusters"` +type JobSubmitInfo interface { + Noop() } -type JobClusterInfo struct { - ClusterID string `json:"clusterID"` - Resources []map[string]interface{} `json:"resources,optional"` - Runtime JobRuntimeInfo `json:"runtime,optional"` +type JobSubmitInfoBase struct { } -type JobRuntimeInfo struct { - Command string `json:"command,optional"` - Envs map[string]interface{} `json:"envs,optional"` - Params map[string]interface{} `json:"params,optional"` -} +var JobSubmitInfoTypeUnion = types.NewTypeUnion[JobSubmitInfo]( + (*PCMJobSubmitInfo)(nil), + (*HPCJobSubmitInfo)(nil), +) -// Engine 定义 engine 结构体 -type Engine struct { - EngineName string `json:"EngineName"` - EngineVersion string `json:"EngineVersion"` - ImageUrl string `json:"ImageUrl"` - InstallSysPackages bool `json:"InstallSysPackages"` -} +var _ = serder.UseTypeUnionInternallyTagged(&JobSubmitInfoTypeUnion, "type") -// bindingCluster 定义根结构体 -type BindingParam struct { - Name string `json:"name"` - Description string `json:"description"` - Engine Engine `json:"engine"` - CodeDir string `json:"codeDir"` - BootFile string `json:"bootFile"` - Branch string `json:"branch"` - Platform string `json:"platform"` - UserID int `json:"userID"` - PackageId int `json:"packageId"` - BucketID int `json:"bucketID"` -} +func (d *JobSubmitInfoBase) Noop() {} -// bindingCluster 定义根结构体 -type BindingCluster struct { - BindingId int `json:"binding_id"` - ClusterId string `json:"cluster_id"` - Status string `json:"status"` - Param string `json:"param"` +type PCMJobSubmitInfo struct { + serder.Metadata `union:"pcm"` + JobSubmitInfoBase + Type string `json:"type"` + Info CreatePCMJobReq `json:"info"` } -/*type TaskResult struct { - Name string `json:"name"` - Description string `json:"description"` - DataDistribute DataDistribute `json:"dataDistributes"` +type HPCJobSubmitInfo struct { + serder.Metadata `union:"hpc"` + JobSubmitInfoBase + Type string `json:"type"` + Info hpc.CreateHPCJobReq `json:"info"` } -type ClusterResources struct { - ClusterId string `json:"clusterId"` -}*/ - type Range struct { Min float64 `json:"min"` Max float64 `json:"max"` From 3077125ab45dcd5717b2edbf679adc203243fd87 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 2 Apr 2025 10:27:00 +0800 Subject: [PATCH 26/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=92=8C=E5=A4=8D=E5=88=B6=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index a20cf30..991a3f6 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -125,6 +125,7 @@ type Package struct { Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` + BindingInfo sch.DataBinding `gorm:"column:binding_info" json:"bindingInfo"` } type PackageDAO struct { @@ -139,6 +140,7 @@ type PackageDAO struct { UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` UploadPriority string `gorm:"column:upload_priority" json:"uploadPriority"` + Param string `gorm:"column:param" json:"param"` } type PackageCloneDAO struct { From e55bb85d3cfbd73a26c7a896f8dc10915bcc2759 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 2 Apr 2025 16:03:41 +0800 Subject: [PATCH 27/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=8A=E9=93=BE?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/blockchain/blockchain.go | 5 ----- sdks/hpc/job.go | 1 + sdks/pcmscheduler/models.go | 2 +- sdks/scheduler/models.go | 1 + 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go index a7cb793..8d67862 100644 --- a/sdks/blockchain/blockchain.go +++ b/sdks/blockchain/blockchain.go @@ -22,11 +22,6 @@ func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { return err } - //token, err := c.getToken() - //if err != nil { - // return err - //} - header := make(map[string]string) header["Content-Type"] = http2.ContentTypeJSON header["Authorization"] = token diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index 2bc4c23..c102fc6 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -11,6 +11,7 @@ import ( type CreateHPCJobReq struct { Name string `json:"name"` + Description string `json:"description"` ClusterID schsdk.ClusterID `json:"clusterId"` Backend string `json:"backend"` App string `json:"app"` diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index c7b22bd..4ba183c 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -442,7 +442,7 @@ type DatasetBinding struct { Category string `json:"category"` PackageID cdssdk.PackageID `json:"packageID"` RepositoryName string `json:"repositoryName"` - ConsumptionPoints float64 `json:"points"` + ConsumptionPoints int64 `json:"points"` } type ModelBinding struct { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index e676645..a2190dd 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -166,6 +166,7 @@ type HPCJobInfo struct { JobInfoBase Type string `json:"type"` Name string `json:"name"` + Description string `json:"description"` ClusterID ClusterID `json:"clusterID"` Backend string `json:"backend"` App string `json:"app"` From fd75ed60427ae44d597fcf289507c294aa391488 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 7 Apr 2025 11:26:27 +0800 Subject: [PATCH 28/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=B0=83=E5=BA=A6?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index a2190dd..57039b1 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -181,6 +181,12 @@ type HPCParameter struct { Ntasks string `json:"ntasks"` Nodes string `json:"nodes"` BamFile string `json:"bamFile"` + HashType string `json:"hashType"` + AttackMode string `json:"attackMode"` + HashInput string `json:"hashInput"` + Mask string `json:"mask"` + Dictionary string `json:"dictionary"` + Dictionary2 string `json:"dictionary2"` HPCBindingFiles []HPCBindingFile `json:"hpcBindingFiles"` } From 62607ad384eac9a2eff942a8009e05c1f8eac45d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 7 Apr 2025 17:11:03 +0800 Subject: [PATCH 29/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=AE=9A=E7=89=88?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 4ba183c..f7062b2 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -72,6 +72,8 @@ const ( ID = "id" Startup = "startup" + + Schedule = "schedule" ) type TaskID int64 @@ -437,6 +439,7 @@ type DatasetBinding struct { DataBindingBase Type string `json:"type"` Name string `json:"name"` + OperateType string `json:"operateType"` ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` Description string `json:"description"` Category string `json:"category"` @@ -450,6 +453,7 @@ type ModelBinding struct { DataBindingBase Type string `json:"type"` Name string `json:"name"` + OperateType string `json:"operateType"` ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` Description string `json:"description"` Category string `json:"category"` @@ -465,6 +469,7 @@ type CodeBinding struct { DataBindingBase Type string `json:"type"` Name string `json:"name"` + OperateType string `json:"operateType"` ClusterID schsdk.ClusterID `json:"clusterID"` Description string `json:"description"` ImageID schsdk.ImageID `json:"imageID"` @@ -491,12 +496,13 @@ type CodeBinding struct { type ImageBinding struct { serder.Metadata `union:"image"` DataBindingBase - Type string `json:"type"` - ID int64 `json:"id"` - Name string `json:"name"` - IDType string `json:"idType"` - ImageID string `json:"imageID"` - ClusterID schsdk.ClusterID `json:"clusterID"` + Type string `json:"type"` + ID int64 `json:"id"` + OperateType string `json:"operateType"` + Name string `json:"name"` + IDType string `json:"idType"` + ImageID string `json:"imageID"` + ClusterID schsdk.ClusterID `json:"clusterID"` } type Image struct { From 9f4cfeed16a406677d91451b72de27cfbc456c8c Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 10 Apr 2025 11:04:59 +0800 Subject: [PATCH 30/73] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=AD=9B=E9=80=89=E6=9D=A1=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index f7062b2..dd717e2 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -415,6 +415,7 @@ type QueryData struct { CurrentPage int `json:"currentPage" binding:"required"` PageSize int `json:"pageSize" binding:"required"` OrderBy string `json:"orderBy" binding:"required"` + PackageName string `json:"packageName"` } type DataBinding interface { From 2f11a0a6202f66285b5f00c7bb9760c9b6575486 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 10 Apr 2025 16:15:40 +0800 Subject: [PATCH 31/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E7=9B=B4=E6=8E=A5?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E6=A8=A1=E5=9E=8B=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 3 +++ sdks/uploader/models.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index dd717e2..ff19a48 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -29,6 +29,9 @@ const ( MODEL = "model" RESULT = "result" + PackageTypeNormal = "normal" + PackageTypeNull = "null" + OrderByName = "name" OrderBySize = "size" OrderByTime = "time" diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 991a3f6..d152928 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -126,6 +126,7 @@ type Package struct { //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` BindingInfo sch.DataBinding `gorm:"column:binding_info" json:"bindingInfo"` + PackageType string `gorm:"column:package_type" json:"packageType"` } type PackageDAO struct { @@ -141,6 +142,7 @@ type PackageDAO struct { Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` UploadPriority string `gorm:"column:upload_priority" json:"uploadPriority"` Param string `gorm:"column:param" json:"param"` + PackageType string `gorm:"column:package_type" json:"packageType"` } type PackageCloneDAO struct { From 7cb79231aec30d4d758db0c0a4f9ef6807796f08 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 11 Apr 2025 10:10:25 +0800 Subject: [PATCH 32/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=88=9B=E5=BB=BApacka?= =?UTF-8?q?ge=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 13 +++++++------ sdks/scheduler/models.go | 17 +++++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index c102fc6..5fc8d08 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -10,12 +10,13 @@ import ( ) type CreateHPCJobReq struct { - Name string `json:"name"` - Description string `json:"description"` - ClusterID schsdk.ClusterID `json:"clusterId"` - Backend string `json:"backend"` - App string `json:"app"` - OperateType string `json:"operateType"` + Name string `json:"name"` + Description string `json:"description"` + ClusterID schsdk.ClusterID `json:"clusterId"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + ScriptContent string `json:"scriptContent"` //Parameters HPCParameter `json:"parameters"` Parameters map[string]string `json:"parameters"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 57039b1..136ded1 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -164,14 +164,15 @@ type ModelBinding struct { type HPCJobInfo struct { serder.Metadata `union:"HPC"` JobInfoBase - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - ClusterID ClusterID `json:"clusterID"` - Backend string `json:"backend"` - App string `json:"app"` - OperateType string `json:"operateType"` - Parameters HPCParameter `json:"parameters"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + ClusterID ClusterID `json:"clusterID"` + Backend string `json:"backend"` + App string `json:"app"` + OperateType string `json:"operateType"` + ScriptContent string `json:"scriptContent"` + Parameters HPCParameter `json:"parameters"` } type HPCParameter struct { From 101de1ad57cf6ac97baf8345d8204e6e22d7b30b Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 11 Apr 2025 15:00:53 +0800 Subject: [PATCH 33/73] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E7=BB=91=E5=AE=9A?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=A2=9E=E5=8A=A0=E7=AD=9B=E9=80=89=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index ff19a48..e5d04ba 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -69,7 +69,7 @@ const ( PlatformPCM = "PCM" PlatformPCMInference = "PCM_Inference" - PlatformHPC = "HPC" + PlatformHPC = "HPCSlurm" URL = "url" ID = "id" From 3323f54d0f2a04472dea59eff0e07cc843ec6071 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 15 Apr 2025 14:31:00 +0800 Subject: [PATCH 34/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E6=8F=90=E4=BA=A4=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/jobflow.go | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 sdks/scheduler/jobflow.go diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go new file mode 100644 index 0000000..7fa61b5 --- /dev/null +++ b/sdks/scheduler/jobflow.go @@ -0,0 +1,66 @@ +package schsdk + +import "time" + +type FlowData struct { + Nodes []Node `json:"nodes"` + Edges []Edge `json:"edges"` +} + +type Node struct { + ID string `json:"id"` + Type string `json:"type"` + X int `json:"x"` + Y int `json:"y"` + Properties JobInfo `json:"properties"` + Text *Text `json:"text,omitempty"` // 有些节点没有 text 字段 +} + +type Edge struct { + ID string `json:"id"` + Type string `json:"type"` + Properties interface{} `json:"properties"` // 为空对象 {} + SourceNodeID string `json:"sourceNodeId"` + TargetNodeID string `json:"targetNodeId"` + SourceAnchorID string `json:"sourceAnchorId"` + TargetAnchorID string `json:"targetAnchorId"` + StartPoint Point `json:"startPoint"` + EndPoint Point `json:"endPoint"` + PointsList []Point `json:"pointsList"` + Text *Text `json:"text,omitempty"` // 有些 edge 有文字标签 +} + +type Point struct { + X int `json:"x"` + Y int `json:"y"` +} + +type Text struct { + X int `json:"x"` + Y int `json:"y"` + Value string `json:"value"` +} + +type JobFlowDAO struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` + UserID int64 `gorm:"column:user_id;not null" json:"UserID"` + Name string `gorm:"column:name;size:255;not null" json:"Name"` + Description string `gorm:"column:description;size:255" json:"Description"` + Content string `gorm:"column:content;type:text" json:"Content"` + Status string `gorm:"column:status;type:enum('pending','running','fail','success');default:'pending'" json:"Status"` + JobType string `gorm:"column:job_type;size:255" json:"JobType"` + UpdatedAt time.Time `gorm:"column:update_at;autoUpdateTime" json:"UpdatedAt"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"CreatedAt"` +} + +type JobFlow struct { + ID int64 `json:"ID"` + UserID int64 `json:"UserID"` + Name string `json:"Name"` + Description string `json:"Description"` + Content FlowData `json:"Content"` + Status string `json:"Status"` + JobType string `json:"JobType"` + UpdatedAt time.Time `json:"UpdatedAt"` + CreatedAt time.Time `json:"CreatedAt"` +} From f8a5e7c84c272b80b70deeea1e2f5b1881b0cda4 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 16 Apr 2025 11:16:51 +0800 Subject: [PATCH 35/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E7=AE=A1=E7=90=86=E7=9B=B8=E5=85=B3=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 1 + sdks/scheduler/jobflow.go | 41 ++++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index e5d04ba..9cdab8e 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -56,6 +56,7 @@ const ( FailedStatus = "failed" SuccessStatus = "success" UploadingStatus = "uploading" + RunningStatus = "running" Query = "query" Delete = "delete" diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index 7fa61b5..d6cf5fb 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -1,6 +1,9 @@ package schsdk -import "time" +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) type FlowData struct { Nodes []Node `json:"nodes"` @@ -42,25 +45,25 @@ type Text struct { } type JobFlowDAO struct { - ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"ID"` - UserID int64 `gorm:"column:user_id;not null" json:"UserID"` - Name string `gorm:"column:name;size:255;not null" json:"Name"` - Description string `gorm:"column:description;size:255" json:"Description"` - Content string `gorm:"column:content;type:text" json:"Content"` - Status string `gorm:"column:status;type:enum('pending','running','fail','success');default:'pending'" json:"Status"` - JobType string `gorm:"column:job_type;size:255" json:"JobType"` - UpdatedAt time.Time `gorm:"column:update_at;autoUpdateTime" json:"UpdatedAt"` - CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"CreatedAt"` + ID int64 `gorm:"column:id;primaryKey;autoIncrement" json:"id"` + UserID cdssdk.UserID `gorm:"column:user_id;not null" json:"userID"` + Name string `gorm:"column:name;size:255;not null" json:"name"` + Description string `gorm:"column:description;size:255" json:"description"` + Content string `gorm:"column:content;type:text" json:"content"` + Status string `gorm:"column:status;type:enum('pending','running','failed','success');default:'pending'" json:"status"` + JobType string `gorm:"column:job_type;size:255" json:"jobType"` + UpdatedAt time.Time `gorm:"column:update_at;autoUpdateTime" json:"updatedAt"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` } type JobFlow struct { - ID int64 `json:"ID"` - UserID int64 `json:"UserID"` - Name string `json:"Name"` - Description string `json:"Description"` - Content FlowData `json:"Content"` - Status string `json:"Status"` - JobType string `json:"JobType"` - UpdatedAt time.Time `json:"UpdatedAt"` - CreatedAt time.Time `json:"CreatedAt"` + ID int64 `json:"id"` + UserID cdssdk.UserID `json:"userID"` + Name string `json:"name"` + Description string `json:"description"` + Content FlowData `json:"content"` + Status string `json:"status"` + JobType string `json:"jobType"` + UpdatedAt time.Time `json:"updatedAt"` + CreatedAt time.Time `json:"createdAt"` } From d8d739b1be137915d878ee69b66c7fd60a0ec97d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 17 Apr 2025 09:56:03 +0800 Subject: [PATCH 36/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=B6=85=E7=AE=97?= =?UTF-8?q?=E6=8F=90=E4=BA=A4=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 136ded1..1c00a7f 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -192,8 +192,9 @@ type HPCParameter struct { } type HPCBindingFile struct { - ParamName string `json:"paramName"` - BindingID int64 `json:"bindingID"` + ParamName string `json:"paramName"` + ObjectID cdssdk.ObjectID `json:"objectID"` + PackageID cdssdk.PackageID `json:"packageID"` } type JobResources struct { From 63b6ead6667fd95c03c7c5d7af9113bf0fa17e53 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 17 Apr 2025 15:51:09 +0800 Subject: [PATCH 37/73] =?UTF-8?q?=E8=B0=83=E6=95=B4HPC=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/blockchain/blockchain.go | 43 ++++++++++++++++++++++++++++++++++- sdks/scheduler/models.go | 6 ++--- utils/http2/http.go | 21 +++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go index 8d67862..5817b54 100644 --- a/sdks/blockchain/blockchain.go +++ b/sdks/blockchain/blockchain.go @@ -14,6 +14,7 @@ type InvokeReq struct { MemberName string `json:"memberName"` Type string `json:"type"` Args []string `json:"args"` + Amount int64 `json:"amount"` } func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { @@ -25,7 +26,6 @@ func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { header := make(map[string]string) header["Content-Type"] = http2.ContentTypeJSON header["Authorization"] = token - println("token: " + token) resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, @@ -50,6 +50,47 @@ func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { return nil } +type UnPledgePointsReq struct { + PrivateKey string `form:"privateKey"` + BusinessCode string `form:"businessCode"` +} + +func (c *Client) UnPledgePoints(req UnPledgePointsReq, token string) error { + targetUrl, err := url.JoinPath(c.baseURL, "/jcc-bcos/points/unPledgePoints") + if err != nil { + return err + } + + header := make(map[string]string) + header["Content-Type"] = http2.ContentTypeJSON + header["Authorization"] = token + + resp, err := http2.PutJSON(targetUrl, http2.RequestParam{ + Query: req, + Header: header, + }) + if err != nil { + println(err) + return err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp response[string] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == 200 { + return nil + } + + return codeResp.ToError() + } + + return nil +} + type TokenReq struct { Username string `json:"username"` Password string `json:"password"` diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 1c00a7f..cfdd689 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -192,9 +192,9 @@ type HPCParameter struct { } type HPCBindingFile struct { - ParamName string `json:"paramName"` - ObjectID cdssdk.ObjectID `json:"objectID"` - PackageID cdssdk.PackageID `json:"packageID"` + ParamName string `json:"paramName"` + ObjectID cdssdk.ObjectID `json:"objectID"` + //PackageID cdssdk.PackageID `json:"packageID"` } type JobResources struct { diff --git a/utils/http2/http.go b/utils/http2/http.go index ab67e21..f57b06f 100644 --- a/utils/http2/http.go +++ b/utils/http2/http.go @@ -119,6 +119,27 @@ func PostJSON(url string, param RequestParam) (*http.Response, error) { return defaultClient.Do(req) } +func PutJSON(url string, param RequestParam) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPut, url, nil) + if err != nil { + return nil, err + } + + if err = prepareQuery(req, param.Query); err != nil { + return nil, err + } + + if err = prepareHeader(req, param.Header); err != nil { + return nil, err + } + + if err = prepareJSONBody(req, param.Body); err != nil { + return nil, err + } + + return defaultClient.Do(req) +} + func PostForm(url string, param RequestParam) (*http.Response, error) { req, err := http.NewRequest(http.MethodPost, url, nil) if err != nil { From b8efb988a75e3fc00b4fc49d18f4573473dba120 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 17 Apr 2025 16:12:55 +0800 Subject: [PATCH 38/73] =?UTF-8?q?=E8=B0=83=E6=95=B4HPC=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=8F=82=E6=95=B02?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index cfdd689..e4e3f38 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -194,7 +194,7 @@ type HPCParameter struct { type HPCBindingFile struct { ParamName string `json:"paramName"` ObjectID cdssdk.ObjectID `json:"objectID"` - //PackageID cdssdk.PackageID `json:"packageID"` + Path string `json:"path"` } type JobResources struct { From 5685f35153601cc2c126f0806624bed4f41577b4 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 18 Apr 2025 09:09:10 +0800 Subject: [PATCH 39/73] =?UTF-8?q?=E7=A7=AF=E5=88=86=E8=A7=A3=E9=99=A4?= =?UTF-8?q?=E8=B4=A8=E6=8A=BC=E4=B8=8A=E9=93=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/job.go | 1 + sdks/pcmscheduler/models.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/sdks/pcmscheduler/job.go b/sdks/pcmscheduler/job.go index fd1997b..3a5d291 100644 --- a/sdks/pcmscheduler/job.go +++ b/sdks/pcmscheduler/job.go @@ -13,6 +13,7 @@ type PCMJob struct { JobSetID schsdk.JobSetID `gorm:"column:jobset_id" json:"jobSetID"` LocalJobID string `gorm:"column:local_job_id" json:"localJobID"` Param string `gorm:"column:param" json:"param"` + Token string `gorm:"column:token" json:"token"` CreatedAt time.Time `gorm:"column:created_at" json:"createdAt"` } diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 9cdab8e..32fa4f2 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -78,6 +78,8 @@ const ( Startup = "startup" Schedule = "schedule" + + BlockChainJobCreatePrefix = "job_create_" ) type TaskID int64 From a7dc1ec0bc5f6bc203701396925c8e2b63c32cb3 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 18 Apr 2025 10:30:40 +0800 Subject: [PATCH 40/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index e4e3f38..9188981 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -192,9 +192,38 @@ type HPCParameter struct { } type HPCBindingFile struct { - ParamName string `json:"paramName"` - ObjectID cdssdk.ObjectID `json:"objectID"` - Path string `json:"path"` + ParamName string `json:"paramName"` + Resource HPCFile `json:"resource"` +} + +type HPCFile interface { + Noop() +} + +var HPCFileTypeUnion = types.NewTypeUnion[HPCFile]( + (*HPCObject)(nil), + (*HPCPath)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&HPCFileTypeUnion, "type") + +type HPCFileBase struct{} + +func (d *HPCFileBase) Noop() {} + +type HPCObject struct { + serder.Metadata `union:"object"` + HPCFileBase + Type string `json:"type"` + ObjectID cdssdk.ObjectID `json:"objectID"` +} + +type HPCPath struct { + serder.Metadata `union:"path"` + HPCFileBase + Type string `json:"type"` + PackageID cdssdk.PackageID `json:"packageID"` + Path string `json:"path"` } type JobResources struct { From 4b575059a0e3a1083d6cb593b5561348710006af Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 21 Apr 2025 10:32:15 +0800 Subject: [PATCH 41/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8A=E9=93=BE?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 10 ++++++++++ sdks/scheduler/models.go | 28 ++++++++++++---------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index 5fc8d08..f7f6710 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -5,6 +5,7 @@ import ( schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" + "io" "net/url" "strings" ) @@ -57,6 +58,15 @@ func (c *Client) CreateJob(req CreateHPCJobReq, token string) (*CreateJobResp, e return nil, err } + // 打印resp.Body内容 + body, err := io.ReadAll(resp.Body) + if err != nil { + println("Error reading response body:", err) + } + + // Print the response body as a string + println("Response Body:", string(body)) + contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { var codeResp respons2[CreateJobResp] diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9188981..c7fa3fc 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -51,7 +51,6 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string GetTargetLocalJobIDs() []string - RemoveTargetLocalJobID(targetID string) } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -70,8 +69,14 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` - TargetLocalJobIDs []string `json:"targetLocalJobIDs"` + LocalJobID string `json:"localJobID"` + //TargetLocalJobIDs []string `json:"targetLocalJobIDs"` + TargetJobInfos []TargetJobInfo `json:"targetJobInfos"` +} + +type TargetJobInfo struct { + TargetJobID string `json:"targetJobID"` + InputParams map[string]string `json:"inputParams"` } func (i *JobInfoBase) GetLocalJobID() string { @@ -79,20 +84,11 @@ func (i *JobInfoBase) GetLocalJobID() string { } func (i *JobInfoBase) GetTargetLocalJobIDs() []string { - return i.TargetLocalJobIDs -} - -func (i *JobInfoBase) GetTargetLocalJobIDs2() []string { - return i.TargetLocalJobIDs -} - -func (i *JobInfoBase) RemoveTargetLocalJobID(targetID string) { - // 从i.TargetLocalJobIDs中删除id - for j, id := range i.TargetLocalJobIDs { - if targetID == id { - i.TargetLocalJobIDs = append(i.TargetLocalJobIDs[:j], i.TargetLocalJobIDs[j+1:]...) - } + var IDs []string + for _, v := range i.TargetJobInfos { + IDs = append(IDs, v.TargetJobID) } + return IDs } type NormalJobInfo struct { From 672cdcf4609f079f9e6a3bbf5c77236dd5ce21e4 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 21 Apr 2025 10:54:15 +0800 Subject: [PATCH 42/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8A=E9=93=BE?= =?UTF-8?q?=E4=BB=A3=E7=A0=812?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index f7f6710..18338bc 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -5,7 +5,6 @@ import ( schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" - "io" "net/url" "strings" ) @@ -59,13 +58,12 @@ func (c *Client) CreateJob(req CreateHPCJobReq, token string) (*CreateJobResp, e } // 打印resp.Body内容 - body, err := io.ReadAll(resp.Body) - if err != nil { - println("Error reading response body:", err) - } - - // Print the response body as a string - println("Response Body:", string(body)) + //body, err := io.ReadAll(resp.Body) + //if err != nil { + // println("Error reading response body:", err) + //} + // + //println("Response Body:", string(body)) contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { From f96bed7ad7562684a2bc496978dbbbec14474a37 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 21 Apr 2025 16:32:38 +0800 Subject: [PATCH 43/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E5=80=BC=E4=BC=A0=E9=80=92=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index c7fa3fc..abe5eaf 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -51,6 +51,7 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string GetTargetLocalJobIDs() []string + GetTargetInputParams(targetID string) map[string]string } var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( @@ -71,7 +72,7 @@ var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { LocalJobID string `json:"localJobID"` //TargetLocalJobIDs []string `json:"targetLocalJobIDs"` - TargetJobInfos []TargetJobInfo `json:"targetJobInfos"` + TargetJob []TargetJobInfo `json:"targetJob"` } type TargetJobInfo struct { @@ -83,9 +84,18 @@ func (i *JobInfoBase) GetLocalJobID() string { return i.LocalJobID } +func (i *JobInfoBase) GetTargetInputParams(targetID string) map[string]string { + for _, v := range i.TargetJob { + if v.TargetJobID == targetID { + return v.InputParams + } + } + return nil +} + func (i *JobInfoBase) GetTargetLocalJobIDs() []string { var IDs []string - for _, v := range i.TargetJobInfos { + for _, v := range i.TargetJob { IDs = append(IDs, v.TargetJobID) } return IDs @@ -357,10 +367,10 @@ type DataPreprocessJobInfo struct { type DataReturnJobInfo struct { serder.Metadata `union:"DataReturn"` JobInfoBase - Type string `json:"type"` - BucketID cdssdk.BucketID `json:"bucketID"` - BindingType string `json:"bindingType"` - TargetLocalJobID string `json:"targetLocalJobID"` + Type string `json:"type"` + BucketID cdssdk.BucketID `json:"bucketID"` + //BindingType string `json:"bindingType"` + TargetLocalJobID string `json:"targetLocalJobID"` } // MultiInstanceJobInfo 多实例(推理任务) From 6d0275324b8611cec87400724d179a28bac4d8e3 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 22 Apr 2025 15:28:30 +0800 Subject: [PATCH 44/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E8=AF=A6=E6=83=85=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/uploader/models.go | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index d152928..35ebf1d 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -112,30 +112,27 @@ func (Cluster) TableName() string { } type Package struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - //JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 - BindingID DataID `gorm:"column:binding_id" json:"bindingID"` - CreateTime time.Time `gorm:"column:create_time" json:"createTime"` - Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` - UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` - Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` - //BlockChain []BlockChain `gorm:"foreignKey:package_id;references:package_id" json:"blockChains"` // 关联 BlockChain 数据 - UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` - BindingInfo sch.DataBinding `gorm:"column:binding_info" json:"bindingInfo"` - PackageType string `gorm:"column:package_type" json:"packageType"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` + BindingID DataID `gorm:"column:binding_id" json:"bindingID"` + CreateTime time.Time `gorm:"column:create_time" json:"createTime"` + Objects []cdssdk.Object `gorm:"column:objects" json:"objects"` + UploadedCluster []Cluster `gorm:"column:uploadedCluster" json:"uploadedCluster"` + Versions []PackageCloneDAO `gorm:"foreignKey:parent_package_id;references:package_id" json:"versions"` + UploadPriority sch.UploadPriority `gorm:"column:upload_priority" json:"uploadPriority"` + BindingInfo sch.DataBinding `gorm:"column:binding_info" json:"bindingInfo"` + PackageType string `gorm:"column:package_type" json:"packageType"` } type PackageDAO struct { - UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` - PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` - PackageName string `gorm:"column:package_name" json:"packageName"` - BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` - DataType string `gorm:"column:data_type" json:"dataType"` - //JsonData string `gorm:"column:json_data" json:"jsonData"` // JSON 数据字段 + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + PackageID cdssdk.PackageID `gorm:"column:package_id" json:"packageID"` + PackageName string `gorm:"column:package_name" json:"packageName"` + BucketID cdssdk.BucketID `gorm:"column:bucket_id" json:"bucketID"` + DataType string `gorm:"column:data_type" json:"dataType"` BindingID DataID `gorm:"column:binding_id" json:"bindingID"` CreateTime time.Time `gorm:"column:create_time" json:"createTime"` UploadedCluster []Cluster `gorm:"foreignKey:package_id;references:package_id" json:"clusters"` // 关联 Cluster 数据 From d612d333d249199b992cf97bd390e0199a1ccadd Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 23 Apr 2025 09:19:58 +0800 Subject: [PATCH 45/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E6=99=BA=E7=AE=97=E4=BB=BB=E5=8A=A1=E7=9A=84output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/job.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/hpc/job.go b/sdks/hpc/job.go index 18338bc..8c9acad 100644 --- a/sdks/hpc/job.go +++ b/sdks/hpc/job.go @@ -62,7 +62,6 @@ func (c *Client) CreateJob(req CreateHPCJobReq, token string) (*CreateJobResp, e //if err != nil { // println("Error reading response body:", err) //} - // //println("Response Body:", string(body)) contType := resp.Header.Get("Content-Type") From 44a51fb2d9adb826b500a0978cd07c6a167463eb Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 23 Apr 2025 10:23:18 +0800 Subject: [PATCH 46/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=99=BA=E7=AE=97?= =?UTF-8?q?=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 7 ++++--- sdks/scheduler/models.go | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 32fa4f2..133b50e 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -68,9 +68,10 @@ const ( PlatformOpenI = "OpenI" PlatformModelArts = "ModelArts" - PlatformPCM = "PCM" - PlatformPCMInference = "PCM_Inference" - PlatformHPC = "HPCSlurm" + PlatformAI = "AI" // 智算 + PlatformCloud = "CLOUD" // 云算 + PlatformCloudInference = "PCM_Inference" + PlatformHPC = "HPCSlurm" //超算 URL = "url" ID = "id" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index abe5eaf..cd182d7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -62,7 +62,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*UpdateMultiInstanceJobInfo)(nil), (*FinetuningJobInfo)(nil), (*DataPreprocessJobInfo)(nil), - (*PCMJobInfo)(nil), + (*AIJobInfo)(nil), (*HPCJobInfo)(nil), (*BindingJobInfo)(nil), (*PCMInferenceJobInfo)(nil), @@ -122,8 +122,8 @@ type PCMInferenceJobInfo struct { JobResources JobResources `json:"jobResources"` } -type PCMJobInfo struct { - serder.Metadata `union:"PCM"` +type AIJobInfo struct { + serder.Metadata `union:"AI"` JobInfoBase Type string `json:"type"` Name string `json:"name"` From d250d681c226bd709129d3d557e11afb8c1e60b0 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 23 Apr 2025 17:06:10 +0800 Subject: [PATCH 47/73] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81X,Y=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/jobflow.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index d6cf5fb..5219b80 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -13,8 +13,8 @@ type FlowData struct { type Node struct { ID string `json:"id"` Type string `json:"type"` - X int `json:"x"` - Y int `json:"y"` + X float64 `json:"x"` + Y float64 `json:"y"` Properties JobInfo `json:"properties"` Text *Text `json:"text,omitempty"` // 有些节点没有 text 字段 } @@ -34,14 +34,14 @@ type Edge struct { } type Point struct { - X int `json:"x"` - Y int `json:"y"` + X float64 `json:"x"` + Y float64 `json:"y"` } type Text struct { - X int `json:"x"` - Y int `json:"y"` - Value string `json:"value"` + X float64 `json:"x"` + Y float64 `json:"y"` + Value string `json:"value"` } type JobFlowDAO struct { From 2c1031f2a8065c7381b6c1bb35103ed74866e52d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 24 Apr 2025 16:47:17 +0800 Subject: [PATCH 48/73] =?UTF-8?q?=E8=A7=A3=E9=99=A4=E8=B4=A8=E6=8A=BC?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BB=93=E6=9D=9F=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/blockchain/blockchain.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdks/blockchain/blockchain.go b/sdks/blockchain/blockchain.go index 5817b54..c905db1 100644 --- a/sdks/blockchain/blockchain.go +++ b/sdks/blockchain/blockchain.go @@ -6,6 +6,7 @@ import ( "gitlink.org.cn/cloudream/common/utils/serder" "net/url" "strings" + "time" ) type InvokeReq struct { @@ -51,8 +52,9 @@ func (c *Client) BlockChainInvoke(req InvokeReq, token string) error { } type UnPledgePointsReq struct { - PrivateKey string `form:"privateKey"` - BusinessCode string `form:"businessCode"` + PrivateKey string `form:"privateKey"` + BusinessCode string `form:"businessCode"` + EndTime time.Time `form:"endTime"` } func (c *Client) UnPledgePoints(req UnPledgePointsReq, token string) error { From c4074cd83e4c4d6e6cc10a62c608f63012d92244 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Sun, 27 Apr 2025 09:47:43 +0800 Subject: [PATCH 49/73] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=9B=E5=BB=BApacka?= =?UTF-8?q?ge=E8=B0=83=E5=BA=A6=E8=B7=AF=E5=BE=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index cd182d7..fc187e7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -156,14 +156,12 @@ func (d *DataBindingBase) Noop() {} type ModelBinding struct { serder.Metadata `union:"model"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - //Description string `json:"description"` - Category string `json:"category"` - ModelType string `json:"modelType"` - Env string `json:"env"` - Version string `json:"version"` - //PackageID cdssdk.PackageID `json:"packageID"` + Type string `json:"type"` + Name string `json:"name"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` RepositoryName string `json:"repositoryName"` } From dd36c63595a7979b23be56b9061b65b7ba52443d Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Sun, 27 Apr 2025 15:06:36 +0800 Subject: [PATCH 50/73] =?UTF-8?q?=E7=BB=91=E5=AE=9A=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=AF=B7=E6=B1=82=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index fc187e7..d545b78 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -156,13 +156,15 @@ func (d *DataBindingBase) Noop() {} type ModelBinding struct { serder.Metadata `union:"model"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - Category string `json:"category"` - ModelType string `json:"modelType"` - Env string `json:"env"` - Version string `json:"version"` - RepositoryName string `json:"repositoryName"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + ClusterIDs []string `json:"clusterIDs"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + RepositoryName string `json:"repositoryName"` } type HPCJobInfo struct { From e50dc3a1796f2892a883fa993a213c1390dffe41 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 28 Apr 2025 16:48:57 +0800 Subject: [PATCH 51/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=99=BA=E7=AE=97?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=8F=90=E4=BA=A4=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index d545b78..36804c6 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -125,11 +125,23 @@ type PCMInferenceJobInfo struct { type AIJobInfo struct { serder.Metadata `union:"AI"` JobInfoBase - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - Files JobFilesInfo `json:"files"` - JobResources JobResources `json:"jobResources"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Files JobFilesInfo `json:"files"` + JobResources JobResources `json:"jobResources"` + ResourceChoice ResourceChoice `json:"resourceChoice"` +} + +type ResourceChoice struct { + Type string `json:"type"` + ResourceScopes []ResourceScope `json:"resourceScopes"` +} + +type ResourceScope struct { + Name string `json:"name"` + Min float64 `json:"min"` + Max float64 `json:"max"` } type BindingJobInfo struct { @@ -443,8 +455,9 @@ func (i *JobFileInfoBase) Noop() {} type BindingJobFileInfo struct { serder.Metadata `union:"Binding"` JobFileInfoBase - Type string `json:"type"` - BindingID int64 `json:"bindingID"` + Type string `json:"type"` + BindingID int64 `json:"bindingID"` + BindingName string `json:"bindingName"` } type PackageJobFileInfo struct { @@ -471,8 +484,9 @@ type DataReturnJobFileInfo struct { type ImageJobFileInfo struct { serder.Metadata `union:"Image"` JobFileInfoBase - Type string `json:"type"` - ImageID ImageID `json:"imageID"` + Type string `json:"type"` + ImageID ImageID `json:"imageID"` + ImageName string `json:"imageName"` } type JobRuntimeInfo struct { From 2edc2ae5e4d16809700522691f19c3ffe3d46113 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 30 Apr 2025 10:27:56 +0800 Subject: [PATCH 52/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E6=8F=90=E4=BA=A4=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 1 + sdks/scheduler/models.go | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 87a6cfe..1057c0d 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -115,6 +115,7 @@ type ModelDistribute struct { type CreateJobResp struct { TaskID TaskID `json:"taskID"` + TaskName string `json:"taskName"` ScheduleDatas []ScheduleData `json:"scheduleDatas"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 36804c6..f19f23a 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -455,8 +455,9 @@ func (i *JobFileInfoBase) Noop() {} type BindingJobFileInfo struct { serder.Metadata `union:"Binding"` JobFileInfoBase - Type string `json:"type"` - BindingID int64 `json:"bindingID"` + Type string `json:"type"` + BindingID int64 `json:"bindingID"` + // 用于参数回显 BindingName string `json:"bindingName"` } @@ -484,9 +485,10 @@ type DataReturnJobFileInfo struct { type ImageJobFileInfo struct { serder.Metadata `union:"Image"` JobFileInfoBase - Type string `json:"type"` - ImageID ImageID `json:"imageID"` - ImageName string `json:"imageName"` + Type string `json:"type"` + ImageID ImageID `json:"imageID"` + // 用于参数回显 + ImageName string `json:"imageName"` } type JobRuntimeInfo struct { From e137e20ff6418f3af95baa1aa9b4f71127f54eef Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 30 Apr 2025 15:52:21 +0800 Subject: [PATCH 53/73] =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E5=AE=8C?= =?UTF-8?q?=E5=96=84=E4=BE=9D=E8=B5=96=E5=85=B3=E7=B3=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index f19f23a..3f35cc3 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -51,6 +51,7 @@ type JobSetInfo struct { type JobInfo interface { GetLocalJobID() string GetTargetLocalJobIDs() []string + SetTargetLocalJob(info TargetJobInfo) GetTargetInputParams(targetID string) map[string]string } @@ -70,9 +71,8 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") type JobInfoBase struct { - LocalJobID string `json:"localJobID"` - //TargetLocalJobIDs []string `json:"targetLocalJobIDs"` - TargetJob []TargetJobInfo `json:"targetJob"` + LocalJobID string `json:"localJobID"` + TargetJob []TargetJobInfo `json:"targetJob"` } type TargetJobInfo struct { @@ -101,6 +101,16 @@ func (i *JobInfoBase) GetTargetLocalJobIDs() []string { return IDs } +func (i *JobInfoBase) SetTargetLocalJob(info TargetJobInfo) { + for _, target := range i.TargetJob { + // 已经存在，则不用再添加 + if target.TargetJobID == info.TargetJobID { + return + } + } + i.TargetJob = append(i.TargetJob, info) +} + type NormalJobInfo struct { serder.Metadata `union:"Normal"` JobInfoBase @@ -379,10 +389,9 @@ type DataPreprocessJobInfo struct { type DataReturnJobInfo struct { serder.Metadata `union:"DataReturn"` JobInfoBase - Type string `json:"type"` - BucketID cdssdk.BucketID `json:"bucketID"` - //BindingType string `json:"bindingType"` - TargetLocalJobID string `json:"targetLocalJobID"` + Type string `json:"type"` + BucketID cdssdk.BucketID `json:"bucketID"` + TargetLocalJobID string `json:"targetLocalJobID"` } // MultiInstanceJobInfo 多实例(推理任务) From af8678becfecb591f77870ccc9b383e98f254c20 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 30 Apr 2025 16:02:50 +0800 Subject: [PATCH 54/73] =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E5=AE=8C?= =?UTF-8?q?=E5=96=84=E4=BE=9D=E8=B5=96=E5=85=B3=E7=B3=BB2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 133b50e..2221f64 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -81,6 +81,9 @@ const ( Schedule = "schedule" BlockChainJobCreatePrefix = "job_create_" + + StartNode = "StartTask" + EndNode = "EndTask" ) type TaskID int64 From 3b89aef0a85d75c22525e131c2c8cb7fc6e5dbac Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 6 May 2025 09:09:49 +0800 Subject: [PATCH 55/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E7=BB=91=E5=AE=9A=E6=A8=A1=E5=9E=8B=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 6 +++--- sdks/pcmscheduler/models.go | 8 +++++--- sdks/scheduler/models.go | 18 +++++++++--------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 1057c0d..fa09286 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -61,7 +61,7 @@ type CreateInferenceJobResp struct { TaskId string `json:"taskId"` } -type CreatePCMJobReq struct { +type CreateAIJobReq struct { Name string `json:"name"` Description string `json:"description"` JobResources schsdk.JobResources `json:"jobResources"` @@ -126,7 +126,7 @@ type ScheduleData struct { ClusterIDs []schsdk.ClusterID `json:"clusterIDs"` } -func (c *Client) CreateInferenceJob(req CreatePCMJobReq, token string) (*CreateInferenceJobResp, error) { +func (c *Client) CreateInferenceJob(req CreateAIJobReq, token string) (*CreateInferenceJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") if err != nil { return nil, err @@ -160,7 +160,7 @@ func (c *Client) CreateInferenceJob(req CreatePCMJobReq, token string) (*CreateI } -func (c *Client) CreateJob(req CreatePCMJobReq, token string) (*CreateJobResp, error) { +func (c *Client) CreateJob(req CreateAIJobReq, token string) (*CreateJobResp, error) { targetUrl, err := url.JoinPath(c.baseURL, "schedule/createTask") if err != nil { return nil, err diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 2221f64..e18f0b6 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -282,8 +282,8 @@ func (d *JobSubmitInfoBase) Noop() {} type PCMJobSubmitInfo struct { serder.Metadata `union:"pcm"` JobSubmitInfoBase - Type string `json:"type"` - Info CreatePCMJobReq `json:"info"` + Type string `json:"type"` + Info CreateAIJobReq `json:"info"` } type HPCJobSubmitInfo struct { @@ -441,7 +441,9 @@ var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") -type DataBindingBase struct{} +type DataBindingBase struct { + RootPath string `json:"rootPath"` +} func (d *DataBindingBase) Noop() {} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 3f35cc3..8ab173b 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -178,15 +178,15 @@ func (d *DataBindingBase) Noop() {} type ModelBinding struct { serder.Metadata `union:"model"` DataBindingBase - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - ClusterIDs []string `json:"clusterIDs"` - Category string `json:"category"` - ModelType string `json:"modelType"` - Env string `json:"env"` - Version string `json:"version"` - RepositoryName string `json:"repositoryName"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + ClusterIDs []ClusterID `json:"clusterIDs"` + Category string `json:"category"` + ModelType string `json:"modelType"` + Env string `json:"env"` + Version string `json:"version"` + RepositoryName string `json:"repositoryName"` } type HPCJobInfo struct { From d7b1738ee986f6b829dece3db5da0439a58f76a3 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 6 May 2025 16:37:27 +0800 Subject: [PATCH 56/73] =?UTF-8?q?=E5=9B=9E=E6=BA=90=E5=90=8E=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E4=B8=BA=E6=A8=A1=E5=9E=8B=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index e18f0b6..cd0b189 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -31,6 +31,7 @@ const ( PackageTypeNormal = "normal" PackageTypeNull = "null" + Null = "null" OrderByName = "name" OrderBySize = "size" From c8e1fcab4558c49af6ddb3b1154511f30934daf1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 7 May 2025 10:49:07 +0800 Subject: [PATCH 57/73] =?UTF-8?q?=E8=A7=84=E8=8C=83mysql=E8=A1=A8=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/hpc/models.go | 4 ++-- sdks/pcmscheduler/models.go | 4 ++-- sdks/uploader/models.go | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sdks/hpc/models.go b/sdks/hpc/models.go index 6511bb9..cee9586 100644 --- a/sdks/hpc/models.go +++ b/sdks/hpc/models.go @@ -466,7 +466,7 @@ type ClusterImage struct { } func (ClusterImage) TableName() string { - return "clusterImage" + return "cluster_image" } type ClusterImageCard struct { @@ -475,7 +475,7 @@ type ClusterImageCard struct { } func (ClusterImageCard) TableName() string { - return "clusterImageCard" + return "cluster_image_card" } type QueryBindingFilters struct { diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index cd0b189..e2c6bc4 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -536,7 +536,7 @@ type ClusterImage struct { } func (ClusterImage) TableName() string { - return "clusterImage" + return "cluster_image" } type ClusterImageCard struct { @@ -545,7 +545,7 @@ type ClusterImageCard struct { } func (ClusterImageCard) TableName() string { - return "clusterImageCard" + return "cluster_image_card" } type QueryBindingFilters struct { diff --git a/sdks/uploader/models.go b/sdks/uploader/models.go index 35ebf1d..9ad3f8d 100644 --- a/sdks/uploader/models.go +++ b/sdks/uploader/models.go @@ -21,7 +21,7 @@ type BlockChain struct { } func (BlockChain) TableName() string { - return "BlockChain" // 确保和数据库中的表名一致 + return "block_chain" // 确保和数据库中的表名一致 } type Binding struct { @@ -73,7 +73,7 @@ type BindingDetail struct { } func (Binding) TableName() string { - return "BindingData" // 确保和数据库中的表名一致 + return "bindings" // 确保和数据库中的表名一致 } type BindingCluster struct { @@ -85,7 +85,7 @@ type BindingCluster struct { } func (BindingCluster) TableName() string { - return "bindingCluster" // 确保和数据库中的表名一致 + return "binding_cluster" // 确保和数据库中的表名一致 } type Folder struct { @@ -108,7 +108,7 @@ type Cluster struct { } func (Cluster) TableName() string { - return "uploadedCluster" // 确保和数据库中的表名一致 + return "uploaded_cluster" // 确保和数据库中的表名一致 } type Package struct { @@ -157,7 +157,7 @@ type PackageCloneDAO struct { } func (PackageCloneDAO) TableName() string { - return "packageClone" // 确保和数据库中的表名一致 + return "package_clone" // 确保和数据库中的表名一致 } type PackageCloneParam struct { @@ -195,7 +195,7 @@ type ClusterMapping struct { } func (ClusterMapping) TableName() string { - return "ClusterMapping" + return "cluster_mapping" } type ScheduleTarget interface { From a059fc72947cbf986e8e3647eaa5b8ed53e8a1d1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 7 May 2025 17:40:23 +0800 Subject: [PATCH 58/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E7=8A=B6=E6=80=81=E6=9F=A5=E8=AF=A2=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 9 +++++++++ sdks/scheduler/jobflow.go | 28 ++++++++++++++++++++++++++++ sdks/scheduler/models.go | 2 ++ 3 files changed, 39 insertions(+) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index e2c6bc4..5b10f04 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -56,6 +56,7 @@ const ( FailedStatus = "failed" SuccessStatus = "success" + SucceededStatus = "succeeded" UploadingStatus = "uploading" RunningStatus = "running" @@ -85,6 +86,14 @@ const ( StartNode = "StartTask" EndNode = "EndTask" + + NodeTypeBinding = "binding" + NodeTypeUpload = "upload" + NodeTypeDataReturn = "data_return" + NodeTypeHPCCreate = "hpc_create" + NodeTypeInference = "inference" + NodeTypeAICreate = "ai_job_create" + NodeTypeAIJobRun = "ai_job_run" ) type TaskID int64 diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index 5219b80..485e9b0 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -67,3 +67,31 @@ type JobFlow struct { UpdatedAt time.Time `json:"updatedAt"` CreatedAt time.Time `json:"createdAt"` } + +type JobFlowRunStatus struct { + RunID JobSetID `gorm:"column:run_id;primaryKey" json:"runID"` + NodeType string `gorm:"column:node_type;size:100" json:"nodeType"` + NodeID string `gorm:"column:node_id;size:255" json:"nodeID"` + Status string `gorm:"column:status;type:enum('pending','running','fail','success')" json:"status"` + RunLog string `gorm:"column:run_log;type:text" json:"runLog"` +} + +type JobFlowRunDAO struct { + ID JobSetID `gorm:"column:id;primaryKey;" json:"runID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name;size:255;not null" json:"name"` + Description string `gorm:"column:description;size:255" json:"description"` + Content string `gorm:"column:content;type:text" json:"content"` + Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` +} + +type JobFlowRun struct { + ID JobSetID `gorm:"column:id;primaryKey;" json:"runID"` + UserID cdssdk.UserID `gorm:"column:user_id" json:"userID"` + Name string `gorm:"column:name;size:255;not null" json:"name"` + Description string `gorm:"column:description;size:255" json:"description"` + Content FlowData `gorm:"column:content;type:text" json:"content"` + Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` +} diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 8ab173b..7e36020 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -31,6 +31,8 @@ type JobID string type JobSetID string +type DataID int64 + type ImageID int64 // 计算中心ID From 7cf21f30837f554da4b31c4e35dc528e3816529e Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Thu, 8 May 2025 16:48:37 +0800 Subject: [PATCH 59/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 7e36020..157f6bc 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -13,6 +13,7 @@ const ( JobTypeInstance = "Instance" JobTypeFinetuning = "Finetuning" JobTypeDataPreprocess = "DataPreprocess" + JobTypeDataReturn = "DataReturn" FileInfoTypePackage = "Package" FileInfoTypeLocalFile = "LocalFile" @@ -132,6 +133,8 @@ type PCMInferenceJobInfo struct { Description string `json:"description"` Files JobFilesInfo `json:"files"` JobResources JobResources `json:"jobResources"` + + BindingName string `json:"bindingName"` } type AIJobInfo struct { @@ -161,6 +164,7 @@ type BindingJobInfo struct { JobInfoBase Type string `json:"type"` Info DataBinding `json:"info"` + Name string `json:"name"` // 临时使用 } type DataBinding interface { @@ -394,6 +398,16 @@ type DataReturnJobInfo struct { Type string `json:"type"` BucketID cdssdk.BucketID `json:"bucketID"` TargetLocalJobID string `json:"targetLocalJobID"` + ReportMessage []ReportMessage `json:"reportMessage"` +} + +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID ClusterID `json:"clusterID"` + Output string `json:"output"` } // MultiInstanceJobInfo 多实例(推理任务) From c8b2ef2afca71759024f0216ba19fb80f77e2301 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 9 May 2025 09:49:40 +0800 Subject: [PATCH 60/73] =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E5=8F=98=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 157f6bc..01a4fcb 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -70,6 +70,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*HPCJobInfo)(nil), (*BindingJobInfo)(nil), (*PCMInferenceJobInfo)(nil), + (*FinishJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -148,6 +149,12 @@ type AIJobInfo struct { ResourceChoice ResourceChoice `json:"resourceChoice"` } +type FinishJobInfo struct { + serder.Metadata `union:"finish"` + JobInfoBase + Type string `json:"type"` +} + type ResourceChoice struct { Type string `json:"type"` ResourceScopes []ResourceScope `json:"resourceScopes"` From 4e1a3cb8807af297316bdc7a36ee3fa1f7e81e16 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 9 May 2025 15:19:39 +0800 Subject: [PATCH 61/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E8=BE=93=E5=87=BA=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/jobflow.go | 13 ++++---- sdks/scheduler/models.go | 62 ++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index 485e9b0..1437598 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -69,11 +69,12 @@ type JobFlow struct { } type JobFlowRunStatus struct { - RunID JobSetID `gorm:"column:run_id;primaryKey" json:"runID"` - NodeType string `gorm:"column:node_type;size:100" json:"nodeType"` - NodeID string `gorm:"column:node_id;size:255" json:"nodeID"` - Status string `gorm:"column:status;type:enum('pending','running','fail','success')" json:"status"` - RunLog string `gorm:"column:run_log;type:text" json:"runLog"` + RunID JobSetID `gorm:"column:run_id;primaryKey" json:"runID"` + NodeType string `gorm:"column:node_type;size:100" json:"nodeType"` + NodeID string `gorm:"column:node_id;size:255" json:"nodeID"` + Status string `gorm:"column:status;type:enum('pending','running','fail','success')" json:"status"` + RunOutput string `gorm:"column:run_output;type:text" json:"runOutput"` + RunLog string `gorm:"column:run_log;type:text" json:"runLog"` } type JobFlowRunDAO struct { @@ -84,6 +85,7 @@ type JobFlowRunDAO struct { Content string `gorm:"column:content;type:text" json:"content"` Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` + FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` } type JobFlowRun struct { @@ -94,4 +96,5 @@ type JobFlowRun struct { Content FlowData `gorm:"column:content;type:text" json:"content"` Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` + FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 01a4fcb..f3a920b 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -134,8 +134,7 @@ type PCMInferenceJobInfo struct { Description string `json:"description"` Files JobFilesInfo `json:"files"` JobResources JobResources `json:"jobResources"` - - BindingName string `json:"bindingName"` + BindingID DataID `json:"bindingID"` } type AIJobInfo struct { @@ -408,15 +407,6 @@ type DataReturnJobInfo struct { ReportMessage []ReportMessage `json:"reportMessage"` } -type ReportMessage struct { - TaskName string `json:"taskName"` - TaskID string `json:"taskID"` - Status bool `json:"status"` - Message string `json:"message"` - ClusterID ClusterID `json:"clusterID"` - Output string `json:"output"` -} - // MultiInstanceJobInfo 多实例(推理任务) type MultiInstanceJobInfo struct { serder.Metadata `union:"MultiInstance"` @@ -629,3 +619,53 @@ type InferencePlatform struct { SimilarityThreshold string `json:"similarityThreshold"` EntriesPerFile string `json:"entriesPerFile"` } + +type JobOutput interface { + Output() +} + +var JobOutputTypeUnion = types.NewTypeUnion[JobOutput]( + (*AIJobOutput)(nil), + (*BindingJobOutput)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&JobOutputTypeUnion, "type") + +type JobOutputBase struct{} + +func (d *JobOutputBase) Output() {} + +type PublicOutput struct { + serder.Metadata `union:"object"` + JobOutputBase + Type string `json:"type"` +} + +type AIJobOutput struct { + serder.Metadata `union:"object"` + JobOutputBase + Type string `json:"type"` +} + +type BindingJobOutput struct { + serder.Metadata `union:"binding"` + JobOutputBase + Type string `json:"type"` + BindingID DataID `json:"bindingID"` +} + +type DataReturnJobOutput struct { + serder.Metadata `union:"DataReturn"` + JobOutputBase + Type string `json:"type"` + ReportMessage []ReportMessage `json:"reportMessage"` +} + +type ReportMessage struct { + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID ClusterID `json:"clusterID"` + Output string `json:"output"` +} From 5a93feb5a028ade3cf0ed03e8bf5d1ae54d00657 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 9 May 2025 15:31:32 +0800 Subject: [PATCH 62/73] fix2 --- sdks/scheduler/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index f3a920b..a5faad7 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -149,7 +149,7 @@ type AIJobInfo struct { } type FinishJobInfo struct { - serder.Metadata `union:"finish"` + serder.Metadata `union:"Finish"` JobInfoBase Type string `json:"type"` } From fda3a4ca9aeedcccddcffb947dbe9d8d66348cf9 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 12 May 2025 10:00:56 +0800 Subject: [PATCH 63/73] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 3 +-- sdks/scheduler/models.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index 5b10f04..ef01c8e 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -84,8 +84,7 @@ const ( BlockChainJobCreatePrefix = "job_create_" - StartNode = "StartTask" - EndNode = "EndTask" + Finish = "Finish" NodeTypeBinding = "binding" NodeTypeUpload = "upload" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index a5faad7..c40c16c 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -70,7 +70,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*HPCJobInfo)(nil), (*BindingJobInfo)(nil), (*PCMInferenceJobInfo)(nil), - (*FinishJobInfo)(nil), + (*CompleteJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -148,7 +148,7 @@ type AIJobInfo struct { ResourceChoice ResourceChoice `json:"resourceChoice"` } -type FinishJobInfo struct { +type CompleteJobInfo struct { serder.Metadata `union:"Finish"` JobInfoBase Type string `json:"type"` From 4be8100d1ab8cb42276ccc48b3a13ad8e2cea8ad Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 12 May 2025 15:32:22 +0800 Subject: [PATCH 64/73] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E5=88=86=E9=A1=B5=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 4 ++-- sdks/scheduler/models.go | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index ef01c8e..fc91a22 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -431,8 +431,8 @@ type QueryData struct { UserID cdssdk.UserID `json:"userID" binding:"required"` Path string `json:"path"` PackageID cdssdk.PackageID `json:"packageID" binding:"required"` - CurrentPage int `json:"currentPage" binding:"required"` - PageSize int `json:"pageSize" binding:"required"` + CurrentPage int `json:"currentPage" binding:"gte=1"` // >= 1 + PageSize int `json:"pageSize" binding:"gte=1,lte=200"` // 1 ~ 200 OrderBy string `json:"orderBy" binding:"required"` PackageName string `json:"packageName"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index c40c16c..2f0ba77 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -129,12 +129,13 @@ type NormalJobInfo struct { type PCMInferenceJobInfo struct { serder.Metadata `union:"PCM_Inference"` JobInfoBase - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - Files JobFilesInfo `json:"files"` - JobResources JobResources `json:"jobResources"` - BindingID DataID `json:"bindingID"` + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description"` + Files JobFilesInfo `json:"files"` + JobResources JobResources `json:"jobResources"` + BindingID DataID `json:"bindingID"` + ResourceChoice ResourceChoice `json:"resourceChoice"` } type AIJobInfo struct { From 7d232ef6258da9af08899021f8e2824cff198679 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 12 May 2025 16:18:42 +0800 Subject: [PATCH 65/73] fix --- sdks/pcmscheduler/models.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index fc91a22..dae0fa3 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -431,8 +431,8 @@ type QueryData struct { UserID cdssdk.UserID `json:"userID" binding:"required"` Path string `json:"path"` PackageID cdssdk.PackageID `json:"packageID" binding:"required"` - CurrentPage int `json:"currentPage" binding:"gte=1"` // >= 1 - PageSize int `json:"pageSize" binding:"gte=1,lte=200"` // 1 ~ 200 + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` OrderBy string `json:"orderBy" binding:"required"` PackageName string `json:"packageName"` } From d01bb92ec90bb4ecbdfc5c732eb77017a6cdd1d9 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 13 May 2025 15:57:57 +0800 Subject: [PATCH 66/73] =?UTF-8?q?=E6=8E=A8=E7=90=86=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=95=B0=E6=8D=AE=E8=B0=83=E5=BA=A6=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/models.go | 2 +- sdks/scheduler/models.go | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/models.go b/sdks/pcmscheduler/models.go index dae0fa3..6f8707f 100644 --- a/sdks/pcmscheduler/models.go +++ b/sdks/pcmscheduler/models.go @@ -84,7 +84,7 @@ const ( BlockChainJobCreatePrefix = "job_create_" - Finish = "Finish" + Complete = "Complete" NodeTypeBinding = "binding" NodeTypeUpload = "upload" diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 2f0ba77..4038cc0 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -71,6 +71,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*BindingJobInfo)(nil), (*PCMInferenceJobInfo)(nil), (*CompleteJobInfo)(nil), + (*StartJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -150,7 +151,13 @@ type AIJobInfo struct { } type CompleteJobInfo struct { - serder.Metadata `union:"Finish"` + serder.Metadata `union:"Complete"` + JobInfoBase + Type string `json:"type"` +} + +type StartJobInfo struct { + serder.Metadata `union:"Start"` JobInfoBase Type string `json:"type"` } From befe803070b17eef3ba9ed50813574f99132e9a1 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 14 May 2025 09:40:53 +0800 Subject: [PATCH 67/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=80=9A=E7=9F=A5?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 52 +++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 4038cc0..9e77968 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -409,10 +409,11 @@ type DataPreprocessJobInfo struct { type DataReturnJobInfo struct { serder.Metadata `union:"DataReturn"` JobInfoBase - Type string `json:"type"` - BucketID cdssdk.BucketID `json:"bucketID"` - TargetLocalJobID string `json:"targetLocalJobID"` - ReportMessage []ReportMessage `json:"reportMessage"` + Type string `json:"type"` + BucketID cdssdk.BucketID `json:"bucketID"` + TargetLocalJobID string `json:"targetLocalJobID"` + Report TrainJobStatusReport `json:"report"` + //ReportMessage []ReportMessage `json:"reportMessage"` } // MultiInstanceJobInfo 多实例(推理任务) @@ -665,8 +666,9 @@ type BindingJobOutput struct { type DataReturnJobOutput struct { serder.Metadata `union:"DataReturn"` JobOutputBase - Type string `json:"type"` - ReportMessage []ReportMessage `json:"reportMessage"` + Type string `json:"type"` + Report TrainJobStatusReport `json:"report"` + //ReportMessage []ReportMessage `json:"reportMessage"` } type ReportMessage struct { @@ -677,3 +679,41 @@ type ReportMessage struct { ClusterID ClusterID `json:"clusterID"` Output string `json:"output"` } + +type JobStatusReport interface { + Report() +} + +var JobStatusReportTypeUnion = types.NewTypeUnion[JobStatusReport]( + (*TrainJobStatusReport)(nil), + (*InferenceJobStatusReport)(nil), +) + +var _ = serder.UseTypeUnionInternallyTagged(&JobStatusReportTypeUnion, "type") + +type JobStatusReportBase struct{} + +func (d *JobStatusReportBase) Report() {} + +type TrainJobStatusReport struct { + serder.Metadata `union:"Train"` + JobStatusReportBase + Type string `json:"type"` + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + ClusterID ClusterID `json:"clusterID"` + Output string `json:"output"` +} + +type InferenceJobStatusReport struct { + serder.Metadata `union:"Inference"` + JobStatusReportBase + Type string `json:"type"` + TaskName string `json:"taskName"` + TaskID string `json:"taskID"` + Status bool `json:"status"` + Message string `json:"message"` + URL string `json:"url"` +} From b3ed5f8140cf95ba06327b9002cafd95093feed0 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 14 May 2025 15:39:02 +0800 Subject: [PATCH 68/73] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9e77968..60261cc 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -666,18 +666,8 @@ type BindingJobOutput struct { type DataReturnJobOutput struct { serder.Metadata `union:"DataReturn"` JobOutputBase - Type string `json:"type"` - Report TrainJobStatusReport `json:"report"` - //ReportMessage []ReportMessage `json:"reportMessage"` -} - -type ReportMessage struct { - TaskName string `json:"taskName"` - TaskID string `json:"taskID"` - Status bool `json:"status"` - Message string `json:"message"` - ClusterID ClusterID `json:"clusterID"` - Output string `json:"output"` + Type string `json:"type"` + ReportMessage TrainJobStatusReport `json:"report"` } type JobStatusReport interface { From 034e9d4768b8103dad3af02bd8527d159fb37597 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Wed, 14 May 2025 17:38:50 +0800 Subject: [PATCH 69/73] fix --- sdks/scheduler/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 60261cc..9dca84d 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -412,7 +412,7 @@ type DataReturnJobInfo struct { Type string `json:"type"` BucketID cdssdk.BucketID `json:"bucketID"` TargetLocalJobID string `json:"targetLocalJobID"` - Report TrainJobStatusReport `json:"report"` + ReportMessage TrainJobStatusReport `json:"report"` //ReportMessage []ReportMessage `json:"reportMessage"` } From b916bbac59373c8d2b4b81ed85b7dc8b5bc7e588 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Fri, 16 May 2025 17:45:28 +0800 Subject: [PATCH 70/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E7=8A=B6=E6=80=81=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/jobflow.go | 2 ++ sdks/scheduler/models.go | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/sdks/scheduler/jobflow.go b/sdks/scheduler/jobflow.go index 1437598..8d78ddf 100644 --- a/sdks/scheduler/jobflow.go +++ b/sdks/scheduler/jobflow.go @@ -84,6 +84,7 @@ type JobFlowRunDAO struct { Description string `gorm:"column:description;size:255" json:"description"` Content string `gorm:"column:content;type:text" json:"content"` Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + Token string `gorm:"column:token;size:255" json:"token"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` } @@ -95,6 +96,7 @@ type JobFlowRun struct { Description string `gorm:"column:description;size:255" json:"description"` Content FlowData `gorm:"column:content;type:text" json:"content"` Status string `gorm:"column:status;type:enum('running','fail','success')" json:"status"` + Token string `gorm:"column:token;size:255" json:"token"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"` FinishAt *time.Time `gorm:"column:finish_at" json:"finishedAt"` } diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 9dca84d..54f5261 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -630,7 +630,7 @@ type InferencePlatform struct { } type JobOutput interface { - Output() + Output2() } var JobOutputTypeUnion = types.NewTypeUnion[JobOutput]( @@ -642,7 +642,7 @@ var _ = serder.UseTypeUnionInternallyTagged(&JobOutputTypeUnion, "type") type JobOutputBase struct{} -func (d *JobOutputBase) Output() {} +func (d *JobOutputBase) Output2() {} type PublicOutput struct { serder.Metadata `union:"object"` @@ -650,10 +650,18 @@ type PublicOutput struct { Type string `json:"type"` } +type HPCOutput struct { + serder.Metadata `union:"HPCSlurm"` + JobOutputBase + Type string `json:"type"` + Output string `json:"output"` +} + type AIJobOutput struct { - serder.Metadata `union:"object"` + serder.Metadata `union:"AI"` JobOutputBase - Type string `json:"type"` + Type string `json:"type"` + Output string `json:"output"` } type BindingJobOutput struct { @@ -668,6 +676,7 @@ type DataReturnJobOutput struct { JobOutputBase Type string `json:"type"` ReportMessage TrainJobStatusReport `json:"report"` + PackageID cdssdk.PackageID `json:"packageID"` } type JobStatusReport interface { From 88484e929d07b15bd4969528d789332df570bb65 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Mon, 19 May 2025 16:27:59 +0800 Subject: [PATCH 71/73] =?UTF-8?q?=E8=B0=83=E7=94=A8C=E7=AB=AF=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E4=BC=A0=E5=85=A5token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/pcmscheduler/jobset.go | 13 +++++++++++-- sdks/scheduler/models.go | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index fa09286..608cfe8 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -19,17 +19,26 @@ type GetClusterInfoReq struct { // TraceId string `json:"traceId"` //} -func (c *Client) GetClusterInfo(req GetClusterInfoReq) ([]ClusterDetail, error) { +func (c *Client) GetClusterInfo(req GetClusterInfoReq, token string) ([]ClusterDetail, error) { targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") if err != nil { return nil, err } //resp, err := http2.PostJSON(targetUrl, http2.RequestParam{Body: req}) - resp, err := http2.PostJSON(targetUrl, http2.RequestParam{}) + header := map[string]string{ + "Authorization": token, + } + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Header: header, + }) if err != nil { return nil, err } + + //all, err := io.ReadAll(resp.Body) + //println(string(all)) + contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 54f5261..911bf2d 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -26,6 +26,10 @@ const ( MemoryUtilization = "MemoryUtilization" GPUUtilization = "GPUUtilization" CPUUtilization = "CPUUtilization" + + MethodPost = "POST" + MethodGet = "GET" + CodeSuccess = 200 ) type JobID string @@ -72,6 +76,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*PCMInferenceJobInfo)(nil), (*CompleteJobInfo)(nil), (*StartJobInfo)(nil), + (*NotifyJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -162,6 +167,16 @@ type StartJobInfo struct { Type string `json:"type"` } +type NotifyJobInfo struct { + serder.Metadata `union:"Notify"` + JobInfoBase + Type string `json:"type"` + RequestType string `json:"requestType"` + Url string `json:"url"` + Body any `json:"body"` + Headers map[string]string `json:"headers"` +} + type ResourceChoice struct { Type string `json:"type"` ResourceScopes []ResourceScope `json:"resourceScopes"` From fe10185b9855bae9079ab72d90612d2bfdd75f16 Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 20 May 2025 10:08:51 +0800 Subject: [PATCH 72/73] fix --- sdks/pcmscheduler/jobset.go | 59 ++++++++++++++++++++++++------------- sdks/scheduler/models.go | 9 +++++- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/sdks/pcmscheduler/jobset.go b/sdks/pcmscheduler/jobset.go index 608cfe8..bcf329c 100644 --- a/sdks/pcmscheduler/jobset.go +++ b/sdks/pcmscheduler/jobset.go @@ -14,31 +14,21 @@ type GetClusterInfoReq struct { IDs []schsdk.ClusterID `json:"clusterIDs"` } -//type GetClusterInfoResp struct { -// Data []ClusterDetail `json:"data"` -// TraceId string `json:"traceId"` -//} - func (c *Client) GetClusterInfo(req GetClusterInfoReq, token string) ([]ClusterDetail, error) { targetUrl, err := url.JoinPath(c.baseURL, "schedule/queryResources") if err != nil { return nil, err } - //resp, err := http2.PostJSON(targetUrl, http2.RequestParam{Body: req}) - header := map[string]string{ - "Authorization": token, - } resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ - Header: header, + Header: map[string]string{ + "Authorization": token, + }, }) if err != nil { return nil, err } - //all, err := io.ReadAll(resp.Body) - //println(string(all)) - contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { @@ -166,7 +156,43 @@ func (c *Client) CreateInferenceJob(req CreateAIJobReq, token string) (*CreateIn } return nil, fmt.Errorf("unknow response content type: %s", contType) +} +type StopInferenceJobReq struct { + TaskId string `json:"taskId"` +} + +func (c *Client) StopInferenceJob(req StopInferenceJobReq, token string) error { + targetUrl, err := url.JoinPath(c.baseURL, "inference/createTask") + if err != nil { + return err + } + + resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ + Body: req, + Header: map[string]string{ + "Authorization": token, + }, + }) + if err != nil { + return err + } + + contType := resp.Header.Get("Content-Type") + if strings.Contains(contType, http2.ContentTypeJSON) { + var codeResp respons2[CreateInferenceJobResp] + if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { + return fmt.Errorf("parsing response: %w", err) + } + + if codeResp.Code == ResponseCodeOK { + return nil + } + + return fmt.Errorf("error: %s", codeResp.Message) + } + + return fmt.Errorf("unknow response content type: %s", contType) } func (c *Client) CreateJob(req CreateAIJobReq, token string) (*CreateJobResp, error) { @@ -175,13 +201,6 @@ func (c *Client) CreateJob(req CreateAIJobReq, token string) (*CreateJobResp, er return nil, err } - // 将req转换成json，并打印 - //req2, err := serder.ObjectToJSONEx(req) - //if err != nil { - // return nil, fmt.Errorf("request to json: %w", err) - //} - //fmt.Println(string(req2)) - resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ Body: req, Header: map[string]string{ diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 911bf2d..2073cf4 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -77,6 +77,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*CompleteJobInfo)(nil), (*StartJobInfo)(nil), (*NotifyJobInfo)(nil), + (*StopInferenceJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -144,6 +145,13 @@ type PCMInferenceJobInfo struct { ResourceChoice ResourceChoice `json:"resourceChoice"` } +type StopInferenceJobInfo struct { + serder.Metadata `union:"StopInference"` + JobInfoBase + Type string `json:"type"` + Url string `json:"url"` +} + type AIJobInfo struct { serder.Metadata `union:"AI"` JobInfoBase @@ -428,7 +436,6 @@ type DataReturnJobInfo struct { BucketID cdssdk.BucketID `json:"bucketID"` TargetLocalJobID string `json:"targetLocalJobID"` ReportMessage TrainJobStatusReport `json:"report"` - //ReportMessage []ReportMessage `json:"reportMessage"` } // MultiInstanceJobInfo 多实例(推理任务) From 921a874200125691488831513b8c1ff710c844cb Mon Sep 17 00:00:00 2001 From: JeshuaRen <270813223@qq.com> Date: Tue, 20 May 2025 17:11:54 +0800 Subject: [PATCH 73/73] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sdks/scheduler/models.go | 31 ++++++++ sdks/uploader/uploader.go | 145 ++------------------------------------ utils/reflect2/assign.go | 29 ++++++++ 3 files changed, 67 insertions(+), 138 deletions(-) create mode 100644 utils/reflect2/assign.go diff --git a/sdks/scheduler/models.go b/sdks/scheduler/models.go index 2073cf4..8262d75 100644 --- a/sdks/scheduler/models.go +++ b/sdks/scheduler/models.go @@ -78,6 +78,7 @@ var JobInfoTypeUnion = types.NewTypeUnion[JobInfo]( (*StartJobInfo)(nil), (*NotifyJobInfo)(nil), (*StopInferenceJobInfo)(nil), + (*UploadJobInfo)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobInfoTypeUnion, "type") @@ -175,6 +176,13 @@ type StartJobInfo struct { Type string `json:"type"` } +type UploadJobInfo struct { + serder.Metadata `union:"Upload"` + JobInfoBase + Type string `json:"type"` + DataType string `json:"dataType"` +} + type NotifyJobInfo struct { serder.Metadata `union:"Notify"` JobInfoBase @@ -210,6 +218,7 @@ type DataBinding interface { var DataBindingTypeUnion = types.NewTypeUnion[DataBinding]( (*ModelBinding)(nil), + (*DatasetBinding)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&DataBindingTypeUnion, "type") @@ -232,6 +241,20 @@ type ModelBinding struct { RepositoryName string `json:"repositoryName"` } +type DatasetBinding struct { + serder.Metadata `union:"dataset"` + DataBindingBase + Type string `json:"type"` + Name string `json:"name"` + OperateType string `json:"operateType"` + ClusterIDs []ClusterID `json:"clusterIDs"` + Description string `json:"description"` + Category string `json:"category"` + PackageID cdssdk.PackageID `json:"packageID"` + RepositoryName string `json:"repositoryName"` + ConsumptionPoints int64 `json:"points"` +} + type HPCJobInfo struct { serder.Metadata `union:"HPC"` JobInfoBase @@ -658,6 +681,7 @@ type JobOutput interface { var JobOutputTypeUnion = types.NewTypeUnion[JobOutput]( (*AIJobOutput)(nil), (*BindingJobOutput)(nil), + (*UploadJobOutput)(nil), ) var _ = serder.UseTypeUnionInternallyTagged(&JobOutputTypeUnion, "type") @@ -693,6 +717,13 @@ type BindingJobOutput struct { BindingID DataID `json:"bindingID"` } +type UploadJobOutput struct { + serder.Metadata `union:"upload"` + JobOutputBase + Type string `json:"type"` + PackageID cdssdk.PackageID `json:"packageID"` +} + type DataReturnJobOutput struct { serder.Metadata `union:"DataReturn"` JobOutputBase diff --git a/sdks/uploader/uploader.go b/sdks/uploader/uploader.go index 54f17df..6cae186 100644 --- a/sdks/uploader/uploader.go +++ b/sdks/uploader/uploader.go @@ -2,152 +2,21 @@ package uploadersdk import ( "fmt" - "gitlink.org.cn/cloudream/common/pkgs/types" - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" "gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/serder" + "mime/multipart" "net/url" "strings" ) -//type DataScheduleReq struct { -// PackageID cdssdk.PackageID `json:"packageID"` -// DataType string `json:"dataType"` -// ScheduleTarget ScheduleTarget `json:"scheduleTarget"` -//} - -//type DataScheduleResp struct { -// Results []sch.DataScheduleResult `json:"data"` -//} - -//type TmpDataScheduleResult struct { -// Cluster sch.DataDetail `json:"cluster"` -// PackageID cdssdk.PackageID `json:"packageID"` -// Status bool `json:"status"` -// Msg string `json:"msg"` -//} - -//func (c *Client) DataSchedule(req DataScheduleReq) ([]sch.DataScheduleResult, error) { -// targetUrl, err := url.JoinPath(c.baseURL, "/dataSchedule") -// if err != nil { -// return nil, err -// } -// -// resp, err := http2.PostJSON(targetUrl, http2.RequestParam{ -// Body: req, -// }) -// if err != nil { -// return nil, err -// } -// println(resp.Body) -// -// contType := resp.Header.Get("Content-Type") -// if strings.Contains(contType, http2.ContentTypeJSON) { -// var codeResp response[[]TmpDataScheduleResult] -// if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { -// return nil, fmt.Errorf("parsing response: %w", err) -// } -// -// if codeResp.Code == ResponseCodeOK { -// var results []sch.DataScheduleResult -// for _, tmpResult := range codeResp.Data { -// result := sch.DataScheduleResult{ -// PackageID: tmpResult.PackageID, -// Status: tmpResult.Status, -// Msg: tmpResult.Msg, -// Clusters: []sch.DataDetail{ -// tmpResult.Cluster, -// }, -// } -// results = append(results, result) -// } -// return results, nil -// } -// -// return nil, codeResp.ToError() -// } -// -// return nil, fmt.Errorf("unknow response content type: %s", contType) -//} - -type UploadReq struct { - DataType string `json:"dataType"` - Source UploadSource `json:"source"` - Target UploadTarget `json:"target"` - //StorageIDs []cdssdk.StorageID `json:"storageIDs"` -} - -type UploadSource interface { - Noop() -} - -var UploadSourceTypeUnion = types.NewTypeUnion[UploadSource]( - (*PackageSource)(nil), - (*UrlSource)(nil), -) - -var _ = serder.UseTypeUnionInternallyTagged(&UploadSourceTypeUnion, "type") - -type PackageSource struct { - serder.Metadata `union:"jcs"` - UploadSourceBase - Type string `json:"type"` - PackageID cdssdk.PackageID `json:"packageID"` -} - -type UrlSource struct { - serder.Metadata `union:"url"` - UploadSourceBase - Type string `json:"type"` - Url string `json:"url"` -} - -type UploadSourceBase struct{} - -func (d *UploadSourceBase) Noop() {} - -type UploadTarget interface { - Noop() -} - -var UploadTargetTypeUnion = types.NewTypeUnion[UploadTarget]( - (*UrlTarget)(nil), - (*ApiTarget)(nil), -) - -var _ = serder.UseTypeUnionInternallyTagged(&UploadTargetTypeUnion, "type") - -type UrlTarget struct { - serder.Metadata `union:"url"` - UploadTargetBase - Type string `json:"type"` - ClusterID ClusterID `json:"clusterId"` - JCSUploadInfo cdsapi.ObjectUploadInfo `form:"JCSUploadInfo"` -} - -type ApiTarget struct { - serder.Metadata `union:"api"` - UploadTargetBase - Type string `json:"type"` - Clusters []ClusterID `json:"clusters"` -} - -type UploadTargetBase struct{} - -func (d *UploadTargetBase) Noop() {} - -type UploadResp struct { - PackageID cdssdk.PackageID `json:"packageID"` - ObjectIDs []cdssdk.ObjectID `json:"objectIDs"` - ClusterID ClusterID `json:"clusterID"` - JsonData string `json:"jsonData"` - Status bool `json:"status"` - Message string `json:"message"` +type ObjectUploadReq struct { + Info cdsapi.ObjectUploadInfo `form:"info" binding:"required"` + Files []*multipart.FileHeader `form:"files"` } -func (c *Client) Upload(req UploadReq) (*UploadResp, error) { - targetUrl, err := url.JoinPath(c.baseURL, "/dataUpload") +func (c *Client) Upload(req ObjectUploadReq) (*cdsapi.ObjectUploadResp, error) { + targetUrl, err := url.JoinPath(c.baseURL, "/object/upload") if err != nil { return nil, err } @@ -161,7 +30,7 @@ func (c *Client) Upload(req UploadReq) (*UploadResp, error) { contType := resp.Header.Get("Content-Type") if strings.Contains(contType, http2.ContentTypeJSON) { - var codeResp response[UploadResp] + var codeResp response[cdsapi.ObjectUploadResp] if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil { return nil, fmt.Errorf("parsing response: %w", err) } diff --git a/utils/reflect2/assign.go b/utils/reflect2/assign.go new file mode 100644 index 0000000..2e8d49d --- /dev/null +++ b/utils/reflect2/assign.go @@ -0,0 +1,29 @@ +package reflect2 + +import "reflect" + +// MergeNonZero 将 src 中非零值字段复制到 dst 中 +func MergeNonZero(dst, src any) { + dstVal := reflect.ValueOf(dst).Elem() + srcVal := reflect.ValueOf(src).Elem() + + for i := 0; i < dstVal.NumField(); i++ { + dstField := dstVal.Field(i) + srcField := srcVal.Field(i) + + // 如果 src 字段是零值，则跳过 + if isZero(srcField) { + continue + } + + // 如果 dst 字段可设置，才设置 + if dstField.CanSet() { + dstField.Set(srcField) + } + } +} + +func isZero(v reflect.Value) bool { + zero := reflect.Zero(v.Type()) + return reflect.DeepEqual(v.Interface(), zero.Interface()) +}