diff --git a/go.mod b/go.mod index bde8a642..0877728d 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( gitlink.org.cn/JointCloud/pcm-hpc v0.0.0-20241125115811-72f3568255a4 gitlink.org.cn/JointCloud/pcm-modelarts v0.0.0-20240918011543-482dcd609877 gitlink.org.cn/JointCloud/pcm-octopus v0.0.0-20240817071412-44397870b110 - gitlink.org.cn/JointCloud/pcm-openi v0.0.0-20241205140615-e1b443983040 + gitlink.org.cn/JointCloud/pcm-openi v0.0.0-20241206092318-9f0c3142fced gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240403033338-e7edabad4203 gitlink.org.cn/JointCloud/pcm-slurm v0.0.0-20240301080743-8b94bbaf57f5 go.opentelemetry.io/otel/trace v1.31.0 diff --git a/go.sum b/go.sum index 61f53d93..12547505 100644 --- a/go.sum +++ b/go.sum @@ -532,8 +532,8 @@ gitlink.org.cn/JointCloud/pcm-modelarts v0.0.0-20240918011543-482dcd609877 h1:a+ gitlink.org.cn/JointCloud/pcm-modelarts v0.0.0-20240918011543-482dcd609877/go.mod h1:/eOmBFZKWGoabG3sRVkVvIbLwsd2631k4jkUBR6x1AA= gitlink.org.cn/JointCloud/pcm-octopus v0.0.0-20240817071412-44397870b110 h1:GaXwr5sgDh0raHjUf9IewTvnRvajYea7zbLsaerYyXo= gitlink.org.cn/JointCloud/pcm-octopus v0.0.0-20240817071412-44397870b110/go.mod h1:QOD5+/l2D+AYBjF2h5T0mdJyfGAmF78QmeKdbBXbjLQ= -gitlink.org.cn/JointCloud/pcm-openi v0.0.0-20241205140615-e1b443983040 h1:dWUf3LDZO3l37FPswxf6vC+hzNT2Ym/G0PXvJDHg+w8= -gitlink.org.cn/JointCloud/pcm-openi v0.0.0-20241205140615-e1b443983040/go.mod h1:oDJrr/TNbUCaVjI+RaOrUtGawD7UPAvp7U/oVgT2Dhc= +gitlink.org.cn/JointCloud/pcm-openi v0.0.0-20241206092318-9f0c3142fced h1:N/UMOMm5fzELf52Neze+2oNMEUejFWWqbFwzhzJvgHc= +gitlink.org.cn/JointCloud/pcm-openi v0.0.0-20241206092318-9f0c3142fced/go.mod h1:oDJrr/TNbUCaVjI+RaOrUtGawD7UPAvp7U/oVgT2Dhc= gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240403033338-e7edabad4203 h1:s6PsZ1+bev294IWdZRlV7mnOwI1+UzFcldVW/BqhQzI= gitlink.org.cn/JointCloud/pcm-openstack v0.0.0-20240403033338-e7edabad4203/go.mod h1:i2rrbMQ+Fve345BY9Heh4MUqVTAimZQElQhzzRee5B8= gitlink.org.cn/JointCloud/pcm-slurm v0.0.0-20240301080743-8b94bbaf57f5 h1:+/5vnzkJBfMRnya1NrhOzlroUtRa5ePiYbPKlHLoLV0= diff --git a/internal/storeLink/openi.go b/internal/storeLink/openi.go index b360679d..616e6e28 100644 --- a/internal/storeLink/openi.go +++ b/internal/storeLink/openi.go @@ -1,10 +1,16 @@ package storeLink import ( + "bytes" "context" + "encoding/json" "gitlink.org.cn/JointCloud/pcm-coordinator/internal/scheduler/service/collector" "gitlink.org.cn/JointCloud/pcm-openi/common" "gitlink.org.cn/JointCloud/pcm-openi/model" + "net/http" + "strconv" + "strings" + "sync" ) const ( @@ -42,31 +48,8 @@ func (o OpenI) GetResourceStats(ctx context.Context) (*collector.ResourceStats, } func (o OpenI) GetDatasetsSpecs(ctx context.Context) ([]*collector.DatasetsSpecs, error) { - var specs []*collector.DatasetsSpecs - var url string - - for _ = range ComputeSource { - param := model.TaskCreationRequiredParam{} - resp := model.TaskCreationRequired{} - go func() { - req := common.GetRestyRequest(common.TIMEOUT) - _, err := req. - SetBody(param). - SetQueryParam(common.ACCESSTOKEN, o.accessToken). - SetResult(&resp). - Get(url) - - if err != nil { - return - } - - spec := &collector.DatasetsSpecs{} - specs = append(specs, spec) - }() - - } - - return specs, nil + //TODO implement me + panic("implement me") } func (o OpenI) GetAlgorithms(ctx context.Context) ([]*collector.Algorithm, error) { @@ -105,6 +88,103 @@ func (o OpenI) GetUserBalance(ctx context.Context) (float64, error) { } func (o OpenI) GetResourceSpecs(ctx context.Context) (*collector.ResourceSpec, error) { - //TODO implement me - panic("implement me") + var resources []interface{} + res := &collector.ResourceSpec{ + ClusterId: strconv.FormatInt(o.participantId, 10), + } + url := o.host + "/api/v1/task/creationRequired" + var wg sync.WaitGroup + var ch = make(chan *collector.Usage) + + for c := range ComputeSource { + wg.Add(1) + i := c + go func() { + defer wg.Done() + param := model.TaskCreationRequiredParam{ + UserName: o.userName, + RepoName: TESTREPO, + JobType: TRAIN, + ComputeSource: ComputeSource[i], + ClusterType: C2NET, + } + + b, _ := json.Marshal(param) + byt := bytes.NewBuffer(b) + + resp := struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data model.TaskCreationRequired `json:"data"` + }{} + + req := common.GetRestyRequest(common.TIMEOUT) + r, _ := http.NewRequest("GET", url, byt) + req.RawRequest = r + req.URL = url + + _, err := req. + SetHeader("Content-Type", "application/json"). + SetQueryParam(common.ACCESSTOKEN, o.accessToken). + SetBody(byt). + SetResult(&resp). + Send() + + if err != nil { + return + } + + if len(resp.Data.Data.Specs.All) == 0 { + return + } + + m := make(map[string]struct { + Id int `json:"id"` + AccCardsNum int `json:"acc_cards_num"` + AccCardType string `json:"acc_card_type"` + CpuCores int `json:"cpu_cores"` + MemGiB int `json:"mem_gi_b"` + GpuMemGiB int `json:"gpu_mem_gi_b"` + ShareMemGiB int `json:"share_mem_gi_b"` + ComputeResource string `json:"compute_resource"` + UnitPrice int `json:"unit_price"` + SourceSpecId string `json:"source_spec_id"` + HasInternet int `json:"has_internet"` + EnableVisualization bool `json:"enable_visualization"` + }) + + for _, s := range resp.Data.Data.Specs.All { + e, ok := m[s.AccCardType] + if ok { + if s.AccCardsNum > e.AccCardsNum { + m[s.AccCardType] = s + } + } else { + m[s.AccCardType] = s + } + } + + for k, v := range m { + u := &collector.Usage{ + Type: strings.ToUpper(k), + Total: v.AccCardsNum, + Available: v.AccCardsNum, + } + ch <- u + } + + }() + } + + go func() { + for v := range ch { + resources = append(resources, v) + } + }() + + wg.Wait() + + res.Resource = resources + + return res, nil }