pcm-coordinator/internal/storeLink/octopusHttp/octopusHttp.go

package octopusHttp

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	common2 "gitlink.org.cn/JointCloud/pcm-coordinator/internal/scheduler/common"
	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/scheduler/entity"
	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/scheduler/schedulers/option"
	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/scheduler/service/collector"
	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/scheduler/service/executor"
	"gitlink.org.cn/JointCloud/pcm-coordinator/internal/scheduler/service/inference"
	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/constants"
	"gitlink.org.cn/JointCloud/pcm-coordinator/pkg/utils"
	omodel "gitlink.org.cn/JointCloud/pcm-octopus/http/model"
	"gitlink.org.cn/JointCloud/pcm-openi/common"
	"mime/multipart"
	"net/http"
	"strconv"
	"strings"
	"time"
)

const (
	Param_Token                = "token"
	Param_Addr                 = "addr"
	Forward_Slash              = "/"
	COMMA                      = ","
	UNDERSCORE                 = "_"
	TASK_NAME_PREFIX           = "trainJob"
	Python                     = "python "
	SemiColon                  = ";"
	BALANCE                    = "balance"
	RATE                       = "rate"
	PERHOUR                    = "per-hour"
	NUMBER                     = "number"
	KILOBYTE                   = "kb"
	GIGABYTE                   = "gb"
	CPUCORE                    = "core"
	STORAGE                    = "STORAGE"
	DISK                       = "disk"
	MEMORY                     = "memory"
	RAM                        = "ram"
	VRAM                       = "vram"
	RMB                        = "rmb"
	POINT                      = "point"
	RUNNINGTASK                = "RUNNING_TASK"
	RUNNING                    = "RUNNING"
	CPU                        = "cpu"
	Gi                         = "Gi"
	AlgorithmRecordOnlyVersion = "V1"
)

const (
	NotImplementError = "not implemented"
)

const (
	MyAlgorithmListUrl = "api/v1/algorithm/myAlgorithmList"
	CreateAlgorithm    = "api/v1/algorithm/create"
	ResourcespecsUrl   = "api/v1/resource/specs"
	CreateTrainJobUrl  = "api/v1/job/create"
	TrainJobDetail     = "api/v1/job/detail"
	TrainJobLog        = "api/v1/job/log"
)

// compute source
var (
	ComputeSourceToCardType = map[string]string{
		"nvidia-a100":     "GPU",
		"nvidia-a100-80g": "GPU",
		"mr-v100":         "ILUVATAR-GPGPU",
		"bi-v100":         "ILUVATAR-GPGPU",
		"MR-V50":          "ILUVATAR-GPGPU",
		"BI-V100":         "ILUVATAR-GPGPU",
		"BI-V150":         "ILUVATAR-GPGPU",
		"MR-V100":         "ILUVATAR-GPGPU",

		"cambricon.com/mlu": "MLU",
		"hygon.com/dcu":     "DCU",

		"huawei.com/Ascend910": "NPU",
		"enflame.com/gcu":      "GCU",
		"ILUVATAR-GPGPU":       "ILUVATAR-GPGPU",
		"MXN260":               "METAX-GPGPU",
	}
)

type OctopusHttp struct {
	server        string
	host          string
	platform      string
	participantId int64
	token         *Token
	resourcePool  string
}

func NewOctopusHttp(id int64, resourcePool, name, server, host, user, pwd string) *OctopusHttp {
	token := &Token{
		user:   user,
		pwd:    pwd,
		server: server,
		host:   host,
	}
	return &OctopusHttp{resourcePool: resourcePool, platform: name, participantId: id, server: server, host: host, token: token}
}

// executor
func (o *OctopusHttp) Execute(ctx context.Context, option *option.AiOption, mode int) (interface{}, error) {
	switch mode {
	case executor.SUBMIT_MODE_JOINT_CLOUD:

	case executor.SUBMIT_MODE_STORAGE_SCHEDULE:
		// cmd
		if option.AlgorithmId == "" {
			return nil, errors.New("algorithmId is empty")
		}
		if option.Cmd != "" {
			option.Cmd = option.Cmd + SemiColon + Python + option.AlgorithmId
		} else {
			option.Cmd = Python + option.AlgorithmId
		}

		// algorithm
		//param := &omodel.CreateMyAlgorithmParam{
		//	AlgorithmName: option.AlgorithmId,
		//	ModelName:     option.AlgorithmId,
		//}
		//algorithm, err := o.createAlgorithm(ctx, param)
		//if err != nil {
		//	return nil, err
		//}
		//if algorithm.Code != http.StatusOK {
		//	if algorithm.Data != nil {
		//		marshal, err := json.Marshal(algorithm.Data)
		//		if err != nil {
		//			return nil, err
		//		}
		//
		//		errormdl := &omodel.Error{}
		//		err = json.Unmarshal(marshal, errormdl)
		//		if err != nil {
		//			return nil, err
		//		}
		//		return nil, errors.New(errormdl.Message)
		//	} else {
		//		return nil, errors.New(algorithm.Msg)
		//	}
		//} else {
		//	if algorithm.Data != nil {
		//		result := &entity.OctCreateAlgorithm{}
		//		marshal, err := json.Marshal(algorithm.Data)
		//		if err != nil {
		//			return nil, err
		//		}
		//		err = json.Unmarshal(marshal, result)
		//		if err != nil {
		//			return nil, err
		//		}
		//		if result.AlgorithmId == "" {
		//			return nil, errors.New("createAlgorithm failed")
		//		}
		//		option.AlgorithmId = result.AlgorithmId
		//	} else {
		//		return nil, errors.New("createAlgorithm failed")
		//	}
		//}

		// resource
		resp, err := o.resourceSpecs(ctx)
		if err != nil {
			return nil, err
		}

		id, err := matchResource(resp, option.ResourcesRequired)
		if err != nil {
			return nil, err
		}

		if id == nil {
			return nil, errors.New("resource id is nil")
		}

		option.ResourceId = *id

		// submit
		task, err := o.SubmitTask(ctx, option.ImageId, option.Cmd, option.Envs, option.Params, option.ResourceId, option.DatasetsId, option.AlgorithmId, option.TaskType)
		if err != nil {
			return nil, err
		}
		return task, nil
	}
	return nil, nil
}

func (o *OctopusHttp) Stop(ctx context.Context, id string) error {
	return nil
}

func (o *OctopusHttp) SubmitTask(ctx context.Context, imageId string, cmd string, envs []string, params []string, resourceId string, datasetsId string, algorithmId string, aiType string) (interface{}, error) {
	// octopus提交任务
	reqUrl := o.server + CreateTrainJobUrl

	token, err := o.token.Get()
	if err != nil {
		return nil, err
	}

	// python参数
	var prms []struct {
		Key   string `json:"key"`
		Value string `json:"value"`
	}
	for _, param := range params {
		var p struct {
			Key   string `json:"key"`
			Value string `json:"value"`
		}
		s := strings.Split(param, COMMA)
		p.Key = s[0]
		p.Value = s[1]
		prms = append(prms, p)
	}

	//环境变量
	envMap := make(map[string]string)
	for _, env := range envs {
		s := strings.Split(env, COMMA)
		envMap[s[0]] = s[1]
	}

	param := &omodel.CreateTrainJobParam{
		//DataSetId:        datasetsId,
		//DataSetVersion:   VERSION,
		//AlgorithmId:      algorithmId,
		//AlgorithmVersion: AlgorithmRecordOnlyVersion,
		Name:          TASK_NAME_PREFIX + UNDERSCORE + utils.RandomString(10),
		ImageId:       imageId,
		IsDistributed: false,
		ResourcePool:  o.resourcePool,
		Config: []*omodel.CreateTrainJobConf{
			{
				Command:               cmd,
				ResourceSpecId:        resourceId,
				MinFailedTaskCount:    1,
				MinSucceededTaskCount: 1,
				TaskNumber:            1,
				//Parameters:            prms,
				Envs: envMap,
			},
		},
	}

	resp := &entity.OctResp{}

	req := common.GetRestyRequest(common.TIMEOUT)
	_, err = req.
		SetHeader("Authorization", "Bearer "+token).
		SetQueryString("token=" + token).
		SetQueryString("addr=" + o.host).
		SetBody(param).
		SetResult(resp).
		Post(reqUrl)

	if err != nil {
		return nil, err
	}
	return resp, nil

}

func (o *OctopusHttp) createAlgorithm(ctx context.Context, param *omodel.CreateMyAlgorithmParam) (*entity.OctResp, error) {
	createAlgorithmUrl := o.server + CreateAlgorithm
	token, err := o.token.Get()
	if err != nil {
		return nil, err
	}
	resp := &entity.OctResp{}

	req := common.GetRestyRequest(common.TIMEOUT)
	_, err = req.
		SetHeader("Authorization", "Bearer "+token).
		SetQueryString("token=" + token).
		SetQueryString("addr=" + o.host).
		SetBody(param).
		SetResult(resp).
		Post(createAlgorithmUrl)

	if err != nil {
		return nil, err
	}

	return resp, nil
}

// collector
func (o *OctopusHttp) resourceSpecs(ctx context.Context) (*entity.OctResp, error) {
	resourcespecsUrl := o.server + ResourcespecsUrl
	token, err := o.token.Get()
	if err != nil {
		return nil, err
	}

	param := omodel.ResourceSpecParam{
		ResourcePool: o.resourcePool,
	}

	b, _ := json.Marshal(param)
	byt := bytes.NewBuffer(b)

	resp := &entity.OctResp{}

	req := common.GetRestyRequest(common.TIMEOUT)
	r, _ := http.NewRequest("GET", resourcespecsUrl, byt)
	req.RawRequest = r
	req.URL = resourcespecsUrl

	_, err = req.
		SetHeader("Content-Type", "application/json").
		SetQueryParam(Param_Token, token).
		SetQueryParam(Param_Addr, o.host).
		SetBody(byt).
		SetResult(resp).
		Send()

	if err != nil {
		return nil, err
	}

	return resp, nil
}

func (o *OctopusHttp) GetResourceStats(ctx context.Context) (*collector.ResourceStats, error) {
	resp, err := o.resourceSpecs(ctx)
	if err != nil {
		return nil, err
	}
	if resp.Code != http.StatusOK {
		if resp.Data != nil {
			marshal, err := json.Marshal(resp.Data)
			if err != nil {
				return nil, err
			}

			errormdl := &omodel.Error{}
			err = json.Unmarshal(marshal, errormdl)
			if err != nil {
				return nil, err
			}
			return nil, errors.New(errormdl.Message)
		}
	} else {
		if resp.Data != nil {
			spec := &entity.OctResourceSpecs{}
			marshal, err := json.Marshal(resp.Data)
			if err != nil {
				return nil, err
			}
			err = json.Unmarshal(marshal, spec)
			if err != nil {
				return nil, err
			}
		}
	}

	return nil, nil
}

func (o *OctopusHttp) GetDatasetsSpecs(ctx context.Context) ([]*collector.DatasetsSpecs, error) {
	return nil, nil
}

func (o *OctopusHttp) GetAlgorithms(ctx context.Context) ([]*collector.Algorithm, error) {
	return nil, errors.New(NotImplementError)
}

func (o *OctopusHttp) GetTrainingTaskLog(ctx context.Context, taskId string, instanceNum string) (string, error) {
	taskDetailsUrl := o.server + TrainJobLog
	token, err := o.token.Get()
	if err != nil {
		return "", err
	}

	param := omodel.TrainJobLog{
		JobId: taskId,
	}

	b, _ := json.Marshal(param)
	byt := bytes.NewBuffer(b)

	resp := &entity.OctResp{}

	req := common.GetRestyRequest(common.TIMEOUT)
	r, _ := http.NewRequest("GET", taskDetailsUrl, byt)
	req.RawRequest = r
	req.URL = taskDetailsUrl

	_, err = req.
		SetHeader("Content-Type", "application/json").
		SetQueryParam(Param_Token, token).
		SetQueryParam(Param_Addr, o.host).
		SetBody(byt).
		SetResult(resp).
		Send()

	if err != nil {
		return "", errors.New("failed to invoke taskDetails")
	}

	if resp.Code != http.StatusOK {
		return "", errors.New("failed to invoke taskDetails")
	}

	var log string
	marshal, err := json.Marshal(resp.Data)
	if err != nil {
		return "", err
	}
	log = string(marshal)

	if strings.Contains(log, "404 Not Found") || log == "" {
		log = "waiting for logs..."
	}

	return log, nil
}

func (o *OctopusHttp) GetTrainingTask(ctx context.Context, taskId string) (*collector.Task, error) {
	if taskId == "" {
		return nil, errors.New("empty taskId")
	}

	resp, err := o.getTrainingTask(ctx, taskId)
	if err != nil {
		return nil, err
	}

	if resp.Code != http.StatusOK {
		if resp.Data != nil {
			marshal, err := json.Marshal(resp.Data)
			if err != nil {
				return nil, err
			}

			errormdl := &omodel.Error{}
			err = json.Unmarshal(marshal, errormdl)
			if err != nil {
				return nil, err
			}
			return nil, errors.New(errormdl.Message)
		}
	} else {
		if resp.Data != nil {
			job := &entity.OctTrainJob{}
			marshal, err := json.Marshal(resp.Data)
			if err != nil {
				return nil, err
			}
			err = json.Unmarshal(marshal, job)
			if err != nil {
				return nil, err
			}

			var task collector.Task
			task.Id = job.TrainJob.Id
			if job.TrainJob.StartedAt != 0 {
				task.Start = time.Unix(int64(job.TrainJob.StartedAt), 0).Format(constants.Layout)
			}
			if job.TrainJob.CompletedAt != 0 {
				task.End = time.Unix(int64(job.TrainJob.CompletedAt), 0).Format(constants.Layout)
			}
			switch job.TrainJob.Status {
			case "succeeded":
				task.Status = constants.Completed
			case "failed":
				task.Status = constants.Failed
			case "running":
				task.Status = constants.Running
			case "stopped":
				task.Status = constants.Stopped
			case "pending":
				task.Status = constants.Pending
			default:
				task.Status = "undefined"
			}

			return &task, nil
		}
	}
	return nil, errors.New("failed to get trainjob")
}

func (o *OctopusHttp) getTrainingTask(ctx context.Context, taskId string) (*entity.OctResp, error) {
	taskDetailsUrl := o.server + TrainJobDetail
	token, err := o.token.Get()
	if err != nil {
		return nil, err
	}

	param := omodel.TrainJobDetailParam{
		JobId: taskId,
	}

	b, _ := json.Marshal(param)
	byt := bytes.NewBuffer(b)

	resp := &entity.OctResp{}

	req := common.GetRestyRequest(common.TIMEOUT)
	r, _ := http.NewRequest("GET", taskDetailsUrl, byt)
	req.RawRequest = r
	req.URL = taskDetailsUrl

	_, err = req.
		SetHeader("Content-Type", "application/json").
		SetQueryParam(Param_Token, token).
		SetQueryParam(Param_Addr, o.host).
		SetBody(byt).
		SetResult(resp).
		Send()

	if err != nil {
		return nil, errors.New("failed to invoke taskDetails")
	}

	return resp, nil
}

func (o *OctopusHttp) DownloadAlgorithmCode(ctx context.Context, resourceType string, card string, taskType string, dataset string, algorithm string) (string, error) {
	return "", errors.New(NotImplementError)
}

func (o *OctopusHttp) UploadAlgorithmCode(ctx context.Context, resourceType string, card string, taskType string, dataset string, algorithm string, code string) error {
	return nil
}

func (o *OctopusHttp) GetComputeCards(ctx context.Context) ([]string, error) {
	return nil, errors.New(NotImplementError)
}

func (o *OctopusHttp) GetUserBalance(ctx context.Context) (float64, error) {
	return 0, errors.New(NotImplementError)
}

func (o *OctopusHttp) GetResourceSpecs(ctx context.Context, resrcType string) (*collector.ResourceSpec, error) {
	resp, err := o.resourceSpecs(ctx)
	if err != nil {
		return nil, err
	}

	res := &collector.ResourceSpec{
		ClusterId: strconv.FormatInt(o.participantId, 10),
		Tag:       resrcType,
	}

	if resp.Code != http.StatusOK {
		if resp.Data != nil {
			marshal, err := json.Marshal(resp.Data)
			if err != nil {
				return nil, err
			}

			errormdl := &omodel.Error{}
			err = json.Unmarshal(marshal, errormdl)
			if err != nil {
				return nil, err
			}
			return nil, errors.New(errormdl.Message)
		}
	} else {
		if resp.Data != nil {
			specs := &entity.OctResourceSpecs{}
			marshal, err := json.Marshal(resp.Data)
			if err != nil {
				return nil, err
			}
			err = json.Unmarshal(marshal, specs)
			if err != nil {
				return nil, err
			}
			clusterResources, err := genSpecs(specs, resrcType, nil)
			if err != nil {
				return nil, err
			}
			res.Resources = clusterResources
		}
	}

	return res, nil
}

func findSpecId(cType string, cNum string, s *omodel.Spec, resourcesRequired []map[string]interface{}) (*string, error) {
	var id string
	for _, res := range resourcesRequired {
		//typeName, ok := res["type"]
		//if !ok {
		//	continue
		//}
		name, ok := res["name"]
		if !ok {
			continue
		}
		if str, ok := name.(string); ok {
			name = strings.ToLower(str)
		} else {
			continue
		}

		num, ok := res["number"]
		if !ok {
			continue
		}
		if str, ok := num.(string); ok {
			num = strings.ToLower(str)
		} else {
			continue
		}

		if cType == name && cNum == num {
			id = s.Id
			return &id, nil
		}
	}

	return nil, nil
}

func matchResource(resp *entity.OctResp, resourcesRequired []map[string]interface{}) (*string, error) {
	if resp.Code != http.StatusOK {
		if resp.Data != nil {
			marshal, err := json.Marshal(resp.Data)
			if err != nil {
				return nil, err
			}

			errormdl := &omodel.Error{}
			err = json.Unmarshal(marshal, errormdl)
			if err != nil {
				return nil, err
			}
			return nil, errors.New(errormdl.Message)
		}
	} else {
		if resp.Data != nil {
			spec := &entity.OctResourceSpecs{}
			marshal, err := json.Marshal(resp.Data)
			if err != nil {
				return nil, err
			}
			err = json.Unmarshal(marshal, spec)
			if err != nil {
				return nil, err
			}

			res, err := genSpecs(spec, "Train", resourcesRequired)
			if err != nil {
				return nil, err
			}

			if len(res) != 1 {
				return nil, errors.New("resource not found")
			}

			if str, ok := res[0].(*string); ok {
				return str, nil
			}
		}
	}

	return nil, errors.New("matchResource failed")
}

func genSpecs(specs *entity.OctResourceSpecs, resrcType string, resourcesRequired []map[string]interface{}) ([]interface{}, error) {
	res := make([]interface{}, 0)
	if resrcType == "Inference" {
		return res, nil
	} else if resrcType == "Train" {
		if specs.MapResourceSpecIdList.Train.ResourceSpecs == nil {
			return res, nil
		} else {
			for _, s := range specs.MapResourceSpecIdList.Train.ResourceSpecs {
				spec := &omodel.Spec{}
				marshal, err := json.Marshal(s)
				if err != nil {
					return nil, err
				}
				err = json.Unmarshal(marshal, spec)
				if err != nil {
					return nil, err
				}

				resType, err := chooseResourceType(spec, resourcesRequired)
				if err != nil {
					return nil, err
				}
				if resType == nil {
					continue
				}
				res = append(res, resType)
			}
		}
	}

	return res, nil
}

func chooseResourceType(spec *omodel.Spec, resourcesRequired []map[string]interface{}) (interface{}, error) {
	if spec.ResourceQuantity.NvidiaA100 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "NvidiaA100")
		if err != nil {
			return nil, err
		}

		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.NvidiaA100, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.NvidiaA100, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}

		return cres, nil
	} else if spec.ResourceQuantity.NvidiaA10080G != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "NvidiaA10080G")
		if err != nil {
			return nil, err
		}

		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.NvidiaA10080G, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.NvidiaA10080G, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.NvidiaA10080G, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.MrV100 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "MrV100")
		if err != nil {
			return nil, err
		}

		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.MrV100, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.MrV100, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.MrV100, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.BiV100 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "BiV100")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.BiV100, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.BiV100, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.BiV100, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.MRV50 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "MRV50")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.MRV50, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.MRV50, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.MRV50, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.BIV100 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "NvidiaA100")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.NvidiaA100, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.NvidiaA100, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.NvidiaA100, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.BIV150 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "BIV150")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.BIV150, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.BIV150, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.BIV150, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.MRV100 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "MRV100")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.MRV100, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.MRV100, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.MRV100, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.CambriconComMlu != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "CambriconComMlu")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.CambriconComMlu, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.CambriconComMlu, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.CambriconComMlu, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.HygonComDcu != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "HygonComDcu")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.HygonComDcu, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.HygonComDcu, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.HygonComDcu, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.HuaweiComAscend910 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "HuaweiComAscend910")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.HuaweiComAscend910, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.HuaweiComAscend910, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.HuaweiComAscend910, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.EnflameComGcu != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "EnflameComGcu")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.EnflameComGcu, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.EnflameComGcu, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.EnflameComGcu, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.MXN260 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "MXN260")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.MXN260, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.MXN260, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.MXN260, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.NvidiaV100 != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "NvidiaV100")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.NvidiaV100, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.NvidiaV100, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.NvidiaV100, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	} else if spec.ResourceQuantity.MetaxTechComGpu != "" {
		tag, err := common2.GetJSONTag(spec.ResourceQuantity, "MetaxTechComGpu")
		if err != nil {
			return nil, err
		}
		var cres interface{}
		if resourcesRequired != nil {
			id, err := findSpecId(tag, spec.ResourceQuantity.MetaxTechComGpu, spec, resourcesRequired)
			if err != nil {
				return nil, err
			}
			if id != nil {
				cres = id
			}
		} else {
			res, err := genClusterResources(tag, spec.ResourceQuantity.MetaxTechComGpu, spec)
			if err != nil {
				return nil, err
			}
			cres = res
		}
		//cres, err := genClusterResources(tag, spec.ResourceQuantity.MetaxTechComGpu, spec)
		//if err != nil {
		//	return nil, err
		//}
		return cres, nil
	}

	return nil, nil
}

func genClusterResources(cType string, cNum string, s *omodel.Spec) (*collector.ClusterResource, error) {
	cres := &collector.ClusterResource{}
	bres := make([]*collector.Usage, 0)

	var cardNum int64
	var cpuCore int64
	var memGi int64

	cardNum, err := strconv.ParseInt(cNum, 10, 64)
	if err != nil {
		cardNum = 0
	}
	cpuCore, err = strconv.ParseInt(s.ResourceQuantity.Cpu, 10, 64)
	if err != nil {
		cpuCore = 0
	}

	if s.ResourceQuantity.Memory != "" {
		gi := strings.Split(s.ResourceQuantity.Memory, Gi)
		if len(gi) != 2 {
			return nil, fmt.Errorf("s.ResourceQuantity.Memory convert error: %s", s.ResourceQuantity.Memory)
		}

		mGi, err := strconv.ParseInt(gi[0], 10, 64)
		if err != nil {
			memGi = 0
		} else {
			memGi = mGi
		}
	} else {
		memGi = 0
	}

	card := &collector.Usage{
		Type:      ComputeSourceToCardType[cType],
		Name:      strings.ToUpper(cType),
		Total:     &collector.UnitValue{Unit: NUMBER, Value: cardNum},
		Available: &collector.UnitValue{Unit: NUMBER, Value: cardNum},
	}
	cpu := &collector.Usage{
		Type:      strings.ToUpper(CPU),
		Name:      strings.ToUpper(CPU),
		Total:     &collector.UnitValue{Unit: CPUCORE, Value: cpuCore},
		Available: &collector.UnitValue{Unit: CPUCORE, Value: cpuCore},
	}
	mem := &collector.Usage{
		Type:      strings.ToUpper(MEMORY),
		Name:      strings.ToUpper(RAM),
		Total:     &collector.UnitValue{Unit: GIGABYTE, Value: memGi},
		Available: &collector.UnitValue{Unit: GIGABYTE, Value: memGi},
	}

	bres = append(bres, cpu)
	bres = append(bres, mem)

	cres.Resource = card
	cres.BaseResources = bres

	return cres, nil
}

// inference
func (o *OctopusHttp) GetClusterInferUrl(ctx context.Context, option *option.InferOption) (*inference.ClusterInferUrl, error) {
	return nil, errors.New(NotImplementError)
}

func (o *OctopusHttp) GetInferDeployInstanceList(ctx context.Context) ([]*inference.DeployInstance, error) {
	return nil, errors.New(NotImplementError)
}

func (o *OctopusHttp) StartInferDeployInstance(ctx context.Context, id string) bool {
	return false
}

func (o *OctopusHttp) StopInferDeployInstance(ctx context.Context, id string) bool {
	return false
}

func (o *OctopusHttp) GetInferDeployInstance(ctx context.Context, id string) (*inference.DeployInstance, error) {
	return nil, errors.New(NotImplementError)
}

func (o *OctopusHttp) CreateInferDeployInstance(ctx context.Context, option *option.InferOption) (string, error) {
	return "", errors.New(NotImplementError)
}

func (o *OctopusHttp) CheckModelExistence(ctx context.Context, modelName string, modelType string) bool {
	return false
}

func (o *OctopusHttp) GetImageInferResult(ctx context.Context, url string, file multipart.File, fileName string) (string, error) {
	return "", errors.New(NotImplementError)
}