Browse Source

update

tags/v1.22.1.2
liuzx 3 years ago
parent
commit
303db82ac0
4 changed files with 87 additions and 87 deletions
  1. +7
    -1
      models/cloudbrain.go
  2. +2
    -0
      modules/auth/modelarts.go
  3. +53
    -52
      modules/modelarts/modelarts.go
  4. +25
    -34
      routers/repo/modelarts.go

+ 7
- 1
models/cloudbrain.go View File

@@ -30,6 +30,7 @@ const (
JobTypeSnn4imagenet JobType = "SNN4IMAGENET"
JobTypeBrainScore JobType = "BRAINSCORE"
JobTypeTrain JobType = "TRAIN"
JobTypeInference JobType = "INFERENCE"

//notebook
ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中
@@ -105,7 +106,7 @@ type Cloudbrain struct {
ComputeResource string //计算资源,例如npu
EngineID int64 //引擎id

TrainUrl string //输出的obs路径
TrainUrl string //输出模型的obs路径
BranchName string //分支名称
Parameters string //传给modelarts的param参数
BootFile string //启动文件
@@ -119,6 +120,11 @@ type Cloudbrain struct {
EngineName string //引擎名称
TotalVersionCount int //任务的所有版本数量,包括删除的

ModelName string //模型名称
ModelVersion string //模型版本
CkptName string //权重文件名称
ResultUrl string //推理结果的obs路径

User *User `xorm:"-"`
Repo *Repository `xorm:"-"`
}


+ 2
- 0
modules/auth/modelarts.go View File

@@ -63,6 +63,8 @@ type CreateModelArtsInferenceJobForm struct {
FlavorName string `form:"flaver_names" binding:"Required"`
EngineName string `form:"engine_names" binding:"Required"`
TrainUrl string `form:"train_url" binding:"Required"`
ModelName string `form:"model_name" binding:"Required"`
ModelVersion string `form:"model_version" binding:"Required"`
CkptName string `form:"ckpt_name" binding:"Required"`
}



+ 53
- 52
modules/modelarts/modelarts.go View File

@@ -46,6 +46,8 @@ const (
Lines = 500
TrainUrl = "train_url"
DataUrl = "data_url"
ResultUrl = "result_url"
CkptName = "ckptName"
PerPage = 10
IsLatestVersion = "1"
NotLatestVersion = "0"
@@ -119,30 +121,29 @@ type GenerateTrainJobVersionReq struct {
}

type GenerateInferenceJobReq struct {
JobName string
Uuid string
Description string
CodeObsPath string
BootFile string
BootFileUrl string
DataUrl string
TrainUrl string
FlavorCode string
LogUrl string
PoolID string
WorkServerNumber int
EngineID int64
Parameters []models.Parameter
CommitID string
IsLatestVersion string
Params string
BranchName string
PreVersionId int64
PreVersionName string
FlavorName string
VersionCount int
EngineName string
TotalVersionCount int
JobName string
Uuid string
Description string
CodeObsPath string
BootFile string
BootFileUrl string
DataUrl string
TrainUrl string
FlavorCode string
LogUrl string
PoolID string
WorkServerNumber int
EngineID int64
Parameters []models.Parameter
CommitID string
Params string
BranchName string
FlavorName string
EngineName string
ModelName string
ModelVersion string
CkptName string
ResultUrl string
}

type VersionInfo struct {
@@ -510,34 +511,34 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e
}

err = models.CreateCloudbrain(&models.Cloudbrain{
Status: TransTrainJobStatus(jobResult.Status),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: strconv.FormatInt(jobResult.JobID, 10),
JobName: req.JobName,
JobType: string(models.JobTypeTrain),
Type: models.TypeCloudBrainTwo,
VersionID: jobResult.VersionID,
VersionName: jobResult.VersionName,
Uuid: req.Uuid,
DatasetName: attach.Name,
CommitID: req.CommitID,
IsLatestVersion: req.IsLatestVersion,
ComputeResource: NPUResource,
EngineID: req.EngineID,
TrainUrl: req.TrainUrl,
BranchName: req.BranchName,
Parameters: req.Params,
BootFile: req.BootFile,
DataUrl: req.DataUrl,
LogUrl: req.LogUrl,
FlavorCode: req.FlavorCode,
Description: req.Description,
WorkServerNumber: req.WorkServerNumber,
FlavorName: req.FlavorName,
EngineName: req.EngineName,
VersionCount: req.VersionCount,
TotalVersionCount: req.TotalVersionCount,
Status: TransTrainJobStatus(jobResult.Status),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: strconv.FormatInt(jobResult.JobID, 10),
JobName: req.JobName,
JobType: string(models.JobTypeInference),
Type: models.TypeCloudBrainTwo,
VersionID: jobResult.VersionID,
VersionName: jobResult.VersionName,
Uuid: req.Uuid,
DatasetName: attach.Name,
CommitID: req.CommitID,
EngineID: req.EngineID,
TrainUrl: req.TrainUrl,
BranchName: req.BranchName,
Parameters: req.Params,
BootFile: req.BootFile,
DataUrl: req.DataUrl,
LogUrl: req.LogUrl,
FlavorCode: req.FlavorCode,
Description: req.Description,
WorkServerNumber: req.WorkServerNumber,
FlavorName: req.FlavorName,
EngineName: req.EngineName,
ModelName: req.ModelName,
ModelVersion: req.ModelVersion,
CkptName: req.CkptName,
ResultUrl: req.ResultUrl,
})

if err != nil {


+ 25
- 34
routers/repo/modelarts.go View File

@@ -1259,7 +1259,7 @@ func TrainJobShow(ctx *context.Context) {
ctx.Data["canNewJob"] = canNewJob

//将运行参数转化为epoch_size = 3, device_target = Ascend的格式
for i, task := range VersionListTasks {
for i, _ := range VersionListTasks {

var parameters models.Parameters

@@ -1280,9 +1280,6 @@ func TrainJobShow(ctx *context.Context) {
} else {
VersionListTasks[i].Parameters = ""
}

VersionListTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
VersionListTasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain)
}

pager := context.NewPagination(VersionListCount, setting.UI.IssuePagingNum, page, 5)
@@ -1518,7 +1515,6 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
flavorCode := form.Flavor
params := form.Params
poolID := form.PoolID
isSaveParam := form.IsSaveParam
repo := ctx.Repo.Repository
codeLocalPath := setting.JobPath + jobName + modelarts.CodePath
codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath
@@ -1526,9 +1522,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath
dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
branch_name := form.BranchName
isLatestVersion := modelarts.IsLatestVersion
FlavorName := form.FlavorName
VersionCount := modelarts.VersionCount
EngineName := form.EngineName
trainUrl := form.TrainUrl
ckptName := form.CkptName
@@ -1608,11 +1602,11 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
var parameters models.Parameters
param := make([]models.Parameter, 0)
param = append(param, models.Parameter{
Label: modelarts.TrainUrl,
Value: trainUrl,
Label: modelarts.ResultUrl,
Value: "s3:/" + resultObsPath,
}, models.Parameter{
Label: modelarts.DataUrl,
Value: dataPath,
Label: modelarts.CkptName,
Value: ckptName,
})
if len(params) != 0 {
err := json.Unmarshal([]byte(params), &parameters)
@@ -1634,28 +1628,25 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
}

req := &modelarts.GenerateInferenceJobReq{
JobName: jobName,
DataUrl: dataPath,
Description: description,
CodeObsPath: codeObsPath,
BootFileUrl: codeObsPath + bootFile,
BootFile: bootFile,
TrainUrl: trainUrl,
FlavorCode: flavorCode,
WorkServerNumber: workServerNumber,
EngineID: int64(engineID),
LogUrl: logObsPath,
PoolID: poolID,
Uuid: uuid,
Parameters: parameters.Parameter,
CommitID: commitID,
IsLatestVersion: isLatestVersion,
BranchName: branch_name,
Params: form.Params,
FlavorName: FlavorName,
EngineName: EngineName,
VersionCount: VersionCount,
TotalVersionCount: modelarts.TotalVersionCount,
JobName: jobName,
DataUrl: dataPath,
Description: description,
CodeObsPath: codeObsPath,
BootFileUrl: codeObsPath + bootFile,
BootFile: bootFile,
TrainUrl: trainUrl,
FlavorCode: flavorCode,
WorkServerNumber: workServerNumber,
EngineID: int64(engineID),
LogUrl: logObsPath,
PoolID: poolID,
Uuid: uuid,
Parameters: param, //modelarts训练时用到
CommitID: commitID,
BranchName: branch_name,
Params: form.Params,
FlavorName: FlavorName,
EngineName: EngineName,
}

//将params转换Parameters.Parameter,出错时返回给前端
@@ -1665,7 +1656,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
return
}

err = modelarts.GenerateTrainJob(ctx, req)
err = modelarts.GenerateInferenceJob(ctx, req)
if err != nil {
log.Error("GenerateTrainJob failed:%v", err.Error())
inferenceJobErrorNewDataPrepare(ctx, form)


Loading…
Cancel
Save