Browse Source

update

tags/v1.22.1.2
liuzx 3 years ago
parent
commit
303db82ac0
4 changed files with 87 additions and 87 deletions
  1. +7
    -1
      models/cloudbrain.go
  2. +2
    -0
      modules/auth/modelarts.go
  3. +53
    -52
      modules/modelarts/modelarts.go
  4. +25
    -34
      routers/repo/modelarts.go

+ 7
- 1
models/cloudbrain.go View File

@@ -30,6 +30,7 @@ const (
JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeSnn4imagenet JobType = "SNN4IMAGENET"
JobTypeBrainScore JobType = "BRAINSCORE" JobTypeBrainScore JobType = "BRAINSCORE"
JobTypeTrain JobType = "TRAIN" JobTypeTrain JobType = "TRAIN"
JobTypeInference JobType = "INFERENCE"


//notebook //notebook
ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中
@@ -105,7 +106,7 @@ type Cloudbrain struct {
ComputeResource string //计算资源,例如npu ComputeResource string //计算资源,例如npu
EngineID int64 //引擎id EngineID int64 //引擎id


TrainUrl string //输出的obs路径
TrainUrl string //输出模型的obs路径
BranchName string //分支名称 BranchName string //分支名称
Parameters string //传给modelarts的param参数 Parameters string //传给modelarts的param参数
BootFile string //启动文件 BootFile string //启动文件
@@ -119,6 +120,11 @@ type Cloudbrain struct {
EngineName string //引擎名称 EngineName string //引擎名称
TotalVersionCount int //任务的所有版本数量,包括删除的 TotalVersionCount int //任务的所有版本数量,包括删除的


ModelName string //模型名称
ModelVersion string //模型版本
CkptName string //权重文件名称
ResultUrl string //推理结果的obs路径

User *User `xorm:"-"` User *User `xorm:"-"`
Repo *Repository `xorm:"-"` Repo *Repository `xorm:"-"`
} }


+ 2
- 0
modules/auth/modelarts.go View File

@@ -63,6 +63,8 @@ type CreateModelArtsInferenceJobForm struct {
FlavorName string `form:"flaver_names" binding:"Required"` FlavorName string `form:"flaver_names" binding:"Required"`
EngineName string `form:"engine_names" binding:"Required"` EngineName string `form:"engine_names" binding:"Required"`
TrainUrl string `form:"train_url" binding:"Required"` TrainUrl string `form:"train_url" binding:"Required"`
ModelName string `form:"model_name" binding:"Required"`
ModelVersion string `form:"model_version" binding:"Required"`
CkptName string `form:"ckpt_name" binding:"Required"` CkptName string `form:"ckpt_name" binding:"Required"`
} }




+ 53
- 52
modules/modelarts/modelarts.go View File

@@ -46,6 +46,8 @@ const (
Lines = 500 Lines = 500
TrainUrl = "train_url" TrainUrl = "train_url"
DataUrl = "data_url" DataUrl = "data_url"
ResultUrl = "result_url"
CkptName = "ckptName"
PerPage = 10 PerPage = 10
IsLatestVersion = "1" IsLatestVersion = "1"
NotLatestVersion = "0" NotLatestVersion = "0"
@@ -119,30 +121,29 @@ type GenerateTrainJobVersionReq struct {
} }


type GenerateInferenceJobReq struct { type GenerateInferenceJobReq struct {
JobName string
Uuid string
Description string
CodeObsPath string
BootFile string
BootFileUrl string
DataUrl string
TrainUrl string
FlavorCode string
LogUrl string
PoolID string
WorkServerNumber int
EngineID int64
Parameters []models.Parameter
CommitID string
IsLatestVersion string
Params string
BranchName string
PreVersionId int64
PreVersionName string
FlavorName string
VersionCount int
EngineName string
TotalVersionCount int
JobName string
Uuid string
Description string
CodeObsPath string
BootFile string
BootFileUrl string
DataUrl string
TrainUrl string
FlavorCode string
LogUrl string
PoolID string
WorkServerNumber int
EngineID int64
Parameters []models.Parameter
CommitID string
Params string
BranchName string
FlavorName string
EngineName string
ModelName string
ModelVersion string
CkptName string
ResultUrl string
} }


type VersionInfo struct { type VersionInfo struct {
@@ -510,34 +511,34 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e
} }


err = models.CreateCloudbrain(&models.Cloudbrain{ err = models.CreateCloudbrain(&models.Cloudbrain{
Status: TransTrainJobStatus(jobResult.Status),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: strconv.FormatInt(jobResult.JobID, 10),
JobName: req.JobName,
JobType: string(models.JobTypeTrain),
Type: models.TypeCloudBrainTwo,
VersionID: jobResult.VersionID,
VersionName: jobResult.VersionName,
Uuid: req.Uuid,
DatasetName: attach.Name,
CommitID: req.CommitID,
IsLatestVersion: req.IsLatestVersion,
ComputeResource: NPUResource,
EngineID: req.EngineID,
TrainUrl: req.TrainUrl,
BranchName: req.BranchName,
Parameters: req.Params,
BootFile: req.BootFile,
DataUrl: req.DataUrl,
LogUrl: req.LogUrl,
FlavorCode: req.FlavorCode,
Description: req.Description,
WorkServerNumber: req.WorkServerNumber,
FlavorName: req.FlavorName,
EngineName: req.EngineName,
VersionCount: req.VersionCount,
TotalVersionCount: req.TotalVersionCount,
Status: TransTrainJobStatus(jobResult.Status),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: strconv.FormatInt(jobResult.JobID, 10),
JobName: req.JobName,
JobType: string(models.JobTypeInference),
Type: models.TypeCloudBrainTwo,
VersionID: jobResult.VersionID,
VersionName: jobResult.VersionName,
Uuid: req.Uuid,
DatasetName: attach.Name,
CommitID: req.CommitID,
EngineID: req.EngineID,
TrainUrl: req.TrainUrl,
BranchName: req.BranchName,
Parameters: req.Params,
BootFile: req.BootFile,
DataUrl: req.DataUrl,
LogUrl: req.LogUrl,
FlavorCode: req.FlavorCode,
Description: req.Description,
WorkServerNumber: req.WorkServerNumber,
FlavorName: req.FlavorName,
EngineName: req.EngineName,
ModelName: req.ModelName,
ModelVersion: req.ModelVersion,
CkptName: req.CkptName,
ResultUrl: req.ResultUrl,
}) })


if err != nil { if err != nil {


+ 25
- 34
routers/repo/modelarts.go View File

@@ -1259,7 +1259,7 @@ func TrainJobShow(ctx *context.Context) {
ctx.Data["canNewJob"] = canNewJob ctx.Data["canNewJob"] = canNewJob


//将运行参数转化为epoch_size = 3, device_target = Ascend的格式 //将运行参数转化为epoch_size = 3, device_target = Ascend的格式
for i, task := range VersionListTasks {
for i, _ := range VersionListTasks {


var parameters models.Parameters var parameters models.Parameters


@@ -1280,9 +1280,6 @@ func TrainJobShow(ctx *context.Context) {
} else { } else {
VersionListTasks[i].Parameters = "" VersionListTasks[i].Parameters = ""
} }

VersionListTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
VersionListTasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain)
} }


pager := context.NewPagination(VersionListCount, setting.UI.IssuePagingNum, page, 5) pager := context.NewPagination(VersionListCount, setting.UI.IssuePagingNum, page, 5)
@@ -1518,7 +1515,6 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
flavorCode := form.Flavor flavorCode := form.Flavor
params := form.Params params := form.Params
poolID := form.PoolID poolID := form.PoolID
isSaveParam := form.IsSaveParam
repo := ctx.Repo.Repository repo := ctx.Repo.Repository
codeLocalPath := setting.JobPath + jobName + modelarts.CodePath codeLocalPath := setting.JobPath + jobName + modelarts.CodePath
codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath
@@ -1526,9 +1522,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath
dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
branch_name := form.BranchName branch_name := form.BranchName
isLatestVersion := modelarts.IsLatestVersion
FlavorName := form.FlavorName FlavorName := form.FlavorName
VersionCount := modelarts.VersionCount
EngineName := form.EngineName EngineName := form.EngineName
trainUrl := form.TrainUrl trainUrl := form.TrainUrl
ckptName := form.CkptName ckptName := form.CkptName
@@ -1608,11 +1602,11 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
var parameters models.Parameters var parameters models.Parameters
param := make([]models.Parameter, 0) param := make([]models.Parameter, 0)
param = append(param, models.Parameter{ param = append(param, models.Parameter{
Label: modelarts.TrainUrl,
Value: trainUrl,
Label: modelarts.ResultUrl,
Value: "s3:/" + resultObsPath,
}, models.Parameter{ }, models.Parameter{
Label: modelarts.DataUrl,
Value: dataPath,
Label: modelarts.CkptName,
Value: ckptName,
}) })
if len(params) != 0 { if len(params) != 0 {
err := json.Unmarshal([]byte(params), &parameters) err := json.Unmarshal([]byte(params), &parameters)
@@ -1634,28 +1628,25 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
} }


req := &modelarts.GenerateInferenceJobReq{ req := &modelarts.GenerateInferenceJobReq{
JobName: jobName,
DataUrl: dataPath,
Description: description,
CodeObsPath: codeObsPath,
BootFileUrl: codeObsPath + bootFile,
BootFile: bootFile,
TrainUrl: trainUrl,
FlavorCode: flavorCode,
WorkServerNumber: workServerNumber,
EngineID: int64(engineID),
LogUrl: logObsPath,
PoolID: poolID,
Uuid: uuid,
Parameters: parameters.Parameter,
CommitID: commitID,
IsLatestVersion: isLatestVersion,
BranchName: branch_name,
Params: form.Params,
FlavorName: FlavorName,
EngineName: EngineName,
VersionCount: VersionCount,
TotalVersionCount: modelarts.TotalVersionCount,
JobName: jobName,
DataUrl: dataPath,
Description: description,
CodeObsPath: codeObsPath,
BootFileUrl: codeObsPath + bootFile,
BootFile: bootFile,
TrainUrl: trainUrl,
FlavorCode: flavorCode,
WorkServerNumber: workServerNumber,
EngineID: int64(engineID),
LogUrl: logObsPath,
PoolID: poolID,
Uuid: uuid,
Parameters: param, //modelarts训练时用到
CommitID: commitID,
BranchName: branch_name,
Params: form.Params,
FlavorName: FlavorName,
EngineName: EngineName,
} }


//将params转换Parameters.Parameter,出错时返回给前端 //将params转换Parameters.Parameter,出错时返回给前端
@@ -1665,7 +1656,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
return return
} }


err = modelarts.GenerateTrainJob(ctx, req)
err = modelarts.GenerateInferenceJob(ctx, req)
if err != nil { if err != nil {
log.Error("GenerateTrainJob failed:%v", err.Error()) log.Error("GenerateTrainJob failed:%v", err.Error())
inferenceJobErrorNewDataPrepare(ctx, form) inferenceJobErrorNewDataPrepare(ctx, form)


Loading…
Cancel
Save