| @@ -30,6 +30,7 @@ const ( | |||||
| JobTypeSnn4imagenet JobType = "SNN4IMAGENET" | JobTypeSnn4imagenet JobType = "SNN4IMAGENET" | ||||
| JobTypeBrainScore JobType = "BRAINSCORE" | JobTypeBrainScore JobType = "BRAINSCORE" | ||||
| JobTypeTrain JobType = "TRAIN" | JobTypeTrain JobType = "TRAIN" | ||||
| JobTypeInference JobType = "INFERENCE" | |||||
| //notebook | //notebook | ||||
| ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 | ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 | ||||
| @@ -105,7 +106,7 @@ type Cloudbrain struct { | |||||
| ComputeResource string //计算资源,例如npu | ComputeResource string //计算资源,例如npu | ||||
| EngineID int64 //引擎id | EngineID int64 //引擎id | ||||
| TrainUrl string //输出的obs路径 | |||||
| TrainUrl string //输出模型的obs路径 | |||||
| BranchName string //分支名称 | BranchName string //分支名称 | ||||
| Parameters string //传给modelarts的param参数 | Parameters string //传给modelarts的param参数 | ||||
| BootFile string //启动文件 | BootFile string //启动文件 | ||||
| @@ -119,6 +120,11 @@ type Cloudbrain struct { | |||||
| EngineName string //引擎名称 | EngineName string //引擎名称 | ||||
| TotalVersionCount int //任务的所有版本数量,包括删除的 | TotalVersionCount int //任务的所有版本数量,包括删除的 | ||||
| ModelName string //模型名称 | |||||
| ModelVersion string //模型版本 | |||||
| CkptName string //权重文件名称 | |||||
| ResultUrl string //推理结果的obs路径 | |||||
| User *User `xorm:"-"` | User *User `xorm:"-"` | ||||
| Repo *Repository `xorm:"-"` | Repo *Repository `xorm:"-"` | ||||
| } | } | ||||
| @@ -63,6 +63,8 @@ type CreateModelArtsInferenceJobForm struct { | |||||
| FlavorName string `form:"flaver_names" binding:"Required"` | FlavorName string `form:"flaver_names" binding:"Required"` | ||||
| EngineName string `form:"engine_names" binding:"Required"` | EngineName string `form:"engine_names" binding:"Required"` | ||||
| TrainUrl string `form:"train_url" binding:"Required"` | TrainUrl string `form:"train_url" binding:"Required"` | ||||
| ModelName string `form:"model_name" binding:"Required"` | |||||
| ModelVersion string `form:"model_version" binding:"Required"` | |||||
| CkptName string `form:"ckpt_name" binding:"Required"` | CkptName string `form:"ckpt_name" binding:"Required"` | ||||
| } | } | ||||
| @@ -46,6 +46,8 @@ const ( | |||||
| Lines = 500 | Lines = 500 | ||||
| TrainUrl = "train_url" | TrainUrl = "train_url" | ||||
| DataUrl = "data_url" | DataUrl = "data_url" | ||||
| ResultUrl = "result_url" | |||||
| CkptName = "ckptName" | |||||
| PerPage = 10 | PerPage = 10 | ||||
| IsLatestVersion = "1" | IsLatestVersion = "1" | ||||
| NotLatestVersion = "0" | NotLatestVersion = "0" | ||||
| @@ -119,30 +121,29 @@ type GenerateTrainJobVersionReq struct { | |||||
| } | } | ||||
| type GenerateInferenceJobReq struct { | type GenerateInferenceJobReq struct { | ||||
| JobName string | |||||
| Uuid string | |||||
| Description string | |||||
| CodeObsPath string | |||||
| BootFile string | |||||
| BootFileUrl string | |||||
| DataUrl string | |||||
| TrainUrl string | |||||
| FlavorCode string | |||||
| LogUrl string | |||||
| PoolID string | |||||
| WorkServerNumber int | |||||
| EngineID int64 | |||||
| Parameters []models.Parameter | |||||
| CommitID string | |||||
| IsLatestVersion string | |||||
| Params string | |||||
| BranchName string | |||||
| PreVersionId int64 | |||||
| PreVersionName string | |||||
| FlavorName string | |||||
| VersionCount int | |||||
| EngineName string | |||||
| TotalVersionCount int | |||||
| JobName string | |||||
| Uuid string | |||||
| Description string | |||||
| CodeObsPath string | |||||
| BootFile string | |||||
| BootFileUrl string | |||||
| DataUrl string | |||||
| TrainUrl string | |||||
| FlavorCode string | |||||
| LogUrl string | |||||
| PoolID string | |||||
| WorkServerNumber int | |||||
| EngineID int64 | |||||
| Parameters []models.Parameter | |||||
| CommitID string | |||||
| Params string | |||||
| BranchName string | |||||
| FlavorName string | |||||
| EngineName string | |||||
| ModelName string | |||||
| ModelVersion string | |||||
| CkptName string | |||||
| ResultUrl string | |||||
| } | } | ||||
| type VersionInfo struct { | type VersionInfo struct { | ||||
| @@ -510,34 +511,34 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e | |||||
| } | } | ||||
| err = models.CreateCloudbrain(&models.Cloudbrain{ | err = models.CreateCloudbrain(&models.Cloudbrain{ | ||||
| Status: TransTrainJobStatus(jobResult.Status), | |||||
| UserID: ctx.User.ID, | |||||
| RepoID: ctx.Repo.Repository.ID, | |||||
| JobID: strconv.FormatInt(jobResult.JobID, 10), | |||||
| JobName: req.JobName, | |||||
| JobType: string(models.JobTypeTrain), | |||||
| Type: models.TypeCloudBrainTwo, | |||||
| VersionID: jobResult.VersionID, | |||||
| VersionName: jobResult.VersionName, | |||||
| Uuid: req.Uuid, | |||||
| DatasetName: attach.Name, | |||||
| CommitID: req.CommitID, | |||||
| IsLatestVersion: req.IsLatestVersion, | |||||
| ComputeResource: NPUResource, | |||||
| EngineID: req.EngineID, | |||||
| TrainUrl: req.TrainUrl, | |||||
| BranchName: req.BranchName, | |||||
| Parameters: req.Params, | |||||
| BootFile: req.BootFile, | |||||
| DataUrl: req.DataUrl, | |||||
| LogUrl: req.LogUrl, | |||||
| FlavorCode: req.FlavorCode, | |||||
| Description: req.Description, | |||||
| WorkServerNumber: req.WorkServerNumber, | |||||
| FlavorName: req.FlavorName, | |||||
| EngineName: req.EngineName, | |||||
| VersionCount: req.VersionCount, | |||||
| TotalVersionCount: req.TotalVersionCount, | |||||
| Status: TransTrainJobStatus(jobResult.Status), | |||||
| UserID: ctx.User.ID, | |||||
| RepoID: ctx.Repo.Repository.ID, | |||||
| JobID: strconv.FormatInt(jobResult.JobID, 10), | |||||
| JobName: req.JobName, | |||||
| JobType: string(models.JobTypeInference), | |||||
| Type: models.TypeCloudBrainTwo, | |||||
| VersionID: jobResult.VersionID, | |||||
| VersionName: jobResult.VersionName, | |||||
| Uuid: req.Uuid, | |||||
| DatasetName: attach.Name, | |||||
| CommitID: req.CommitID, | |||||
| EngineID: req.EngineID, | |||||
| TrainUrl: req.TrainUrl, | |||||
| BranchName: req.BranchName, | |||||
| Parameters: req.Params, | |||||
| BootFile: req.BootFile, | |||||
| DataUrl: req.DataUrl, | |||||
| LogUrl: req.LogUrl, | |||||
| FlavorCode: req.FlavorCode, | |||||
| Description: req.Description, | |||||
| WorkServerNumber: req.WorkServerNumber, | |||||
| FlavorName: req.FlavorName, | |||||
| EngineName: req.EngineName, | |||||
| ModelName: req.ModelName, | |||||
| ModelVersion: req.ModelVersion, | |||||
| CkptName: req.CkptName, | |||||
| ResultUrl: req.ResultUrl, | |||||
| }) | }) | ||||
| if err != nil { | if err != nil { | ||||
| @@ -1259,7 +1259,7 @@ func TrainJobShow(ctx *context.Context) { | |||||
| ctx.Data["canNewJob"] = canNewJob | ctx.Data["canNewJob"] = canNewJob | ||||
| //将运行参数转化为epoch_size = 3, device_target = Ascend的格式 | //将运行参数转化为epoch_size = 3, device_target = Ascend的格式 | ||||
| for i, task := range VersionListTasks { | |||||
| for i, _ := range VersionListTasks { | |||||
| var parameters models.Parameters | var parameters models.Parameters | ||||
| @@ -1280,9 +1280,6 @@ func TrainJobShow(ctx *context.Context) { | |||||
| } else { | } else { | ||||
| VersionListTasks[i].Parameters = "" | VersionListTasks[i].Parameters = "" | ||||
| } | } | ||||
| VersionListTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) | |||||
| VersionListTasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) | |||||
| } | } | ||||
| pager := context.NewPagination(VersionListCount, setting.UI.IssuePagingNum, page, 5) | pager := context.NewPagination(VersionListCount, setting.UI.IssuePagingNum, page, 5) | ||||
| @@ -1518,7 +1515,6 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference | |||||
| flavorCode := form.Flavor | flavorCode := form.Flavor | ||||
| params := form.Params | params := form.Params | ||||
| poolID := form.PoolID | poolID := form.PoolID | ||||
| isSaveParam := form.IsSaveParam | |||||
| repo := ctx.Repo.Repository | repo := ctx.Repo.Repository | ||||
| codeLocalPath := setting.JobPath + jobName + modelarts.CodePath | codeLocalPath := setting.JobPath + jobName + modelarts.CodePath | ||||
| codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath | codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath | ||||
| @@ -1526,9 +1522,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference | |||||
| logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath | logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath | ||||
| dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" | dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" | ||||
| branch_name := form.BranchName | branch_name := form.BranchName | ||||
| isLatestVersion := modelarts.IsLatestVersion | |||||
| FlavorName := form.FlavorName | FlavorName := form.FlavorName | ||||
| VersionCount := modelarts.VersionCount | |||||
| EngineName := form.EngineName | EngineName := form.EngineName | ||||
| trainUrl := form.TrainUrl | trainUrl := form.TrainUrl | ||||
| ckptName := form.CkptName | ckptName := form.CkptName | ||||
| @@ -1608,11 +1602,11 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference | |||||
| var parameters models.Parameters | var parameters models.Parameters | ||||
| param := make([]models.Parameter, 0) | param := make([]models.Parameter, 0) | ||||
| param = append(param, models.Parameter{ | param = append(param, models.Parameter{ | ||||
| Label: modelarts.TrainUrl, | |||||
| Value: trainUrl, | |||||
| Label: modelarts.ResultUrl, | |||||
| Value: "s3:/" + resultObsPath, | |||||
| }, models.Parameter{ | }, models.Parameter{ | ||||
| Label: modelarts.DataUrl, | |||||
| Value: dataPath, | |||||
| Label: modelarts.CkptName, | |||||
| Value: ckptName, | |||||
| }) | }) | ||||
| if len(params) != 0 { | if len(params) != 0 { | ||||
| err := json.Unmarshal([]byte(params), ¶meters) | err := json.Unmarshal([]byte(params), ¶meters) | ||||
| @@ -1634,28 +1628,25 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference | |||||
| } | } | ||||
| req := &modelarts.GenerateInferenceJobReq{ | req := &modelarts.GenerateInferenceJobReq{ | ||||
| JobName: jobName, | |||||
| DataUrl: dataPath, | |||||
| Description: description, | |||||
| CodeObsPath: codeObsPath, | |||||
| BootFileUrl: codeObsPath + bootFile, | |||||
| BootFile: bootFile, | |||||
| TrainUrl: trainUrl, | |||||
| FlavorCode: flavorCode, | |||||
| WorkServerNumber: workServerNumber, | |||||
| EngineID: int64(engineID), | |||||
| LogUrl: logObsPath, | |||||
| PoolID: poolID, | |||||
| Uuid: uuid, | |||||
| Parameters: parameters.Parameter, | |||||
| CommitID: commitID, | |||||
| IsLatestVersion: isLatestVersion, | |||||
| BranchName: branch_name, | |||||
| Params: form.Params, | |||||
| FlavorName: FlavorName, | |||||
| EngineName: EngineName, | |||||
| VersionCount: VersionCount, | |||||
| TotalVersionCount: modelarts.TotalVersionCount, | |||||
| JobName: jobName, | |||||
| DataUrl: dataPath, | |||||
| Description: description, | |||||
| CodeObsPath: codeObsPath, | |||||
| BootFileUrl: codeObsPath + bootFile, | |||||
| BootFile: bootFile, | |||||
| TrainUrl: trainUrl, | |||||
| FlavorCode: flavorCode, | |||||
| WorkServerNumber: workServerNumber, | |||||
| EngineID: int64(engineID), | |||||
| LogUrl: logObsPath, | |||||
| PoolID: poolID, | |||||
| Uuid: uuid, | |||||
| Parameters: param, //modelarts训练时用到 | |||||
| CommitID: commitID, | |||||
| BranchName: branch_name, | |||||
| Params: form.Params, | |||||
| FlavorName: FlavorName, | |||||
| EngineName: EngineName, | |||||
| } | } | ||||
| //将params转换Parameters.Parameter,出错时返回给前端 | //将params转换Parameters.Parameter,出错时返回给前端 | ||||
| @@ -1665,7 +1656,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference | |||||
| return | return | ||||
| } | } | ||||
| err = modelarts.GenerateTrainJob(ctx, req) | |||||
| err = modelarts.GenerateInferenceJob(ctx, req) | |||||
| if err != nil { | if err != nil { | ||||
| log.Error("GenerateTrainJob failed:%v", err.Error()) | log.Error("GenerateTrainJob failed:%v", err.Error()) | ||||
| inferenceJobErrorNewDataPrepare(ctx, form) | inferenceJobErrorNewDataPrepare(ctx, form) | ||||