From e24b67cfa97bfca382801751eb4f3e9c38904482 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 13 Sep 2022 15:16:39 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 38 ++++----- modules/cloudbrain/cloudbrain.go | 6 +- modules/grampus/grampus.go | 84 +++++++++--------- modules/modelarts/modelarts.go | 142 +++++++++++++++---------------- routers/repo/cloudbrain.go | 2 +- routers/repo/grampus.go | 4 +- routers/repo/modelarts.go | 9 +- 7 files changed, 146 insertions(+), 139 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 12c76ce57..f93b653e1 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -171,25 +171,25 @@ type Cloudbrain struct { ImageID string //grampus image_id AiCenter string //grampus ai center: center_id+center_name - TrainUrl string //输出模型的obs路径 - BranchName string //分支名称 - Parameters string //传给modelarts的param参数 - BootFile string //启动文件 - DataUrl string //数据集的obs路径 - LogUrl string //日志输出的obs路径 - PreVersionId int64 //父版本的版本id - FlavorCode string //modelarts上的规格id - Description string `xorm:"varchar(256)"` //描述 - WorkServerNumber int //节点数 - FlavorName string //规格名称 - EngineName string //引擎名称 - TotalVersionCount int //任务的所有版本数量，包括删除的 - LabelName string //标签名称 - ModelName string //模型名称 - ModelVersion string //模型版本 - CkptName string //权重文件名称 - PreTrainingModelUrl string //预训练模型地址 - ResultUrl string //推理结果的obs路径 + TrainUrl string //输出模型的obs路径 + BranchName string //分支名称 + Parameters string //传给modelarts的param参数 + BootFile string //启动文件 + DataUrl string //数据集的obs路径 + LogUrl string //日志输出的obs路径 + PreVersionId int64 //父版本的版本id + FlavorCode string //modelarts上的规格id + Description string `xorm:"varchar(256)"` //描述 + WorkServerNumber int //节点数 + FlavorName string //规格名称 + EngineName string //引擎名称 + TotalVersionCount int //任务的所有版本数量，包括删除的 + LabelName string //标签名称 + ModelName string //模型名称 + ModelVersion string //模型版本 + CkptName string //权重文件名称 + PreTrainModelUrl string //预训练模型地址 + ResultUrl string //推理结果的obs路径 User *User `xorm:"-"` Repo *Repository `xorm:"-"` diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 5a4d2fe05..4e527b6bf 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -79,7 +79,7 @@ type GenerateCloudBrainTaskReq struct { CkptName string LabelName string PreTrainModelPath string - PreTrainingModelUrl string + PreTrainModelUrl string Spec *models.Specification } @@ -279,7 +279,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { }, } - if req.PreTrainingModelUrl != "" { //预训练 + if req.PreTrainModelUrl != "" { //预训练 volumes = append(volumes, models.Volume{ HostPath: models.StHostPath{ Path: req.PreTrainModelPath, @@ -372,7 +372,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { CkptName: req.CkptName, ResultUrl: req.ResultPath, LabelName: req.LabelName, - PreTrainingModelUrl: req.PreTrainingModelUrl, + PreTrainModelUrl: req.PreTrainModelUrl, CreatedUnix: createTime, UpdatedUnix: createTime, CommitID: req.CommitID, diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index f434a484c..45c127141 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -63,16 +63,16 @@ type GenerateTrainJobReq struct { ComputeResource string ProcessType string - DatasetNames string - DatasetInfos map[string]models.DatasetInfo - Params string - ModelName string - LabelName string - CkptName string - ModelVersion string - PreTrainModelPath string - PreTrainingModelUrl string - Spec *models.Specification + DatasetNames string + DatasetInfos map[string]models.DatasetInfo + Params string + ModelName string + LabelName string + CkptName string + ModelVersion string + PreTrainModelPath string + PreTrainModelUrl string + Spec *models.Specification } func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error) { @@ -102,38 +102,38 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error jobID := jobResult.JobInfo.JobID err = models.CreateCloudbrain(&models.Cloudbrain{ - Status: TransTrainJobStatus(jobResult.JobInfo.Status), - UserID: ctx.User.ID, - RepoID: ctx.Repo.Repository.ID, - JobID: jobID, - JobName: req.JobName, - DisplayJobName: req.DisplayJobName, - JobType: string(models.JobTypeTrain), - Type: models.TypeC2Net, - Uuid: req.Uuid, - DatasetName: req.DatasetNames, - CommitID: req.CommitID, - IsLatestVersion: req.IsLatestVersion, - ComputeResource: req.ComputeResource, - ImageID: req.ImageId, - TrainUrl: req.TrainUrl, - BranchName: req.BranchName, - Parameters: req.Params, - BootFile: req.BootFile, - DataUrl: req.DataUrl, - Description: req.Description, - WorkServerNumber: req.WorkServerNumber, - EngineName: req.EngineName, - VersionCount: req.VersionCount, - TotalVersionCount: req.TotalVersionCount, - CreatedUnix: createTime, - UpdatedUnix: createTime, - Spec: req.Spec, - ModelName: req.ModelName, - ModelVersion: req.ModelVersion, - LabelName: req.LabelName, - PreTrainingModelUrl: req.PreTrainingModelUrl, - CkptName: req.CkptName, + Status: TransTrainJobStatus(jobResult.JobInfo.Status), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: jobID, + JobName: req.JobName, + DisplayJobName: req.DisplayJobName, + JobType: string(models.JobTypeTrain), + Type: models.TypeC2Net, + Uuid: req.Uuid, + DatasetName: req.DatasetNames, + CommitID: req.CommitID, + IsLatestVersion: req.IsLatestVersion, + ComputeResource: req.ComputeResource, + ImageID: req.ImageId, + TrainUrl: req.TrainUrl, + BranchName: req.BranchName, + Parameters: req.Params, + BootFile: req.BootFile, + DataUrl: req.DataUrl, + Description: req.Description, + WorkServerNumber: req.WorkServerNumber, + EngineName: req.EngineName, + VersionCount: req.VersionCount, + TotalVersionCount: req.TotalVersionCount, + CreatedUnix: createTime, + UpdatedUnix: createTime, + Spec: req.Spec, + ModelName: req.ModelName, + ModelVersion: req.ModelVersion, + LabelName: req.LabelName, + PreTrainModelUrl: req.PreTrainModelUrl, + CkptName: req.CkptName, }) if err != nil { diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index ead824b60..f35601191 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -75,40 +75,40 @@ var ( ) type GenerateTrainJobReq struct { - JobName string - DisplayJobName string - Uuid string - Description string - CodeObsPath string - BootFile string - BootFileUrl string - DataUrl string - TrainUrl string - LogUrl string - PoolID string - WorkServerNumber int - EngineID int64 - Parameters []models.Parameter - CommitID string - IsLatestVersion string - Params string - BranchName string - PreVersionId int64 - PreVersionName string - FlavorCode string - FlavorName string - VersionCount int - EngineName string - TotalVersionCount int - UserImageUrl string - UserCommand string - DatasetName string - Spec *models.Specification - ModelName string - LabelName string - CkptName string - ModelVersion string - PreTrainingModelUrl string + JobName string + DisplayJobName string + Uuid string + Description string + CodeObsPath string + BootFile string + BootFileUrl string + DataUrl string + TrainUrl string + LogUrl string + PoolID string + WorkServerNumber int + EngineID int64 + Parameters []models.Parameter + CommitID string + IsLatestVersion string + Params string + BranchName string + PreVersionId int64 + PreVersionName string + FlavorCode string + FlavorName string + VersionCount int + EngineName string + TotalVersionCount int + UserImageUrl string + UserCommand string + DatasetName string + Spec *models.Specification + ModelName string + LabelName string + CkptName string + ModelVersion string + PreTrainModelUrl string } type GenerateInferenceJobReq struct { @@ -412,43 +412,43 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error } jobId := strconv.FormatInt(jobResult.JobID, 10) createErr = models.CreateCloudbrain(&models.Cloudbrain{ - Status: TransTrainJobStatus(jobResult.Status), - UserID: ctx.User.ID, - RepoID: ctx.Repo.Repository.ID, - JobID: jobId, - JobName: req.JobName, - DisplayJobName: req.DisplayJobName, - JobType: string(models.JobTypeTrain), - Type: models.TypeCloudBrainTwo, - VersionID: jobResult.VersionID, - VersionName: jobResult.VersionName, - Uuid: req.Uuid, - DatasetName: req.DatasetName, - CommitID: req.CommitID, - IsLatestVersion: req.IsLatestVersion, - ComputeResource: models.NPUResource, - EngineID: req.EngineID, - TrainUrl: req.TrainUrl, - BranchName: req.BranchName, - Parameters: req.Params, - BootFile: req.BootFile, - DataUrl: req.DataUrl, - LogUrl: req.LogUrl, - FlavorCode: req.Spec.SourceSpecId, - Description: req.Description, - WorkServerNumber: req.WorkServerNumber, - FlavorName: req.FlavorName, - EngineName: req.EngineName, - VersionCount: req.VersionCount, - TotalVersionCount: req.TotalVersionCount, - CreatedUnix: createTime, - UpdatedUnix: createTime, - Spec: req.Spec, - ModelName: req.ModelName, - ModelVersion: req.ModelVersion, - LabelName: req.LabelName, - PreTrainingModelUrl: req.PreTrainingModelUrl, - CkptName: req.CkptName, + Status: TransTrainJobStatus(jobResult.Status), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: jobId, + JobName: req.JobName, + DisplayJobName: req.DisplayJobName, + JobType: string(models.JobTypeTrain), + Type: models.TypeCloudBrainTwo, + VersionID: jobResult.VersionID, + VersionName: jobResult.VersionName, + Uuid: req.Uuid, + DatasetName: req.DatasetName, + CommitID: req.CommitID, + IsLatestVersion: req.IsLatestVersion, + ComputeResource: models.NPUResource, + EngineID: req.EngineID, + TrainUrl: req.TrainUrl, + BranchName: req.BranchName, + Parameters: req.Params, + BootFile: req.BootFile, + DataUrl: req.DataUrl, + LogUrl: req.LogUrl, + FlavorCode: req.Spec.SourceSpecId, + Description: req.Description, + WorkServerNumber: req.WorkServerNumber, + FlavorName: req.FlavorName, + EngineName: req.EngineName, + VersionCount: req.VersionCount, + TotalVersionCount: req.TotalVersionCount, + CreatedUnix: createTime, + UpdatedUnix: createTime, + Spec: req.Spec, + ModelName: req.ModelName, + ModelVersion: req.ModelVersion, + LabelName: req.LabelName, + PreTrainModelUrl: req.PreTrainModelUrl, + CkptName: req.CkptName, }) if createErr != nil { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 56a485b66..8cd45e06f 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -334,7 +334,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { req.CkptName = form.CkptName req.ModelVersion = form.ModelVersion req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl - req.PreTrainingModelUrl = form.PreTrainModelUrl + req.PreTrainModelUrl = form.PreTrainModelUrl } diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index b32070a84..ed869e76e 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -389,7 +389,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain req.LabelName = form.LabelName req.CkptName = form.CkptName req.ModelVersion = form.ModelVersion - req.PreTrainingModelUrl = form.PreTrainModelUrl + req.PreTrainModelUrl = form.PreTrainModelUrl } @@ -617,7 +617,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain req.LabelName = form.LabelName req.CkptName = form.CkptName req.ModelVersion = form.ModelVersion - req.PreTrainingModelUrl = form.PreTrainModelUrl + req.PreTrainModelUrl = form.PreTrainModelUrl } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 121c4fd78..cb4b2c1cc 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1010,6 +1010,13 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { ctx.Data["engine_id"] = task.EngineID ctx.Data["datasetType"] = models.TypeCloudBrainTwo + //pretrain model + ctx.Data["model_name"] = task.ModelName + ctx.Data["model_version"] = task.ModelVersion + ctx.Data["ckpt_name"] = task.CkptName + ctx.Data["label_names"] = task.LabelName + ctx.Data["pre_train_model_url"] = task.PreTrainModelUrl + configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) if err != nil { ctx.ServerError("getConfigList failed:", err) @@ -1370,7 +1377,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) req.LabelName = form.LabelName req.CkptName = form.CkptName req.ModelVersion = form.ModelVersion - req.PreTrainingModelUrl = form.PreTrainModelUrl + req.PreTrainModelUrl = form.PreTrainModelUrl }