From 6af7c8b7672ddc76199e73bd85c6307088af6ace Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 15 Feb 2022 15:33:30 +0800 Subject: [PATCH] update --- modules/modelarts/modelarts.go | 20 +++---- routers/repo/modelarts.go | 106 ++++++++++++++++----------------- 2 files changed, 62 insertions(+), 64 deletions(-) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index d07a2551c..aa7896990 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -65,7 +65,7 @@ var ( ) type GenerateTrainJobReq struct { - JobName string + OpeniJobName string ApiJobName string Uuid string Description string @@ -118,7 +118,7 @@ type GenerateTrainJobVersionReq struct { } type GenerateInferenceJobReq struct { - JobName string + OpeniJobName string ApiJobName string Uuid string Description string @@ -301,7 +301,8 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error UserID: ctx.User.ID, RepoID: ctx.Repo.Repository.ID, JobID: strconv.FormatInt(jobResult.JobID, 10), - JobName: req.JobName, + JobName: req.OpeniJobName, + ApiJobName: req.ApiJobName, JobType: string(models.JobTypeTrain), Type: models.TypeCloudBrainTwo, VersionID: jobResult.VersionID, @@ -325,11 +326,10 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error EngineName: req.EngineName, VersionCount: req.VersionCount, TotalVersionCount: req.TotalVersionCount, - ApiJobName: jobResult.JobName, }) if err != nil { - log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error()) + log.Error("CreateCloudbrain(%s) failed:%v", req.OpeniJobName, err.Error()) return err } @@ -386,7 +386,7 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job UserID: ctx.User.ID, RepoID: ctx.Repo.Repository.ID, JobID: strconv.FormatInt(jobResult.JobID, 10), - JobName: req.JobName, + JobName: req.OpeniJobName, ApiJobName: req.ApiJobName, JobType: string(models.JobTypeTrain), Type: models.TypeCloudBrainTwo, @@ -415,7 +415,7 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job VersionCount: VersionListCount + 1, }) if err != nil { - log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error()) + log.Error("CreateCloudbrain(%s) failed:%v", req.OpeniJobName, err.Error()) return err } @@ -523,7 +523,8 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e UserID: ctx.User.ID, RepoID: ctx.Repo.Repository.ID, JobID: strconv.FormatInt(jobResult.JobID, 10), - JobName: req.JobName, + JobName: req.OpeniJobName, + ApiJobName: jobResult.JobName, JobType: string(models.JobTypeInference), Type: models.TypeCloudBrainTwo, VersionID: jobResult.VersionID, @@ -551,11 +552,10 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e ModelVersion: req.ModelVersion, CkptName: req.CkptName, ResultUrl: req.ResultUrl, - ApiJobName: jobResult.JobName, }) if err != nil { - log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error()) + log.Error("CreateCloudbrain(%s) failed:%v", req.OpeniJobName, err.Error()) return err } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 718be3367..130dbacdc 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -790,6 +790,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) VersionOutputPath := modelarts.GetOutputPathByCount(modelarts.TotalVersionCount) openiJobName := form.JobName apiJobName := util.ConvertToApiJobName(openiJobName) + jobName := apiJobName uuid := form.Attachment description := form.Description workServerNumber := form.WorkServerNumber @@ -800,10 +801,10 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) poolID := form.PoolID isSaveParam := form.IsSaveParam repo := ctx.Repo.Repository - codeLocalPath := setting.JobPath + apiJobName + modelarts.CodePath - codeObsPath := "/" + setting.Bucket + modelarts.JobPath + apiJobName + modelarts.CodePath - outputObsPath := "/" + setting.Bucket + modelarts.JobPath + apiJobName + modelarts.OutputPath + VersionOutputPath + "/" - logObsPath := "/" + setting.Bucket + modelarts.JobPath + apiJobName + modelarts.LogPath + VersionOutputPath + "/" + codeLocalPath := setting.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" branch_name := form.BranchName isLatestVersion := modelarts.IsLatestVersion @@ -883,14 +884,14 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) } //todo: upload code (send to file_server todo this work?) - if err := obsMkdir(setting.CodePathPrefix + apiJobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobNew, &form) return } - if err := obsMkdir(setting.CodePathPrefix + apiJobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobNew, &form) @@ -898,7 +899,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) } // parentDir := VersionOutputPath + "/" - if err := uploadCodeToObs(codeLocalPath, apiJobName, ""); err != nil { + if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { // if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) trainJobErrorNewDataPrepare(ctx, form) @@ -971,8 +972,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) } req := &modelarts.GenerateTrainJobReq{ - JobName: openiJobName, - ApiJobName: apiJobName, + OpeniJobName: openiJobName, + ApiJobName: jobName, DataUrl: dataPath, Description: description, CodeObsPath: codeObsPath, @@ -1039,8 +1040,30 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ } VersionOutputPath := modelarts.GetOutputPathByCount(latestTask.TotalVersionCount + 1) - openijobName := form.JobName - apiJobName := util.ConvertToApiJobName(openijobName) + //判断权限 + canNewJob, _ := canUserCreateTrainJobVersion(ctx, latestTask.UserID) + if !canNewJob { + ctx.RenderWithErr("user cann't new trainjob", tplModelArtsTrainJobVersionNew, &form) + return + } + + if err := paramCheckCreateTrainJob(form); err != nil { + log.Error("paramCheckCreateTrainJob failed:(%v)", err) + versionErrorDataPrepare(ctx, form) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) + return + } + PreVersionName := form.VersionName + + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, PreVersionName) + if err != nil { + log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", jobID, err.Error()) + ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) + return + } + + apiJobName := task.ApiJobName + jobName := apiJobName uuid := form.Attachment description := form.Description workServerNumber := form.WorkServerNumber @@ -1051,40 +1074,20 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ poolID := form.PoolID isSaveParam := form.IsSaveParam repo := ctx.Repo.Repository - codeLocalPath := setting.JobPath + apiJobName + modelarts.CodePath - codeObsPath := "/" + setting.Bucket + modelarts.JobPath + apiJobName + modelarts.CodePath + VersionOutputPath + "/" - outputObsPath := "/" + setting.Bucket + modelarts.JobPath + apiJobName + modelarts.OutputPath + VersionOutputPath + "/" - logObsPath := "/" + setting.Bucket + modelarts.JobPath + apiJobName + modelarts.LogPath + VersionOutputPath + "/" + codeLocalPath := setting.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/" + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" branch_name := form.BranchName - PreVersionName := form.VersionName FlavorName := form.FlavorName EngineName := form.EngineName isLatestVersion := modelarts.IsLatestVersion - //判断权限 - canNewJob, _ := canUserCreateTrainJobVersion(ctx, latestTask.UserID) - if !canNewJob { - ctx.RenderWithErr("user cann't new trainjob", tplModelArtsTrainJobVersionNew, &form) - return - } - - if err := paramCheckCreateTrainJob(form); err != nil { - log.Error("paramCheckCreateTrainJob failed:(%v)", err) - versionErrorDataPrepare(ctx, form) - ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) - return - } - //todo: del the codeLocalPath _, err = ioutil.ReadDir(codeLocalPath) if err == nil { os.RemoveAll(codeLocalPath) - } else { - log.Error("创建任务失败，原代码还未删除，请重试！: %s (%v)", repo.FullName(), err) - versionErrorDataPrepare(ctx, form) - ctx.RenderWithErr("创建任务失败，原代码还未删除，请重试！", tplModelArtsTrainJobVersionNew, &form) - return } // os.RemoveAll(codeLocalPath) @@ -1100,14 +1103,14 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ } //todo: upload code (send to file_server todo this work?) - if err := obsMkdir(setting.CodePathPrefix + apiJobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobVersionNew, &form) return } - if err := obsMkdir(setting.CodePathPrefix + apiJobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobVersionNew, &form) @@ -1117,7 +1120,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ parentDir := VersionOutputPath + "/" // parentDir := "" // if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { - if err := uploadCodeToObs(codeLocalPath, apiJobName, parentDir); err != nil { + if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobVersionNew, &form) @@ -1194,15 +1197,9 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ return } - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, PreVersionName) - if err != nil { - log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", jobID, err.Error()) - ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) - return - } req := &modelarts.GenerateTrainJobReq{ - JobName: task.JobName, - ApiJobName: apiJobName, + OpeniJobName: task.JobName, + ApiJobName: task.ApiJobName, DataUrl: dataPath, Description: description, CodeObsPath: codeObsPath, @@ -1629,6 +1626,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference VersionOutputPath := modelarts.GetOutputPathByCount(modelarts.TotalVersionCount) openiJobName := form.JobName apiJobName := util.ConvertToApiJobName(openiJobName) + jobName := apiJobName uuid := form.Attachment description := form.Description workServerNumber := form.WorkServerNumber @@ -1638,10 +1636,10 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference params := form.Params poolID := form.PoolID repo := ctx.Repo.Repository - codeLocalPath := setting.JobPath + apiJobName + modelarts.CodePath - codeObsPath := "/" + setting.Bucket + modelarts.JobPath + apiJobName + modelarts.CodePath - resultObsPath := "/" + setting.Bucket + modelarts.JobPath + apiJobName + modelarts.ResultPath + VersionOutputPath + "/" - logObsPath := "/" + setting.Bucket + modelarts.JobPath + apiJobName + modelarts.LogPath + VersionOutputPath + "/" + codeLocalPath := setting.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + resultObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.ResultPath + VersionOutputPath + "/" + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" branch_name := form.BranchName FlavorName := form.FlavorName @@ -1725,21 +1723,21 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference } //todo: upload code (send to file_server todo this work?) - if err := obsMkdir(setting.CodePathPrefix + apiJobName + modelarts.ResultPath + VersionOutputPath + "/"); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.ResultPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_result: %s (%v)", repo.FullName(), err) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_result", tplModelArtsInferenceJobNew, &form) return } - if err := obsMkdir(setting.CodePathPrefix + apiJobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsInferenceJobNew, &form) return } - if err := uploadCodeToObs(codeLocalPath, apiJobName, ""); err != nil { + if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsInferenceJobNew, &form) @@ -1776,7 +1774,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference } req := &modelarts.GenerateInferenceJobReq{ - JobName: openiJobName, + OpeniJobName: openiJobName, ApiJobName: apiJobName, DataUrl: dataPath, Description: description,