From be19ee26f154ada0fe55c14dae1be53f9f76ab90 Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 22 Nov 2021 11:56:42 +0800 Subject: [PATCH] update --- models/cloudbrain.go | 4 +- modules/modelarts/modelarts.go | 40 +++---- modules/storage/obs.go | 59 ---------- routers/api/v1/repo/modelarts.go | 11 +- routers/repo/modelarts.go | 113 +++++--------------- routers/routes/routes.go | 2 - templates/repo/modelarts/trainjob/show.tmpl | 4 +- 7 files changed, 56 insertions(+), 177 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 18b9c83ad..89e0f9014 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -59,8 +59,8 @@ type Cloudbrain struct { SubTaskName string ContainerID string ContainerIp string - CreatedUnix timeutil.TimeStamp - UpdatedUnix timeutil.TimeStamp + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` Duration int64 TrainJobDuration string DeletedAt time.Time `xorm:"deleted"` diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 70283a2fe..35ba508e5 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -2,6 +2,7 @@ package modelarts import ( "encoding/json" + "fmt" "path" "strconv" @@ -258,22 +259,21 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error } err = models.CreateCloudbrain(&models.Cloudbrain{ - Status: TransTrainJobStatus(jobResult.Status), - UserID: ctx.User.ID, - RepoID: ctx.Repo.Repository.ID, - JobID: strconv.FormatInt(jobResult.JobID, 10), - JobName: req.JobName, - JobType: string(models.JobTypeTrain), - Type: models.TypeCloudBrainTwo, - VersionID: jobResult.VersionID, - VersionName: jobResult.VersionName, - Uuid: req.Uuid, - DatasetName: attach.Name, - CommitID: req.CommitID, - IsLatestVersion: req.IsLatestVersion, - ComputeResource: ComputeResource, - EngineID: req.EngineID, - // FatherVersionName: req.FatherVersionName, + Status: TransTrainJobStatus(jobResult.Status), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: strconv.FormatInt(jobResult.JobID, 10), + JobName: req.JobName, + JobType: string(models.JobTypeTrain), + Type: models.TypeCloudBrainTwo, + VersionID: jobResult.VersionID, + VersionName: jobResult.VersionName, + Uuid: req.Uuid, + DatasetName: attach.Name, + CommitID: req.CommitID, + IsLatestVersion: req.IsLatestVersion, + ComputeResource: ComputeResource, + EngineID: req.EngineID, TrainUrl: req.TrainUrl, BranchName: req.BranchName, Parameters: req.Params, @@ -340,7 +340,7 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobVersionR Uuid: req.Uuid, DatasetName: attach.Name, CommitID: req.CommitID, - FatherVersionName: req.FatherVersionName, + PreVersionName: req.PreVersionName, ComputeResource: ComputeResource, EngineID: req.EngineID, TrainUrl: req.TrainUrl, @@ -454,6 +454,10 @@ func TransTrainJobStatus(status int) string { default: return strconv.Itoa(status) } +} - return "" +func GetVersionOutputPathByTotalVersionCount(TotalVersionCount int) (VersionOutputPath string) { + talVersionCountToString := fmt.Sprintf("%04d", TotalVersionCount) + VersionOutputPath = "V" + talVersionCountToString + return VersionOutputPath } diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 584181dbe..7a5b47099 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -235,65 +235,6 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { } } -func GetObsListObjectVersion(jobName, parentDir string, VersionOutputPath string) ([]FileInfo, error) { - input := &obs.ListObjectsInput{} - input.Bucket = setting.Bucket - input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionOutputPath, parentDir), "/") - strPrefix := strings.Split(input.Prefix, "/") - output, err := ObsCli.ListObjects(input) - fileInfos := make([]FileInfo, 0) - if err == nil { - for _, val := range output.Contents { - str1 := strings.Split(val.Key, "/") - var isDir bool - var fileName, nextParentDir string - if strings.HasSuffix(val.Key, "/") { - //dirs in next level dir - if len(str1)-len(strPrefix) > 2 { - continue - } - fileName = str1[len(str1)-2] - isDir = true - if parentDir == "" { - nextParentDir = fileName - } else { - nextParentDir = parentDir + "/" + fileName - } - - if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath { - continue - } - } else { - //files in next level dir - if len(str1)-len(strPrefix) > 1 { - continue - } - fileName = str1[len(str1)-1] - isDir = false - nextParentDir = parentDir - } - - fileInfo := FileInfo{ - ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), - FileName: fileName, - Size: val.Size, - IsDir: isDir, - ParenDir: nextParentDir, - } - fileInfos = append(fileInfos, fileInfo) - } - sort.Slice(fileInfos, func(i, j int) bool { - return fileInfos[i].ModTime > fileInfos[j].ModTime - }) - return fileInfos, err - } else { - if obsError, ok := err.(obs.ObsError); ok { - log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) - } - return nil, err - } -} - func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 32ef1a0fe..cf72caf1a 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -314,8 +314,9 @@ func ModelList(ctx *context.APIContext) { log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) return } - VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount) - models, err := storage.GetObsListObjectVersion(task.JobName, parentDir, VersionOutputPath) + VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount) + parentDir = VersionOutputPath + "/" + parentDir + models, err := storage.GetObsListObject(task.JobName, parentDir) if err != nil { log.Info("get TrainJobListModel failed:", err) ctx.ServerError("GetObsListObject:", err) @@ -347,9 +348,9 @@ func ModelDownload(ctx *context.APIContext) { log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) return } - VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount) - - url, err := storage.GetObsCreateVersionSignedUrl(task.JobName, parentDir, fileName, VersionOutputPath) + VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount) + parentDir = VersionOutputPath + "/" + parentDir + url, err := storage.GetObsCreateSignedUrl(task.JobName, parentDir, fileName) if err != nil { log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) ctx.ServerError("GetObsCreateSignedUrl", err) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index ff4a7b88c..ffb97344c 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -3,7 +3,6 @@ package repo import ( "encoding/json" "errors" - "fmt" "io" "io/ioutil" "net/http" @@ -289,17 +288,6 @@ func TrainJobIndex(ctx *context.Context) { return } - // for i, task := range tasks { - // result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10)) - // if err != nil { - // log.Error("GetJob(%s) failed:%v", task.JobID, err.Error()) - // return - // } - // // tasks[i].Status = modelarts.TransTrainJobStatus(result.Status) - // tasks[i].Status = result.Status - // tasks[i].Duration = result.Duration - // } - pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager @@ -492,8 +480,9 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) { ctx.Data["PageIsTrainJob"] = true - StringTotalVersionCount := fmt.Sprintf("%04d", modelarts.TotalVersionCount) - VersionOutputPath := "V" + StringTotalVersionCount + VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(modelarts.TotalVersionCount) + // StringTotalVersionCount := fmt.Sprintf("%04d", modelarts.TotalVersionCount) + // VersionOutputPath := "V" + StringTotalVersionCount jobName := form.JobName uuid := form.Attachment description := form.Description @@ -644,25 +633,24 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) } req := &modelarts.GenerateTrainJobReq{ - JobName: jobName, - DataUrl: dataPath, - Description: description, - CodeObsPath: codeObsPath, - BootFileUrl: codeObsPath + bootFile, - BootFile: bootFile, - TrainUrl: outputObsPath, - FlavorCode: flavorCode, - WorkServerNumber: workServerNumber, - EngineID: int64(engineID), - LogUrl: logObsPath, - PoolID: poolID, - Uuid: uuid, - Parameters: parameters.Parameter, - CommitID: commitID, - IsLatestVersion: isLatestVersion, - BranchName: branch_name, - Params: form.Params, - // FatherVersionName: InitVersionName, + JobName: jobName, + DataUrl: dataPath, + Description: description, + CodeObsPath: codeObsPath, + BootFileUrl: codeObsPath + bootFile, + BootFile: bootFile, + TrainUrl: outputObsPath, + FlavorCode: flavorCode, + WorkServerNumber: workServerNumber, + EngineID: int64(engineID), + LogUrl: logObsPath, + PoolID: poolID, + Uuid: uuid, + Parameters: parameters.Parameter, + CommitID: commitID, + IsLatestVersion: isLatestVersion, + BranchName: branch_name, + Params: form.Params, FlavorName: FlavorName, EngineName: EngineName, VersionCount: VersionCount, @@ -693,8 +681,9 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ ctx.ServerError("GetCloudbrainByJobIDAndIsLatestVersion faild:", err) return } - StringTotalVersionCount := fmt.Sprintf("%04d", latestTask.TotalVersionCount+1) - VersionOutputPath := "V" + StringTotalVersionCount + // StringTotalVersionCount := fmt.Sprintf("%04d", latestTask.TotalVersionCount+1) + // VersionOutputPath := "V" + StringTotalVersionCount + VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(latestTask.TotalVersionCount + 1) jobName := form.JobName uuid := form.Attachment @@ -1245,37 +1234,6 @@ func TrainJobShowModels(ctx *context.Context) { ctx.HTML(200, tplModelArtsTrainJobShowModels) } -func TrainJobVersionShowModels(ctx *context.Context) { - ctx.Data["PageIsCloudBrain"] = true - - jobID := ctx.Params(":jobid") - parentDir := ctx.Query("parentDir") - versionName := ctx.Query("version_name") - // dirArray := strings.Split(parentDir, "/") - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - if err != nil { - log.Error("no such job!", ctx.Data["msgID"]) - ctx.ServerError("no such job:", err) - return - } - // parentDir = versionName - VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount) - dirArray := strings.Split(VersionOutputPath, "/") - models, err := storage.GetObsListObjectVersion(task.JobName, parentDir, VersionOutputPath) - if err != nil { - log.Info("get TrainJobListModel failed:", err) - ctx.ServerError("GetVersionObsListObject:", err) - return - } - - ctx.Data["Path"] = dirArray - // ctx.Data["Path"] = VersionOutputPath - ctx.Data["Dirs"] = models - ctx.Data["task"] = task - ctx.Data["JobID"] = jobID - ctx.HTML(200, tplModelArtsTrainJobShowModels) -} - func TrainJobDownloadModel(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") @@ -1288,26 +1246,3 @@ func TrainJobDownloadModel(ctx *context.Context) { } http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } - -func TrainJobVersionDownloadModel(ctx *context.Context) { - var jobID = ctx.Params(":jobid") - - parentDir := ctx.Query("parentDir") - fileName := ctx.Query("fileName") - versionName := ctx.Query("version_name") - - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - if err != nil { - log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) - return - } - VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount) - - url, err := storage.GetObsCreateVersionSignedUrl(task.JobName, parentDir, fileName, VersionOutputPath) - if err != nil { - log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) - ctx.ServerError("GetObsCreateSignedUrl", err) - return - } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) -} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index d236bf85e..fe2588b25 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -989,8 +989,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow) m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) - m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobVersionShowModels) - m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobVersionDownloadModel) m.Get("/create_version", reqRepoCloudBrainReader, repo.TrainJobNewVersion) m.Post("/create_version", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) }) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index cb92e3f30..4cea862d6 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -194,9 +194,9 @@ td, th {
- 2021/11/08 19:35:19 + {{.Cloudbrain.CreatedUnix}} 当前版本：{{.VersionName}} - 父版本：{{.FatherVersionName}} + 父版本：{{.PreVersionName}} 状态： {{.Status}}