Browse Source

update

tags/v1.21.12.1
liuzx 4 years ago
parent
commit
1dd0a3f515
5 changed files with 24 additions and 51 deletions
  1. +2
    -11
      models/cloudbrain.go
  2. +1
    -1
      modules/modelarts/modelarts.go
  3. +0
    -21
      modules/storage/obs.go
  4. +9
    -8
      routers/api/v1/repo/modelarts.go
  5. +12
    -10
      routers/repo/modelarts.go

+ 2
- 11
models/cloudbrain.go View File

@@ -69,7 +69,7 @@ type Cloudbrain struct {
Type int

VersionID int64 //版本id
VersionName string //当前版本
VersionName string `xorm:"INDEX"` //当前版本
Uuid string //数据集id
DatasetName string
VersionCount int //任务的当前版本数量,不包括删除的
@@ -1029,7 +1029,7 @@ func SetTrainJobStatusByJobID(jobID string, status string, duration int64, train
return
}

func SetVersionCountAndLatestVersionByJobIDAndVersionName(jobID string, versionName string, versionCount int, isLatestVersion string, totalVersionCount int) (err error) {
func SetVersionCountAndLatestVersion(jobID string, versionName string, versionCount int, isLatestVersion string, totalVersionCount int) (err error) {
cb := &Cloudbrain{JobID: jobID, VersionName: versionName, VersionCount: versionCount, IsLatestVersion: isLatestVersion, TotalVersionCount: totalVersionCount}
_, err = x.Cols("version_Count", "is_latest_version", "total_version_count").Where("cloudbrain.job_id=? AND cloudbrain.version_name=?", jobID, versionName).Update(cb)
return
@@ -1066,15 +1066,6 @@ func deleteJob(e Engine, job *Cloudbrain) error {
return err
}

func DeleteJobVersion(job *Cloudbrain) error {
return deleteJobVersion(x, job)
}

func deleteJobVersion(e Engine, job *Cloudbrain) error {
_, err := e.ID(job.ID).Delete(job)
return err
}

func GetCloudbrainByName(jobName string) (*Cloudbrain, error) {
cb := &Cloudbrain{JobName: jobName}
return getRepoCloudBrain(cb)


+ 1
- 1
modules/modelarts/modelarts.go View File

@@ -379,7 +379,7 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job
}

//将训练任务的上一版本的isLatestVersion设置为"0"
err = models.SetVersionCountAndLatestVersionByJobIDAndVersionName(strconv.FormatInt(jobResult.JobID, 10), VersionTaskList[0].VersionName, VersionCount, NotLatestVersion, TotalVersionCount)
err = models.SetVersionCountAndLatestVersion(strconv.FormatInt(jobResult.JobID, 10), VersionTaskList[0].VersionName, VersionCount, NotLatestVersion, TotalVersionCount)
if err != nil {
ctx.ServerError("Update IsLatestVersion failed", err)
return err


+ 0
- 21
modules/storage/obs.go View File

@@ -279,27 +279,6 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error)
return output.SignedUrl, nil
}

func GetObsCreateVersionSignedUrl(jobName, parentDir, fileName string, VersionOutputPath string) (string, error) {
input := &obs.CreateSignedUrlInput{}
input.Bucket = setting.Bucket
input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionOutputPath, parentDir, fileName), "/")

input.Expires = 60 * 60
input.Method = obs.HttpMethodGet

reqParams := make(map[string]string)
fileName = url.QueryEscape(fileName)
reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\""
input.QueryParams = reqParams
output, err := ObsCli.CreateSignedUrl(input)
if err != nil {
log.Error("CreateSignedUrl failed:", err.Error())
return "", err
}

return output.SignedUrl, nil
}

func ObsGetPreSignedUrl(uuid, fileName string) (string, error) {
input := &obs.CreateSignedUrlInput{}
input.Method = obs.HttpMethodGet


+ 9
- 8
routers/api/v1/repo/modelarts.go View File

@@ -212,17 +212,18 @@ func DelTrainJobVersion(ctx *context.APIContext) {
return
}

//删除数据库记录
err = models.DeleteJobVersion(task)
//删除modelarts上的记录
_, err = modelarts.DelTrainJobVersion(jobID, strconv.FormatInt(task.VersionID, 10))
if err != nil {
ctx.ServerError("DeleteJobVersion failed", err)
log.Error("DelTrainJobVersion(%s) failed:%v", task.JobName, err.Error())
ctx.NotFound(err)
return
}
//删除modelarts上的记录
_, err = modelarts.DelTrainJobVersion(jobID, strconv.FormatInt(task.VersionID, 10))

//删除数据库记录
err = models.DeleteJob(task)
if err != nil {
log.Error("DelTrainJobVersion(%s) failed:%v", task.JobName, err.Error())
ctx.ServerError("DeleteJob failed", err)
ctx.NotFound(err)
return
}
@@ -242,13 +243,13 @@ func DelTrainJobVersion(ctx *context.APIContext) {

// 判断当前删掉的任务是否是最新版本,若是,将排序后的TotalVersionCount置为删掉的最新版本的TotalVersionCount,若不是,按时间排序后的版本列表的第一个版本设置为最新版本,TotalVersionCount不变
if task.IsLatestVersion == modelarts.IsLatestVersion {
err = models.SetVersionCountAndLatestVersionByJobIDAndVersionName(jobID, VersionTaskList[0].Cloudbrain.VersionName, VersionListCount, modelarts.IsLatestVersion, task.TotalVersionCount)
err = models.SetVersionCountAndLatestVersion(jobID, VersionTaskList[0].Cloudbrain.VersionName, VersionListCount, modelarts.IsLatestVersion, task.TotalVersionCount)
if err != nil {
ctx.ServerError("UpdateJobVersionCount failed", err)
return
}
} else {
err = models.SetVersionCountAndLatestVersionByJobIDAndVersionName(jobID, VersionTaskList[0].VersionName, VersionListCount, modelarts.IsLatestVersion, VersionTaskList[0].Cloudbrain.TotalVersionCount)
err = models.SetVersionCountAndLatestVersion(jobID, VersionTaskList[0].VersionName, VersionListCount, modelarts.IsLatestVersion, VersionTaskList[0].Cloudbrain.TotalVersionCount)
if err != nil {
ctx.ServerError("UpdateJobVersionCount failed", err)
return


+ 12
- 10
routers/repo/modelarts.go View File

@@ -527,7 +527,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{
Branch: branch_name,
}); err != nil {
log.Error("创建任务失败,任务名称已存在!: %s (%v)", repo.FullName(), err)
log.Error("创建任务失败,服务器超时!: %s (%v)", repo.FullName(), err)
trainJobNewDataPrepare(ctx)

ctx.Data["bootFile"] = form.BootFile
@@ -536,7 +536,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
ctx.Data["params"] = form.Params
ctx.Data["branch_name"] = branch_name
trainJobNewDataPrepare(ctx)
ctx.RenderWithErr("创建任务失败,任务名称已存在!", tplModelArtsTrainJobNew, &form)
ctx.RenderWithErr("创建任务失败,服务器超时!", tplModelArtsTrainJobNew, &form)
return
}

@@ -1086,14 +1086,7 @@ func TrainJobDel(ctx *context.Context) {
ctx.ServerError("get VersionListTasks failed", err)
return
}
//删除数据库Cloudbrain表的记录
for _, task := range VersionListTasks {
err = models.DeleteJobVersion(&task.Cloudbrain)
if err != nil {
ctx.ServerError("DeleteJobVersion failed", err)
return
}
}

//删除modelarts上的任务记录
_, err = modelarts.DelTrainJob(jobID)
if err != nil {
@@ -1102,6 +1095,15 @@ func TrainJobDel(ctx *context.Context) {
return
}

//删除数据库Cloudbrain表的记录
for _, task := range VersionListTasks {
err = models.DeleteJob(&task.Cloudbrain)
if err != nil {
ctx.ServerError("DeleteJob failed", err)
return
}
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
}



Loading…
Cancel
Save