From 0b2af71870e956c9d4a473de35dd36cc4357f89b Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 14 Oct 2021 16:50:18 +0800 Subject: [PATCH 1/2] add modelarts --- routers/repo/modelarts.go | 2 ++ templates/repo/cloudbrain/index.tmpl | 2 +- templates/repo/modelarts/index.tmpl | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 2dc327d33..080c36377 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -57,6 +57,8 @@ func ModelArtsIndex(ctx *context.Context) { } else { ciTasks[i].CanDebug = false } + + ciTasks[i].CanDel = models.CanDelJob(ctx.IsSigned, ctx.User, task) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index 200f5e133..0626f57c6 100755 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -352,7 +352,7 @@ - +
{{$.CsrfTokenHtml}} diff --git a/templates/repo/modelarts/index.tmpl b/templates/repo/modelarts/index.tmpl index fb499bdca..8c4d5a47f 100755 --- a/templates/repo/modelarts/index.tmpl +++ b/templates/repo/modelarts/index.tmpl @@ -312,9 +312,9 @@ - + {{$.CsrfTokenHtml}} - + 删除
From deaf44866875e577bc0a3127495b020470feddc8 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 14 Oct 2021 17:18:51 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix-333=E4=BC=98=E5=8C=96=EF=BC=8C=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E9=94=99=E8=AF=AF=E9=87=8D=E8=AF=95=EF=BC=8C=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E7=AD=89=E5=BE=85=E7=9A=84=E4=BA=91=E8=84=91=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 4 ++-- routers/repo/cloudbrain.go | 25 +++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 4f615ed00..dd4e9a6f4 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -629,13 +629,13 @@ func GetCloudbrainByJobID(jobID string) (*Cloudbrain, error) { func GetCloudbrainsNeededStopByUserID(userID int64) ([]*Cloudbrain, error) { cloudBrains := make([]*Cloudbrain, 0) - err := x.Cols("job_id", "status", "type").Where("user_id=? AND (status =? OR status=?)", userID, string(JobRunning), string(JobWaiting)).Find(&cloudBrains) + err := x.Cols("job_id", "status", "type").Where("user_id=? AND status !=?", userID, string(JobStopped)).Find(&cloudBrains) return cloudBrains, err } func GetCloudbrainsNeededStopByRepoID(repoID int64) ([]*Cloudbrain, error) { cloudBrains := make([]*Cloudbrain, 0) - err := x.Cols("job_id", "status", "type").Where("repo_id=? AND (status =? OR status=?)", repoID, string(JobRunning), string(JobWaiting)).Find(&cloudBrains) + err := x.Cols("job_id", "status", "type").Where("repo_id=? AND status !=?", repoID, string(JobStopped)).Find(&cloudBrains) return cloudBrains, err } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index ccd51a89f..0bafe24aa 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/json" "errors" + "fmt" "io" "net/http" "os" @@ -391,19 +392,39 @@ func StopJobs(cloudBrains []*models.Cloudbrain) { for _, taskInfo := range cloudBrains { if taskInfo.Type == models.TypeCloudBrainOne { - err := cloudbrain.StopJob(taskInfo.JobID) + err := retry(3, time.Second*30, func() error { + return cloudbrain.StopJob(taskInfo.JobID) + }) + logErrorAndUpdateJobStatus(err, taskInfo) } else { param := models.NotebookAction{ Action: models.ActionStop, } - _, err := modelarts.StopJob(taskInfo.JobID, param) + err := retry(3, time.Second*30, func() error { + _, err := modelarts.StopJob(taskInfo.JobID, param) + return err + }) logErrorAndUpdateJobStatus(err, taskInfo) } } } +func retry(attempts int, sleep time.Duration, f func() error) (err error) { + for i := 0; i < attempts; i++ { + if i > 0 { + log.Warn("retrying after error:", err) + time.Sleep(sleep) + } + err = f() + if err == nil { + return nil + } + } + return fmt.Errorf("after %d attempts, last error: %s", attempts, err) +} + func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) { if err != nil { log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)