Merge branch 'V20211018' of git.openi.org.cn:OpenI/aiforge into repo-statistic

models/cloudbrain.go

@@ -631,13 +631,13 @@ func GetCloudbrainByJobID(jobID string) (*Cloudbrain, error) {
 
 func GetCloudbrainsNeededStopByUserID(userID int64) ([]*Cloudbrain, error) {
 	cloudBrains := make([]*Cloudbrain, 0)
-	err := x.Cols("job_id", "status", "type").Where("user_id=? AND (status =? OR status=?)", userID, string(JobRunning), string(JobWaiting)).Find(&cloudBrains)
+	err := x.Cols("job_id", "status", "type").Where("user_id=? AND status !=?", userID, string(JobStopped)).Find(&cloudBrains)
 	return cloudBrains, err
 }
 
 func GetCloudbrainsNeededStopByRepoID(repoID int64) ([]*Cloudbrain, error) {
 	cloudBrains := make([]*Cloudbrain, 0)
-	err := x.Cols("job_id", "status", "type").Where("repo_id=? AND (status =? OR status=?)", repoID, string(JobRunning), string(JobWaiting)).Find(&cloudBrains)
+	err := x.Cols("job_id", "status", "type").Where("repo_id=? AND status !=?", repoID, string(JobStopped)).Find(&cloudBrains)
 	return cloudBrains, err
 }
 

routers/repo/cloudbrain.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
@@ -392,19 +393,39 @@ func StopJobs(cloudBrains []*models.Cloudbrain) {
 	for _, taskInfo := range cloudBrains {
 
 		if taskInfo.Type == models.TypeCloudBrainOne {
-			err := cloudbrain.StopJob(taskInfo.JobID)
+			err := retry(3, time.Second*30, func() error {
+				return cloudbrain.StopJob(taskInfo.JobID)
+			})
+
 			logErrorAndUpdateJobStatus(err, taskInfo)
 		} else {
 			param := models.NotebookAction{
 				Action: models.ActionStop,
 			}
-			_, err := modelarts.StopJob(taskInfo.JobID, param)
+			err := retry(3, time.Second*30, func() error {
+				_, err := modelarts.StopJob(taskInfo.JobID, param)
+				return err
+			})
 			logErrorAndUpdateJobStatus(err, taskInfo)
 		}
 
 	}
 }
 
+func retry(attempts int, sleep time.Duration, f func() error) (err error) {
+	for i := 0; i < attempts; i++ {
+		if i > 0 {
+			log.Warn("retrying after error:", err)
+			time.Sleep(sleep)
+		}
+		err = f()
+		if err == nil {
+			return nil
+		}
+	}
+	return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
+}
+
 func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
 	if err != nil {
 		log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)

routers/repo/modelarts.go

@@ -57,6 +57,8 @@ func ModelArtsIndex(ctx *context.Context) {
 		} else {
 			ciTasks[i].CanDebug = false
 		}
+
+		ciTasks[i].CanDel = models.CanDelJob(ctx.IsSigned, ctx.User, task)
 	}
 
 	pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)

templates/repo/cloudbrain/index.tmpl

@@ -352,7 +352,7 @@
 
                                     </div>
                                     
-                                    <!-- 删除镜像 -->
+                                    <!-- 删除任务 -->
                                     <form class="ui compact buttons" id="delForm-{{.JobID}}" action="{{if not .CanDel}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
                                         {{$.CsrfTokenHtml}}
                                         <a class="ui compact {{if not .CanDel}}disabled {{else}}red {{end}}button" onclick="assertDelete(this)" style="border-radius: .28571429rem;">

templates/repo/modelarts/index.tmpl

@@ -312,9 +312,9 @@
                                     </div>
 
                                     <!-- 删除任务 -->
-                                    <form class="ui compact buttons" id="delForm-{{.JobID}}" action="{{if ne .Status "STOPPED"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
+                                    <form class="ui compact buttons" id="delForm-{{.JobID}}" action="{{if not .CanDel}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
                                         {{$.CsrfTokenHtml}}
-                                        <a class="ui compact {{if ne .Status "STOPPED"}}disabled {{else}}red {{end}}button" onclick="assertDelete(this)" style="border-radius: .28571429rem;">
+                                        <a class="ui compact {{if not .CanDel}}disabled {{else}}red {{end}}button" onclick="assertDelete(this)" style="border-radius: .28571429rem;">
                                             删除
                                         </a>
                                     </form>