Merge pull request 'bench-mark' (#141) from bench-mark into develop

Reviewed-by: yuyuanshifu <747342561@qq.com>
Reviewed-by: 林嘉怡 <2441898885@qq.com>

custom/conf/app.ini.sample

@@ -1056,6 +1056,11 @@ USER     = cW4cMtH24eoWPE7X
 PWD      = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DA.C
 JOB_TYPE = debug_openi
 
+[benchmark]
+ENABLED = true
+BENCHMARKCODE = https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git
+HOST = http://192.168.202.90:3366/
+
 [decompress]
 HOST     = http://192.168.207.34:39987
 USER     = cW4cMtH24eoWPE7X

models/attachment.go

@@ -384,6 +384,7 @@ func getPrivateAttachments(e Engine, userID int64) ([]*AttachmentUsername, error
 	return attachments, nil
 }
 
+/*
 func GetAllUserAttachments(userID int64) ([]*AttachmentUsername, error) {
 	attachsPub, err := getAllPublicAttachments(x)
 	if err != nil {
@@ -400,3 +401,17 @@ func GetAllUserAttachments(userID int64) ([]*AttachmentUsername, error) {
 	return append(attachsPub, attachsPri...), nil
 }
 
+ */
+
+func getAllUserAttachments(e Engine, userID int64) ([]*AttachmentUsername, error) {
+	attachments := make([]*AttachmentUsername, 0, 10)
+	if err := e.Table("attachment").Join("LEFT", "`user`", "attachment.uploader_id " +
+		"= `user`.id").Where("decompress_state= ? and (uploader_id= ? or is_private = ?)", DecompressStateDone, userID, false).Find(&attachments); err != nil {
+		return nil, err
+	}
+	return attachments, nil
+}
+
+func GetAllUserAttachments(userID int64) ([]*AttachmentUsername, error) {
+	return getAllUserAttachments(x, userID)
+}

models/cloudbrain.go

@@ -13,6 +13,7 @@ import (
 )
 
 type CloudbrainStatus string
+type JobType string
 
 const (
 	JobWaiting   CloudbrainStatus = "WAITING"
@@ -20,11 +21,15 @@ const (
 	JobSucceeded CloudbrainStatus = "SUCCEEDED"
 	JobFailed    CloudbrainStatus = "FAILED"
 	JobRunning   CloudbrainStatus = "RUNNING"
+
+	JobTypeDebug		JobType = "DEBUG"
+	JobTypeBenchmark	JobType = "BENCHMARK"
 )
 
 type Cloudbrain struct {
 	ID          int64  `xorm:"pk autoincr"`
 	JobID       string `xorm:"INDEX NOT NULL"`
+	JobType		string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"`
 	JobName     string			   `xorm:"INDEX"`
 	Status      string             `xorm:"INDEX"`
 	UserID      int64              `xorm:"INDEX"`
@@ -131,6 +136,13 @@ type TaskPod struct {
 	} `json:"taskStatuses"`
 }
 
+
+type TaskInfo struct {
+	Username	string	`json:"username"`
+	TaskName	string	`json:"task_name"`
+	CodeName	string	`json:"code_name"`
+}
+
 func ConvertToTaskPod(input map[string]interface{}) (TaskPod, error) {
 	data, _ := json.Marshal(input)
 	var taskPod TaskPod

modules/auth/cloudbrain.go

@@ -11,6 +11,7 @@ type CreateCloudBrainForm struct {
 	Image   string `form:"image" binding:"Required"`
 	Command string `form:"command" binding:"Required"`
 	Attachment string `form:"attachment" binding:"Required"`
+	JobType string `form:"job_type" binding:"Required"`
 }
 
 type CommitImageCloudBrainForm struct {

modules/cloudbrain/cloudbrain.go

@@ -14,13 +14,15 @@ const (
 	CodeMountPath =	"/code"
 	DataSetMountPath = "/dataset"
 	ModelMountPath = "/model"
+	BenchMarkMountPath = "/benchmark"
+	TaskInfoName = "/taskInfo"
 
 	SubTaskName = "task1"
 
 	Success = "S000"
 )
 
-func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath string) error {
+func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, jobType string) error {
 	dataActualPath := setting.Attachment.Minio.RealPath +
 		setting.Attachment.Minio.Bucket + "/" +
 		setting.Attachment.Minio.BasePath +
@@ -69,6 +71,13 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath,
 					ReadOnly: 			false,
 				},
 			},
+			{
+				HostPath: models.StHostPath{
+					Path:				benchmarkPath,
+					MountPath:			BenchMarkMountPath,
+					ReadOnly: 			true,
+				},
+			},
 		},
 	})
 	if err != nil {
@@ -88,6 +97,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath,
 		JobID:   jobID,
 		JobName: jobName,
 		SubTaskName: SubTaskName,
+		JobType: jobType,
 	})
 
 	if err != nil {

modules/setting/setting.go

@@ -438,6 +438,11 @@ var (
 	JobPath					string
 	JobType					string
 	DebugServerHost			string
+
+	//benchmark config
+	IsBenchmarkEnabled		bool
+	BenchmarkCode			string
+	BenchmarkServerHost		string
 )
 
 // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@@ -1113,6 +1118,11 @@ func NewContext() {
 	JobPath = sec.Key("JOB_PATH").MustString("/datasets/minio/data/opendata/jobs/")
 	DebugServerHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73")
 	JobType = sec.Key("JOB_TYPE").MustString("debug_openi")
+
+	sec = Cfg.Section("benchmark")
+	IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)
+	BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git")
+	BenchmarkServerHost = sec.Key("HOST").MustString("http://192.168.202.90:3366/")
 }
 
 func loadInternalToken(sec *ini.Section) string {

routers/repo/cloudbrain.go

@@ -2,8 +2,10 @@ package repo
 
 import (
 	"code.gitea.io/gitea/modules/git"
+	"encoding/json"
 	"errors"
 	"os"
+	"os/exec"
 	"strconv"
 	"strings"
 	"time"
@@ -109,6 +111,8 @@ func CloudBrainNew(ctx *context.Context) {
 	ctx.Data["code_path"] = cloudbrain.CodeMountPath
 	ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
 	ctx.Data["model_path"] = cloudbrain.ModelMountPath
+	ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath
+	ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled
 	ctx.HTML(200, tplCloudBrainNew)
 }
 
@@ -118,18 +122,24 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 	image := form.Image
 	command := form.Command
 	uuid := form.Attachment
+	jobType := form.JobType
 	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
 	repo := ctx.Repo.Repository
 	downloadCode(repo, codePath)
 
-	modelPath := setting.JobPath + jobName + "/model"
+	modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath
 	err := os.MkdirAll(modelPath, os.ModePerm)
 	if err != nil {
 		ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
 		return
 	}
 
-	err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath)
+	benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
+	if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) {
+		downloadBenchmarkCode(repo, jobName, benchmarkPath)
+	}
+
+	err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, jobType)
 	if err != nil {
 		ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
 		return
@@ -270,6 +280,17 @@ func CloudBrainDel(ctx *context.Context) {
 	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain")
 }
 
+func CloudBrainBenchmark(ctx *context.Context) {
+	var jobID = ctx.Params(":jobid")
+	_, err := models.GetCloudbrainByJobID(jobID)
+	if err != nil {
+		ctx.ServerError("GetCloudbrainByJobID failed", err)
+		return
+	}
+
+	ctx.Redirect(setting.BenchmarkServerHost)
+}
+
 func downloadCode(repo *models.Repository, codePath string) error {
 	if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil {
 		log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
@@ -278,3 +299,47 @@ func downloadCode(repo *models.Repository, codePath string) error {
 
 	return nil
 }
+
+func downloadBenchmarkCode(repo *models.Repository, taskName string, benchmarkPath string) error {
+	err := os.MkdirAll(benchmarkPath, os.ModePerm)
+	if err != nil {
+		log.Error("mkdir benchmarkPath failed", err.Error())
+		return err
+	}
+
+	command := "git clone " + setting.BenchmarkCode + " " + benchmarkPath
+	cmd := exec.Command("/bin/bash", "-c", command)
+	output, err := cmd.Output()
+	log.Info(string(output))
+	if err != nil {
+		log.Error("exec.Command(%s) failed:%v", command, err)
+		return err
+	}
+
+	fileName := benchmarkPath + cloudbrain.TaskInfoName
+	f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
+	if err != nil {
+		log.Error("OpenFile failed", err.Error())
+		return err
+	}
+
+	defer f.Close()
+
+	data, err := json.Marshal(models.TaskInfo{
+		Username: repo.Owner.Name,
+		TaskName: taskName,
+		CodeName: repo.Name,
+	})
+	if err != nil {
+		log.Error("json.Marshal failed", err.Error())
+		return err
+	}
+
+	_, err = f.Write(data)
+	if err != nil {
+		log.Error("WriteString failed", err.Error())
+		return err
+	}
+
+	return nil
+}

routers/routes/routes.go

@@ -900,6 +900,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 				m.Post("/commit_image", reqRepoCloudBrainWriter, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage)
 				m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop)
 				m.Post("/del", reqRepoCloudBrainWriter, repo.CloudBrainDel)
+				m.Get("/benchmark", reqRepoCloudBrainWriter, repo.CloudBrainBenchmark)
 			})
 			m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew)
 			m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate)

templates/repo/cloudbrain/index.tmpl

@@ -169,6 +169,11 @@
     .dis {
         margin-bottom: 20px;
     }
+    
+    .disabled {
+        cursor: pointer;
+        pointer-events: none;
+    }
 </style>
 
 <!-- 弹窗 -->
@@ -221,8 +226,8 @@
                             <div class="column right aligned">
                                 <div class="ui right dropdown type jump item">
                                     <span class="text">
-                                                    {{.i18n.Tr "repo.issues.filter_sort"}}<i class="dropdown icon"></i>
-                                                </span>
+                                        {{.i18n.Tr "repo.issues.filter_sort"}}<i class="dropdown icon"></i>
+                                    </span>
                                 </div>
                             </div>
                         </div>
@@ -235,7 +240,7 @@
                             <div class="row">
 
                                 <!-- 任务名 -->
-                                <div class="five wide column">
+                                <div class="four wide column">
                                     <a class="title" href="{{$.Link}}/{{.JobID}}">
                                         <span class="fitted">{{svg "octicon-tasklist" 16}}</span>
                                         <span class="fitted">{{.JobName}}</span>
@@ -255,8 +260,17 @@
                                 <!-- 查看 -->
                                 <div class="one wide column">
                                     <span class="ui text center clipboard">
-                                                    <a class="title" href="{{$.Link}}/{{.JobID}}">
-                                                        <span class="fitted">查看</span>
+                                        <a class="title" href="{{$.Link}}/{{.JobID}}">
+                                            <span class="fitted">查看</span>
+                                    </a>
+                                    </span>
+                                </div>
+
+                                <!-- 评分 -->
+                                <div class="one wide column">
+                                    <span class="ui text center clipboard">
+                                        <a class="title" onclick="stop(this)" href="{{if and (ne .Status "WAITING") (eq .JobType "BENCHMARK")}}{{$.Link}}/{{.JobID}}/benchmark{{else}}javascript:void(0);{{end}}" style="{{if and (ne .Status "WAITING") (eq .JobType "BENCHMARK")}}{{else}}color:#CCCCCC{{end}}">
+                                            <span class="fitted">评分</span>
                                     </a>
                                     </span>
                                 </div>
@@ -264,19 +278,18 @@
                                 <!-- 删除镜像 -->
                                 <div class="one wide column">
                                     <span class="ui text center clipboard">
-                                                    <form id="delForm-{{.JobID}}" action="{{if ne .Status "STOPPED"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
-                                                        {{$.CsrfTokenHtml}}
-                                                        <!-- <a class="fitted" onclick="document.getElementById('delForm-{{.JobID}}').submit();" style="{{if ne .Status "STOPPED"}}color:#CCCCCC{{end}}; font-size:16px; font-weight:bold">删除</a> -->
-                                                        <a class="fitted" onclick="assertDelete(this)" style="{{if ne .Status "STOPPED"}}color:#CCCCCC{{end}}; font-size:16px; font-weight:bold">删除</a>
-                                                    </form>
-                                                </span>
+                                        <form id="delForm-{{.JobID}}" action="{{if ne .Status "STOPPED"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
+                                            {{$.CsrfTokenHtml}}
+                                            <a class="fitted" onclick="assertDelete(this)" style="{{if ne .Status "STOPPED"}}color:#CCCCCC{{end}}; font-size:16px; font-weight:bold">删除</a>
+                                        </form>
+                                    </span>
                                 </div>
 
                                 <!-- 调试 -->
                                 <div class="one wide column">
                                     <span class="ui text center clipboard">
-                                                    <a class="title" href="{{if not .CanDebug}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/debug{{end}}" style="{{if not .CanDebug}}color:#CCCCCC{{end}}">
-                                                        <span class="fitted">调试</span>
+                                        <a class="title" onclick="stop(this)" href="{{if not .CanDebug}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/debug{{end}}" style="{{if not .CanDebug}}color:#CCCCCC{{end}}">
+                                            <span class="fitted">调试</span>
                                     </a>
                                     </span>
                                 </div>
@@ -284,11 +297,11 @@
                                 <!-- 停止 -->
                                 <div class="one wide column">
                                     <span class="ui text center clipboard">
-                                                    <form id="stopForm-{{.JobID}}" action="{{if ne .Status "RUNNING"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post">
-                                                        {{$.CsrfTokenHtml}}
-                                                        <a class="fitted" onclick="document.getElementById('stopForm-{{.JobID}}').submit();" style="{{if ne .Status "RUNNING"}}color:#CCCCCC{{end}}; font-size:16px; font-weight:bold">停止</a>
-                                                    </form>
-                                                </span>
+                                        <form id="stopForm-{{.JobID}}" action="{{if ne .Status "RUNNING"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post">
+                                            {{$.CsrfTokenHtml}}
+                                            <a class="fitted" onclick="document.getElementById('stopForm-{{.JobID}}').submit();" style="{{if ne .Status "RUNNING"}}color:#CCCCCC{{end}}; font-size:16px; font-weight:bold">停止</a>
+                                        </form>
+                                    </span>
                                 </div>
 
                                 <!-- 接收结果 -->
@@ -325,8 +338,8 @@
                                             <div class="inline field">
                                                 <label></label>
                                                 <button class="ui green button" onclick="showmask()">
-                                                                {{$.i18n.Tr "repo.cloudbrain.commit_image"}}
-                                                            </button>
+                                                    {{$.i18n.Tr "repo.cloudbrain.commit_image"}}
+                                                </button>
                                             </div>
                                         </form>
                                     </div>
@@ -373,6 +386,15 @@
 {{template "base/footer" .}}
 
 <script>
+    // 调试和评分新开窗口
+    function stop(obj) {
+        if (obj.style.color != "rgb(204, 204, 204)") {
+            obj.target = '_blank'
+        } else {
+            return
+        }
+    }
+
     // 删除时用户确认
     function assertDelete(obj) {
         if (obj.style.color == "rgb(204, 204, 204)") {

templates/repo/cloudbrain/new.tmpl

@@ -80,6 +80,10 @@
             -webkit-transform: scaleY(1.0);
         }
     }
+    
+    .inline.required.field.cloudbrain_benchmark {
+        display: none;
+    }
 </style>
 
 <div id="mask">
@@ -102,28 +106,38 @@
                     {{.i18n.Tr "repo.cloudbrain.new"}}
                 </h3>
                 <div class="ui attached segment">
-                    <br>
+                    <!-- <br> -->
                     <div class="inline required field">
                         <label>任务名称</label>
                         <input name="job_name" id="cloudbrain_job_name" placeholder="任务名称" value="{{.job_name}}" tabindex="3" autofocus required maxlength="255">
                     </div>
-                    <br>
+
+                    <div class="inline required field" style="{{if .is_benchmark_enabled}}display:block;{{else}}display:none;{{end}}">
+                        <label>任务类型</label>
+                        <select id="cloudbrain_job_type" class="ui search dropdown" placeholder="选择任务类型" style='width:385px' name="job_type">
+                            <option name="job_type" value="DEBUG">DEBUG</option>
+                            <option name="job_type" value="BENCHMARK">BENCHMARK</option>
+                        </select>
+                    </div>
+
                     <div class="inline required field">
                         <label>镜像</label>
                         <select class="ui search dropdown" id="cloudbrain_image" placeholder="选择镜像" style='width:385px' name="image">
-              {{range .images}}
-                <option name="image" value="{{.Place}}">{{.PlaceView}}</option>
-              {{end}}
-            </select>
+                            {{range .images}}
+                                <option name="image" value="{{.Place}}">{{.PlaceView}}</option>
+                            {{end}}
+                        </select>
                     </div>
+
                     <div class="inline required field">
                         <label>数据集(只有zip格式的数据集才能发起云脑任务)</label>
                         <select id="cloudbrain_dataset" class="ui search dropdown" placeholder="选择数据集" style='width:385px' name="attachment">
-              {{range .attachments}}
-                <option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>
-              {{end}}
-            </select>
+                            {{range .attachments}}
+                                <option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>
+                            {{end}}
+                        </select>
                     </div>
+
                     <div class="inline required field">
                         <label>数据集存放路径</label>
                         <input name="dataset_path" id="cloudbrain_dataset_path" value="{{.dataset_path}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
@@ -136,6 +150,10 @@
                         <label>代码存放路径</label>
                         <input name="code_path" id="cloudbrain_code_path" value="{{.code_path}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
                     </div>
+                    <div class="inline required field cloudbrain_benchmark">
+                        <label>benchmark脚本存放路径</label>
+                        <input name="benchmark_path" id="cloudbrain_benchmark_path" value="{{.benchmark_path}}" tabindex="3" autofocus required maxlength="255" readonly="readonly">
+                    </div>
                     <div class="inline required field" hidden>
                         <label>启动命令</label>
                         <textarea name="command" rows="10" readonly="readonly">{{.command}}</textarea>
@@ -166,4 +184,17 @@
             document.getElementById("mask").style.display = "none"
         }
     }
+
+    $('select.dropdown')
+        .dropdown();
+
+    $(function() {
+        $("#cloudbrain_job_type").change(function() {
+            if ($(this).val() == 'BENCHMARK') {
+                $(".cloudbrain_benchmark").show();
+            } else {
+                $(".cloudbrain_benchmark").hide();
+            }
+        })
+    })
 </script>