Merge branch 'V20211213' into zouap

modules/setting/setting.go

@@ -451,18 +451,21 @@ var (
 
 	//benchmark config
 	IsBenchmarkEnabled  bool
-	BenchmarkCode       string
+	BenchmarkOwner       string
+	BenchmarkName       string
 	BenchmarkServerHost string
 	BenchmarkCategory   string
 
 	//snn4imagenet config
 	IsSnn4imagenetEnabled  bool
-	Snn4imagenetCode       string
+	Snn4imagenetOwner       string
+	Snn4imagenetName       string
 	Snn4imagenetServerHost string
 
 	//snn4imagenet config
 	IsBrainScoreEnabled  bool
-	BrainScoreCode       string
+	BrainScoreOwner       string
+	BrainScoreName       string
 	BrainScoreServerHost string
 
 	//blockchain config
@@ -1238,18 +1241,21 @@ func NewContext() {
 
 	sec = Cfg.Section("benchmark")
 	IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)
-	BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("")
+	BenchmarkOwner = sec.Key("OWNER").MustString("")
+	BenchmarkName = sec.Key("NAME").MustString("")
 	BenchmarkServerHost = sec.Key("HOST").MustString("")
 	BenchmarkCategory = sec.Key("CATEGORY").MustString("")
 
 	sec = Cfg.Section("snn4imagenet")
 	IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false)
-	Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("")
+	Snn4imagenetOwner = sec.Key("OWNER").MustString("")
+	Snn4imagenetName = sec.Key("NAME").MustString("")
 	Snn4imagenetServerHost = sec.Key("HOST").MustString("")
 
 	sec = Cfg.Section("brainscore")
 	IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false)
-	BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("")
+	BrainScoreOwner = sec.Key("OWNER").MustString("")
+	BrainScoreName = sec.Key("NAME").MustString("")
 	BrainScoreServerHost = sec.Key("HOST").MustString("")
 
 	sec = Cfg.Section("blockchain")

routers/repo/cloudbrain.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"os/exec"
 	"regexp"
 	"sort"
 	"strconv"
@@ -251,13 +250,9 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 	downloadCode(repo, codePath)
 	uploadCodeToMinio(codePath + "/", jobName, "/code/")
 
-	modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath
-	err = os.MkdirAll(modelPath, os.ModePerm)
-	if err != nil {
-		cloudBrainNewDataPrepare(ctx)
-		ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
-		return
-	}
+	modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/"
+	mkModelPath(modelPath)
+	uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath + "/")
 
 	benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
 	if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) {
@@ -267,23 +262,25 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 				gpuType = gpuInfo.Value
 			}
 		}
-		downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType)
+		downloadRateCode(repo, jobName, setting.BenchmarkOwner, setting.BrainScoreName, benchmarkPath, form.BenchmarkCategory, gpuType)
 		uploadCodeToMinio(benchmarkPath + "/", jobName, cloudbrain.BenchMarkMountPath + "/")
 	}
 
 	snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
 	if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
-		downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "")
+		downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "")
 		uploadCodeToMinio(snn4imagenetPath + "/", jobName, cloudbrain.Snn4imagenetMountPath + "/")
 	}
 
 	brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
 	if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
-		downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "")
+		downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "")
 		uploadCodeToMinio(brainScorePath + "/", jobName, cloudbrain.BrainScoreMountPath + "/")
 	}
 
-	err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId)
+	err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, getMinioPath(jobName, cloudbrain.ModelMountPath + "/"),
+		getMinioPath(jobName, cloudbrain.BenchMarkMountPath + "/"), getMinioPath(jobName, cloudbrain.Snn4imagenetMountPath + "/"),
+		getMinioPath(jobName, cloudbrain.BrainScoreMountPath + "/"), jobType, gpuQueue, resourceSpecId)
 	if err != nil {
 		cloudBrainNewDataPrepare(ctx)
 		ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form)
@@ -607,7 +604,7 @@ func getImages(ctx *context.Context, imageType string) {
 
 func GetModelDirs(jobName string, parentDir string) (string, error) {
 	var req string
-	modelActualPath := setting.JobPath + jobName + "/model/"
+	modelActualPath := getMinioPath(jobName, cloudbrain.ModelMountPath + "/")
 	if parentDir == "" {
 		req = "baseDir=" + modelActualPath
 	} else {
@@ -617,6 +614,10 @@ func GetModelDirs(jobName string, parentDir string) (string, error) {
 	return getDirs(req)
 }
 
+func getMinioPath(jobName, suffixPath string) string {
+	return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + jobName + suffixPath
+}
+
 func CloudBrainDownloadModel(ctx *context.Context) {
 	parentDir := ctx.Query("parentDir")
 	fileName := ctx.Query("fileName")
@@ -698,19 +699,21 @@ func downloadCode(repo *models.Repository, codePath string) error {
 	return nil
 }
 
-func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error {
+func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType string) error {
 	err := os.MkdirAll(codePath, os.ModePerm)
 	if err != nil {
 		log.Error("mkdir codePath failed", err.Error())
 		return err
 	}
 
-	command := "git clone " + gitPath + " " + codePath
-	cmd := exec.Command("/bin/bash", "-c", command)
-	_, err = cmd.Output()
-
+	repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName)
 	if err != nil {
-		log.Error("exec.Command(%s) failed:%v", command, err)
+		log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error())
+		return err
+	}
+
+	if err := git.Clone(repoExt.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil {
+		log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err)
 		return err
 	}
 
@@ -772,6 +775,32 @@ func uploadCodeToMinio(codePath, jobName, parentDir string) error {
 	return nil
 }
 
+func mkModelPath(modelPath string) error {
+	err := os.MkdirAll(modelPath, os.ModePerm)
+	if err != nil {
+		log.Error("MkdirAll(%s) failed:%v", modelPath, err)
+		return err
+	}
+
+	fileName := modelPath + "README"
+	f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
+	if err != nil {
+		log.Error("OpenFile failed", err.Error())
+		return err
+	}
+
+	defer f.Close()
+
+	_, err = f.WriteString("You can put the model file into this directory and download it by the web page.")
+	if err != nil {
+		log.Error("WriteString failed", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+
 func SyncCloudbrainStatus() {
 	cloudBrains, err := models.GetCloudBrainUnStoppedJob()
 	if err != nil {