isolation

custom/conf/app.ini.sample

@@ -1059,50 +1059,50 @@ RESULT_BACKEND = redis://localhost:6379
 [cloudbrain]
 USERNAME =
 PASSWORD =
-REST_SERVER_HOST = http://192.168.202.73
-JOB_PATH = /datasets/minio/data/opendata/jobs/
-DEBUG_SERVER_HOST = http://192.168.202.73/
+REST_SERVER_HOST =
+JOB_PATH =
+DEBUG_SERVER_HOST =
 ; cloudbrain visit opendata
-USER     = cW4cMtH24eoWPE7X
-PWD      = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DA.C
-GPU_TYPE_DEFAULT = openidebug
-GPU_TYPES = {"gpu_type":[{"id":1,"queue":"openidebug","value":"T4"},{"id":2,"queue":"openidgx","value":"V100"}]}
+USER     =
+PWD      =
+GPU_TYPE_DEFAULT =
+GPU_TYPES =
 
 [benchmark]
-ENABLED = true
-BENCHMARKCODE = https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git
-HOST = http://192.168.202.90:3366/
+ENABLED =
+BENCHMARKCODE =
+HOST =
 
 [snn4imagenet]
-ENABLED = true
-SNN4IMAGENETCODE = https://yult:eh2Ten4iLYjFkbj@git.openi.org.cn/ylt/snn4imagenet.git
-HOST = http://192.168.202.90:3366/
+ENABLED =
+SNN4IMAGENETCODE =
+HOST =
 
 [decompress]
-HOST     = http://192.168.207.34:39987
-USER     = cW4cMtH24eoWPE7X
-PASSWORD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC
+HOST     =
+USER     =
+PASSWORD =
 
 [blockchain]
-HOST     = http://192.168.207.84:3002/
-COMMIT_VALID_DATE = 2021-01-15
+HOST     =
+COMMIT_VALID_DATE =
 
 [obs]
-ENDPOINT = https://obs.cn-south-222.ai.pcl.cn
-ACCESS_KEY_ID = FDP3LRMHLB9S77VWEHE3
-SECRET_ACCESS_KEY = LyM82Wk80pgjhs2z7AdDcsdpCWhbsJtSzQ7hkESN
-BUCKET = testopendata
-LOCATION = cn-south-222
-BASE_PATH = attachment/
+ENDPOINT =
+ACCESS_KEY_ID =
+SECRET_ACCESS_KEY =
+BUCKET =
+LOCATION =
+BASE_PATH =
 
 [modelarts]
-ORGANIZATION = modelarts
-ENDPOINT = https://modelarts.cn-south-222.ai.pcl.cn
-PROJECT_ID = edfccf24aace4e17a56da6bcbb55a5aa
-PROJECT_NAME = cn-south-222_test
-USERNAME = test1
-PASSWORD = Qizhi@test.
-DOMAIN = cn-south-222
+ORGANIZATION =
+ENDPOINT =
+PROJECT_ID =
+PROJECT_NAME =
+USERNAME =
+PASSWORD =
+DOMAIN =
 
 [radar_map]
 impact=0.3

modules/setting/setting.go

@@ -439,14 +439,15 @@ var (
 	DecompressOBSTaskName   string
 
 	//cloudbrain config
-	CBAuthUser      string
-	CBAuthPassword  string
-	RestServerHost  string
-	JobPath         string
-	JobType         string
-	GpuTypes        string
-	DebugServerHost string
-	ResourceSpecs   string
+	CBAuthUser       string
+	CBAuthPassword   string
+	RestServerHost   string
+	JobPath          string
+	CBCodePathPrefix string
+	JobType          string
+	GpuTypes         string
+	DebugServerHost  string
+	ResourceSpecs    string
 
 	//benchmark config
 	IsBenchmarkEnabled  bool
@@ -549,7 +550,7 @@ var (
 		RecordBeginTime    string
 		IgnoreMirrorRepo   bool
 	}{}
-	
+
 	Warn_Notify_Mails []string
 )
 
@@ -1216,8 +1217,8 @@ func NewContext() {
 
 	sec = Cfg.Section("decompress")
 	DecompressAddress = sec.Key("HOST").MustString("http://192.168.207.34:39987")
-	AuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X")
-	AuthPassword = sec.Key("PASSWORD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC")
+	AuthUser = sec.Key("USER").MustString("")
+	AuthPassword = sec.Key("PASSWORD").MustString("")
 
 	sec = Cfg.Section("labelsystem")
 	LabelTaskName = sec.Key("LabelTaskName").MustString("LabelRedisQueue")
@@ -1225,10 +1226,11 @@ func NewContext() {
 	DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue")
 
 	sec = Cfg.Section("cloudbrain")
-	CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X")
-	CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC")
+	CBAuthUser = sec.Key("USER").MustString("")
+	CBAuthPassword = sec.Key("PWD").MustString("")
 	RestServerHost = sec.Key("REST_SERVER_HOST").MustString("http://192.168.202.73")
 	JobPath = sec.Key("JOB_PATH").MustString("/datasets/minio/data/opendata/jobs/")
+	CBCodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("jobs/")
 	DebugServerHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73")
 	JobType = sec.Key("GPU_TYPE_DEFAULT").MustString("openidebug")
 	GpuTypes = sec.Key("GPU_TYPES").MustString("")
@@ -1236,31 +1238,31 @@ func NewContext() {
 
 	sec = Cfg.Section("benchmark")
 	IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)
-	BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git")
-	BenchmarkServerHost = sec.Key("HOST").MustString("http://192.168.202.90:3366/")
+	BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("")
+	BenchmarkServerHost = sec.Key("HOST").MustString("")
 	BenchmarkCategory = sec.Key("CATEGORY").MustString("")
 
 	sec = Cfg.Section("snn4imagenet")
 	IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false)
-	Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/snn4imagenet_script.git")
-	Snn4imagenetServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/")
+	Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("")
+	Snn4imagenetServerHost = sec.Key("HOST").MustString("")
 
 	sec = Cfg.Section("brainscore")
 	IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false)
-	BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/brainscore_script.git")
-	BrainScoreServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/")
+	BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("")
+	BrainScoreServerHost = sec.Key("HOST").MustString("")
 
 	sec = Cfg.Section("blockchain")
 	BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/")
 	CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15")
 
 	sec = Cfg.Section("obs")
-	Endpoint = sec.Key("ENDPOINT").MustString("112.95.163.82")
+	Endpoint = sec.Key("ENDPOINT").MustString("")
 	AccessKeyID = sec.Key("ACCESS_KEY_ID").MustString("")
 	SecretAccessKey = sec.Key("SECRET_ACCESS_KEY").MustString("")
-	Bucket = sec.Key("BUCKET").MustString("testopendata")
-	Location = sec.Key("LOCATION").MustString("cn-south-222")
-	BasePath = sec.Key("BASE_PATH").MustString("attachment/")
+	Bucket = sec.Key("BUCKET").MustString("")
+	Location = sec.Key("LOCATION").MustString("")
+	BasePath = sec.Key("BASE_PATH").MustString("")
 	TrainJobModelPath = sec.Key("TrainJobModel_Path").MustString("job/")
 	OutPutPath = sec.Key("Output_Path").MustString("output/")
 	CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/")
@@ -1268,17 +1270,17 @@ func NewContext() {
 	PROXYURL = sec.Key("PROXY_URL").MustString("")
 
 	sec = Cfg.Section("modelarts")
-	ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80")
-	IamHost = sec.Key("IAMHOST").MustString("112.95.163.80")
+	ModelArtsHost = sec.Key("ENDPOINT").MustString("")
+	IamHost = sec.Key("IAMHOST").MustString("")
 	ProjectID = sec.Key("PROJECT_ID").MustString("")
 	ProjectName = sec.Key("PROJECT_NAME").MustString("")
 	ModelArtsUsername = sec.Key("USERNAME").MustString("")
 	ModelArtsPassword = sec.Key("PASSWORD").MustString("")
-	ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222")
+	ModelArtsDomain = sec.Key("DOMAIN").MustString("")
 	AllowedOrg = sec.Key("ORGANIZATION").MustString("")
 	ProfileID = sec.Key("PROFILE_ID").MustString("")
 	PoolInfos = sec.Key("POOL_INFOS").MustString("")
-	Flavor = sec.Key("FLAVOR").MustString("modelarts.bm.910.arm.public.2")
+	Flavor = sec.Key("FLAVOR").MustString("")
 	ResourcePools = sec.Key("Resource_Pools").MustString("")
 	Engines = sec.Key("Engines").MustString("")
 	EngineVersions = sec.Key("Engine_Versions").MustString("")
@@ -1286,10 +1288,10 @@ func NewContext() {
 	TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("")
 
 	sec = Cfg.Section("elk")
-	ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch")
-	ElkUser = sec.Key("ELKUSER").MustString("Qizhi")
-	ElkPassword = sec.Key("ELKPASSWORD").MustString("Pcl2020")
-	Index = sec.Key("INDEX").MustString("filebeat-7.3.2*")
+	ElkUrl = sec.Key("ELKURL").MustString("")
+	ElkUser = sec.Key("ELKUSER").MustString("")
+	ElkPassword = sec.Key("ELKPASSWORD").MustString("")
+	Index = sec.Key("INDEX").MustString("")
 	TimeField = sec.Key("TIMEFIELD").MustString(" @timestamptest")
 	ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time")
 

modules/storage/local.go

@@ -76,3 +76,7 @@ func (l *LocalStorage) PresignedPutURL(path string) (string, error) {
 func (l *LocalStorage) HasObject(path string) (bool, error) {
 	return false, nil
 }
+
+func (l *LocalStorage) UploadObject(fileName, filePath string) error {
+	return nil
+}

modules/storage/minio.go

@@ -122,3 +122,9 @@ func (m *MinioStorage) HasObject(path string) (bool, error) {
 
 	return hasObject, nil
 }
+
+//upload object
+func (m *MinioStorage) UploadObject(fileName, filePath string) error {
+	_, err := m.client.FPutObject(m.bucket, fileName, filePath, minio.PutObjectOptions{})
+	return err
+}

modules/storage/storage.go

@@ -26,6 +26,7 @@ type ObjectStorage interface {
 	PresignedGetURL(path string, fileName string) (string, error)
 	PresignedPutURL(path string) (string, error)
 	HasObject(path string) (bool, error)
+	UploadObject(fileName, filePath string) error
 }
 
 // Copy copys a file from source ObjectStorage to dest ObjectStorage

routers/repo/cloudbrain.go

@@ -249,6 +249,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 	}
 	repo := ctx.Repo.Repository
 	downloadCode(repo, codePath)
+	uploadCodeToMinio(codePath + "/", jobName, "/code/")
 
 	modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath
 	err = os.MkdirAll(modelPath, os.ModePerm)
@@ -267,16 +268,19 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 			}
 		}
 		downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType)
+		uploadCodeToMinio(benchmarkPath + "/", jobName, cloudbrain.BenchMarkMountPath + "/")
 	}
 
 	snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
 	if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
 		downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "")
+		uploadCodeToMinio(snn4imagenetPath + "/", jobName, cloudbrain.Snn4imagenetMountPath + "/")
 	}
 
 	brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
 	if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
 		downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "")
+		uploadCodeToMinio(brainScorePath + "/", jobName, cloudbrain.BrainScoreMountPath + "/")
 	}
 
 	err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId)
@@ -703,8 +707,8 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc
 
 	command := "git clone " + gitPath + " " + codePath
 	cmd := exec.Command("/bin/bash", "-c", command)
-	output, err := cmd.Output()
-	log.Info(string(output))
+	_, err = cmd.Output()
+
 	if err != nil {
 		log.Error("exec.Command(%s) failed:%v", command, err)
 		return err
@@ -741,6 +745,33 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc
 	return nil
 }
 
+func uploadCodeToMinio(codePath, jobName, parentDir string) error {
+	files, err := readDir(codePath)
+	if err != nil {
+		log.Error("readDir(%s) failed: %s", codePath, err.Error())
+		return err
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			if err = uploadCodeToMinio(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil {
+				log.Error("uploadCodeToMinio(%s) failed: %s", file.Name(), err.Error())
+				return err
+			}
+		} else {
+			destObject := setting.CBCodePathPrefix + jobName + parentDir + file.Name()
+			sourceFile := codePath + file.Name()
+			err = storage.Attachments.UploadObject(destObject, sourceFile)
+			if err != nil {
+				log.Error("UploadObject(%s) failed: %s", file.Name(), err.Error())
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 func SyncCloudbrainStatus() {
 	cloudBrains, err := models.GetCloudBrainUnStoppedJob()
 	if err != nil {