From 9ab213bc6acd9dd5bc0843eb6471c259ac505d5b Mon Sep 17 00:00:00 2001 From: Gitea Date: Mon, 6 Dec 2021 19:23:37 +0800 Subject: [PATCH 1/3] iso --- custom/conf/app.ini.sample | 62 ++++++++++++++++++------------------ modules/setting/setting.go | 64 ++++++++++++++++++++------------------ modules/storage/local.go | 4 +++ modules/storage/minio.go | 6 ++++ modules/storage/storage.go | 1 + routers/repo/cloudbrain.go | 35 +++++++++++++++++++-- 6 files changed, 108 insertions(+), 64 deletions(-) diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 8bc971f2c..d294c8823 100755 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -1059,50 +1059,50 @@ RESULT_BACKEND = redis://localhost:6379 [cloudbrain] USERNAME = PASSWORD = -REST_SERVER_HOST = http://192.168.202.73 -JOB_PATH = /datasets/minio/data/opendata/jobs/ -DEBUG_SERVER_HOST = http://192.168.202.73/ +REST_SERVER_HOST = +JOB_PATH = +DEBUG_SERVER_HOST = ; cloudbrain visit opendata -USER = cW4cMtH24eoWPE7X -PWD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DA.C -GPU_TYPE_DEFAULT = openidebug -GPU_TYPES = {"gpu_type":[{"id":1,"queue":"openidebug","value":"T4"},{"id":2,"queue":"openidgx","value":"V100"}]} +USER = +PWD = +GPU_TYPE_DEFAULT = +GPU_TYPES = [benchmark] -ENABLED = true -BENCHMARKCODE = https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git -HOST = http://192.168.202.90:3366/ +ENABLED = +BENCHMARKCODE = +HOST = [snn4imagenet] -ENABLED = true -SNN4IMAGENETCODE = https://yult:eh2Ten4iLYjFkbj@git.openi.org.cn/ylt/snn4imagenet.git -HOST = http://192.168.202.90:3366/ +ENABLED = +SNN4IMAGENETCODE = +HOST = [decompress] -HOST = http://192.168.207.34:39987 -USER = cW4cMtH24eoWPE7X -PASSWORD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC +HOST = +USER = +PASSWORD = [blockchain] -HOST = http://192.168.207.84:3002/ -COMMIT_VALID_DATE = 2021-01-15 +HOST = +COMMIT_VALID_DATE = [obs] -ENDPOINT = https://obs.cn-south-222.ai.pcl.cn -ACCESS_KEY_ID = FDP3LRMHLB9S77VWEHE3 -SECRET_ACCESS_KEY = LyM82Wk80pgjhs2z7AdDcsdpCWhbsJtSzQ7hkESN -BUCKET = testopendata -LOCATION = cn-south-222 -BASE_PATH = attachment/ +ENDPOINT = +ACCESS_KEY_ID = +SECRET_ACCESS_KEY = +BUCKET = +LOCATION = +BASE_PATH = [modelarts] -ORGANIZATION = modelarts -ENDPOINT = https://modelarts.cn-south-222.ai.pcl.cn -PROJECT_ID = edfccf24aace4e17a56da6bcbb55a5aa -PROJECT_NAME = cn-south-222_test -USERNAME = test1 -PASSWORD = Qizhi@test. -DOMAIN = cn-south-222 +ORGANIZATION = +ENDPOINT = +PROJECT_ID = +PROJECT_NAME = +USERNAME = +PASSWORD = +DOMAIN = [radar_map] impact=0.3 diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 92eae63b0..3f7345900 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -439,14 +439,15 @@ var ( DecompressOBSTaskName string //cloudbrain config - CBAuthUser string - CBAuthPassword string - RestServerHost string - JobPath string - JobType string - GpuTypes string - DebugServerHost string - ResourceSpecs string + CBAuthUser string + CBAuthPassword string + RestServerHost string + JobPath string + CBCodePathPrefix string + JobType string + GpuTypes string + DebugServerHost string + ResourceSpecs string //benchmark config IsBenchmarkEnabled bool @@ -549,7 +550,7 @@ var ( RecordBeginTime string IgnoreMirrorRepo bool }{} - + Warn_Notify_Mails []string ) @@ -1216,8 +1217,8 @@ func NewContext() { sec = Cfg.Section("decompress") DecompressAddress = sec.Key("HOST").MustString("http://192.168.207.34:39987") - AuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") - AuthPassword = sec.Key("PASSWORD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") + AuthUser = sec.Key("USER").MustString("") + AuthPassword = sec.Key("PASSWORD").MustString("") sec = Cfg.Section("labelsystem") LabelTaskName = sec.Key("LabelTaskName").MustString("LabelRedisQueue") @@ -1225,10 +1226,11 @@ func NewContext() { DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") sec = Cfg.Section("cloudbrain") - CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") - CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") + CBAuthUser = sec.Key("USER").MustString("") + CBAuthPassword = sec.Key("PWD").MustString("") RestServerHost = sec.Key("REST_SERVER_HOST").MustString("http://192.168.202.73") JobPath = sec.Key("JOB_PATH").MustString("/datasets/minio/data/opendata/jobs/") + CBCodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("jobs/") DebugServerHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73") JobType = sec.Key("GPU_TYPE_DEFAULT").MustString("openidebug") GpuTypes = sec.Key("GPU_TYPES").MustString("") @@ -1236,31 +1238,31 @@ func NewContext() { sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) - BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git") - BenchmarkServerHost = sec.Key("HOST").MustString("http://192.168.202.90:3366/") + BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("") + BenchmarkServerHost = sec.Key("HOST").MustString("") BenchmarkCategory = sec.Key("CATEGORY").MustString("") sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) - Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/snn4imagenet_script.git") - Snn4imagenetServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") + Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("") + Snn4imagenetServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("brainscore") IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false) - BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/brainscore_script.git") - BrainScoreServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") + BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("") + BrainScoreServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("blockchain") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") sec = Cfg.Section("obs") - Endpoint = sec.Key("ENDPOINT").MustString("112.95.163.82") + Endpoint = sec.Key("ENDPOINT").MustString("") AccessKeyID = sec.Key("ACCESS_KEY_ID").MustString("") SecretAccessKey = sec.Key("SECRET_ACCESS_KEY").MustString("") - Bucket = sec.Key("BUCKET").MustString("testopendata") - Location = sec.Key("LOCATION").MustString("cn-south-222") - BasePath = sec.Key("BASE_PATH").MustString("attachment/") + Bucket = sec.Key("BUCKET").MustString("") + Location = sec.Key("LOCATION").MustString("") + BasePath = sec.Key("BASE_PATH").MustString("") TrainJobModelPath = sec.Key("TrainJobModel_Path").MustString("job/") OutPutPath = sec.Key("Output_Path").MustString("output/") CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/") @@ -1268,17 +1270,17 @@ func NewContext() { PROXYURL = sec.Key("PROXY_URL").MustString("") sec = Cfg.Section("modelarts") - ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80") - IamHost = sec.Key("IAMHOST").MustString("112.95.163.80") + ModelArtsHost = sec.Key("ENDPOINT").MustString("") + IamHost = sec.Key("IAMHOST").MustString("") ProjectID = sec.Key("PROJECT_ID").MustString("") ProjectName = sec.Key("PROJECT_NAME").MustString("") ModelArtsUsername = sec.Key("USERNAME").MustString("") ModelArtsPassword = sec.Key("PASSWORD").MustString("") - ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222") + ModelArtsDomain = sec.Key("DOMAIN").MustString("") AllowedOrg = sec.Key("ORGANIZATION").MustString("") ProfileID = sec.Key("PROFILE_ID").MustString("") PoolInfos = sec.Key("POOL_INFOS").MustString("") - Flavor = sec.Key("FLAVOR").MustString("modelarts.bm.910.arm.public.2") + Flavor = sec.Key("FLAVOR").MustString("") ResourcePools = sec.Key("Resource_Pools").MustString("") Engines = sec.Key("Engines").MustString("") EngineVersions = sec.Key("Engine_Versions").MustString("") @@ -1286,10 +1288,10 @@ func NewContext() { TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("") sec = Cfg.Section("elk") - ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch") - ElkUser = sec.Key("ELKUSER").MustString("Qizhi") - ElkPassword = sec.Key("ELKPASSWORD").MustString("Pcl2020") - Index = sec.Key("INDEX").MustString("filebeat-7.3.2*") + ElkUrl = sec.Key("ELKURL").MustString("") + ElkUser = sec.Key("ELKUSER").MustString("") + ElkPassword = sec.Key("ELKPASSWORD").MustString("") + Index = sec.Key("INDEX").MustString("") TimeField = sec.Key("TIMEFIELD").MustString(" @timestamptest") ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time") diff --git a/modules/storage/local.go b/modules/storage/local.go index c462dcd9e..d46a5528d 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -76,3 +76,7 @@ func (l *LocalStorage) PresignedPutURL(path string) (string, error) { func (l *LocalStorage) HasObject(path string) (bool, error) { return false, nil } + +func (l *LocalStorage) UploadObject(fileName, filePath string) error { + return nil +} diff --git a/modules/storage/minio.go b/modules/storage/minio.go index b14442d56..664e58d1b 100755 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -122,3 +122,9 @@ func (m *MinioStorage) HasObject(path string) (bool, error) { return hasObject, nil } + +//upload object +func (m *MinioStorage) UploadObject(fileName, filePath string) error { + _, err := m.client.FPutObject(m.bucket, fileName, filePath, minio.PutObjectOptions{}) + return err +} diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 191871d93..d364346f0 100755 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -26,6 +26,7 @@ type ObjectStorage interface { PresignedGetURL(path string, fileName string) (string, error) PresignedPutURL(path string) (string, error) HasObject(path string) (bool, error) + UploadObject(fileName, filePath string) error } // Copy copys a file from source ObjectStorage to dest ObjectStorage diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index b2ded0752..a3af588fb 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -249,6 +249,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } repo := ctx.Repo.Repository downloadCode(repo, codePath) + uploadCodeToMinio(codePath + "/", jobName, "/code/") modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath err = os.MkdirAll(modelPath, os.ModePerm) @@ -267,16 +268,19 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } } downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) + uploadCodeToMinio(benchmarkPath + "/", jobName, cloudbrain.BenchMarkMountPath + "/") } snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") + uploadCodeToMinio(snn4imagenetPath + "/", jobName, cloudbrain.Snn4imagenetMountPath + "/") } brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "") + uploadCodeToMinio(brainScorePath + "/", jobName, cloudbrain.BrainScoreMountPath + "/") } err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId) @@ -703,8 +707,8 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc command := "git clone " + gitPath + " " + codePath cmd := exec.Command("/bin/bash", "-c", command) - output, err := cmd.Output() - log.Info(string(output)) + _, err = cmd.Output() + if err != nil { log.Error("exec.Command(%s) failed:%v", command, err) return err @@ -741,6 +745,33 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc return nil } +func uploadCodeToMinio(codePath, jobName, parentDir string) error { + files, err := readDir(codePath) + if err != nil { + log.Error("readDir(%s) failed: %s", codePath, err.Error()) + return err + } + + for _, file := range files { + if file.IsDir() { + if err = uploadCodeToMinio(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil { + log.Error("uploadCodeToMinio(%s) failed: %s", file.Name(), err.Error()) + return err + } + } else { + destObject := setting.CBCodePathPrefix + jobName + parentDir + file.Name() + sourceFile := codePath + file.Name() + err = storage.Attachments.UploadObject(destObject, sourceFile) + if err != nil { + log.Error("UploadObject(%s) failed: %s", file.Name(), err.Error()) + return err + } + } + } + + return nil +} + func SyncCloudbrainStatus() { cloudBrains, err := models.GetCloudBrainUnStoppedJob() if err != nil { From 3641ea9c0f435de7b86faf109b240ba33d006959 Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 7 Dec 2021 11:01:38 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E7=82=B9=E5=87=BB=E4=B8=AA=E4=BA=BA?= =?UTF-8?q?=E4=B8=AD=E5=BF=83=E8=B0=83=E8=BD=AC=EF=BC=8C=E5=B9=B6=E4=B8=94?= =?UTF-8?q?=E4=B8=AA=E4=BA=BA=E4=B8=AD=E5=BF=83=E5=92=8C=E6=8E=A2=E7=B4=A2?= =?UTF-8?q?=E4=BA=A4=E4=BA=92=E4=BF=9D=E6=8C=81=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/base/head_navbar.tmpl | 48 +++++++++++++++------------ templates/base/head_navbar_fluid.tmpl | 41 +++++++++++++---------- templates/base/head_navbar_home.tmpl | 44 +++++++++++++----------- web_src/js/index.js | 9 +++++ 4 files changed, 85 insertions(+), 57 deletions(-) diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 299832f2d..c87524afb 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -17,19 +17,23 @@ {{if .IsSigned}} - -
- {{.i18n.Tr "index"}} - -
- {{.i18n.Tr "issues"}} - {{.i18n.Tr "pull_requests"}} - {{.i18n.Tr "milestones"}} +
+
+ + {{.i18n.Tr "index"}}   + + +
+ {{.i18n.Tr "issues"}} + {{.i18n.Tr "pull_requests"}} + {{.i18n.Tr "milestones"}} +
+ {{.i18n.Tr "custom.head.project"}} {{.i18n.Tr "custom.head.dataset"}} -
+ {{else if .IsLandingPageHome}} - - -
- {{.i18n.Tr "home"}} - -
- - {{.i18n.Tr "issues"}} - {{.i18n.Tr "pull_requests"}} - {{.i18n.Tr "milestones"}} +
+
+ + {{.i18n.Tr "home"}}   + + +
+ {{.i18n.Tr "issues"}} + {{.i18n.Tr "pull_requests"}} + {{.i18n.Tr "milestones"}} +
+ {{.i18n.Tr "custom.head.project"}} {{.i18n.Tr "custom.head.dataset"}} -
+