diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 4cd3539d7..e058c0df8 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -955,6 +955,8 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, return err } userNewAddActivity := make(map[int64]map[int64]int64) + userAcitvateJsonMap := make(map[int64]map[int64]int64) + userCurrentDayRegistMap := make(map[int64]map[int64]int64) ParaWeight := getParaWeight() userMetrics := make(map[string]int) var indexTotal int64 @@ -1028,7 +1030,10 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("has activity." + userRecord.Name) addUserToMap(userNewAddActivity, userRecord.CreatedUnix, dateRecord.ID) } - + if userRecord.IsActive { + addUserToMap(userAcitvateJsonMap, userRecord.CreatedUnix, dateRecord.ID) + } + addUserToMap(userCurrentDayRegistMap, userRecord.CreatedUnix, dateRecord.ID) } indexTotal += PAGE_SIZE @@ -1064,36 +1069,61 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, } statictisSess.Insert(&useMetrics) //update new user activity - updateNewUserAcitivity(userNewAddActivity, statictisSess) + updateNewUserAcitivity(userNewAddActivity, userAcitvateJsonMap, userCurrentDayRegistMap, statictisSess) return nil } -func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, statictisSess *xorm.Session) { - for key, value := range currentUserActivity { +func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, userAcitvateJsonMap map[int64]map[int64]int64, userCurrentDayRegistMap map[int64]map[int64]int64, statictisSess *xorm.Session) { + for key, value := range userCurrentDayRegistMap { useMetrics := &UserMetrics{CountDate: key} + userAcitvateValue := userAcitvateJsonMap[key] + HuodongValue := currentUserActivity[key] has, err := statictisSess.Get(useMetrics) if err == nil && has { - userIdArrays := strings.Split(useMetrics.HasActivityUserJson, ",") - for _, userIdStr := range userIdArrays { - userIdInt, err := strconv.ParseInt(userIdStr, 10, 64) - if err == nil { - value[userIdInt] = userIdInt - } - } - userIdArray := "" - for _, tmpValue := range value { - userIdArray += fmt.Sprint(tmpValue) + "," - } - useMetrics.HasActivityUser = len(value) - if len(userIdArray) > 0 { - useMetrics.HasActivityUserJson = userIdArray[0 : len(userIdArray)-1] - } - updateSql := "update public.user_metrics set has_activity_user_json='" + useMetrics.HasActivityUserJson + "',regist_activity_user=" + fmt.Sprint(useMetrics.HasActivityUser) + " where count_date=" + fmt.Sprint(key) + ActivityUserArray, HuodongTotal := setUniqueUserId(useMetrics.HasActivityUserJson, HuodongValue) + useMetrics.HasActivityUser = HuodongTotal + useMetrics.HasActivityUserJson = ActivityUserArray + + useMetrics.CurrentDayRegistUser = len(value) + + RegistUserArray, lenRegistUser := setUniqueUserId(useMetrics.ActivityUserJson, userAcitvateValue) + useMetrics.ActivityUserJson = RegistUserArray + useMetrics.ActivateRegistUser = lenRegistUser + + updateSql := "update public.user_metrics set has_activity_user_json='" + useMetrics.HasActivityUserJson + + "',regist_activity_user=" + fmt.Sprint(useMetrics.HasActivityUser) + + ",activity_user_json='" + useMetrics.ActivityUserJson + "'" + + ",activate_regist_user=" + fmt.Sprint(useMetrics.ActivateRegistUser) + + ",not_activate_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser-useMetrics.ActivateRegistUser) + + ",current_day_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser) + + " where count_date=" + fmt.Sprint(key) + statictisSess.Exec(updateSql) } } } +func setUniqueUserId(jsonString string, value map[int64]int64) (string, int) { + if value == nil { + value = make(map[int64]int64, 0) + } + userIdArrays := strings.Split(jsonString, ",") + for _, userIdStr := range userIdArrays { + userIdInt, err := strconv.ParseInt(userIdStr, 10, 64) + if err == nil { + value[userIdInt] = userIdInt + } + } + userIdArray := "" + for _, tmpValue := range value { + userIdArray += fmt.Sprint(tmpValue) + "," + } + if len(userIdArray) > 0 { + return userIdArray[0 : len(userIdArray)-1], len(value) + } + return userIdArray, len(value) +} + func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate timeutil.TimeStamp, userId int64) { CountDateTime := time.Date(registDate.Year(), registDate.AsTime().Month(), registDate.AsTime().Day(), 0, 1, 0, 0, registDate.AsTime().Location()) CountDate := CountDateTime.Unix() @@ -1104,7 +1134,6 @@ func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate time } else { currentUserActivity[CountDate][userId] = userId } - } func setUserMetrics(userMetrics map[string]int, user *User, start_time int64, end_time int64, dateRecord UserBusinessAnalysis) { diff --git a/models/user_business_struct.go b/models/user_business_struct.go index fec361bca..870a64bc7 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -467,11 +467,11 @@ type UserAnalysisPara struct { type UserMetrics struct { CountDate int64 `xorm:"pk"` - ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` - NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` - ActivateIndex float64 `xorm:"NOT NULL DEFAULT 0"` - RegistActivityUser int `xorm:"NOT NULL DEFAULT 0"` - HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` + ActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` //当天激活用户 + NotActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` //当天未激活用户 + ActivateIndex float64 `xorm:"NOT NULL DEFAULT 0"` //激活比率 + RegistActivityUser int `xorm:"NOT NULL DEFAULT 0"` //当天注册激活的人中，有贡献活动的人 + HasActivityUser int `xorm:"NOT NULL DEFAULT 0"` //当天有贡献活动的人 TotalUser int `xorm:"NOT NULL DEFAULT 0"` TotalRegistUser int `xorm:"-"` TotalActivateRegistUser int `xorm:"NOT NULL DEFAULT 0"` @@ -480,5 +480,7 @@ type UserMetrics struct { DisplayDate string `xorm:"-"` DataDate string `xorm:"NULL"` DaysForMonth int `xorm:"NOT NULL DEFAULT 0"` - HasActivityUserJson string `xorm:"text NULL"` + HasActivityUserJson string `xorm:"text NULL"` //贡献活动用户列表 + ActivityUserJson string `xorm:"text NULL"` //激活用户列表 + CurrentDayRegistUser int `xorm:"NOT NULL DEFAULT 0"` //当天注册用户 } diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index 4d87bb607..e83ccccc6 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -19,12 +19,14 @@ const ( ProcessorTypeNPU = "npu.huawei.com/NPU" ProcessorTypeGPU = "nvidia.com/gpu" - CommandPrepareScript = "pwd;cd /cache;mkdir -p output;mkdir -p code;mkdir -p dataset;echo \"start loading script\";wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" + + GpuWorkDir = "/tmp/" + NpuWorkDir = "/cache/" + + CommandPrepareScript = ";mkdir -p output;mkdir -p code;mkdir -p dataset;echo \"start loading script\";wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" + "echo \"finish loading script\";unzip -q master.zip;cd script_for_grampus;chmod 777 downloader_for_obs uploader_for_obs downloader_for_minio uploader_for_minio;" + //CommandPrepareScript = "pwd;cd /cache;mkdir -p output;mkdir -p code;mkdir -p dataset;echo \"start loading script\";wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" + + // "echo \"finish loading script\";unzip -q master.zip;cd script_for_grampus;chmod 777 downloader_for_obs uploader_for_obs downloader_for_minio uploader_for_minio;" - //CommandPrepareScript = "bash;pwd;apt-get -y update;apt-get -y upgrade;apt-get -y install wget;apt-get -y install unzip;" + - // "cd /tmp;mkdir -p output;mkdir -p code;mkdir -p dataset;wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" + - // "unzip -q master.zip;cd script_for_grampus;chmod 777 downloader_for_obs uploader_for_obs downloader_for_minio uploader_for_minio;" CodeArchiveName = "master.zip" ) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index cdfedc862..66af19931 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1178,6 +1178,7 @@ model.manage.model_accuracy = Model Accuracy grampus.train_job.ai_center = AI Center grampus.dataset_path_rule = The code is storaged in /cache/code;the dataset is storaged in /cache/dataset;and please put your model into /cache/output, then you can download it online。 +grampus.gpu_dataset_path_rule = The code is storaged in /tmp/code;the dataset is storaged in /tmp/dataset;and please put your model into /tmp/output, then you can download it online。 grampus.no_operate_right = You have no right to do this operation. template.items = Template Items diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index de15dd98f..d74e08899 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1192,6 +1192,7 @@ model.manage.model_accuracy = 模型精度 grampus.train_job.ai_center=智算中心 grampus.dataset_path_rule = 训练脚本存储在/cache/code中，数据集存储在/cache/dataset中，训练输出请存储在/cache/output中以供后续下载。 +grampus.gpu_dataset_path_rule = 训练脚本存储在/tmp/code中，数据集存储在/tmp/dataset中，训练输出请存储在/tmp/output中以供后续下载。 grampus.no_operate_right = 您没有权限创建这类任务。 diff --git a/public/home/home.js b/public/home/home.js index f772403f1..95ea3da4c 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -156,7 +156,7 @@ document.onreadystatechange = function () { html += recordPrefix + actionName; html += " " + getRepotext(record) + "" } - else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30" || record.OpType == "31"){ + else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30" || record.OpType == "31" || record.OpType == "32" || record.OpType == "33"){ html += recordPrefix + actionName; html += " " + record.RefName + "" } @@ -201,6 +201,8 @@ function getTaskLink(record){ re = re + "/modelmanage/show_model_info?name=" + record.RefName; }else if(record.OpType == 31){ re = re + "/cloudbrain/train-job/" + record.Content; + }else if(record.OpType == 32 || record.OpType == 33){ + re = re + "/grampus/train-job/" + record.Content; } re = encodeURI(re); return re; @@ -374,7 +376,9 @@ var actionNameZH={ "28":"创建了推理任务", "29":"创建了评测任务", "30":"导入了新模型", - "31":"创建了CPU/GPU类型训练任务" + "31":"创建了CPU/GPU类型训练任务", + "32":"创建了NPU类型训练任务", + "33":"创建了CPU/GPU类型训练任务" }; var actionNameEN={ @@ -401,6 +405,8 @@ var actionNameEN={ "29":" created profiling task", "30":" created new model", "31":" created CPU/GPU type training task", + "32":" created NPU type training task", + "33":" created CPU/GPU type training task" }; var repoAndOrgZH={ diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 0481e6743..8cfe10795 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -43,12 +43,6 @@ func CloudBrains(ctx *context.Context) { if page <= 0 { page = 1 } - debugType := models.TypeCloudBrainAll - if listType == models.GPUResource { - debugType = models.TypeCloudBrainOne - } else if listType == models.NPUResource { - debugType = models.TypeCloudBrainTwo - } var jobTypes []string jobTypeNot := false @@ -77,13 +71,14 @@ func CloudBrains(ctx *context.Context) { PageSize: setting.UI.IssuePagingNum, }, Keyword: keyword, - Type: debugType, JobTypeNot: jobTypeNot, JobStatusNot: jobStatusNot, JobStatus: jobStatuses, JobTypes: jobTypes, NeedRepoInfo: true, IsLatestVersion: modelarts.IsLatestVersion, + ComputeResource: listType, + Type: models.TypeCloudBrainAll, }) if err != nil { ctx.ServerError("Get job failed:", err) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 06b25b57f..52f803d1d 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -664,7 +664,12 @@ func GrampusGetLog(ctx *context.Context) { func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName string) (string, error) { var command string - command += grampus.CommandPrepareScript + workDir := grampus.NpuWorkDir + if processorType == grampus.ProcessorTypeGPU { + workDir = grampus.GpuWorkDir + } + + command += "pwd;cd " + workDir + grampus.CommandPrepareScript //download code & dataset if processorType == grampus.ProcessorTypeNPU { commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " " + datasetName + ";" @@ -683,7 +688,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo if strings.HasSuffix(datasetName, ".tar.gz") { toolUnzip = "tar -zxvf " } - commandUnzip := "cd /cache/code;unzip -q master.zip;echo \"start to unzip dataset\";cd /cache/dataset;" + toolUnzip + datasetName + ";" + commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + toolUnzip + datasetName + ";" command += commandUnzip //check unzip result @@ -712,7 +717,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo } } - commandCode := "cd /cache/code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" + commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";" command += commandCode //get exec result @@ -721,10 +726,10 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo //upload models if processorType == grampus.ProcessorTypeNPU { - commandUpload := "cd /cache/script_for_grampus/;./uploader_for_obs " + setting.Bucket + " " + outputRemotePath + " " + "/cache/output/;" + commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_obs " + setting.Bucket + " " + outputRemotePath + " " + workDir + "output/;" command += commandUpload } else if processorType == grampus.ProcessorTypeGPU { - commandUpload := "cd /cache/script_for_grampus/;./uploader_for_minio " + setting.Grampus.Env + " " + outputRemotePath + " " + "/cache/output/;" + commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_minio " + setting.Grampus.Env + " " + outputRemotePath + " " + workDir + "output/;" command += commandUpload } diff --git a/services/socketwrap/clientManager.go b/services/socketwrap/clientManager.go index 6ffa96933..98bcc8a85 100755 --- a/services/socketwrap/clientManager.go +++ b/services/socketwrap/clientManager.go @@ -10,7 +10,7 @@ import ( "github.com/elliotchance/orderedmap" ) -var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} +var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33} type ClientsManager struct { Clients *orderedmap.OrderedMap diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index 4c63b167a..347b5658d 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -102,7 +102,7 @@ {{else if eq .JobType "TRAIN"}} {{.DisplayJobName}} @@ -204,7 +204,7 @@ {{else}} {{$.i18n.Tr "repo.stop"}} @@ -212,7 +212,7 @@
{{$.CsrfTokenHtml}} {{.i18n.Tr "repo.grampus.dataset_path_rule"}} + {{.i18n.Tr "repo.grampus.gpu_dataset_path_rule"}}
{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}