merge

models/user_business_analysis.go

@@ -955,6 +955,8 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
 		return err
 	}
 	userNewAddActivity := make(map[int64]map[int64]int64)
+	userAcitvateJsonMap := make(map[int64]map[int64]int64)
+	userCurrentDayRegistMap := make(map[int64]map[int64]int64)
 	ParaWeight := getParaWeight()
 	userMetrics := make(map[string]int)
 	var indexTotal int64
@@ -1028,7 +1030,10 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
 				log.Info("has activity." + userRecord.Name)
 				addUserToMap(userNewAddActivity, userRecord.CreatedUnix, dateRecord.ID)
 			}
-
+			if userRecord.IsActive {
+				addUserToMap(userAcitvateJsonMap, userRecord.CreatedUnix, dateRecord.ID)
+			}
+			addUserToMap(userCurrentDayRegistMap, userRecord.CreatedUnix, dateRecord.ID)
 		}
 
 		indexTotal += PAGE_SIZE
@@ -1064,36 +1069,61 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
 	}
 	statictisSess.Insert(&useMetrics)
 	//update new user activity
-	updateNewUserAcitivity(userNewAddActivity, statictisSess)
+	updateNewUserAcitivity(userNewAddActivity, userAcitvateJsonMap, userCurrentDayRegistMap, statictisSess)
 	return nil
 }
 
-func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, statictisSess *xorm.Session) {
-	for key, value := range currentUserActivity {
+func updateNewUserAcitivity(currentUserActivity map[int64]map[int64]int64, userAcitvateJsonMap map[int64]map[int64]int64, userCurrentDayRegistMap map[int64]map[int64]int64, statictisSess *xorm.Session) {
+	for key, value := range userCurrentDayRegistMap {
 		useMetrics := &UserMetrics{CountDate: key}
+		userAcitvateValue := userAcitvateJsonMap[key]
+		HuodongValue := currentUserActivity[key]
 		has, err := statictisSess.Get(useMetrics)
 		if err == nil && has {
-			userIdArrays := strings.Split(useMetrics.HasActivityUserJson, ",")
-			for _, userIdStr := range userIdArrays {
-				userIdInt, err := strconv.ParseInt(userIdStr, 10, 64)
-				if err == nil {
-					value[userIdInt] = userIdInt
-				}
-			}
-			userIdArray := ""
-			for _, tmpValue := range value {
-				userIdArray += fmt.Sprint(tmpValue) + ","
-			}
-			useMetrics.HasActivityUser = len(value)
-			if len(userIdArray) > 0 {
-				useMetrics.HasActivityUserJson = userIdArray[0 : len(userIdArray)-1]
-			}
-			updateSql := "update public.user_metrics set has_activity_user_json='" + useMetrics.HasActivityUserJson + "',regist_activity_user=" + fmt.Sprint(useMetrics.HasActivityUser) + " where count_date=" + fmt.Sprint(key)
+			ActivityUserArray, HuodongTotal := setUniqueUserId(useMetrics.HasActivityUserJson, HuodongValue)
+			useMetrics.HasActivityUser = HuodongTotal
+			useMetrics.HasActivityUserJson = ActivityUserArray
+
+			useMetrics.CurrentDayRegistUser = len(value)
+
+			RegistUserArray, lenRegistUser := setUniqueUserId(useMetrics.ActivityUserJson, userAcitvateValue)
+			useMetrics.ActivityUserJson = RegistUserArray
+			useMetrics.ActivateRegistUser = lenRegistUser
+
+			updateSql := "update public.user_metrics set has_activity_user_json='" + useMetrics.HasActivityUserJson +
+				"',regist_activity_user=" + fmt.Sprint(useMetrics.HasActivityUser) +
+				",activity_user_json='" + useMetrics.ActivityUserJson + "'" +
+				",activate_regist_user=" + fmt.Sprint(useMetrics.ActivateRegistUser) +
+				",not_activate_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser-useMetrics.ActivateRegistUser) +
+				",current_day_regist_user=" + fmt.Sprint(useMetrics.CurrentDayRegistUser) +
+				" where count_date=" + fmt.Sprint(key)
+
 			statictisSess.Exec(updateSql)
 		}
 	}
 }
 
+func setUniqueUserId(jsonString string, value map[int64]int64) (string, int) {
+	if value == nil {
+		value = make(map[int64]int64, 0)
+	}
+	userIdArrays := strings.Split(jsonString, ",")
+	for _, userIdStr := range userIdArrays {
+		userIdInt, err := strconv.ParseInt(userIdStr, 10, 64)
+		if err == nil {
+			value[userIdInt] = userIdInt
+		}
+	}
+	userIdArray := ""
+	for _, tmpValue := range value {
+		userIdArray += fmt.Sprint(tmpValue) + ","
+	}
+	if len(userIdArray) > 0 {
+		return userIdArray[0 : len(userIdArray)-1], len(value)
+	}
+	return userIdArray, len(value)
+}
+
 func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate timeutil.TimeStamp, userId int64) {
 	CountDateTime := time.Date(registDate.Year(), registDate.AsTime().Month(), registDate.AsTime().Day(), 0, 1, 0, 0, registDate.AsTime().Location())
 	CountDate := CountDateTime.Unix()
@@ -1104,7 +1134,6 @@ func addUserToMap(currentUserActivity map[int64]map[int64]int64, registDate time
 	} else {
 		currentUserActivity[CountDate][userId] = userId
 	}
-
 }
 
 func setUserMetrics(userMetrics map[string]int, user *User, start_time int64, end_time int64, dateRecord UserBusinessAnalysis) {

models/user_business_struct.go

@@ -467,11 +467,11 @@ type UserAnalysisPara struct {
 
 type UserMetrics struct {
 	CountDate                  int64   `xorm:"pk"`
-	ActivateRegistUser         int     `xorm:"NOT NULL DEFAULT 0"`
-	NotActivateRegistUser      int     `xorm:"NOT NULL DEFAULT 0"`
-	ActivateIndex              float64 `xorm:"NOT NULL DEFAULT 0"`
-	RegistActivityUser         int     `xorm:"NOT NULL DEFAULT 0"`
-	HasActivityUser            int     `xorm:"NOT NULL DEFAULT 0"`
+	ActivateRegistUser         int     `xorm:"NOT NULL DEFAULT 0"` //当天激活用户
+	NotActivateRegistUser      int     `xorm:"NOT NULL DEFAULT 0"` //当天未激活用户
+	ActivateIndex              float64 `xorm:"NOT NULL DEFAULT 0"` //激活比率
+	RegistActivityUser         int     `xorm:"NOT NULL DEFAULT 0"` //当天注册激活的人中，有贡献活动的人
+	HasActivityUser            int     `xorm:"NOT NULL DEFAULT 0"` //当天有贡献活动的人
 	TotalUser                  int     `xorm:"NOT NULL DEFAULT 0"`
 	TotalRegistUser            int     `xorm:"-"`
 	TotalActivateRegistUser    int     `xorm:"NOT NULL DEFAULT 0"`
@@ -480,5 +480,7 @@ type UserMetrics struct {
 	DisplayDate                string  `xorm:"-"`
 	DataDate                   string  `xorm:"NULL"`
 	DaysForMonth               int     `xorm:"NOT NULL DEFAULT 0"`
-	HasActivityUserJson        string  `xorm:"text NULL"`
+	HasActivityUserJson        string  `xorm:"text NULL"`          //贡献活动用户列表
+	ActivityUserJson           string  `xorm:"text NULL"`          //激活用户列表
+	CurrentDayRegistUser       int     `xorm:"NOT NULL DEFAULT 0"` //当天注册用户
 }

modules/grampus/grampus.go

@@ -19,12 +19,14 @@ const (
 	ProcessorTypeNPU = "npu.huawei.com/NPU"
 	ProcessorTypeGPU = "nvidia.com/gpu"
 
-	CommandPrepareScript = "pwd;cd /cache;mkdir -p output;mkdir -p code;mkdir -p dataset;echo \"start loading script\";wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" +
+	GpuWorkDir = "/tmp/"
+	NpuWorkDir = "/cache/"
+
+	CommandPrepareScript = ";mkdir -p output;mkdir -p code;mkdir -p dataset;echo \"start loading script\";wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" +
 		"echo \"finish loading script\";unzip -q master.zip;cd script_for_grampus;chmod 777 downloader_for_obs uploader_for_obs downloader_for_minio uploader_for_minio;"
+	//CommandPrepareScript = "pwd;cd /cache;mkdir -p output;mkdir -p code;mkdir -p dataset;echo \"start loading script\";wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" +
+	//	"echo \"finish loading script\";unzip -q master.zip;cd script_for_grampus;chmod 777 downloader_for_obs uploader_for_obs downloader_for_minio uploader_for_minio;"
 
-	//CommandPrepareScript = "bash;pwd;apt-get -y update;apt-get -y upgrade;apt-get -y install wget;apt-get -y install unzip;" +
-	//	"cd /tmp;mkdir -p output;mkdir -p code;mkdir -p dataset;wget -q https://git.openi.org.cn/OpenIOSSG/script_for_grampus/archive/master.zip;" +
-	//	"unzip -q master.zip;cd script_for_grampus;chmod 777 downloader_for_obs uploader_for_obs downloader_for_minio uploader_for_minio;"
 	CodeArchiveName = "master.zip"
 )
 

options/locale/locale_en-US.ini

@@ -1178,6 +1178,7 @@ model.manage.model_accuracy = Model Accuracy
 
 grampus.train_job.ai_center = AI Center
 grampus.dataset_path_rule = The code is storaged in /cache/code;the dataset is storaged in /cache/dataset;and please put your model into /cache/output, then you can download it online。
+grampus.gpu_dataset_path_rule = The code is storaged in /tmp/code;the dataset is storaged in /tmp/dataset;and please put your model into /tmp/output, then you can download it online。
 grampus.no_operate_right = You have no right to do this operation.
 
 template.items = Template Items

options/locale/locale_zh-CN.ini

@@ -1192,6 +1192,7 @@ model.manage.model_accuracy = 模型精度
 
 grampus.train_job.ai_center=智算中心
 grampus.dataset_path_rule = 训练脚本存储在/cache/code中，数据集存储在/cache/dataset中，训练输出请存储在/cache/output中以供后续下载。
+grampus.gpu_dataset_path_rule = 训练脚本存储在/tmp/code中，数据集存储在/tmp/dataset中，训练输出请存储在/tmp/output中以供后续下载。
 
 grampus.no_operate_right = 您没有权限创建这类任务。
 

public/home/home.js

@@ -156,7 +156,7 @@ document.onreadystatechange = function () {
                     html += recordPrefix + actionName;
                     html += " <a href=\"" + getRepoLink(record) + "\" rel=\"nofollow\">" + getRepotext(record) + "</a>"
                 } 
-                else if(record.OpType == "24"  ||  record.OpType == "26" || record.OpType == "27" || record.OpType == "28"  || record.OpType == "30" || record.OpType == "31"){
+                else if(record.OpType == "24"  ||  record.OpType == "26" || record.OpType == "27" || record.OpType == "28"  || record.OpType == "30" || record.OpType == "31" || record.OpType == "32" || record.OpType == "33"){
                     html += recordPrefix + actionName;
                     html += " <a href=\"" + getTaskLink(record) + "\" rel=\"nofollow\">" + record.RefName + "</a>"
                 }
@@ -201,6 +201,8 @@ function getTaskLink(record){
         re = re + "/modelmanage/show_model_info?name=" + record.RefName;
     }else if(record.OpType == 31){
         re = re + "/cloudbrain/train-job/" + record.Content;
+    }else if(record.OpType == 32 || record.OpType == 33){
+        re = re + "/grampus/train-job/" + record.Content;
     }
     re = encodeURI(re);
     return re;
@@ -374,7 +376,9 @@ var actionNameZH={
     "28":"创建了推理任务",
     "29":"创建了评测任务",
     "30":"导入了新模型",
-    "31":"创建了CPU/GPU类型训练任务"
+    "31":"创建了CPU/GPU类型训练任务",
+    "32":"创建了NPU类型训练任务",
+    "33":"创建了CPU/GPU类型训练任务"
 };
 
 var actionNameEN={
@@ -401,6 +405,8 @@ var actionNameEN={
     "29":" created profiling task",
     "30":" created new model",
     "31":" created CPU/GPU type training task",
+    "32":" created NPU type training task",
+    "33":" created CPU/GPU type training task"
 };
 
 var repoAndOrgZH={

routers/admin/cloudbrains.go

@@ -43,12 +43,6 @@ func CloudBrains(ctx *context.Context) {
 	if page <= 0 {
 		page = 1
 	}
-	debugType := models.TypeCloudBrainAll
-	if listType == models.GPUResource {
-		debugType = models.TypeCloudBrainOne
-	} else if listType == models.NPUResource {
-		debugType = models.TypeCloudBrainTwo
-	}
 
 	var jobTypes []string
 	jobTypeNot := false
@@ -77,13 +71,14 @@ func CloudBrains(ctx *context.Context) {
 			PageSize: setting.UI.IssuePagingNum,
 		},
 		Keyword:         keyword,
-		Type:            debugType,
 		JobTypeNot:      jobTypeNot,
 		JobStatusNot:    jobStatusNot,
 		JobStatus:       jobStatuses,
 		JobTypes:        jobTypes,
 		NeedRepoInfo:    true,
 		IsLatestVersion: modelarts.IsLatestVersion,
+		ComputeResource: listType,
+		Type:            models.TypeCloudBrainAll,
 	})
 	if err != nil {
 		ctx.ServerError("Get job failed:", err)

routers/repo/grampus.go

@@ -664,7 +664,12 @@ func GrampusGetLog(ctx *context.Context) {
 func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName string) (string, error) {
 	var command string
 
-	command += grampus.CommandPrepareScript
+	workDir := grampus.NpuWorkDir
+	if processorType == grampus.ProcessorTypeGPU {
+		workDir = grampus.GpuWorkDir
+	}
+
+	command += "pwd;cd " + workDir + grampus.CommandPrepareScript
 	//download code & dataset
 	if processorType == grampus.ProcessorTypeNPU {
 		commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " " + datasetName + ";"
@@ -683,7 +688,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
 	if strings.HasSuffix(datasetName, ".tar.gz") {
 		toolUnzip = "tar -zxvf "
 	}
-	commandUnzip := "cd /cache/code;unzip -q master.zip;echo \"start to unzip dataset\";cd /cache/dataset;" + toolUnzip + datasetName + ";"
+	commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + toolUnzip + datasetName + ";"
 	command += commandUnzip
 
 	//check unzip result
@@ -712,7 +717,7 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
 		}
 	}
 
-	commandCode := "cd /cache/code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";"
+	commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";"
 	command += commandCode
 
 	//get exec result
@@ -721,10 +726,10 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
 
 	//upload models
 	if processorType == grampus.ProcessorTypeNPU {
-		commandUpload := "cd /cache/script_for_grampus/;./uploader_for_obs " + setting.Bucket + " " + outputRemotePath + " " + "/cache/output/;"
+		commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_obs " + setting.Bucket + " " + outputRemotePath + " " + workDir + "output/;"
 		command += commandUpload
 	} else if processorType == grampus.ProcessorTypeGPU {
-		commandUpload := "cd /cache/script_for_grampus/;./uploader_for_minio " + setting.Grampus.Env + " " + outputRemotePath + " " + "/cache/output/;"
+		commandUpload := "cd " + workDir + "script_for_grampus/;./uploader_for_minio " + setting.Grampus.Env + " " + outputRemotePath + " " + workDir + "output/;"
 		command += commandUpload
 	}
 

services/socketwrap/clientManager.go

@@ -10,7 +10,7 @@ import (
 	"github.com/elliotchance/orderedmap"
 )
 
-var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}
+var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33}
 
 type ClientsManager struct {
 	Clients    *orderedmap.OrderedMap

templates/admin/cloudbrain/list.tmpl

@@ -102,7 +102,7 @@
                                     </a>
                                     {{else if eq .JobType "TRAIN"}}
                                     <a class="title"
-                                        href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts{{end}}/train-job/{{$JobID}}"
+                                        href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 0}}/cloudbrain{{else if eq .Cloudbrain.Type 1}}/modelarts{{else if eq .Cloudbrain.Type 2}}/grampus{{end}}/train-job/{{$JobID}}"
                                         title="{{.DisplayJobName}}" style="font-size: 14px;">
                                         <span class="fitted"
                                             style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
@@ -204,7 +204,7 @@
                                         {{else}}
                                         <a style="padding: 0.5rem 1rem;" id="ai-stop-{{$JobID}}"
                                             class="ui basic ai_stop_version {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "SUCCEEDED" "STOPPED"}}disabled {{else}} blue {{end}}button"
-                                            data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain/train-job{{else}}/modelarts/{{if eq .JobType "INFERENCE"}}inference-job{{else}}train-job{{end}}{{end}}"
+                                            data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{else}}/modelarts/{{if eq .JobType "INFERENCE"}}inference-job{{else}}train-job{{end}}{{end}}"
                                             data-jobid="{{$JobID}}" data-version="{{.VersionName}}">
                                             {{$.i18n.Tr "repo.stop"}}
                                         </a>
@@ -212,7 +212,7 @@
                                     </div>
                                     <!-- 删除任务 -->
                                     <form class="ui compact buttons" id="delForm-{{$JobID}}"
-                                        action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}/train-job{{end}}/{{$JobID}}/del?isadminpage=true'
+                                        action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}{{end}}/{{$JobID}}/del?isadminpage=true'
                                         method="post">
                                         {{$.CsrfTokenHtml}}
                                         <a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}"

templates/repo/grampus/trainjob/gpu/new.tmpl

@@ -167,7 +167,7 @@
 
                 
                 {{template "custom/select_dataset_train" .}}
-                <span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 2rem;">{{.i18n.Tr "repo.grampus.dataset_path_rule"}}</span>
+                <span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 2rem;">{{.i18n.Tr "repo.grampus.gpu_dataset_path_rule"}}</span>
                 <div class="inline min_title field">
                     <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
                     <span id="add_run_para" style="margin-left: 0.5rem;cursor:pointer;color: rgba(3, 102, 214, 100);font-size: 14px;line-height: 26px;font-family: SourceHanSansSC-medium;"><i class="plus square outline icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>