|
|
@@ -407,63 +407,132 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi |
|
|
|
return userBusinessAnalysisReturnList, allCount |
|
|
|
} |
|
|
|
|
|
|
|
func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { |
|
|
|
log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) |
|
|
|
statictisSess := xStatistic.NewSession() |
|
|
|
defer statictisSess.Close() |
|
|
|
var cond = builder.NewCond() |
|
|
|
cond = cond.And( |
|
|
|
builder.Gte{"count_date": opts.StartTime}, |
|
|
|
) |
|
|
|
cond = cond.And( |
|
|
|
builder.Lte{"count_date": opts.EndTime}, |
|
|
|
) |
|
|
|
count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis)) |
|
|
|
func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wikiCountMap map[string]int) ([]*UserBusinessAnalysis, int64) { |
|
|
|
log.Info("start to count other user info data") |
|
|
|
sess := x.NewSession() |
|
|
|
defer sess.Close() |
|
|
|
|
|
|
|
currentTimeNow := time.Now() |
|
|
|
log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) |
|
|
|
|
|
|
|
start_unix := opts.StartTime |
|
|
|
|
|
|
|
end_unix := opts.EndTime |
|
|
|
CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) |
|
|
|
|
|
|
|
CodeMergeCountMap := queryPullRequest(start_unix, end_unix) |
|
|
|
CommitCountMap := queryCommitAction(start_unix, end_unix, 5) |
|
|
|
IssueCountMap := queryCreateIssue(start_unix, end_unix) |
|
|
|
|
|
|
|
CommentCountMap := queryComment(start_unix, end_unix) |
|
|
|
FocusRepoCountMap := queryWatch(start_unix, end_unix) |
|
|
|
StarRepoCountMap := queryStar(start_unix, end_unix) |
|
|
|
WatchedCountMap, WatchOtherMap := queryFollow(start_unix, end_unix) |
|
|
|
|
|
|
|
CommitCodeSizeMap, err := GetAllUserKPIStats() |
|
|
|
if err != nil { |
|
|
|
log.Info("query error." + err.Error()) |
|
|
|
return nil, 0 |
|
|
|
log.Info("query commit code errr.") |
|
|
|
} else { |
|
|
|
log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) |
|
|
|
CommitCodeSizeMapJson, _ := json.Marshal(CommitCodeSizeMap) |
|
|
|
log.Info("CommitCodeSizeMapJson=" + string(CommitCodeSizeMapJson)) |
|
|
|
} |
|
|
|
resultMap := make(map[int64]*UserBusinessAnalysis) |
|
|
|
if count > 0 { |
|
|
|
pageSize := 1000 |
|
|
|
totalPage := int(count) / pageSize |
|
|
|
for i := 0; i <= int(totalPage); i++ { |
|
|
|
userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) |
|
|
|
if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). |
|
|
|
Find(&userBusinessAnalysisList); err != nil { |
|
|
|
return nil, 0 |
|
|
|
CommitDatasetSizeMap, CommitDatasetNumMap := queryDatasetSize(start_unix, end_unix) |
|
|
|
SolveIssueCountMap := querySolveIssue(start_unix, end_unix) |
|
|
|
CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) |
|
|
|
LoginCountMap := queryLoginCount(start_unix, end_unix) |
|
|
|
OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) |
|
|
|
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) |
|
|
|
AiModelManageMap := queryUserModel(start_unix, end_unix) |
|
|
|
|
|
|
|
CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) |
|
|
|
RecommendDataset := queryRecommedDataSet(start_unix, end_unix) |
|
|
|
CollectImage, CollectedImage := queryImageStars(start_unix, end_unix) |
|
|
|
RecommendImage := queryRecommedImage(start_unix, end_unix) |
|
|
|
|
|
|
|
statictisSess := xStatistic.NewSession() |
|
|
|
defer statictisSess.Close() |
|
|
|
|
|
|
|
cond := "type != 1 and and is_active=true" |
|
|
|
count, err := sess.Where(cond).Count(new(User)) |
|
|
|
|
|
|
|
ParaWeight := getParaWeight() |
|
|
|
ResultList := make([]*UserBusinessAnalysis, 0) |
|
|
|
var indexTotal int64 |
|
|
|
indexTotal = 0 |
|
|
|
for { |
|
|
|
sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) |
|
|
|
userList := make([]*User, 0) |
|
|
|
sess.Find(&userList) |
|
|
|
|
|
|
|
for i, userRecord := range userList { |
|
|
|
var dateRecord UserBusinessAnalysis |
|
|
|
dateRecord.ID = userRecord.ID |
|
|
|
log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) |
|
|
|
dateRecord.CountDate = CountDate.Unix() |
|
|
|
|
|
|
|
dateRecord.Email = userRecord.Email |
|
|
|
dateRecord.RegistDate = userRecord.CreatedUnix |
|
|
|
dateRecord.Name = userRecord.Name |
|
|
|
dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) |
|
|
|
|
|
|
|
dateRecord.CodeMergeCount = getMapValue(dateRecord.ID, CodeMergeCountMap) |
|
|
|
dateRecord.CommitCount = getMapValue(dateRecord.ID, CommitCountMap) |
|
|
|
dateRecord.IssueCount = getMapValue(dateRecord.ID, IssueCountMap) |
|
|
|
dateRecord.CommentCount = getMapValue(dateRecord.ID, CommentCountMap) |
|
|
|
dateRecord.FocusRepoCount = getMapValue(dateRecord.ID, FocusRepoCountMap) |
|
|
|
dateRecord.StarRepoCount = getMapValue(dateRecord.ID, StarRepoCountMap) |
|
|
|
dateRecord.WatchedCount = getMapValue(dateRecord.ID, WatchedCountMap) |
|
|
|
dateRecord.FocusOtherUser = getMapValue(dateRecord.ID, WatchOtherMap) |
|
|
|
if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { |
|
|
|
dateRecord.CommitCodeSize = 0 |
|
|
|
} else { |
|
|
|
dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) |
|
|
|
} |
|
|
|
log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) |
|
|
|
for _, userRecord := range userBusinessAnalysisList { |
|
|
|
if _, ok := resultMap[userRecord.ID]; !ok { |
|
|
|
resultMap[userRecord.ID] = userRecord |
|
|
|
} else { |
|
|
|
resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount |
|
|
|
resultMap[userRecord.ID].CommitCount += userRecord.CommitCount |
|
|
|
resultMap[userRecord.ID].IssueCount += userRecord.IssueCount |
|
|
|
resultMap[userRecord.ID].CommentCount += userRecord.CommentCount |
|
|
|
resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount |
|
|
|
resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount |
|
|
|
resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount |
|
|
|
resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize |
|
|
|
resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize |
|
|
|
resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum |
|
|
|
resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount |
|
|
|
resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount |
|
|
|
resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount |
|
|
|
resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount |
|
|
|
resultMap[userRecord.ID].LoginCount += userRecord.LoginCount |
|
|
|
} |
|
|
|
dateRecord.CommitDatasetSize = getMapValue(dateRecord.ID, CommitDatasetSizeMap) |
|
|
|
dateRecord.CommitDatasetNum = getMapValue(dateRecord.ID, CommitDatasetNumMap) |
|
|
|
dateRecord.SolveIssueCount = getMapValue(dateRecord.ID, SolveIssueCountMap) |
|
|
|
|
|
|
|
dateRecord.EncyclopediasCount = getMapKeyStringValue(dateRecord.Name, wikiCountMap) |
|
|
|
|
|
|
|
dateRecord.CreateRepoCount = getMapValue(dateRecord.ID, CreateRepoCountMap) |
|
|
|
|
|
|
|
dateRecord.LoginCount = getMapValue(dateRecord.ID, LoginCountMap) |
|
|
|
|
|
|
|
if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { |
|
|
|
dateRecord.OpenIIndex = 0 |
|
|
|
} else { |
|
|
|
dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] |
|
|
|
} |
|
|
|
|
|
|
|
dateRecord.CloudBrainTaskNum = getMapValue(dateRecord.ID, CloudBrainTaskMap) |
|
|
|
dateRecord.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuDebugJob", CloudBrainTaskItemMap) |
|
|
|
dateRecord.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuDebugJob", CloudBrainTaskItemMap) |
|
|
|
dateRecord.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuTrainJob", CloudBrainTaskItemMap) |
|
|
|
dateRecord.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuTrainJob", CloudBrainTaskItemMap) |
|
|
|
dateRecord.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap) |
|
|
|
dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) |
|
|
|
dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) |
|
|
|
dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) |
|
|
|
|
|
|
|
dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) |
|
|
|
dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) |
|
|
|
dateRecord.RecommendDataset = getMapValue(dateRecord.ID, RecommendDataset) |
|
|
|
dateRecord.CollectImage = getMapValue(dateRecord.ID, CollectImage) |
|
|
|
dateRecord.CollectedImage = getMapValue(dateRecord.ID, CollectedImage) |
|
|
|
dateRecord.RecommendImage = getMapValue(dateRecord.ID, RecommendImage) |
|
|
|
|
|
|
|
dateRecord.UserIndexPrimitive = getUserIndex(dateRecord, ParaWeight) |
|
|
|
ResultList = append(ResultList, &dateRecord) |
|
|
|
} |
|
|
|
|
|
|
|
indexTotal += PAGE_SIZE |
|
|
|
if indexTotal >= count { |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
userBusinessAnalysisReturnList := UserBusinessAnalysisList{} |
|
|
|
for _, v := range resultMap { |
|
|
|
userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) |
|
|
|
} |
|
|
|
sort.Sort(userBusinessAnalysisReturnList) |
|
|
|
log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) |
|
|
|
return userBusinessAnalysisReturnList, count |
|
|
|
|
|
|
|
return ResultList, int64(len(ResultList)) |
|
|
|
} |
|
|
|
|
|
|
|
func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { |
|
|
@@ -792,7 +861,12 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, userMetrics map[stri |
|
|
|
pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset) |
|
|
|
refreshUserStaticTable(wikiCountMap, "user_business_analysis_current_week", pageStartTime, pageEndTime, userMetrics) |
|
|
|
|
|
|
|
pageEndTime = pageStartTime |
|
|
|
pageStartTime = pageStartTime.AddDate(0, 0, -7) |
|
|
|
refreshUserStaticTable(wikiCountMap, "user_business_analysis_last_week", pageStartTime, pageEndTime, userMetrics) |
|
|
|
|
|
|
|
pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -30) |
|
|
|
pageEndTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) |
|
|
|
refreshUserStaticTable(wikiCountMap, "user_business_analysis_last30_day", pageStartTime, pageEndTime, userMetrics) |
|
|
|
|
|
|
|
pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -1) |
|
|
@@ -937,7 +1011,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, |
|
|
|
log.Info("has activity." + userRecord.Name) |
|
|
|
addUserToMap(userNewAddActivity, userRecord.CreatedUnix, dateRecord.ID) |
|
|
|
} |
|
|
|
if userRecord.IsActive { |
|
|
|
if !userRecord.IsActive { |
|
|
|
continue |
|
|
|
} |
|
|
|
statictisSess.Delete(&dateRecord) |
|
|
|