提交代码。

Signed-off-by: zouap <zouap@pcl.ac.cn>

models/models.go

@@ -143,7 +143,7 @@ func init() {
 		new(SummaryStatistic),
 		new(UserBusinessAnalysis),
 		new(UserBusinessAnalysisAll),
-		new(UserBusinessAnalysisYear),
+		new(UserBusinessAnalysisCurrentYear),
 		new(UserBusinessAnalysisLast30Day),
 		new(UserBusinessAnalysisLastMonth),
 		new(UserBusinessAnalysisCurrentMonth),

models/user_business_analysis.go

@@ -10,10 +10,12 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/timeutil"
 	"xorm.io/builder"
+	"xorm.io/xorm"
 )
 
 const (
-	Page_SIZE = 2000
+	PAGE_SIZE         = 2000
+	BATCH_INSERT_SIZE = 50
 )
 
 type UserBusinessAnalysisAll struct {
@@ -163,14 +165,6 @@ func (ulist UserBusinessAnalysisList) Less(i, j int) bool {
 	return ulist[i].ID > ulist[j].ID
 }
 
-type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll
-
-func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] }
-func (ulist UserBusinessAnalysisAllList) Len() int      { return len(ulist) }
-func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool {
-	return ulist[i].ID > ulist[j].ID
-}
-
 func getLastCountDate() int64 {
 	statictisSess := xStatistic.NewSession()
 	defer statictisSess.Close()
@@ -202,9 +196,9 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi
 	}
 	log.Info("query return total:" + fmt.Sprint(allCount))
 
-	pageSize := 1000
+	pageSize := PAGE_SIZE
 	totalPage := int(allCount) / pageSize
-	userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{}
+	userBusinessAnalysisReturnList := make([]*UserBusinessAnalysisAll, 0)
 	for i := 0; i <= int(totalPage); i++ {
 		userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0)
 		if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize).
@@ -217,7 +211,6 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi
 		}
 	}
 
-	sort.Sort(userBusinessAnalysisReturnList)
 	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
 	return userBusinessAnalysisReturnList, allCount
 }
@@ -337,28 +330,24 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus
 	return userBusinessAnalysisReturnList, count
 }
 
-func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) {
-
+func refreshUserStaticTable(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats, tableName string, pageStartTime time.Time, pageEndTime time.Time) {
 	sess := x.NewSession()
 	defer sess.Close()
 
 	statictisSess := xStatistic.NewSession()
 	defer statictisSess.Close()
-	log.Info("truncate all data from table: user_business_analysis_all")
-	statictisSess.Exec("TRUNCATE TABLE user_business_analysis_all")
-
-	currentTimeNow := time.Now()
-
-	startTime := currentTimeNow.AddDate(0, 0, -1)
+	log.Info("truncate all data from table: " + tableName)
+	statictisSess.Exec("TRUNCATE TABLE " + tableName)
 
-	pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location())
 	log.Info("pageStartTime:" + pageStartTime.Format("2006-01-02 15:04:05"))
-	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location())
 	log.Info("pageEndTime time:" + pageEndTime.Format("2006-01-02 15:04:05"))
 
 	start_unix := pageStartTime.Unix()
 	end_unix := pageEndTime.Unix()
 
+	currentTimeNow := time.Now()
+	startTime := currentTimeNow.AddDate(0, 0, -1)
+
 	CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
 	CommitCountMap := queryCommitAction(start_unix, end_unix, 5)
 	IssueCountMap := queryCreateIssue(start_unix, end_unix)
@@ -385,12 +374,14 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap ma
 	}
 	var indexTotal int64
 	indexTotal = 0
+	insertCount := 0
+	dateRecordBatch := make([]UserBusinessAnalysisAll, 0)
 	for {
-		sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		userList := make([]*User, 0)
 		sess.Find(&userList)
-		for i, userRecord := range userList {
-			log.Info("insert all static, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name)
+
+		for _, userRecord := range userList {
 			var dateRecordAll UserBusinessAnalysisAll
 			dateRecordAll.ID = userRecord.ID
 			dateRecordAll.Email = userRecord.Email
@@ -484,18 +475,81 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap ma
 			}
 
 			dateRecordAll.CommitModelCount = 0
-			_, err = statictisSess.Insert(&dateRecordAll)
-			if err != nil {
-				log.Info("insert all data failed." + err.Error())
+
+			dateRecordBatch = append(dateRecordBatch, dateRecordAll)
+			if len(dateRecordBatch) >= BATCH_INSERT_SIZE {
+				insertTable(dateRecordBatch, tableName, statictisSess)
+				insertCount += BATCH_INSERT_SIZE
+				if err != nil {
+					log.Info("insert all data failed." + err.Error())
+				}
+				dateRecordBatch = make([]UserBusinessAnalysisAll, 0)
 			}
 		}
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
 	}
+	if len(dateRecordBatch) > 0 {
+		insertTable(dateRecordBatch, tableName, statictisSess)
+		insertCount += len(dateRecordBatch)
+		if err != nil {
+			log.Info("insert all data failed." + err.Error())
+		}
+	}
+
+	log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount))
+}
+
+func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, statictisSess *xorm.Session) {
+
+	insertBatchSql := "INSERT INTO public." + tableName +
+		"(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " +
+		"commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date) " +
+		"VALUES"
 
+	for _, record := range dateRecords {
+		insertBatchSql += "(" + fmt.Sprint(record.ID) + ", " + fmt.Sprint(record.CountDate) + ", " + fmt.Sprint(record.CodeMergeCount) + ", " + fmt.Sprint(record.CommitCount) +
+			", " + fmt.Sprint(record.IssueCount) + ", " + fmt.Sprint(record.CommentCount) + ", " + fmt.Sprint(record.FocusRepoCount) + ", " + fmt.Sprint(record.StarRepoCount) +
+			", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) +
+			", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) +
+			", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "')"
+	}
+
+	statictisSess.Insert(insertBatchSql)
+}
+
+func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) {
+	currentTimeNow := time.Now()
+	pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location())
+	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location())
+	refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_all", pageStartTime, pageEndTime)
 	log.Info("refresh all data finished.")
+
+	pageStartTime = time.Date(currentTimeNow.Year(), 0, 1, 0, 0, 0, 0, currentTimeNow.Location())
+	refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_year", pageStartTime, pageEndTime)
+
+	thisMonth := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location())
+	refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_month", thisMonth, pageEndTime)
+
+	offset := int(time.Monday - currentTimeNow.Weekday())
+	if offset > 0 {
+		offset = -6
+	}
+	pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset)
+	refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_week", pageStartTime, pageEndTime)
+
+	pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -30)
+	refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_last30_day", pageStartTime, pageEndTime)
+
+	pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+	refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_yesterday", pageStartTime, pageEndTime)
+
+	pageStartTime = thisMonth.AddDate(0, -1, 0)
+	pageEndTime = thisMonth.AddDate(0, 0, -1)
+	refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_last_month", pageStartTime, pageEndTime)
+
 }
 
 func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error {
@@ -550,7 +604,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		userList := make([]*User, 0)
 		sess.Find(&userList)
 
@@ -660,7 +714,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
 			}
 		}
 
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -700,7 +754,7 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int {
 		issueAssigneesList := make([]*IssueAssignees, 0)
 		sess.Select("issue_assignees.*").Table("issue_assignees").
 			Join("inner", "issue", "issue.id=issue_assignees.issue_id").
-			Where(cond).OrderBy("issue_assignees.id asc").Limit(Page_SIZE, int(indexTotal))
+			Where(cond).OrderBy("issue_assignees.id asc").Limit(PAGE_SIZE, int(indexTotal))
 
 		sess.Find(&issueAssigneesList)
 
@@ -712,7 +766,7 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int {
 				resultMap[issueAssigneesRecord.AssigneeID] += 1
 			}
 		}
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -735,7 +789,7 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int {
 	indexTotal = 0
 	for {
 		issueList := make([]*Issue, 0)
-		sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).OrderBy("issue.id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).OrderBy("issue.id asc").Limit(PAGE_SIZE, int(indexTotal))
 		sess.Find(&issueList)
 		log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList)))
 		for _, issueRecord := range issueList {
@@ -745,7 +799,7 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int {
 				resultMap[issueRecord.PosterID] += 1
 			}
 		}
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -768,7 +822,7 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[i
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		actionList := make([]*Action, 0)
 		sess.Find(&actionList)
 
@@ -781,7 +835,7 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[i
 			}
 		}
 
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -805,7 +859,7 @@ func queryCreateIssue(start_unix int64, end_unix int64) map[int64]int {
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		sess.Select("id,poster_id").Table("issue").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("id,poster_id").Table("issue").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		issueList := make([]*Issue, 0)
 		sess.Find(&issueList)
 		log.Info("query issue size=" + fmt.Sprint(len(issueList)))
@@ -816,7 +870,7 @@ func queryCreateIssue(start_unix int64, end_unix int64) map[int64]int {
 				resultMap[issueRecord.PosterID] += 1
 			}
 		}
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -839,7 +893,7 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int {
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		sess.Select("id,type,poster_id").Table("comment").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("id,type,poster_id").Table("comment").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		commentList := make([]*Comment, 0)
 		sess.Find(&commentList)
 		log.Info("query Comment size=" + fmt.Sprint(len(commentList)))
@@ -850,7 +904,7 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int {
 				resultMap[commentRecord.PosterID] += 1
 			}
 		}
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -875,7 +929,7 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int {
 	indexTotal = 0
 	for {
 		watchList := make([]*Watch, 0)
-		sess.Select("id,user_id,repo_id").Table("watch").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("id,user_id,repo_id").Table("watch").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		sess.Find(&watchList)
 
 		log.Info("query Watch size=" + fmt.Sprint(len(watchList)))
@@ -887,7 +941,7 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int {
 			}
 		}
 
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -913,7 +967,7 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int {
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		sess.Select("id,uid,repo_id").Table("star").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("id,uid,repo_id").Table("star").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		starList := make([]*Star, 0)
 		sess.Find(&starList)
 
@@ -926,7 +980,7 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int {
 			}
 		}
 
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -949,7 +1003,7 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int {
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		sess.Select("id,user_id,follow_id").Table("follow").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("id,user_id,follow_id").Table("follow").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		followList := make([]*Follow, 0)
 		sess.Find(&followList)
 
@@ -962,7 +1016,7 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int {
 			}
 		}
 
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -985,7 +1039,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int {
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		sess.Select("id,uploader_id,size").Table("attachment").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("id,uploader_id,size").Table("attachment").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		attachmentList := make([]*Attachment, 0)
 		sess.Find(&attachmentList)
 
@@ -998,7 +1052,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int {
 			}
 		}
 
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -1021,7 +1075,7 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int {
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		sess.Select("id,owner_id,name").Table("repository").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		sess.Select("id,owner_id,name").Table("repository").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		repoList := make([]*Repository, 0)
 		sess.Find(&repoList)
 		log.Info("query Repository size=" + fmt.Sprint(len(repoList)))
@@ -1032,7 +1086,7 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int {
 				resultMap[repoRecord.OwnerID] += 1
 			}
 		}
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}
@@ -1111,7 +1165,7 @@ func queryLoginCount(start_unix int64, end_unix int64) map[int64]int {
 	var indexTotal int64
 	indexTotal = 0
 	for {
-		statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal))
+		statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
 		userLoginLogList := make([]*UserLoginLog, 0)
 		statictisSess.Find(&userLoginLogList)
 		log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList)))
@@ -1122,7 +1176,7 @@ func queryLoginCount(start_unix int64, end_unix int64) map[int64]int {
 				resultMap[loginRecord.UId] += 1
 			}
 		}
-		indexTotal += Page_SIZE
+		indexTotal += PAGE_SIZE
 		if indexTotal >= count {
 			break
 		}

models/user_business_struct.go

@@ -2,7 +2,7 @@ package models
 
 import "code.gitea.io/gitea/modules/timeutil"
 
-type UserBusinessAnalysisYear struct {
+type UserBusinessAnalysisCurrentYear struct {
 	ID        int64 `xorm:"pk"`
 	CountDate int64 `xorm:"pk"`
 	//action :ActionMergePullRequest                         // 11