aiforge/routers/repo/user_data_analysis.go

package repo

import (
	"fmt"
	"net/http"
	"net/url"
	"time"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/services/mailer"
	"github.com/360EntSecGroup-Skylar/excelize/v2"
)

const (
	PAGE_SIZE = 2000
)

func queryUserDataPage(ctx *context.Context, tableName string, queryObj interface{}) {
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}
	pageSize := ctx.QueryInt("pageSize")
	if pageSize <= 0 {
		pageSize = setting.UI.IssuePagingNum
	}
	userName := ctx.Query("userName")
	IsReturnFile := ctx.QueryBool("IsReturnFile")

	if IsReturnFile {
		//writer exec file.
		xlsx := excelize.NewFile()
		sheetName := ctx.Tr("user.static.sheetname")
		index := xlsx.NewSheet(sheetName)
		xlsx.DeleteSheet("Sheet1")
		dataHeader := map[string]string{
			"A1": ctx.Tr("user.static.id"),
			"B1": ctx.Tr("user.static.name"),
			"C1": ctx.Tr("user.static.codemergecount"),
			"D1": ctx.Tr("user.static.commitcount"),
			"E1": ctx.Tr("user.static.issuecount"),
			"F1": ctx.Tr("user.static.commentcount"),
			"G1": ctx.Tr("user.static.focusrepocount"),
			"H1": ctx.Tr("user.static.starrepocount"),
			"I1": ctx.Tr("user.static.logincount"),
			"J1": ctx.Tr("user.static.watchedcount"),
			"K1": ctx.Tr("user.static.commitcodesize"),
			"L1": ctx.Tr("user.static.solveissuecount"),
			"M1": ctx.Tr("user.static.encyclopediascount"),
			"N1": ctx.Tr("user.static.createrepocount"),
			"O1": ctx.Tr("user.static.openiindex"),
			"P1": ctx.Tr("user.static.registdate"),
			"Q1": ctx.Tr("user.static.CloudBrainTaskNum"),
			"R1": ctx.Tr("user.static.CloudBrainRunTime"),
			"S1": ctx.Tr("user.static.CommitDatasetNum"),
			"T1": ctx.Tr("user.static.CommitModelCount"),
			"U1": ctx.Tr("user.static.UserIndex"),
			"V1": ctx.Tr("user.static.countdate"),
		}
		for k, v := range dataHeader {
			//设置单元格的值
			xlsx.SetCellValue(sheetName, k, v)
		}
		_, count := models.QueryUserStaticDataByTableName(1, 1, tableName, queryObj, userName)
		var indexTotal int64
		indexTotal = 0
		row := 1
		for {
			re, _ := models.QueryUserStaticDataByTableName(int(indexTotal), PAGE_SIZE, tableName, queryObj, "")
			log.Info("return count=" + fmt.Sprint(count))
			for _, userRecord := range re {
				row++
				rows := fmt.Sprint(row)
				xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID)
				xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name)
				xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount)
				xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CommitCount)
				xlsx.SetCellValue(sheetName, "E"+rows, userRecord.IssueCount)
				xlsx.SetCellValue(sheetName, "F"+rows, userRecord.CommentCount)
				xlsx.SetCellValue(sheetName, "G"+rows, userRecord.FocusRepoCount)
				xlsx.SetCellValue(sheetName, "H"+rows, userRecord.StarRepoCount)
				xlsx.SetCellValue(sheetName, "I"+rows, userRecord.LoginCount)
				xlsx.SetCellValue(sheetName, "J"+rows, userRecord.WatchedCount)
				xlsx.SetCellValue(sheetName, "K"+rows, userRecord.CommitCodeSize)
				xlsx.SetCellValue(sheetName, "L"+rows, userRecord.SolveIssueCount)
				xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount)
				xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount)
				xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex))

				formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05")
				xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3])

				xlsx.SetCellValue(sheetName, "Q"+rows, userRecord.CloudBrainTaskNum)
				xlsx.SetCellValue(sheetName, "R"+rows, fmt.Sprintf("%.2f", float64(userRecord.CloudBrainRunTime)/3600))
				xlsx.SetCellValue(sheetName, "S"+rows, userRecord.CommitDatasetNum)
				xlsx.SetCellValue(sheetName, "T"+rows, userRecord.CommitModelCount)
				xlsx.SetCellValue(sheetName, "U"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex))

				formatTime = userRecord.DataDate
				xlsx.SetCellValue(sheetName, "V"+rows, formatTime)
			}

			indexTotal += PAGE_SIZE
			if indexTotal >= count {
				break
			}
		}
		//设置默认打开的表单
		xlsx.SetActiveSheet(index)
		filename := sheetName + "_" + ctx.Tr("user.static."+tableName) + ".xlsx"
		ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename))
		ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
		if _, err := xlsx.WriteTo(ctx.Resp); err != nil {
			log.Info("writer exel error." + err.Error())
		}
	} else {
		re, count := models.QueryUserStaticDataByTableName((page-1)*pageSize, pageSize, tableName, queryObj, userName)
		mapInterface := make(map[string]interface{})
		mapInterface["data"] = re
		mapInterface["count"] = count
		ctx.JSON(http.StatusOK, mapInterface)
	}
}

func QueryMetrics(ctx *context.Context) {
	startDate := ctx.Query("startDate")
	endDate := ctx.Query("endDate")
	startTime, _ := time.ParseInLocation("2006-01-02", startDate, time.Local)
	endTime, _ := time.ParseInLocation("2006-01-02", endDate, time.Local)
	result, count := models.QueryMetrics(startTime.Unix(), endTime.Unix())
	mapInterface := make(map[string]interface{})
	mapInterface["data"] = result
	mapInterface["count"] = count
	ctx.JSON(http.StatusOK, mapInterface)
}

func QueryRankingList(ctx *context.Context) {
	key := ctx.Query("key")
	tableName := ctx.Query("tableName")
	limit := ctx.QueryInt("limit")

	result, count := models.QueryRankList(key, tableName, limit)
	mapInterface := make(map[string]interface{})
	mapInterface["data"] = result
	mapInterface["count"] = count
	ctx.JSON(http.StatusOK, mapInterface)
}

func QueryUserStaticCurrentMonth(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_current_month", new(models.UserBusinessAnalysisCurrentMonth))
}

func QueryUserStaticCurrentWeek(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_current_week", new(models.UserBusinessAnalysisCurrentWeek))
}

func QueryUserStaticCurrentYear(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_current_year", new(models.UserBusinessAnalysisCurrentYear))
}

func QueryUserStaticLast30Day(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_last30_day", new(models.UserBusinessAnalysisLast30Day))
}

func QueryUserStaticLastMonth(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_last_month", new(models.UserBusinessAnalysisLastMonth))
}

func QueryUserStaticYesterday(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_yesterday", new(models.UserBusinessAnalysisYesterday))
}

func QueryUserStaticAll(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_all", new(models.UserBusinessAnalysisAll))
}

func QueryUserStaticDataPage(ctx *context.Context) {
	startDate := ctx.Query("startDate")
	endDate := ctx.Query("endDate")
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}
	pageSize := ctx.QueryInt("pageSize")
	if pageSize <= 0 {
		pageSize = setting.UI.IssuePagingNum
	}
	userName := ctx.Query("userName")
	IsReturnFile := ctx.QueryBool("IsReturnFile")
	log.Info("startDate=" + startDate + "  endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page))
	var startTime time.Time
	var endTime time.Time
	var isAll bool
	if startDate == "all" {
		isAll = true
		startTime = time.Now()
		endTime = time.Now()
	} else {
		startTime, _ = time.ParseInLocation("2006-01-02", startDate, time.Local)
		startTime = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 12, 0, 0, 0, startTime.Location())
		settingStartTime, _ := time.Parse("2006-01-02", setting.RadarMap.RecordBeginTime)
		if startTime.Unix() < settingStartTime.Unix() {
			startTime = settingStartTime
			startDate = settingStartTime.Format("2006-01-02")
		}
		endTime, _ = time.ParseInLocation("2006-01-02", endDate, time.Local)
		endTime = endTime.AddDate(0, 0, 1)
		endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 23, 59, 59, 0, startTime.Location())

		isAll = false
		log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + "  endDate=" + fmt.Sprint(endTime.Unix()))
	}

	if IsReturnFile {
		page = -1
		pageSize = -1
	}

	pageOpts := &models.UserBusinessAnalysisQueryOptions{
		ListOptions: models.ListOptions{
			Page:     page,
			PageSize: pageSize,
		},
		UserName:  userName,
		StartTime: startTime.Unix(),
		EndTime:   endTime.Unix(),
		IsAll:     isAll,
	}

	if IsReturnFile {
		re, count := models.QueryUserStaticDataAll(pageOpts)
		log.Info("return count=" + fmt.Sprint(count))
		//writer exec file.
		xlsx := excelize.NewFile()
		sheetName := ctx.Tr("user.static.sheetname")
		index := xlsx.NewSheet(sheetName)
		xlsx.DeleteSheet("Sheet1")
		dataHeader := map[string]string{
			"A1": ctx.Tr("user.static.id"),
			"B1": ctx.Tr("user.static.name"),
			"C1": ctx.Tr("user.static.codemergecount"),
			"D1": ctx.Tr("user.static.commitcount"),
			"E1": ctx.Tr("user.static.issuecount"),
			"F1": ctx.Tr("user.static.commentcount"),
			"G1": ctx.Tr("user.static.focusrepocount"),
			"H1": ctx.Tr("user.static.starrepocount"),
			"I1": ctx.Tr("user.static.logincount"),
			"J1": ctx.Tr("user.static.watchedcount"),
			"K1": ctx.Tr("user.static.commitcodesize"),
			"L1": ctx.Tr("user.static.solveissuecount"),
			"M1": ctx.Tr("user.static.encyclopediascount"),
			"N1": ctx.Tr("user.static.createrepocount"),
			"O1": ctx.Tr("user.static.openiindex"),
			"P1": ctx.Tr("user.static.registdate"),
			"Q1": ctx.Tr("user.static.CloudBrainTaskNum"),
			"R1": ctx.Tr("user.static.CloudBrainRunTime"),
			"S1": ctx.Tr("user.static.CommitDatasetNum"),
			"T1": ctx.Tr("user.static.CommitModelCount"),
			"U1": ctx.Tr("user.static.UserIndex"),
			"V1": ctx.Tr("user.static.countdate"),
		}
		for k, v := range dataHeader {
			//设置单元格的值
			xlsx.SetCellValue(sheetName, k, v)
		}

		for i, userRecord := range re {
			rows := fmt.Sprint(i + 2)

			xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID)
			xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name)
			xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount)
			xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CommitCount)
			xlsx.SetCellValue(sheetName, "E"+rows, userRecord.IssueCount)
			xlsx.SetCellValue(sheetName, "F"+rows, userRecord.CommentCount)
			xlsx.SetCellValue(sheetName, "G"+rows, userRecord.FocusRepoCount)
			xlsx.SetCellValue(sheetName, "H"+rows, userRecord.StarRepoCount)
			xlsx.SetCellValue(sheetName, "I"+rows, userRecord.LoginCount)
			xlsx.SetCellValue(sheetName, "J"+rows, userRecord.WatchedCount)
			xlsx.SetCellValue(sheetName, "K"+rows, userRecord.CommitCodeSize)
			xlsx.SetCellValue(sheetName, "L"+rows, userRecord.SolveIssueCount)
			xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount)
			xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount)
			xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex))

			formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05")
			xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3])
			xlsx.SetCellValue(sheetName, "Q"+rows, userRecord.CloudBrainTaskNum)
			xlsx.SetCellValue(sheetName, "R"+rows, fmt.Sprintf("%.2f", float64(userRecord.CloudBrainRunTime)/3600))
			xlsx.SetCellValue(sheetName, "S"+rows, userRecord.CommitDatasetNum)
			xlsx.SetCellValue(sheetName, "T"+rows, userRecord.CommitModelCount)
			xlsx.SetCellValue(sheetName, "U"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex))
			formatTime = userRecord.DataDate
			xlsx.SetCellValue(sheetName, "V"+rows, formatTime)
		}

		//设置默认打开的表单
		xlsx.SetActiveSheet(index)

		filename := sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx"

		ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename))
		ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
		if _, err := xlsx.WriteTo(ctx.Resp); err != nil {
			log.Info("writer exel error." + err.Error())
		}

	} else {
		mapInterface := make(map[string]interface{})
		re, count := models.QueryUserStaticDataPage(pageOpts)
		mapInterface["data"] = re
		mapInterface["count"] = count
		ctx.JSON(http.StatusOK, mapInterface)
	}
}

func TimingCountDataByDateAndReCount(date string, isReCount bool) {

	t, _ := time.Parse("2006-01-02", date)
	startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())

	endTime := time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location())

	//query wiki data
	log.Info("start to time count data")
	wikiMap := make(map[string]int)
	warnEmailMessage := "用户统计信息入库失败，请尽快定位。"
	repoList, err := models.GetAllRepositories()
	if err != nil {
		log.Error("query repo error." + err.Error())
		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
		return
	}
	log.Info("start to query wiki data")
	for _, repoRecord := range repoList {
		wikiPath := models.WikiPath(repoRecord.OwnerName, repoRecord.Name)
		time, err := git.GetLatestCommitTime(wikiPath)
		if err == nil {
			log.Info("last commit time:" + time.Format("2006-01-02 15:04:05") + " wikiPath=" + wikiPath)
			if time.After(startTime) {
				wikiRepo, _, err := FindWikiRepoCommitByWikiPath(wikiPath)
				if err != nil {
					log.Error("wiki not exist. wikiPath=" + wikiPath)
				} else {
					log.Info("wiki exist, wikiPath=" + wikiPath)
					list, err := wikiRepo.GetCommitByPathAndDays(wikiPath, 1)
					if err != nil {
						log.Info("err,err=v%", err)
					} else {
						for logEntry := list.Front(); logEntry != nil; logEntry = logEntry.Next() {
							commit := logEntry.Value.(*git.Commit)
							log.Info("commit msg=" + commit.CommitMessage + " time=" + commit.Committer.When.Format("2006-01-02 15:04:05") + " user=" + commit.Committer.Name)
							if _, ok := wikiMap[commit.Committer.Name]; !ok {
								wikiMap[commit.Committer.Name] = 1
							} else {
								wikiMap[commit.Committer.Name] += 1
							}
						}
					}

				}
			}
		}
	}
	//other user info data
	err = models.CounDataByDateAndReCount(wikiMap, startTime, endTime, isReCount)
	if err != nil {
		log.Error("count user info error." + err.Error())
		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
	}
	log.Info("start to count all user info data")
	//models.RefreshUserStaticAllTabel(wikiMap)
	log.Info("end to count all user info data")
}

func TimingCountDataByDate(date string) {
	TimingCountDataByDateAndReCount(date, true)
}

func TimingCountData() {
	log.Info("start to time count data")
	currentTimeNow := time.Now()
	log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05"))
	startTime := currentTimeNow.AddDate(0, 0, -1).Format("2006-01-02")
	TimingCountDataByDateAndReCount(startTime, false)
}