项目看板所有周期数据导出随着网站发展记录数可能很大，改为后端支持导出

modules/cron/tasks_basic.go

@@ -11,6 +11,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/migrations"
 	repository_service "code.gitea.io/gitea/modules/repository"
+	api_repo "code.gitea.io/gitea/routers/api/v1/repo"
 	"code.gitea.io/gitea/routers/repo"
 	mirror_service "code.gitea.io/gitea/services/mirror"
 )
@@ -195,6 +196,17 @@ func registerHandleUserStatistic() {
 	})
 }
 
+func registerHandleClearRepoStatisticFile() {
+	RegisterTaskFatal("handle_repo_clear_statistic_file", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "@daily",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		api_repo.ClearUnusedStatisticsFile()
+		return nil
+	})
+}
+
 func initBasicTasks() {
 	registerUpdateMirrorTask()
 	registerRepoHealthCheck()

modules/setting/setting.go

@@ -545,6 +545,7 @@ var (
 		GrowthCommit       float64
 		GrowthComments     float64
 		RecordBeginTime    string
+		Path               string
 	}{}
 )
 
@@ -1326,6 +1327,7 @@ func SetRadarMapConfig() {
 	RadarMap.GrowthCommit = sec.Key("growth_commit").MustFloat64(0.2)
 	RadarMap.GrowthComments = sec.Key("growth_comments").MustFloat64(0.2)
 	RadarMap.RecordBeginTime = sec.Key("record_beigin_time").MustString("2021-11-05")
+	RadarMap.Path = sec.Key("PATH").MustString("data/projectborad")
 
 }
 

options/locale/locale_en-US.ini

@@ -2157,6 +2157,19 @@ repos.stars = Stars
 repos.forks = Forks
 repos.issues = Issues
 repos.size = Size
+repos.id=ID
+repos.projectName=Project Name
+repos.isPrivate=Private
+repos.openi=OpenI
+repos.visit=Visit
+repos.download=Code Download
+repos.pr=PR
+repos.commit=Commit
+repos.closedIssues=Closed Issue
+repos.contributor=Contributor
+repos.yes=Yes
+repos.no=No
+
 
 datasets.dataset_manage_panel= Dataset Manage
 datasets.owner=Owner

options/locale/locale_zh-CN.ini

@@ -2159,6 +2159,19 @@ repos.stars=点赞数
 repos.forks=派生数
 repos.issues=任务数
 repos.size=大小
+repos.id=ID
+repos.projectName=项目名称
+repos.isPrivate=私有
+repos.openi=OpenI指数
+repos.visit=浏览量
+repos.download=代码下载量
+repos.pr=PR数
+repos.commit=Commit数
+repos.closedIssues=已解决任务数
+repos.contributor=贡献者数
+repos.yes=是
+repos.no=否
+
 
 datasets.dataset_manage_panel=数据集管理
 datasets.owner=所有者

routers/api/v1/api.go

@@ -531,6 +531,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 
 			m.Group("/project", func() {
 				m.Get("", adminReq, repo.GetAllProjectsPeriodStatistics)
+				m.Get("/download", adminReq, repo.ServeAllProjectsPeriodStatisticsFile)
 				m.Group("/:id", func() {
 					m.Get("", adminReq, repo.GetProjectLatestStatistics)
 					m.Get("/period", adminReq, repo.GetProjectPeriodStatistics)

routers/api/v1/repo/repo_dashbord.go

@@ -1,8 +1,12 @@
 package repo
 
 import (
+	"encoding/csv"
 	"fmt"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"path"
 	"strconv"
 	"time"
 
@@ -117,6 +121,125 @@ func GetAllProjectsPeriodStatistics(ctx *context.Context) {
 
 }
 
+func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) {
+
+	recordBeginTime, err := getRecordBeginTime()
+	if err != nil {
+		log.Error("Can not get record begin time", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
+		return
+	}
+	beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime)
+	if err != nil {
+		log.Error("Parameter is wrong", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong"))
+		return
+	}
+	q := ctx.QueryTrim("q")
+	page := ctx.QueryInt("page")
+	if page <= 0 {
+		page = 1
+	}
+	pageSize := 1000
+	orderBy := getOrderBy(ctx)
+
+	_, latestDate, err := models.GetRepoStatLastUpdatedTime()
+	if err != nil {
+		log.Error("Can not query the last updated time.", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error"))
+		return
+	}
+
+	countSql := generateCountSql(beginTime, endTime, latestDate, q)
+	total, err := models.CountRepoStatByRawSql(countSql)
+	if err != nil {
+		log.Error("Can not query total count.", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error"))
+		return
+	}
+
+	fileName := getFileName(ctx, beginTime, endTime)
+
+	if err := os.MkdirAll(setting.RadarMap.Path, os.ModePerm); err != nil {
+		ctx.Error(http.StatusBadRequest, fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err).Error())
+	}
+
+	totalPage := getTotalPage(total, pageSize)
+
+	f, e := os.Create(fileName)
+	defer f.Close()
+	if e != nil {
+		log.Warn("Failed to create file", e)
+	}
+	writer := csv.NewWriter(f)
+	writer.Write(allProjectsPeroidHeader(ctx))
+	for i := 0; i <= totalPage; i++ {
+
+		pageRecords := models.GetRepoStatisticByRawSql(generatePageSql(beginTime, endTime, latestDate, q, orderBy, i+1, pageSize))
+		for _, record := range pageRecords {
+			e = writer.Write(allProjectsPeroidValues(record, ctx))
+			if e != nil {
+				log.Warn("Failed to write record", e)
+			}
+		}
+		writer.Flush()
+
+	}
+
+	ctx.ServeFile(fileName)
+
+}
+
+func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time) string {
+	baseName := setting.RadarMap.Path + "/"
+	if ctx.QueryTrim("type") != "" {
+		baseName = baseName + ctx.QueryTrim("type") + "_"
+	}
+	if ctx.QueryTrim("q") != "" {
+		baseName = baseName + ctx.QueryTrim("q") + "_"
+	}
+	baseName = baseName + beginTime.Format(DATE_FORMAT) + "_to_" + endTime.Format(DATE_FORMAT) + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".csv"
+	return baseName
+}
+
+func ClearUnusedStatisticsFile() {
+	fileInfos, err := ioutil.ReadDir(setting.RadarMap.Path)
+	if err != nil {
+		log.Warn("can not read dir: "+setting.RadarMap.Path, err)
+		return
+	}
+
+	for _, fileInfo := range fileInfos {
+		if !fileInfo.IsDir() && fileInfo.ModTime().Before(time.Now().AddDate(0, 0, -1)) {
+			os.Remove(path.Join(setting.RadarMap.Path, fileInfo.Name()))
+		}
+	}
+
+}
+
+func allProjectsPeroidHeader(ctx *context.Context) []string {
+
+	return []string{ctx.Tr("repos.id"), ctx.Tr("repos.projectName"), ctx.Tr("repos.isPrivate"), ctx.Tr("repos.openi"), ctx.Tr("repos.visit"), ctx.Tr("repos.download"), ctx.Tr("repos.pr"), ctx.Tr("repos.commit"),
+		ctx.Tr("repos.watches"), ctx.Tr("repos.stars"), ctx.Tr("repos.forks"), ctx.Tr("repos.issues"), ctx.Tr("repos.closedIssues"), ctx.Tr("repos.contributor")}
+
+}
+
+func allProjectsPeroidValues(rs *models.RepoStatistic, ctx *context.Context) []string {
+	return []string{strconv.FormatInt(rs.RepoID, 10), rs.Name, getIsPrivateDisplay(rs.IsPrivate, ctx), strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64),
+		strconv.FormatInt(rs.NumVisits, 10), strconv.FormatInt(rs.NumDownloads, 10), strconv.FormatInt(rs.NumPulls, 10), strconv.FormatInt(rs.NumCommits, 10),
+		strconv.FormatInt(rs.NumWatches, 10), strconv.FormatInt(rs.NumStars, 10), strconv.FormatInt(rs.NumForks, 10), strconv.FormatInt(rs.NumIssues, 10),
+		strconv.FormatInt(rs.NumClosedIssues, 10), strconv.FormatInt(rs.NumContributor, 10),
+	}
+}
+
+func getIsPrivateDisplay(private bool, ctx *context.Context) string {
+	if private {
+		return ctx.Tr("repos.yes")
+	} else {
+		return ctx.Tr("repos.no")
+	}
+}
+
 func GetProjectLatestStatistics(ctx *context.Context) {
 	repoId := ctx.Params(":id")
 	recordBeginTime, err := getRecordBeginTime()