From 1361b3915fb1aa83d857fbc0a5b4d23448cb4a3e Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 28 Nov 2022 16:18:32 +0800 Subject: [PATCH] #3188 update --- models/list_options.go | 20 ++++ models/repo.go | 13 ++- models/repo_list.go | 54 ++++++----- models/repo_statistic.go | 20 ++++ models/repo_tag.go | 31 ++++++ modules/setting/setting.go | 8 ++ routers/admin/resources.go | 34 +++++++ routers/home.go | 60 +++++++++++- routers/private/internal.go | 1 + routers/repo/cloudbrain_statistic.go | 6 ++ routers/repo/repo_statistic.go | 20 ++++ routers/routes/routes.go | 6 +- services/repository/square.go | 135 +++++++++++++++++++++++++++ 13 files changed, 381 insertions(+), 27 deletions(-) create mode 100644 services/repository/square.go diff --git a/models/list_options.go b/models/list_options.go index 0946917fe..29144ba22 100644 --- a/models/list_options.go +++ b/models/list_options.go @@ -10,6 +10,26 @@ import ( "xorm.io/xorm" ) +type SelectedPageSize int + +const ( + PageSize15 SelectedPageSize = 15 + PageSize30 SelectedPageSize = 30 + PageSize50 SelectedPageSize = 50 +) + +func (s SelectedPageSize) IsLegal() bool { + switch s { + case PageSize30, PageSize50, PageSize15: + return true + } + return false +} + +func (s SelectedPageSize) Int() int { + return int(s) +} + // ListOptions options to paginate results type ListOptions struct { PageSize int diff --git a/models/repo.go b/models/repo.go index f0760108b..e1085a429 100755 --- a/models/repo.go +++ b/models/repo.go @@ -231,10 +231,15 @@ type Repository struct { CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Hot int64 `xorm:"-"` - Active int64 `xorm:"-"` - Alias string `xorm:"INDEX"` - LowerAlias string `xorm:"INDEX"` + Hot int64 `xorm:"-"` + Active int64 `xorm:"-"` + Alias string `xorm:"INDEX"` + LowerAlias string `xorm:"INDEX"` + AiTaskCnt int64 `xorm:"NOT NULL DEFAULT 0"` + ModelCnt int64 `xorm:"NOT NULL DEFAULT 0"` + DatasetCnt int64 `xorm:"NOT NULL DEFAULT 0"` + LastMonthVisits int64 `xorm:"NOT NULL DEFAULT 0"` + LastFourMonthCommits int64 `xorm:"NOT NULL DEFAULT 0"` } type RepositoryShow struct { diff --git a/models/repo_list.go b/models/repo_list.go index 92654c11c..6ec9bbe8f 100755 --- a/models/repo_list.go +++ b/models/repo_list.go @@ -201,29 +201,41 @@ func (s SearchOrderBy) String() string { return string(s) } +type FindReposResponse struct { + Repos RepositoryList + Page int + PageSize int + Total int64 +} + // Strings for sorting result const ( - SearchOrderByAlphabetically SearchOrderBy = "name ASC" - SearchOrderByAlphabeticallyReverse SearchOrderBy = "name DESC" - SearchOrderByLeastUpdated SearchOrderBy = "updated_unix ASC" - SearchOrderByRecentUpdated SearchOrderBy = "updated_unix DESC" - SearchOrderByOldest SearchOrderBy = "created_unix ASC" - SearchOrderByNewest SearchOrderBy = "created_unix DESC" - SearchOrderBySize SearchOrderBy = "size ASC" - SearchOrderBySizeReverse SearchOrderBy = "size DESC" - SearchOrderByID SearchOrderBy = "id ASC" - SearchOrderByIDReverse SearchOrderBy = "id DESC" - SearchOrderByStars SearchOrderBy = "num_stars ASC" - SearchOrderByStarsReverse SearchOrderBy = "num_stars DESC" - SearchOrderByForks SearchOrderBy = "num_forks ASC" - SearchOrderByForksReverse SearchOrderBy = "num_forks DESC" - SearchOrderByDownloadTimes SearchOrderBy = "download_times DESC" - SearchOrderByUseCount SearchOrderBy = "use_count ASC" - SearchOrderByUseCountReverse SearchOrderBy = "use_count DESC" - SearchOrderByHot SearchOrderBy = "(num_watches + num_stars + num_forks + clone_cnt) DESC" - SearchOrderByActive SearchOrderBy = "(num_issues + num_pulls + num_commit) DESC" - SearchOrderByWatches SearchOrderBy = "num_watches DESC" - SearchOrderByDefault SearchOrderBy = "recommend desc,num_stars DESC,updated_unix DESC" + SearchOrderByAlphabetically SearchOrderBy = "name ASC" + SearchOrderByAlphabeticallyReverse SearchOrderBy = "name DESC" + SearchOrderByLeastUpdated SearchOrderBy = "updated_unix ASC" + SearchOrderByRecentUpdated SearchOrderBy = "updated_unix DESC" + SearchOrderByOldest SearchOrderBy = "created_unix ASC" + SearchOrderByNewest SearchOrderBy = "created_unix DESC" + SearchOrderBySize SearchOrderBy = "size ASC" + SearchOrderBySizeReverse SearchOrderBy = "size DESC" + SearchOrderByID SearchOrderBy = "id ASC" + SearchOrderByIDReverse SearchOrderBy = "id DESC" + SearchOrderByStars SearchOrderBy = "num_stars ASC" + SearchOrderByStarsReverse SearchOrderBy = "num_stars DESC" + SearchOrderByForks SearchOrderBy = "num_forks ASC" + SearchOrderByForksReverse SearchOrderBy = "num_forks DESC" + SearchOrderByDownloadTimes SearchOrderBy = "download_times DESC" + SearchOrderByUseCount SearchOrderBy = "use_count ASC" + SearchOrderByUseCountReverse SearchOrderBy = "use_count DESC" + SearchOrderByHot SearchOrderBy = "(num_watches + num_stars + num_forks + clone_cnt) DESC" + SearchOrderByActive SearchOrderBy = "(num_issues + num_pulls + num_commit) DESC" + SearchOrderByWatches SearchOrderBy = "num_watches DESC" + SearchOrderByDefault SearchOrderBy = "recommend desc,num_stars DESC,updated_unix DESC" + SearchOrderByAiTaskCntReverse SearchOrderBy = "ai_task_cnt desc" + SearchOrderByModelCntReverse SearchOrderBy = "model_cnt desc" + SearchOrderByDatasetCntReverse SearchOrderBy = "dataset_cnt desc" + SearchOrderByLastMonthVisitsReverse SearchOrderBy = "last_month_visits desc" + SearchOrderByLastFourMonthCommitsReverse SearchOrderBy = "last_four_month_commits desc" ) // SearchRepositoryCondition creates a query condition according search repository options diff --git a/models/repo_statistic.go b/models/repo_statistic.go index 4f8f13ed7..ecdd77e57 100755 --- a/models/repo_statistic.go +++ b/models/repo_statistic.go @@ -200,3 +200,23 @@ func UpdateRepoStatVisits(repoStat *RepoStatistic) error { _, err := xStatistic.Exec(sql, repoStat.NumVisits, repoStat.RepoID, repoStat.Date) return err } + +func SumRepoStatColumn(begin, end time.Time, repoId int64, columnName string) (int64, error) { + res, err := xStatistic.Where("created_unix <= ? and created_unix >= ? and repo_id = ? ", end.Unix(), begin.Unix(), repoId).Sum(&RepoStatistic{}, columnName) + if err != nil { + return 0, err + } + return int64(res), nil +} + +func SumLastMonthNumVisits(repoId int64) (int64, error) { + end := time.Now() + begin := end.AddDate(0, 0, -30) + return SumRepoStatColumn(begin, end, repoId, "num_visits") +} + +func SumLastFourMonthNumCommits(repoId int64) (int64, error) { + end := time.Now() + begin := end.AddDate(0, 0, -120) + return SumRepoStatColumn(begin, end, repoId, "num_commits_added") +} diff --git a/models/repo_tag.go b/models/repo_tag.go index 730eb3f2a..e4e37fbd2 100644 --- a/models/repo_tag.go +++ b/models/repo_tag.go @@ -4,6 +4,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "fmt" + "xorm.io/builder" ) type OfficialTag struct { @@ -166,3 +167,33 @@ func GetAllOfficialTags() ([]OfficialTag, error) { } return o, nil } + +type FindSelectedReposOpts struct { + ListOptions + OrgId int64 + OnlyPublic bool +} + +func GetSelectedRepos(opts FindSelectedReposOpts) ([]Repository, error) { + if opts.Page < 1 { + opts.Page = 1 + } + var cond = builder.NewCond() + cond = cond.And(builder.Eq{"official_tag.code": "selected"}) + if opts.OrgId > 0 { + cond = cond.And(builder.Eq{"official_tag_repos.org_id": opts.OrgId}) + } + if opts.OnlyPublic { + cond = cond.And(builder.Eq{"repository.is_private": false}) + } + t := make([]Repository, 0) + err := x.Join("inner", "official_tag_repos", "repository.id = official_tag_repos.repo_id"). + Join("inner", "official_tag", "official_tag.id = official_tag_repos.tag_id"). + Where(cond).OrderBy("repository.updated_unix desc").Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).Find(&t) + + if err != nil { + return nil, err + } + + return t, nil +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index e237d5a8a..0d623eb3b 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -664,6 +664,10 @@ var ( CloudbrainStoppedTitle string CloudbrainStoppedRemark string + //repo square config + IncubationSourceOrgId int64 + PaperSourceOwnerId int64 + //nginx proxy PROXYURL string RadarMap = struct { @@ -1557,6 +1561,10 @@ func NewContext() { CloudbrainStoppedTitle = sec.Key("CLOUDBRAIN_STOPPED_TITLE").MustString("您好，您申请的算力资源已结束使用，任务已完成运行，状态为%s，请您关注运行结果") CloudbrainStoppedRemark = sec.Key("CLOUDBRAIN_STOPPED_REMARK").MustString("感谢您的耐心等待。") + sec = Cfg.Section("repo-square") + IncubationSourceOrgId = sec.Key("INCUBATION_ORG_ID").MustInt64(9) + PaperSourceOwnerId = sec.Key("PAPER_OWNER_ID").MustInt64(36008) + sec = Cfg.Section("point") CloudBrainPaySwitch = sec.Key("CLOUDBRAIN_PAY_SWITCH").MustBool(false) CloudBrainPayDelay = sec.Key("CLOUDBRAIN_PAY_DELAY").MustDuration(30 * time.Minute) diff --git a/routers/admin/resources.go b/routers/admin/resources.go index feea7b69b..8db958ef5 100644 --- a/routers/admin/resources.go +++ b/routers/admin/resources.go @@ -307,3 +307,37 @@ func RefreshHistorySpec(ctx *context.Context) { r["total"] = total ctx.JSON(http.StatusOK, response.SuccessWithData(r)) } + +func RefreshReposHistoryCnt(ctx *context.Context) { + scope := ctx.Query("scope") + list := ctx.Query("list") + + var scopeAll = false + if scope == "all" { + scopeAll = true + } + var ids = make([]int64, 0) + if list != "" { + strs := strings.Split(list, "|") + for _, s := range strs { + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + ids = append(ids, i) + } + + } + + total, success, err := resource.RefreshHistorySpec(scopeAll, ids) + if err != nil { + log.Error("RefreshHistorySpec error. %v", err) + ctx.JSON(http.StatusOK, response.ServerError(err.Error())) + return + } + r := make(map[string]interface{}, 0) + r["success"] = success + r["total"] = total + ctx.JSON(http.StatusOK, response.SuccessWithData(r)) +} diff --git a/routers/home.go b/routers/home.go index aab760611..7c13cb05e 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,6 +7,7 @@ package routers import ( "bytes" + "code.gitea.io/gitea/routers/response" "net/http" "strconv" "strings" @@ -295,6 +296,63 @@ func ExploreRepos(ctx *context.Context) { }) } +func RepoSquare(ctx *context.Context) { + var result []models.Repository + var err error + switch ctx.Query("type") { + case "preferred": + result, err = repository.GetPreferredRepos() + case "incubation": + result, err = repository.GetIncubationRepos() + case "hot-paper": + result, err = repository.GetHotPaperRepos() + default: + result, err = repository.GetPreferredRepos() + } + if err != nil { + ctx.JSON(http.StatusOK, response.ResponseError(err)) + return + } + resultMap := make(map[string]interface{}, 0) + resultMap["Repos"] = result + ctx.JSON(http.StatusOK, response.SuccessWithData(resultMap)) +} +func RepoFind(ctx *context.Context) { + keyword := strings.Trim(ctx.Query("q"), " ") + topic := strings.Trim(ctx.Query("topic"), " ") + sort := strings.Trim(ctx.Query("sort"), " ") + page := ctx.QueryInt("page") + pageSize := models.SelectedPageSize(ctx.QueryInt("pageSize")) + if !pageSize.IsLegal() { + ctx.JSON(http.StatusOK, response.ServerError("pageSize illegal")) + return + } + if page <= 0 { + page = 1 + } + + var ownerID int64 + if ctx.User != nil && !ctx.User.IsAdmin { + ownerID = ctx.User.ID + } + + result, err := repository.FindRepos(repository.FindReposOptions{ + ListOptions: models.ListOptions{Page: page, PageSize: pageSize.Int()}, + Actor: ctx.User, + Sort: sort, + Keyword: keyword, + Topic: topic, + Private: ctx.User != nil, + OwnerID: ownerID, + }) + if err != nil { + log.Error("RepoFind error. %v", err) + ctx.JSON(http.StatusOK, response.ResponseError(err)) + return + } + ctx.JSON(http.StatusOK, response.SuccessWithData(result)) +} + func ExploreDatasets(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("explore") ctx.Data["PageIsExplore"] = true @@ -824,4 +882,4 @@ func HomePrivacy(ctx *context.Context) { func HomeResoruceDesc(ctx *context.Context) { ctx.HTML(200, tplResoruceDesc) -} \ No newline at end of file +} diff --git a/routers/private/internal.go b/routers/private/internal.go index 14b0f05de..eae3713a9 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -55,6 +55,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/task/history_handle/duration", repo.HandleTaskWithNoDuration) m.Post("/task/history_handle/aicenter", repo.HandleTaskWithAiCenter) m.Post("/resources/specification/handle_historical_task", admin.RefreshHistorySpec) + m.Post("/repos/cnt_stat/handle_historical_task", admin.RefreshHistorySpec) m.Post("/duration_statisctic/history_handle", repo.CloudbrainUpdateHistoryData) }, CheckInternalToken) diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 4476b3e45..8084614eb 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -13,6 +13,12 @@ import ( ) func CloudbrainDurationStatisticHour() { + defer func() { + err := recover() + if err == nil { + return + } + }() var statisticTime time.Time var count int64 recordDurationUpdateTime, err := models.GetDurationRecordUpdateTime() diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 468e6fa85..dd73a9f3f 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -166,6 +166,8 @@ func RepoStatisticDaily(date string) { repoStat.NumIssuesGrowth = repoStat.NumIssues - repoStatisticFourMonthsAgo.NumIssues } + SyncStatDataToRepo(repo) + if _, err = models.InsertRepoStat(&repoStat); err != nil { log.Error("InsertRepoStat failed(%s): %v", projectName, err) log.Error("failed statistic: %s", projectName) @@ -284,6 +286,24 @@ func RepoStatisticDaily(date string) { } +func SyncStatDataToRepo(repo *models.Repository) { + changed := false + //Save the visit number of repository in the last month + if lv, err := models.SumLastMonthNumVisits(repo.ID); err == nil { + repo.LastMonthVisits = lv + changed = true + } + //Save the commits number of repository in the last four month + if lc, err := models.SumLastFourMonthNumCommits(repo.ID); err == nil { + repo.LastFourMonthCommits = lc + changed = true + } + + if changed { + models.UpdateRepository(repo, false) + } +} + func getDistinctProjectName(repo *models.Repository) string { return repo.OwnerName + "/" + repo.Alias } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 2b361b507..e902d10be 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -370,7 +370,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/images/custom", repo.GetCustomImages) m.Get("/images/star", repo.GetStarImages) - m.Get("/repos", routers.ExploreRepos) + m.Group("/repos", func() { + m.Get("", routers.ExploreRepos) + m.Get("/square", routers.RepoSquare) + m.Get("/search", routers.RepoFind) + }) m.Get("/datasets", routers.ExploreDatasets) m.Get("/users", routers.ExploreUsers) m.Get("/organizations", routers.ExploreOrganizations) diff --git a/services/repository/square.go b/services/repository/square.go new file mode 100644 index 000000000..45416667c --- /dev/null +++ b/services/repository/square.go @@ -0,0 +1,135 @@ +package repository + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +func GetPreferredRepos() ([]models.Repository, error) { + return models.GetSelectedRepos(models.FindSelectedReposOpts{ + ListOptions: models.ListOptions{ + PageSize: 10, + Page: 1, + }, + OnlyPublic: true, + }) +} + +func GetIncubationRepos() ([]models.Repository, error) { + return models.GetSelectedRepos(models.FindSelectedReposOpts{ + ListOptions: models.ListOptions{ + PageSize: 10, + Page: 1, + }, + OrgId: setting.IncubationSourceOrgId, + OnlyPublic: true, + }) +} + +func GetHotPaperRepos() ([]models.Repository, error) { + rlist, _, err := models.SearchRepository(&models.SearchRepoOptions{ + ListOptions: models.ListOptions{ + Page: 1, + PageSize: 10, + }, + OrderBy: models.SearchOrderByLastMonthVisitsReverse + "," + models.SearchOrderByRecentUpdated, + TopicOnly: true, + TopicName: "openi-paper", + AllPublic: true, + }) + if err != nil { + return nil, err + } + result := make([]models.Repository, len(rlist)) + for i, r := range rlist { + result[i] = *r + } + return result, nil +} + +type FindReposOptions struct { + models.ListOptions + Actor *models.User + Sort string + Keyword string + Topic string + Private bool + OwnerID int64 +} + +func FindRepos(opts FindReposOptions) (*models.FindReposResponse, error) { + + var ( + repos []*models.Repository + count int64 + err error + orderBy models.SearchOrderBy + ) + + switch opts.Sort { + //1.近期热门：按最近1个月浏览量倒序排序，最近1个月浏览量>最近更新>项目名称升序 + //2.近期活跃：按提交增长量（最近4个月commit数）倒序排序，提交增长量>最近更新>项目名称升序。 + //3.最近更新：按最近更新>项目名称升序排序。 + //4.最近创建：按项目创建时间排序，最近的排前面。最近创建>项目名称升序。 + //5.点赞最多：按点赞数倒序排序。点赞数>最近更新>项目名称升序。 + //6.派生最多：按派生数倒序排序。派生数>最近更新>项目名称升序。 + //7.数据集最多：按项目包含的数据集文件数量倒序排序，数据集文件数>最近更新>项目名称升序。 + //8.AI任务最多：按项目包含的AI任务数量倒序排序，AI任务数>最近更新>项目名称升序。 + //9.模型最多：按项目包含的模型数量倒序排序，模型大小为0则不统计。模型数>最近更新>项目名称升序。\ + + //1.近期热门 + case "most_popular": + orderBy = models.SearchOrderByLastMonthVisitsReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically + //2.近期活跃 + case "most_active": + orderBy = models.SearchOrderByLastFourMonthCommitsReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically + //3.最近更新 + case "recent_update": + orderBy = models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically + //4.最近创建 + case "newest": + orderBy = models.SearchOrderByNewest + "," + models.SearchOrderByAlphabetically + //5.点赞最多 + case "most_stars": + orderBy = models.SearchOrderByStarsReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically + //6.派生最多 + case "most_forks": + orderBy = models.SearchOrderByForksReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically + //7.数据集最多 + case "most_datasets": + orderBy = models.SearchOrderByDatasetCntReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically + //8.AI任务最多 + case "most_ai_tasks": + orderBy = models.SearchOrderByAiTaskCntReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically + //9.模型最多 + case "most_models": + orderBy = models.SearchOrderByModelCntReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically + + default: + orderBy = models.SearchOrderByLastMonthVisitsReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically + } + + repos, count, err = models.SearchRepository(&models.SearchRepoOptions{ + ListOptions: opts.ListOptions, + Actor: opts.Actor, + OrderBy: orderBy, + Private: opts.Private, + Keyword: opts.Keyword, + OwnerID: opts.OwnerID, + AllPublic: true, + AllLimited: true, + TopicName: opts.Topic, + IncludeDescription: setting.UI.SearchRepoDescription, + }) + if err != nil { + log.Error("FindRepos error when SearchRepository.%v", err) + return nil, err + } + return &models.FindReposResponse{ + Repos: repos, + Total: count, + Page: opts.Page, + PageSize: opts.PageSize, + }, nil +}