Merge pull request '获取分支列表、标签列表优化' (#1018) from fix-965 into V20211213.patch

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/1018
Reviewed-by: avadesian <xuchx@pcl.ac.cn>

modules/context/repo.go

@@ -524,7 +524,7 @@ func RepoAssignment() macaron.Handler {
 		}
 		ctx.Data["Tags"] = tags
 
-		brs, err := ctx.Repo.GitRepo.GetBranches()
+		brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
 		if err != nil {
 			ctx.ServerError("GetBranches", err)
 			return
@@ -712,7 +712,7 @@ func RepoRefByType(refType RepoRefType) macaron.Handler {
 			refName = ctx.Repo.Repository.DefaultBranch
 			ctx.Repo.BranchName = refName
 			if !ctx.Repo.GitRepo.IsBranchExist(refName) {
-				brs, err := ctx.Repo.GitRepo.GetBranches()
+				brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
 				if err != nil {
 					ctx.ServerError("GetBranches", err)
 					return

modules/git/repo_branch.go

@@ -6,7 +6,9 @@
 package git
 
 import (
+	"bufio"
 	"fmt"
+	"io"
 	"strings"
 
 	"github.com/go-git/go-git/v5/plumbing"
@@ -74,25 +76,6 @@ func (repo *Repository) SetDefaultBranch(name string) error {
 	return err
 }
 
-// GetBranches returns all branches of the repository.
-func (repo *Repository) GetBranches() ([]string, error) {
-	var branchNames []string
-
-	branches, err := repo.gogitRepo.Branches()
-	if err != nil {
-		return nil, err
-	}
-
-	_ = branches.ForEach(func(branch *plumbing.Reference) error {
-		branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix))
-		return nil
-	})
-
-	// TODO: Sort?
-
-	return branchNames, nil
-}
-
 // GetBranch returns a branch by it's name
 func (repo *Repository) GetBranch(branch string) (*Branch, error) {
 	if !repo.IsBranchExist(branch) {
@@ -106,16 +89,16 @@ func (repo *Repository) GetBranch(branch string) (*Branch, error) {
 }
 
 // GetBranchesByPath returns a branch by it's path
-func GetBranchesByPath(path string) ([]*Branch, error) {
+func GetBranchesByPath(path string, skip, limit int) ([]*Branch, int, error) {
 	gitRepo, err := OpenRepository(path)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 	defer gitRepo.Close()
 
-	brs, err := gitRepo.GetBranches()
+	brs, countAll, err := gitRepo.GetBranches(skip, limit)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	branches := make([]*Branch, len(brs))
@@ -127,7 +110,7 @@ func GetBranchesByPath(path string) ([]*Branch, error) {
 		}
 	}
 
-	return branches, nil
+	return branches, countAll, nil
 }
 
 // DeleteBranchOptions Option(s) for delete branch
@@ -183,3 +166,91 @@ func (repo *Repository) RemoveRemote(name string) error {
 func (branch *Branch) GetCommit() (*Commit, error) {
 	return branch.gitRepo.GetBranchCommit(branch.Name)
 }
+
+// GetBranches returns branches from the repository, skipping skip initial branches and
+// returning at most limit branches, or all branches if limit is 0.
+func (repo *Repository) GetBranches(skip, limit int) ([]string, int, error) {
+	return callShowRef(repo.Path, BranchPrefix, "--heads", skip, limit)
+}
+
+// callShowRef return refs, if limit = 0 it will not limit
+func callShowRef(repoPath, prefix, arg string, skip, limit int) (branchNames []string, countAll int, err error) {
+	stdoutReader, stdoutWriter := io.Pipe()
+	defer func() {
+		_ = stdoutReader.Close()
+		_ = stdoutWriter.Close()
+	}()
+
+	go func() {
+		stderrBuilder := &strings.Builder{}
+		err := NewCommand("show-ref", arg).RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder)
+		if err != nil {
+			if stderrBuilder.Len() == 0 {
+				_ = stdoutWriter.Close()
+				return
+			}
+			_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
+		} else {
+			_ = stdoutWriter.Close()
+		}
+	}()
+
+	i := 0
+	bufReader := bufio.NewReader(stdoutReader)
+	for i < skip {
+		_, isPrefix, err := bufReader.ReadLine()
+		if err == io.EOF {
+			return branchNames, i, nil
+		}
+		if err != nil {
+			return nil, 0, err
+		}
+		if !isPrefix {
+			i++
+		}
+	}
+	for limit == 0 || i < skip+limit {
+		// The output of show-ref is simply a list:
+		// <sha> SP <ref> LF
+		_, err := bufReader.ReadSlice(' ')
+		for err == bufio.ErrBufferFull {
+			// This shouldn't happen but we'll tolerate it for the sake of peace
+			_, err = bufReader.ReadSlice(' ')
+		}
+		if err == io.EOF {
+			return branchNames, i, nil
+		}
+		if err != nil {
+			return nil, 0, err
+		}
+
+		branchName, err := bufReader.ReadString('\n')
+		if err == io.EOF {
+			// This shouldn't happen... but we'll tolerate it for the sake of peace
+			return branchNames, i, nil
+		}
+		if err != nil {
+			return nil, i, err
+		}
+		branchName = strings.TrimPrefix(branchName, prefix)
+		if len(branchName) > 0 {
+			branchName = branchName[:len(branchName)-1]
+		}
+		branchNames = append(branchNames, branchName)
+		i++
+	}
+	// count all refs
+	for limit != 0 {
+		_, isPrefix, err := bufReader.ReadLine()
+		if err == io.EOF {
+			return branchNames, i, nil
+		}
+		if err != nil {
+			return nil, 0, err
+		}
+		if !isPrefix {
+			i++
+		}
+	}
+	return branchNames, i, nil
+}

modules/git/repo_tag.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 
 	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/mcuadros/go-version"
 )
 
 // TagPrefix tags prefix path on the repository
@@ -225,29 +224,35 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) {
 	return tags, nil
 }
 
-// GetTags returns all tags of the repository.
-func (repo *Repository) GetTags() ([]string, error) {
-	var tagNames []string
-
-	tags, err := repo.gogitRepo.Tags()
-	if err != nil {
-		return nil, err
-	}
+//// GetTags returns all tags of the repository.
+//func (repo *Repository) GetTags() ([]string, error) {
+//	var tagNames []string
+//
+//	tags, err := repo.gogitRepo.Tags()
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	_ = tags.ForEach(func(tag *plumbing.Reference) error {
+//		tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
+//		return nil
+//	})
+//
+//	version.Sort(tagNames)
+//
+//	// Reverse order
+//	for i := 0; i < len(tagNames)/2; i++ {
+//		j := len(tagNames) - i - 1
+//		tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
+//	}
+//
+//	return tagNames, nil
+//}
 
-	_ = tags.ForEach(func(tag *plumbing.Reference) error {
-		tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
-		return nil
-	})
-
-	version.Sort(tagNames)
-
-	// Reverse order
-	for i := 0; i < len(tagNames)/2; i++ {
-		j := len(tagNames) - i - 1
-		tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
-	}
-
-	return tagNames, nil
+// GetTags returns all tags of the repository.
+func (repo *Repository) GetTags() (tags []string, err error) {
+	tags, _, err = callShowRef(repo.Path, TagPrefix, "--tags", 0, 0)
+	return
 }
 
 // GetTagType gets the type of the tag, either commit (simple) or tag (annotated)

modules/git/utils.go

@@ -140,3 +140,11 @@ func ParseBool(value string) (result bool, valid bool) {
 	}
 	return intValue != 0, true
 }
+
+// ConcatenateError concatenats an error with stderr string
+func ConcatenateError(err error, stderr string) error {
+	if len(stderr) == 0 {
+		return err
+	}
+	return fmt.Errorf("%w - %s", err, stderr)
+}

modules/repository/branch.go

@@ -23,9 +23,10 @@ func GetBranch(repo *models.Repository, branch string) (*git.Branch, error) {
 	return gitRepo.GetBranch(branch)
 }
 
-// GetBranches returns all the branches of a repository
-func GetBranches(repo *models.Repository) ([]*git.Branch, error) {
-	return git.GetBranchesByPath(repo.RepoPath())
+// GetBranches returns branches from the repository, skipping skip initial branches and
+// returning at most limit branches, or all branches if limit is 0.
+func GetBranches(repo *models.Repository, skip, limit int) ([]*git.Branch, int, error) {
+	return git.GetBranchesByPath(repo.RepoPath(), skip, limit)
 }
 
 // checkBranchName validates branch name with existing repository branches
@@ -36,7 +37,7 @@ func checkBranchName(repo *models.Repository, name string) error {
 	}
 	defer gitRepo.Close()
 
-	branches, err := GetBranches(repo)
+	branches, _, err := GetBranches(repo, 0, 0)
 	if err != nil {
 		return err
 	}

routers/api/v1/repo/branch.go

@@ -204,7 +204,7 @@ func ListBranches(ctx *context.APIContext) {
 	//   "200":
 	//     "$ref": "#/responses/BranchList"
 
-	branches, err := repo_module.GetBranches(ctx.Repo.Repository)
+	branches, _, err := repo_module.GetBranches(ctx.Repo.Repository,0,0)
 	if err != nil {
 		ctx.Error(http.StatusInternalServerError, "GetBranches", err)
 		return

routers/repo/branch.go

@@ -181,7 +181,7 @@ func deleteBranch(ctx *context.Context, branchName string) error {
 }
 
 func loadBranches(ctx *context.Context) []*Branch {
-	rawBranches, err := repo_module.GetBranches(ctx.Repo.Repository)
+	rawBranches, _, err := repo_module.GetBranches(ctx.Repo.Repository, 0, 0)
 	if err != nil {
 		ctx.ServerError("GetBranches", err)
 		return nil

routers/repo/compare.go

@@ -507,7 +507,7 @@ func getBranchesForRepo(user *models.User, repo *models.Repository) (bool, []str
 	}
 	defer gitRepo.Close()
 
-	branches, err := gitRepo.GetBranches()
+	branches, _, err := gitRepo.GetBranches(0, 0)
 	if err != nil {
 		return false, nil, err
 	}
@@ -528,7 +528,7 @@ func CompareDiff(ctx *context.Context) {
 	}
 
 	if ctx.Data["PageIsComparePull"] == true {
-		headBranches, err := headGitRepo.GetBranches()
+		headBranches, _, err := headGitRepo.GetBranches(0,0)
 		if err != nil {
 			ctx.ServerError("GetBranches", err)
 			return

routers/repo/issue.go

@@ -424,7 +424,7 @@ func RetrieveRepoMetas(ctx *context.Context, repo *models.Repository, isPull boo
 		return nil
 	}
 
-	brs, err := ctx.Repo.GitRepo.GetBranches()
+	brs, _, err := ctx.Repo.GitRepo.GetBranches(0,0)
 	if err != nil {
 		ctx.ServerError("GetBranches", err)
 		return nil

routers/repo/modelarts.go

@@ -377,13 +377,6 @@ func trainJobNewDataPrepare(ctx *context.Context) error {
 	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
 	ctx.Data["train_url"] = outputObsPath
 
-	Branches, err := ctx.Repo.GitRepo.GetBranches()
-	if err != nil {
-		ctx.ServerError("GetBranches error:", err)
-		return err
-	}
-	ctx.Data["Branches"] = Branches
-	ctx.Data["BranchesCount"] = len(Branches)
 	ctx.Data["params"] = ""
 	ctx.Data["BranchName"] = ctx.Repo.BranchName
 
@@ -454,14 +447,6 @@ func ErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJob
 	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
 	ctx.Data["train_url"] = outputObsPath
 
-	Branches, err := ctx.Repo.GitRepo.GetBranches()
-	if err != nil {
-		ctx.ServerError("GetBranches error:", err)
-		return err
-	}
-	ctx.Data["Branches"] = Branches
-	ctx.Data["BranchesCount"] = len(Branches)
-
 	configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom)
 	if err != nil {
 		ctx.ServerError("getConfigList failed:", err)
@@ -557,13 +542,13 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error {
 	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
 	ctx.Data["train_url"] = outputObsPath
 
-	Branches, err := ctx.Repo.GitRepo.GetBranches()
+	branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
 	if err != nil {
 		ctx.ServerError("GetBranches error:", err)
 		return err
 	}
 
-	ctx.Data["branches"] = Branches
+	ctx.Data["branches"] = branches
 	ctx.Data["branch_name"] = task.BranchName
 	ctx.Data["description"] = task.Description
 	ctx.Data["boot_file"] = task.BootFile
@@ -646,12 +631,12 @@ func VersionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrai
 	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
 	ctx.Data["train_url"] = outputObsPath
 
-	Branches, err := ctx.Repo.GitRepo.GetBranches()
+	branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
 	if err != nil {
 		ctx.ServerError("GetBranches error:", err)
 		return err
 	}
-	ctx.Data["branches"] = Branches
+	ctx.Data["branches"] = branches
 	ctx.Data["description"] = form.Description
 	ctx.Data["dataset_name"] = task.DatasetName
 	ctx.Data["work_server_number"] = form.WorkServerNumber

services/mirror/mirror.go

@@ -252,7 +252,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) {
 		}
 	}
 
-	branches, err := repo_module.GetBranches(m.Repo)
+	branches, _, err := repo_module.GetBranches(m.Repo,0,0)
 	if err != nil {
 		log.Error("GetBranches: %v", err)
 		return nil, false

services/pull/pull.go

@@ -452,7 +452,7 @@ func CloseBranchPulls(doer *models.User, repoID int64, branch string) error {
 
 // CloseRepoBranchesPulls close all pull requests which head branches are in the given repository
 func CloseRepoBranchesPulls(doer *models.User, repo *models.Repository) error {
-	branches, err := git.GetBranchesByPath(repo.RepoPath())
+	branches, _, err := git.GetBranchesByPath(repo.RepoPath(), 0, 0)
 	if err != nil {
 		return err
 	}