提交代码

models/dataset.go

@@ -121,7 +121,7 @@ func (datasets DatasetList) loadAttachmentAttributes(opts *SearchDatasetOptions)
 			if attachment.DatasetID == datasets[i].ID {
 				if opts.StarByMe {
 
-					permission,ok := permissionMap[datasets[i].ID];
+					permission, ok := permissionMap[datasets[i].ID]
 					if !ok {
 
 						permission = false
@@ -159,8 +159,8 @@ func (datasets DatasetList) loadAttachmentAttributes(opts *SearchDatasetOptions)
 	}
 
 	for i := range datasets {
-		if datasets[i].Attachments==nil{
-			datasets[i].Attachments=[]*Attachment{}
+		if datasets[i].Attachments == nil {
+			datasets[i].Attachments = []*Attachment{}
 		}
 		datasets[i].Repo.Owner = nil
 	}
@@ -178,7 +178,7 @@ type SearchDatasetOptions struct {
 	Category      string
 	Task          string
 	License       string
-	DatasetIDs    []int64 // 目前只在StarByMe为true时起作用
+	DatasetIDs    []int64
 	ListOptions
 	SearchOrderBy
 	IsOwner              bool

models/dataset_reference.go

@@ -0,0 +1,42 @@
+package models
+
+import "code.gitea.io/gitea/modules/timeutil"
+
+type DatasetReference struct {
+	ID          int64              `xorm:"pk autoincr"`
+	RepoID      int64              `xorm:"INDEX unique(repo_dataset)"`
+	DatasetID   int64              `xorm:"INDEX  unique(repo_dataset)"`
+	CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
+}
+
+func GetDatasetIdsByRepoID(repoID int64) []int64 {
+	var datasets []int64
+	_ = x.Table("dataset_reference").Where("repo_id=?", repoID).
+		Cols("dataset_reference.dataset_id").Find(&datasets)
+	return datasets
+}
+
+func DeleteDatasetIdsByRepoID(repoID int64, datasetIds []int64) error {
+	if len(datasetIds) == 0 {
+		return nil
+	}
+	_, err := x.In("dataset_id", datasetIds).And("repo_id", repoID).Delete(new(DatasetReference))
+	return err
+}
+
+func NewDatasetIdsByRepoID(repoID int64, datasetIds []int64) error {
+	var datasetReference []DatasetReference
+	for _, datasetId := range datasetIds {
+		datasetReference = append(datasetReference, DatasetReference{
+			DatasetID: datasetId,
+			RepoID:    repoID,
+		})
+	}
+	_, err := x.Insert(datasetReference)
+	return err
+}
+
+func DeleteReferenceDatasetByDatasetID(datasetId int64) error {
+	_, err := x.Delete(&DatasetReference{DatasetID: datasetId})
+	return err
+}

modules/auth/dataset.go

@@ -44,3 +44,11 @@ type EditAttachmentForm struct {
 func (f *EditAttachmentForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
 	return validate(errs, ctx.Data, f, ctx.Locale)
 }
+
+type ReferenceDatasetForm struct {
+	DatasetID []int64 `binding:"Required"`
+}
+
+func (f *ReferenceDatasetForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
+	return validate(errs, ctx.Data, f, ctx.Locale)
+}

modules/dataset/dataset.go

@@ -1,5 +1,7 @@
 package dataset
 
+import "code.gitea.io/gitea/models"
+
 func GetResourceType(cloudbrainType int) string {
 	if cloudbrainType == 0 {
 		return "CPU/GPU"
@@ -15,3 +17,19 @@ func GetStatusText(isPrivate bool) string {
 		return "dataset.public"
 	}
 }
+
+func IsShowDataSetOfCurrentRepo(repoID int64) bool {
+	repo := models.Repository{
+		ID: repoID,
+	}
+
+	dataset, _ := models.GetDatasetByRepo(&repo)
+	if dataset != nil {
+		return true
+	}
+	if len(models.GetDatasetIdsByRepoID(repoID)) > 0 {
+		return false
+	}
+	return true
+
+}

modules/setting/repository.go

@@ -193,8 +193,9 @@ var (
 			Wiki:          []string{"never"},
 		},
 	}
-	RepoRootPath string
-	ScriptType   = "bash"
+	RepoRootPath               string
+	RepoMaxReferenceDatasetNum int
+	ScriptType                 = "bash"
 )
 
 func newRepository() {
@@ -210,6 +211,8 @@ func newRepository() {
 	Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool()
 	Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1)
 	RepoRootPath = sec.Key("ROOT").MustString(path.Join(homeDir, "gitea-repositories"))
+	RepoMaxReferenceDatasetNum = sec.Key("MAX_REF_DATASET_NUM").MustInt(20)
+
 	forcePathSeparator(RepoRootPath)
 	if !filepath.IsAbs(RepoRootPath) {
 		RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath)

modules/templates/helper.go

@@ -97,23 +97,24 @@ func NewFuncMap() []template.FuncMap {
 		"AllowedReactions": func() []string {
 			return setting.UI.Reactions
 		},
-		"AvatarLink":             models.AvatarLink,
-		"Safe":                   Safe,
-		"SafeJS":                 SafeJS,
-		"Str2html":               Str2html,
-		"subOne":                 subOne,
-		"TimeSince":              timeutil.TimeSince,
-		"TimeSinceUnix":          timeutil.TimeSinceUnix,
-		"TimeSinceUnix1":         timeutil.TimeSinceUnix1,
-		"AttachmentResourceType": dataset.GetResourceType,
-		"AttachmentStatus":       dataset.GetStatusText,
-		"TimeSinceUnixShort":     timeutil.TimeSinceUnixShort,
-		"RawTimeSince":           timeutil.RawTimeSince,
-		"FileSize":               base.FileSize,
-		"PrettyNumber":           base.PrettyNumber,
-		"Subtract":               base.Subtract,
-		"EntryIcon":              base.EntryIcon,
-		"MigrationIcon":          MigrationIcon,
+		"AvatarLink":                 models.AvatarLink,
+		"Safe":                       Safe,
+		"SafeJS":                     SafeJS,
+		"Str2html":                   Str2html,
+		"subOne":                     subOne,
+		"TimeSince":                  timeutil.TimeSince,
+		"TimeSinceUnix":              timeutil.TimeSinceUnix,
+		"TimeSinceUnix1":             timeutil.TimeSinceUnix1,
+		"AttachmentResourceType":     dataset.GetResourceType,
+		"AttachmentStatus":           dataset.GetStatusText,
+		"IsShowDataSetOfCurrentRepo": dataset.IsShowDataSetOfCurrentRepo,
+		"TimeSinceUnixShort":         timeutil.TimeSinceUnixShort,
+		"RawTimeSince":               timeutil.RawTimeSince,
+		"FileSize":                   base.FileSize,
+		"PrettyNumber":               base.PrettyNumber,
+		"Subtract":                   base.Subtract,
+		"EntryIcon":                  base.EntryIcon,
+		"MigrationIcon":              MigrationIcon,
 		"Add": func(a, b int) int {
 			return a + b
 		},
@@ -357,13 +358,15 @@ func NewTextFuncMap() []texttmpl.FuncMap {
 		"AppDomain": func() string {
 			return setting.Domain
 		},
-		"TimeSince":              timeutil.TimeSince,
-		"TimeSinceUnix":          timeutil.TimeSinceUnix,
-		"TimeSinceUnix1":         timeutil.TimeSinceUnix1,
-		"TimeSinceUnixShort":     timeutil.TimeSinceUnixShort,
-		"RawTimeSince":           timeutil.RawTimeSince,
-		"AttachmentResourceType": dataset.GetResourceType,
-		"AttachmentStatus":       dataset.GetStatusText,
+		"TimeSince":                  timeutil.TimeSince,
+		"TimeSinceUnix":              timeutil.TimeSinceUnix,
+		"TimeSinceUnix1":             timeutil.TimeSinceUnix1,
+		"TimeSinceUnixShort":         timeutil.TimeSinceUnixShort,
+		"RawTimeSince":               timeutil.RawTimeSince,
+		"AttachmentResourceType":     dataset.GetResourceType,
+		"AttachmentStatus":           dataset.GetStatusText,
+		"IsShowDataSetOfCurrentRepo": dataset.IsShowDataSetOfCurrentRepo,
+
 		"DateFmtLong": func(t time.Time) string {
 			return t.Format(time.RFC1123Z)
 		},

options/locale/locale_en-US.ini

@@ -832,6 +832,8 @@ category = Category
 no_category = No Category
 task = Task
 no_task = No Task
+reference_dataset_fail=Failed to reference dataset, please try again later.
+cancel_reference_dataset_fail=Failed to cancel reference dataset, please try again later.
 license = License
 no_license = No License
 file = Dataset File

options/locale/locale_zh-CN.ini

@@ -830,6 +830,8 @@ create_dataset=创建数据集
 create_dataset_fail=创建数据集失败。
 query_dataset_fail=查询数据集失败。
 edit_attachment_fail=修改描述失败。
+reference_dataset_fail=关联数据集失败，请稍后再试。
+cancel_reference_dataset_fail=取消关联数据集失败，请稍后再试。
 
 show_dataset=数据集
 edit_dataset=编辑数据集

routers/home.go

@@ -296,11 +296,10 @@ func ExploreDatasets(ctx *context.Context) {
 	// ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
 
 	var (
-		datasets         []*models.Dataset
-		datasetsWithStar []*models.DatasetWithStar
-		count            int64
-		err              error
-		orderBy          models.SearchOrderBy
+		datasets []*models.Dataset
+		count    int64
+		err      error
+		orderBy  models.SearchOrderBy
 	)
 	page := ctx.QueryInt("page")
 	if page <= 0 {
@@ -379,14 +378,6 @@ func ExploreDatasets(ctx *context.Context) {
 		ctx.ServerError("SearchDatasets", err)
 		return
 	}
-	for _, dataset := range datasets {
-		if !ctx.IsSigned {
-			datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: false})
-		} else {
-			datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: models.IsDatasetStaring(ctx.User.ID, dataset.ID)})
-		}
-
-	}
 
 	pager := context.NewPagination(int(count), opts.PageSize, page, 5)
 	ctx.Data["Keyword"] = opts.Keyword
@@ -397,7 +388,7 @@ func ExploreDatasets(ctx *context.Context) {
 	pager.SetDefaultParams(ctx)
 	ctx.Data["Page"] = pager
 
-	ctx.Data["Datasets"] = datasetsWithStar
+	ctx.Data["Datasets"] = repository.ConvertToDatasetWithStar(ctx, datasets)
 	ctx.Data["Total"] = count
 	ctx.Data["PageIsDatasets"] = true
 	ctx.HTML(200, tplExploreDataset)

routers/repo/dataset.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 	"unicode/utf8"
 
+	"code.gitea.io/gitea/services/repository"
+
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
 	"code.gitea.io/gitea/modules/base"
@@ -22,6 +24,7 @@ const (
 	tplDatasetCreate base.TplName = "repo/datasets/create"
 	tplDatasetEdit   base.TplName = "repo/datasets/edit"
 	taskstplIndex    base.TplName = "repo/datasets/tasks/index"
+	tplReference     base.TplName = "repo/datasets/reference"
 )
 
 // MustEnableDataset check if repository enable internal dataset
@@ -267,6 +270,59 @@ func CreateDatasetPost(ctx *context.Context, form auth.CreateDatasetForm) {
 	}
 
 }
+func ReferenceDatasetDelete(ctx *context.Context) {
+	repoID := ctx.Repo.Repository.ID
+	datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64)
+
+	err := models.DeleteDatasetIdsByRepoID(repoID, []int64{datasetId})
+	if err != nil {
+		ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.cancel_reference_dataset_fail"))
+	}
+	ctx.JSON(http.StatusOK, models.BaseOKMessage)
+
+}
+
+func ReferenceDatasetPost(ctx *context.Context, form auth.ReferenceDatasetForm) {
+	repoID := ctx.Repo.Repository.ID
+	datasetsID := models.GetDatasetIdsByRepoID(repoID)
+
+	var newDataset []int64
+	var deletedDataset []int64
+
+	for _, id := range datasetsID {
+		if !contains(form.DatasetID, id) {
+			deletedDataset = append(deletedDataset, id)
+		}
+	}
+
+	for _, id := range form.DatasetID {
+		if !contains(datasetsID, id) {
+			newDataset = append(newDataset, id)
+		}
+	}
+
+	err := models.DeleteDatasetIdsByRepoID(repoID, deletedDataset)
+
+	if err != nil {
+		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.reference_dataset_fail")))
+	}
+	err = models.NewDatasetIdsByRepoID(repoID, newDataset)
+	if err != nil {
+		ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.reference_dataset_fail"))
+	}
+
+	ctx.JSON(http.StatusOK, models.BaseOKMessage)
+
+}
+
+func contains(s []int64, e int64) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}
 
 func EditDatasetPost(ctx *context.Context, form auth.EditDatasetForm) {
 	ctx.Data["PageIsDataset"] = true
@@ -411,24 +467,13 @@ func MyDatasets(ctx *context.Context) {
 }
 
 func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) {
-	page := ctx.QueryInt("page")
-	cloudbrainType := ctx.QueryInt("type")
-	keyword := strings.Trim(ctx.Query("q"), " ")
-	orderBy := models.SearchOrderByRecentUpdated
-	opts.Keyword = keyword
-	opts.SearchOrderBy = orderBy
-	opts.RecommendOnly = ctx.QueryBool("recommend")
-	opts.CloudBrainType = cloudbrainType
-	opts.ListOptions = models.ListOptions{
-		Page:     page,
-		PageSize: setting.UI.DatasetPagingNum,
-	}
-	opts.NeedAttachment = true
-	opts.JustNeedZipFile = true
-	opts.User = ctx.User
+	datasets, count, err := datasetMultipleGet(ctx, opts)
 
-	datasets, count, err := models.SearchDataset(opts)
+	datasetMultipleResult(ctx, err, datasets, count)
+
+}
 
+func datasetMultipleResult(ctx *context.Context, err error, datasets models.DatasetList, count int64) {
 	if err != nil {
 		ctx.ServerError("datasets", err)
 		return
@@ -449,15 +494,53 @@ func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) {
 		"data":        string(data),
 		"count":       strconv.FormatInt(count, 10),
 	})
+}
 
+func datasetMultipleGet(ctx *context.Context, opts *models.SearchDatasetOptions) (models.DatasetList, int64, error) {
+	page := ctx.QueryInt("page")
+	keyword := strings.Trim(ctx.Query("q"), " ")
+	orderBy := models.SearchOrderByRecentUpdated
+	opts.Keyword = keyword
+	opts.SearchOrderBy = orderBy
+	opts.RecommendOnly = ctx.QueryBool("recommend")
+	opts.ListOptions = models.ListOptions{
+		Page:     page,
+		PageSize: setting.UI.DatasetPagingNum,
+	}
+	opts.JustNeedZipFile = true
+	opts.User = ctx.User
+
+	datasets, count, err := models.SearchDataset(opts)
+	return datasets, count, err
 }
 
 func CurrentRepoDatasetMultiple(ctx *context.Context) {
 
 	opts := &models.SearchDatasetOptions{
-		RepoID: ctx.Repo.Repository.ID,
+		RepoID:         ctx.Repo.Repository.ID,
+		NeedAttachment: true,
+		CloudBrainType: ctx.QueryInt("type"),
+		DatasetIDs:     models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID),
+	}
+	datasetList, count, err := datasetMultipleGet(ctx, opts)
+
+	if len(datasetList) > 0 {
+		var convertDatasetList models.DatasetList
+
+		for _, dataset := range datasetList {
+			if dataset.RepoID == ctx.Repo.Repository.ID && len(convertDatasetList) == 0 {
+				convertDatasetList = append(convertDatasetList, dataset)
+			}
+		}
+		for _, dataset := range datasetList {
+			if dataset.RepoID != ctx.Repo.Repository.ID {
+				convertDatasetList = append(convertDatasetList, dataset)
+			}
+		}
+		datasetMultipleResult(ctx, err, convertDatasetList, count)
+	} else {
+		datasetMultipleResult(ctx, err, datasetList, count)
 	}
-	datasetMultiple(ctx, opts)
 
 }
 
@@ -465,6 +548,19 @@ func MyDatasetsMultiple(ctx *context.Context) {
 
 	opts := &models.SearchDatasetOptions{
 		UploadAttachmentByMe: true,
+		NeedAttachment:       true,
+		CloudBrainType:       ctx.QueryInt("type"),
+	}
+	datasetMultiple(ctx, opts)
+
+}
+
+func ReferenceDatasetAvailable(ctx *context.Context) {
+
+	opts := &models.SearchDatasetOptions{
+		PublicOnly:     true,
+		NeedAttachment: false,
+		CloudBrainType: models.TypeCloudBrainAll,
 	}
 	datasetMultiple(ctx, opts)
 
@@ -473,7 +569,9 @@ func MyDatasetsMultiple(ctx *context.Context) {
 func PublicDatasetMultiple(ctx *context.Context) {
 
 	opts := &models.SearchDatasetOptions{
-		PublicOnly: true,
+		PublicOnly:     true,
+		NeedAttachment: true,
+		CloudBrainType: ctx.QueryInt("type"),
 	}
 	datasetMultiple(ctx, opts)
 
@@ -482,11 +580,38 @@ func PublicDatasetMultiple(ctx *context.Context) {
 func MyFavoriteDatasetMultiple(ctx *context.Context) {
 
 	opts := &models.SearchDatasetOptions{
-		StarByMe:   true,
-		DatasetIDs: models.GetDatasetIdsStarByUser(ctx.User.ID),
+		StarByMe:       true,
+		DatasetIDs:     models.GetDatasetIdsStarByUser(ctx.User.ID),
+		NeedAttachment: true,
+		CloudBrainType: ctx.QueryInt("type"),
 	}
 	datasetMultiple(ctx, opts)
 }
+func ReferenceDataset(ctx *context.Context) {
+	MustEnableDataset(ctx)
+	opts := &models.SearchDatasetOptions{
+		DatasetIDs:     models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID),
+		NeedAttachment: false,
+		CloudBrainType: models.TypeCloudBrainAll,
+		ListOptions: models.ListOptions{
+			Page:     1,
+			PageSize: setting.RepoMaxReferenceDatasetNum,
+		},
+		SearchOrderBy: models.SearchOrderByRecentUpdated,
+	}
+	datasets, count, err := models.SearchDataset(opts)
+
+	if err != nil {
+		ctx.ServerError("SearchDatasets", err)
+		return
+	}
+
+	ctx.Data["Datasets"] = repository.ConvertToDatasetWithStar(ctx, datasets)
+	ctx.Data["PageIsDataset"] = true
+	ctx.Data["Total"] = count
+	ctx.HTML(200, tplReference)
+
+}
 
 func PublicDataset(ctx *context.Context) {
 	page := ctx.QueryInt("page")

routers/routes/routes.go

@@ -1032,10 +1032,13 @@ func RegisterRoutes(m *macaron.Macaron) {
 
 		m.Group("/datasets", func() {
 			m.Get("", reqRepoDatasetReader, repo.DatasetIndex)
+			m.Get("/reference_datasets", repo.ReferenceDataset)
+			m.Delete("/reference_datasets/:id", repo.ReferenceDatasetDelete)
 			m.Put("/:id/:action", reqRepoDatasetReader, repo.DatasetAction)
 			m.Get("/create", reqRepoDatasetWriter, repo.CreateDataset)
 			m.Post("/create", reqRepoDatasetWriter, bindIgnErr(auth.CreateDatasetForm{}), repo.CreateDatasetPost)
 			m.Get("/edit/:id", reqRepoDatasetWriter, repo.EditDataset)
+			m.Post("/reference_datasets", reqRepoDatasetWriter, bindIgnErr(auth.ReferenceDatasetForm{}), repo.ReferenceDatasetPost)
 			m.Post("/edit", reqRepoDatasetWriter, bindIgnErr(auth.EditDatasetForm{}), repo.EditDatasetPost)
 			m.Get("/current_repo", repo.CurrentRepoDataset)
 			m.Get("/my_datasets", repo.MyDatasets)
@@ -1045,6 +1048,8 @@ func RegisterRoutes(m *macaron.Macaron) {
 			m.Get("/current_repo_m", repo.CurrentRepoDatasetMultiple)
 			m.Get("/my_datasets_m", repo.MyDatasetsMultiple)
 			m.Get("/public_datasets_m", repo.PublicDatasetMultiple)
+
+			m.Get("/reference_datasets_available", repo.ReferenceDatasetAvailable)
 			m.Get("/my_favorite_m", repo.MyFavoriteDatasetMultiple)
 
 			m.Group("/status", func() {

services/repository/dataset.go

@@ -0,0 +1,19 @@
+package repository
+
+import (
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/context"
+)
+
+func ConvertToDatasetWithStar(ctx *context.Context, datasets []*models.Dataset) []*models.DatasetWithStar {
+	var datasetsWithStar []*models.DatasetWithStar
+	for _, dataset := range datasets {
+		if !ctx.IsSigned {
+			datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: false})
+		} else {
+			datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: models.IsDatasetStaring(ctx.User.ID, dataset.ID)})
+		}
+
+	}
+	return datasetsWithStar
+}