diff --git a/models/dataset.go b/models/dataset.go index 7f049f068..45dfad735 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -121,7 +121,7 @@ func (datasets DatasetList) loadAttachmentAttributes(opts *SearchDatasetOptions) if attachment.DatasetID == datasets[i].ID { if opts.StarByMe { - permission,ok := permissionMap[datasets[i].ID]; + permission, ok := permissionMap[datasets[i].ID] if !ok { permission = false @@ -159,8 +159,8 @@ func (datasets DatasetList) loadAttachmentAttributes(opts *SearchDatasetOptions) } for i := range datasets { - if datasets[i].Attachments==nil{ - datasets[i].Attachments=[]*Attachment{} + if datasets[i].Attachments == nil { + datasets[i].Attachments = []*Attachment{} } datasets[i].Repo.Owner = nil } @@ -178,7 +178,7 @@ type SearchDatasetOptions struct { Category string Task string License string - DatasetIDs []int64 // 目前只在StarByMe为true时起作用 + DatasetIDs []int64 ListOptions SearchOrderBy IsOwner bool diff --git a/models/dataset_reference.go b/models/dataset_reference.go new file mode 100644 index 000000000..f3083ca32 --- /dev/null +++ b/models/dataset_reference.go @@ -0,0 +1,42 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type DatasetReference struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX unique(repo_dataset)"` + DatasetID int64 `xorm:"INDEX unique(repo_dataset)"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` +} + +func GetDatasetIdsByRepoID(repoID int64) []int64 { + var datasets []int64 + _ = x.Table("dataset_reference").Where("repo_id=?", repoID). + Cols("dataset_reference.dataset_id").Find(&datasets) + return datasets +} + +func DeleteDatasetIdsByRepoID(repoID int64, datasetIds []int64) error { + if len(datasetIds) == 0 { + return nil + } + _, err := x.In("dataset_id", datasetIds).And("repo_id", repoID).Delete(new(DatasetReference)) + return err +} + +func NewDatasetIdsByRepoID(repoID int64, datasetIds []int64) error { + var datasetReference []DatasetReference + for _, datasetId := range datasetIds { + datasetReference = append(datasetReference, DatasetReference{ + DatasetID: datasetId, + RepoID: repoID, + }) + } + _, err := x.Insert(datasetReference) + return err +} + +func DeleteReferenceDatasetByDatasetID(datasetId int64) error { + _, err := x.Delete(&DatasetReference{DatasetID: datasetId}) + return err +} diff --git a/modules/auth/dataset.go b/modules/auth/dataset.go index 71b5ac938..8aed3a8c2 100755 --- a/modules/auth/dataset.go +++ b/modules/auth/dataset.go @@ -44,3 +44,11 @@ type EditAttachmentForm struct { func (f *EditAttachmentForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { return validate(errs, ctx.Data, f, ctx.Locale) } + +type ReferenceDatasetForm struct { + DatasetID []int64 `binding:"Required"` +} + +func (f *ReferenceDatasetForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { + return validate(errs, ctx.Data, f, ctx.Locale) +} diff --git a/modules/dataset/dataset.go b/modules/dataset/dataset.go index a180af184..f79f0d1c2 100644 --- a/modules/dataset/dataset.go +++ b/modules/dataset/dataset.go @@ -1,5 +1,7 @@ package dataset +import "code.gitea.io/gitea/models" + func GetResourceType(cloudbrainType int) string { if cloudbrainType == 0 { return "CPU/GPU" @@ -15,3 +17,19 @@ func GetStatusText(isPrivate bool) string { return "dataset.public" } } + +func IsShowDataSetOfCurrentRepo(repoID int64) bool { + repo := models.Repository{ + ID: repoID, + } + + dataset, _ := models.GetDatasetByRepo(&repo) + if dataset != nil { + return true + } + if len(models.GetDatasetIdsByRepoID(repoID)) > 0 { + return false + } + return true + +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index ee4f8b379..1eb1875b2 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -193,8 +193,9 @@ var ( Wiki: []string{"never"}, }, } - RepoRootPath string - ScriptType = "bash" + RepoRootPath string + RepoMaxReferenceDatasetNum int + ScriptType = "bash" ) func newRepository() { @@ -210,6 +211,8 @@ func newRepository() { Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool() Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) RepoRootPath = sec.Key("ROOT").MustString(path.Join(homeDir, "gitea-repositories")) + RepoMaxReferenceDatasetNum = sec.Key("MAX_REF_DATASET_NUM").MustInt(20) + forcePathSeparator(RepoRootPath) if !filepath.IsAbs(RepoRootPath) { RepoRootPath = filepath.Join(AppWorkPath, RepoRootPath) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 857e365f8..8e7723e75 100755 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -97,23 +97,24 @@ func NewFuncMap() []template.FuncMap { "AllowedReactions": func() []string { return setting.UI.Reactions }, - "AvatarLink": models.AvatarLink, - "Safe": Safe, - "SafeJS": SafeJS, - "Str2html": Str2html, - "subOne": subOne, - "TimeSince": timeutil.TimeSince, - "TimeSinceUnix": timeutil.TimeSinceUnix, - "TimeSinceUnix1": timeutil.TimeSinceUnix1, - "AttachmentResourceType": dataset.GetResourceType, - "AttachmentStatus": dataset.GetStatusText, - "TimeSinceUnixShort": timeutil.TimeSinceUnixShort, - "RawTimeSince": timeutil.RawTimeSince, - "FileSize": base.FileSize, - "PrettyNumber": base.PrettyNumber, - "Subtract": base.Subtract, - "EntryIcon": base.EntryIcon, - "MigrationIcon": MigrationIcon, + "AvatarLink": models.AvatarLink, + "Safe": Safe, + "SafeJS": SafeJS, + "Str2html": Str2html, + "subOne": subOne, + "TimeSince": timeutil.TimeSince, + "TimeSinceUnix": timeutil.TimeSinceUnix, + "TimeSinceUnix1": timeutil.TimeSinceUnix1, + "AttachmentResourceType": dataset.GetResourceType, + "AttachmentStatus": dataset.GetStatusText, + "IsShowDataSetOfCurrentRepo": dataset.IsShowDataSetOfCurrentRepo, + "TimeSinceUnixShort": timeutil.TimeSinceUnixShort, + "RawTimeSince": timeutil.RawTimeSince, + "FileSize": base.FileSize, + "PrettyNumber": base.PrettyNumber, + "Subtract": base.Subtract, + "EntryIcon": base.EntryIcon, + "MigrationIcon": MigrationIcon, "Add": func(a, b int) int { return a + b }, @@ -357,13 +358,15 @@ func NewTextFuncMap() []texttmpl.FuncMap { "AppDomain": func() string { return setting.Domain }, - "TimeSince": timeutil.TimeSince, - "TimeSinceUnix": timeutil.TimeSinceUnix, - "TimeSinceUnix1": timeutil.TimeSinceUnix1, - "TimeSinceUnixShort": timeutil.TimeSinceUnixShort, - "RawTimeSince": timeutil.RawTimeSince, - "AttachmentResourceType": dataset.GetResourceType, - "AttachmentStatus": dataset.GetStatusText, + "TimeSince": timeutil.TimeSince, + "TimeSinceUnix": timeutil.TimeSinceUnix, + "TimeSinceUnix1": timeutil.TimeSinceUnix1, + "TimeSinceUnixShort": timeutil.TimeSinceUnixShort, + "RawTimeSince": timeutil.RawTimeSince, + "AttachmentResourceType": dataset.GetResourceType, + "AttachmentStatus": dataset.GetStatusText, + "IsShowDataSetOfCurrentRepo": dataset.IsShowDataSetOfCurrentRepo, + "DateFmtLong": func(t time.Time) string { return t.Format(time.RFC1123Z) }, diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 0fecf7de2..eda1af4ce 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -832,6 +832,8 @@ category = Category no_category = No Category task = Task no_task = No Task +reference_dataset_fail=Failed to reference dataset, please try again later. +cancel_reference_dataset_fail=Failed to cancel reference dataset, please try again later. license = License no_license = No License file = Dataset File diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 2878627e2..ac38820b1 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -830,6 +830,8 @@ create_dataset=创建数据集 create_dataset_fail=创建数据集失败。 query_dataset_fail=查询数据集失败。 edit_attachment_fail=修改描述失败。 +reference_dataset_fail=关联数据集失败，请稍后再试。 +cancel_reference_dataset_fail=取消关联数据集失败，请稍后再试。 show_dataset=数据集 edit_dataset=编辑数据集 diff --git a/routers/home.go b/routers/home.go index 1a697946a..026491156 100755 --- a/routers/home.go +++ b/routers/home.go @@ -296,11 +296,10 @@ func ExploreDatasets(ctx *context.Context) { // ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled var ( - datasets []*models.Dataset - datasetsWithStar []*models.DatasetWithStar - count int64 - err error - orderBy models.SearchOrderBy + datasets []*models.Dataset + count int64 + err error + orderBy models.SearchOrderBy ) page := ctx.QueryInt("page") if page <= 0 { @@ -379,14 +378,6 @@ func ExploreDatasets(ctx *context.Context) { ctx.ServerError("SearchDatasets", err) return } - for _, dataset := range datasets { - if !ctx.IsSigned { - datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: false}) - } else { - datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: models.IsDatasetStaring(ctx.User.ID, dataset.ID)}) - } - - } pager := context.NewPagination(int(count), opts.PageSize, page, 5) ctx.Data["Keyword"] = opts.Keyword @@ -397,7 +388,7 @@ func ExploreDatasets(ctx *context.Context) { pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager - ctx.Data["Datasets"] = datasetsWithStar + ctx.Data["Datasets"] = repository.ConvertToDatasetWithStar(ctx, datasets) ctx.Data["Total"] = count ctx.Data["PageIsDatasets"] = true ctx.HTML(200, tplExploreDataset) diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 4dde646a6..e987e4573 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -9,6 +9,8 @@ import ( "strings" "unicode/utf8" + "code.gitea.io/gitea/services/repository" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -22,6 +24,7 @@ const ( tplDatasetCreate base.TplName = "repo/datasets/create" tplDatasetEdit base.TplName = "repo/datasets/edit" taskstplIndex base.TplName = "repo/datasets/tasks/index" + tplReference base.TplName = "repo/datasets/reference" ) // MustEnableDataset check if repository enable internal dataset @@ -267,6 +270,59 @@ func CreateDatasetPost(ctx *context.Context, form auth.CreateDatasetForm) { } } +func ReferenceDatasetDelete(ctx *context.Context) { + repoID := ctx.Repo.Repository.ID + datasetId, _ := strconv.ParseInt(ctx.Params(":id"), 10, 64) + + err := models.DeleteDatasetIdsByRepoID(repoID, []int64{datasetId}) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.cancel_reference_dataset_fail")) + } + ctx.JSON(http.StatusOK, models.BaseOKMessage) + +} + +func ReferenceDatasetPost(ctx *context.Context, form auth.ReferenceDatasetForm) { + repoID := ctx.Repo.Repository.ID + datasetsID := models.GetDatasetIdsByRepoID(repoID) + + var newDataset []int64 + var deletedDataset []int64 + + for _, id := range datasetsID { + if !contains(form.DatasetID, id) { + deletedDataset = append(deletedDataset, id) + } + } + + for _, id := range form.DatasetID { + if !contains(datasetsID, id) { + newDataset = append(newDataset, id) + } + } + + err := models.DeleteDatasetIdsByRepoID(repoID, deletedDataset) + + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.reference_dataset_fail"))) + } + err = models.NewDatasetIdsByRepoID(repoID, newDataset) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessage("dataset.reference_dataset_fail")) + } + + ctx.JSON(http.StatusOK, models.BaseOKMessage) + +} + +func contains(s []int64, e int64) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} func EditDatasetPost(ctx *context.Context, form auth.EditDatasetForm) { ctx.Data["PageIsDataset"] = true @@ -411,24 +467,13 @@ func MyDatasets(ctx *context.Context) { } func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) { - page := ctx.QueryInt("page") - cloudbrainType := ctx.QueryInt("type") - keyword := strings.Trim(ctx.Query("q"), " ") - orderBy := models.SearchOrderByRecentUpdated - opts.Keyword = keyword - opts.SearchOrderBy = orderBy - opts.RecommendOnly = ctx.QueryBool("recommend") - opts.CloudBrainType = cloudbrainType - opts.ListOptions = models.ListOptions{ - Page: page, - PageSize: setting.UI.DatasetPagingNum, - } - opts.NeedAttachment = true - opts.JustNeedZipFile = true - opts.User = ctx.User + datasets, count, err := datasetMultipleGet(ctx, opts) - datasets, count, err := models.SearchDataset(opts) + datasetMultipleResult(ctx, err, datasets, count) + +} +func datasetMultipleResult(ctx *context.Context, err error, datasets models.DatasetList, count int64) { if err != nil { ctx.ServerError("datasets", err) return @@ -449,15 +494,53 @@ func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) { "data": string(data), "count": strconv.FormatInt(count, 10), }) +} +func datasetMultipleGet(ctx *context.Context, opts *models.SearchDatasetOptions) (models.DatasetList, int64, error) { + page := ctx.QueryInt("page") + keyword := strings.Trim(ctx.Query("q"), " ") + orderBy := models.SearchOrderByRecentUpdated + opts.Keyword = keyword + opts.SearchOrderBy = orderBy + opts.RecommendOnly = ctx.QueryBool("recommend") + opts.ListOptions = models.ListOptions{ + Page: page, + PageSize: setting.UI.DatasetPagingNum, + } + opts.JustNeedZipFile = true + opts.User = ctx.User + + datasets, count, err := models.SearchDataset(opts) + return datasets, count, err } func CurrentRepoDatasetMultiple(ctx *context.Context) { opts := &models.SearchDatasetOptions{ - RepoID: ctx.Repo.Repository.ID, + RepoID: ctx.Repo.Repository.ID, + NeedAttachment: true, + CloudBrainType: ctx.QueryInt("type"), + DatasetIDs: models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID), + } + datasetList, count, err := datasetMultipleGet(ctx, opts) + + if len(datasetList) > 0 { + var convertDatasetList models.DatasetList + + for _, dataset := range datasetList { + if dataset.RepoID == ctx.Repo.Repository.ID && len(convertDatasetList) == 0 { + convertDatasetList = append(convertDatasetList, dataset) + } + } + for _, dataset := range datasetList { + if dataset.RepoID != ctx.Repo.Repository.ID { + convertDatasetList = append(convertDatasetList, dataset) + } + } + datasetMultipleResult(ctx, err, convertDatasetList, count) + } else { + datasetMultipleResult(ctx, err, datasetList, count) } - datasetMultiple(ctx, opts) } @@ -465,6 +548,19 @@ func MyDatasetsMultiple(ctx *context.Context) { opts := &models.SearchDatasetOptions{ UploadAttachmentByMe: true, + NeedAttachment: true, + CloudBrainType: ctx.QueryInt("type"), + } + datasetMultiple(ctx, opts) + +} + +func ReferenceDatasetAvailable(ctx *context.Context) { + + opts := &models.SearchDatasetOptions{ + PublicOnly: true, + NeedAttachment: false, + CloudBrainType: models.TypeCloudBrainAll, } datasetMultiple(ctx, opts) @@ -473,7 +569,9 @@ func MyDatasetsMultiple(ctx *context.Context) { func PublicDatasetMultiple(ctx *context.Context) { opts := &models.SearchDatasetOptions{ - PublicOnly: true, + PublicOnly: true, + NeedAttachment: true, + CloudBrainType: ctx.QueryInt("type"), } datasetMultiple(ctx, opts) @@ -482,11 +580,38 @@ func PublicDatasetMultiple(ctx *context.Context) { func MyFavoriteDatasetMultiple(ctx *context.Context) { opts := &models.SearchDatasetOptions{ - StarByMe: true, - DatasetIDs: models.GetDatasetIdsStarByUser(ctx.User.ID), + StarByMe: true, + DatasetIDs: models.GetDatasetIdsStarByUser(ctx.User.ID), + NeedAttachment: true, + CloudBrainType: ctx.QueryInt("type"), } datasetMultiple(ctx, opts) } +func ReferenceDataset(ctx *context.Context) { + MustEnableDataset(ctx) + opts := &models.SearchDatasetOptions{ + DatasetIDs: models.GetDatasetIdsByRepoID(ctx.Repo.Repository.ID), + NeedAttachment: false, + CloudBrainType: models.TypeCloudBrainAll, + ListOptions: models.ListOptions{ + Page: 1, + PageSize: setting.RepoMaxReferenceDatasetNum, + }, + SearchOrderBy: models.SearchOrderByRecentUpdated, + } + datasets, count, err := models.SearchDataset(opts) + + if err != nil { + ctx.ServerError("SearchDatasets", err) + return + } + + ctx.Data["Datasets"] = repository.ConvertToDatasetWithStar(ctx, datasets) + ctx.Data["PageIsDataset"] = true + ctx.Data["Total"] = count + ctx.HTML(200, tplReference) + +} func PublicDataset(ctx *context.Context) { page := ctx.QueryInt("page") diff --git a/routers/routes/routes.go b/routers/routes/routes.go index a4bc09472..d54606581 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1032,10 +1032,13 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/datasets", func() { m.Get("", reqRepoDatasetReader, repo.DatasetIndex) + m.Get("/reference_datasets", repo.ReferenceDataset) + m.Delete("/reference_datasets/:id", repo.ReferenceDatasetDelete) m.Put("/:id/:action", reqRepoDatasetReader, repo.DatasetAction) m.Get("/create", reqRepoDatasetWriter, repo.CreateDataset) m.Post("/create", reqRepoDatasetWriter, bindIgnErr(auth.CreateDatasetForm{}), repo.CreateDatasetPost) m.Get("/edit/:id", reqRepoDatasetWriter, repo.EditDataset) + m.Post("/reference_datasets", reqRepoDatasetWriter, bindIgnErr(auth.ReferenceDatasetForm{}), repo.ReferenceDatasetPost) m.Post("/edit", reqRepoDatasetWriter, bindIgnErr(auth.EditDatasetForm{}), repo.EditDatasetPost) m.Get("/current_repo", repo.CurrentRepoDataset) m.Get("/my_datasets", repo.MyDatasets) @@ -1045,6 +1048,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/current_repo_m", repo.CurrentRepoDatasetMultiple) m.Get("/my_datasets_m", repo.MyDatasetsMultiple) m.Get("/public_datasets_m", repo.PublicDatasetMultiple) + + m.Get("/reference_datasets_available", repo.ReferenceDatasetAvailable) m.Get("/my_favorite_m", repo.MyFavoriteDatasetMultiple) m.Group("/status", func() { diff --git a/services/repository/dataset.go b/services/repository/dataset.go new file mode 100644 index 000000000..ffe3c5466 --- /dev/null +++ b/services/repository/dataset.go @@ -0,0 +1,19 @@ +package repository + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" +) + +func ConvertToDatasetWithStar(ctx *context.Context, datasets []*models.Dataset) []*models.DatasetWithStar { + var datasetsWithStar []*models.DatasetWithStar + for _, dataset := range datasets { + if !ctx.IsSigned { + datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: false}) + } else { + datasetsWithStar = append(datasetsWithStar, &models.DatasetWithStar{Dataset: *dataset, IsStaring: models.IsDatasetStaring(ctx.User.ID, dataset.ID)}) + } + + } + return datasetsWithStar +}