Browse Source

Merge branch 'fix-323' of https://git.openi.org.cn/OpenI/aiforge into fix-323

tags/v1.22.4.1^2
zhoupzh 3 years ago
parent
commit
3cfc0fb5ce
10 changed files with 372 additions and 95 deletions
  1. +135
    -66
      models/cloudbrain_image.go
  2. +2
    -0
      models/models.go
  3. +10
    -0
      modules/convert/convert.go
  4. +8
    -0
      modules/structs/repo_topic.go
  5. +3
    -0
      options/locale/locale_en-US.ini
  6. +3
    -0
      options/locale/locale_zh-CN.ini
  7. +3
    -0
      routers/api/v1/api.go
  8. +60
    -0
      routers/api/v1/repo/topic.go
  9. +146
    -28
      routers/repo/cloudbrain.go
  10. +2
    -1
      routers/routes/routes.go

+ 135
- 66
models/cloudbrain_image.go View File

@@ -13,20 +13,23 @@ const RECOMMOND_TYPE = 5
const NORMAL_TYPE = 0

type Image struct {
ID int64 `xorm:"pk autoincr"`
Type int `xorm:"INDEX NOT NULL"` //0 normal 5官方推荐,中间值保留为后续扩展
CloudbrainType int `xorm:"INDEX NOT NULL"` //0 云脑一 1云脑二
UID int64 `xorm:"INDEX NOT NULL"`
IsPrivate bool `xorm:"INDEX NOT NULL"`
Tag string `xorm:"varchar(100) UNIQUE"`
Description string `xorm:"varchar(765)"`
Topics []string `xorm:"TEXT JSON"`
Place string `xorm:"varchar(300)"`
NumStars int `xorm:"NOT NULL DEFAULT 0"`
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
ID int64 `xorm:"pk autoincr" json:"id"`
Type int `xorm:"INDEX NOT NULL" json:"type"` //0 normal 5官方推荐,中间值保留为后续扩展
CloudbrainType int `xorm:"INDEX NOT NULL" json:"cloudbrainType"` //0 云脑一 1云脑二
UID int64 `xorm:"INDEX NOT NULL" json:"uid"`
IsPrivate bool `xorm:"INDEX NOT NULL" json:"isPrivate"`
Tag string `xorm:"varchar(100) UNIQUE" json:"tag"`
Description string `xorm:"varchar(765)" json:"description"`
Topics []string `xorm:"TEXT JSON" json:"topics"`
Place string `xorm:"varchar(300)" json:"place"`
NumStars int `xorm:"NOT NULL DEFAULT 0" json:"numStars"`
Creator *User `xorm:"-" json:"creator"`
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"createdUnix"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"updatedUnix"`
}

type ImageList []*Image

type ImageStar struct {
ID int64 `xorm:"pk autoincr"`
UID int64 `xorm:"UNIQUE(s)"`
@@ -48,14 +51,15 @@ type ImageTopicRelation struct {
}

type SearchImageOptions struct {
Keyword string
UID int64
IncludePublic bool
IncludeStarByMe bool
IncludeCustom bool
CodeLanguage string
Framework string
CudaVersion string
Keyword string
UID int64
IncludePublicOnly bool
IncludeOfficialOnly bool
IncludePrivateOnly bool
IncludeStarByMe bool
IncludeCustom bool
IncludeOwnerOnly bool
Topics string
ListOptions
SearchOrderBy
}
@@ -63,6 +67,11 @@ type ErrorImageTagExist struct {
Tag string
}

type ImagesPageResult struct {
Count int64 `json:"count"`
Images []*Image `json:"images"`
}

func (err ErrorImageTagExist) Error() string {
return fmt.Sprintf("Image already exists [tag: %s]", err.Tag)
}
@@ -251,58 +260,83 @@ func removeTopicFromImage(e Engine, imageId int64, topic *ImageTopic) error {
return nil
}

/*func SearchImage(opts *SearchImageOptions) ([]*Image, int64, error) {
func SearchImage(opts *SearchImageOptions) (ImageList, int64, error) {
cond := SearchImageCondition(opts)
return SearchImageByCondition(opts, cond)
}*/
}

func SearchImageCondition(opts *SearchImageOptions) builder.Cond {
var cond = builder.NewCond()

if len(opts.Keyword) > 0 {
cond = cond.And(builder.Or(builder.Like{"image.tag", opts.Keyword}, builder.Like{"image.description", opts.Keyword}))
}

if len(opts.CudaVersion) > 0 {
cond = cond.And(builder.Eq{"image.cuda_version": opts.CudaVersion})
var subQueryCond = builder.NewCond()
for _, v := range strings.Split(opts.Keyword, ",") {

subQueryCond = subQueryCond.Or(builder.Like{"LOWER(image_topic.name)", strings.ToLower(v)})

}
subQuery := builder.Select("image_topic_relation.image_id").From("image_topic_relation").
Join("INNER", "image_topic", "image_topic.id = image_topic_relation.image_id").
Where(subQueryCond).
GroupBy("image_topic_relation.image_id")
var keywordCond = builder.In("id", subQuery)

var likes = builder.NewCond()
for _, v := range strings.Split(opts.Keyword, ",") {
likes = likes.Or(builder.Like{"LOWER(tag)", strings.ToLower(v)})

likes = likes.Or(builder.Like{"LOWER(description)", strings.ToLower(v)})

}
keywordCond = keywordCond.Or(likes)

cond = cond.And(keywordCond)

}
if len(opts.Topics) > 0 { //标签精确匹配
var subQueryCond = builder.NewCond()
for _, v := range strings.Split(opts.Keyword, ",") {

subQueryCond = subQueryCond.Or(builder.Eq{"LOWER(image_topic.name)": strings.ToLower(v)})
subQuery := builder.Select("image_topic_relation.image_id").From("image_topic_relation").
Join("INNER", "image_topic", "image_topic.id = image_topic_relation.image_id").
Where(subQueryCond).
GroupBy("image_topic_relation.image_id")
var topicCond = builder.In("id", subQuery)
cond = cond.And(topicCond)
}
}

if len(opts.CodeLanguage) > 0 {
cond = cond.And(builder.Eq{"image.code_language": opts.CodeLanguage})
if opts.IncludePublicOnly {
cond = cond.And(builder.Eq{"is_private": false})
}
if len(opts.Framework) > 0 {
cond = cond.And(builder.Eq{"image.framework": opts.Framework})

if opts.IncludePrivateOnly {
cond = cond.And(builder.Eq{"is_private": true})
}

if opts.IncludePublic {
cond = cond.And(builder.Eq{"image.is_private": false})
if opts.IncludeOwnerOnly {

cond = cond.And(builder.Eq{"uid": opts.UID})
}
/**
if opts.IncludeOfficialOnly {
cond = cond.And(builder.Eq{"type": RECOMMOND_TYPE})
}

if opts.IncludeStarByMe {
cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic})
cond = cond.And(builder.Eq{"attachment.is_private": false})
if opts.OwnerID > 0 {
if len(opts.Keyword) == 0 {
cond = cond.Or(builder.Eq{"repository.owner_id": opts.OwnerID})
} else {
subCon := builder.NewCond()
subCon = subCon.And(builder.Eq{"repository.owner_id": opts.OwnerID}, builder.Or(builder.Like{"dataset.title", opts.Keyword}, builder.Like{"dataset.description", opts.Keyword}))
cond = cond.Or(subCon)

}
}
} else if opts.OwnerID > 0 {
cond = cond.And(builder.Eq{"repository.owner_id": opts.OwnerID})
if !opts.IsOwner {
cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic})
cond = cond.And(builder.Eq{"attachment.is_private": false})
}
}*/
subQuery := builder.Select("image_id").From("image_star").
Where(builder.Eq{"uid": opts.UID})
var starCond = builder.In("id", subQuery)
cond = cond.And(starCond)

}

return cond
}

/*func SearchImageByCondition(opts *SearchImageOptions, cond builder.Cond) ([]*Image, int64, error) {
func SearchImageByCondition(opts *SearchImageOptions, cond builder.Cond) (ImageList, int64, error) {
if opts.Page <= 0 {
opts.Page = 1
}
@@ -311,34 +345,57 @@ func SearchImageCondition(opts *SearchImageOptions) builder.Cond {
sess := x.NewSession()
defer sess.Close()

datasets := make(DatasetList, 0, opts.PageSize)
selectColumnsSql := "distinct dataset.id,dataset.title, dataset.status, dataset.category, dataset.description, dataset.download_times, dataset.license, dataset.task, dataset.release_id, dataset.user_id, dataset.repo_id, dataset.created_unix,dataset.updated_unix,dataset.num_stars"

count, err := sess.Distinct("dataset.id").Join("INNER", "repository", "repository.id = dataset.repo_id").
Join("INNER", "attachment", "attachment.dataset_id=dataset.id").
Where(cond).Count(new(Dataset))
images := make(ImageList, 0, opts.PageSize)
count, err := sess.Where(cond).Count(new(Image))

if err != nil {
return nil, 0, fmt.Errorf("Count: %v", err)
}

sess.Select(selectColumnsSql).Join("INNER", "repository", "repository.id = dataset.repo_id").
Join("INNER", "attachment", "attachment.dataset_id=dataset.id").
Where(cond).OrderBy(opts.SearchOrderBy.String())
sess.Where(cond).OrderBy(opts.SearchOrderBy.String())

if opts.PageSize > 0 {
sess.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize)
}
if err = sess.Find(&datasets); err != nil {
return nil, 0, fmt.Errorf("Dataset: %v", err)
if err = sess.Find(&images); err != nil {
return nil, 0, fmt.Errorf("Images: %v", err)
}

if err = datasets.loadAttributes(sess); err != nil {
if err = images.loadAttributes(sess); err != nil {
return nil, 0, fmt.Errorf("LoadAttributes: %v", err)
}

return datasets, count, nil
}*/
return images, count, nil
}

func (images ImageList) loadAttributes(e Engine) error {
if len(images) == 0 {
return nil
}

set := make(map[int64]struct{})

for i := range images {
set[images[i].UID] = struct{}{}
}

// Load creators.
users := make(map[int64]*User, len(set))
if err := e.Table("\"user\"").
Cols("name", "lower_name", "avatar", "email").
Where("id > 0").
In("id", keysInt64(set)).
Find(&users); err != nil {
return fmt.Errorf("find users: %v", err)
}

for i := range images {
images[i].Creator = users[images[i].UID]

}

return nil
}

func CreateLocalImage(image *Image) error {

@@ -346,6 +403,18 @@ func CreateLocalImage(image *Image) error {
return err
}

func UpdateLocalImage(image *Image) error {

_, err := x.ID(image.ID).Update(image)
return err
}

func DeleteLocalImage(id int64) error {
image := new(Image)
_, err := x.ID(id).Delete(image)
return err
}

//star or unstar Image
func StarImage(userID, imageID int64, star bool) error {
sess := x.NewSession()


+ 2
- 0
models/models.go View File

@@ -133,6 +133,8 @@ func init() {
new(Cloudbrain),
new(Image),
new(ImageStar),
new(ImageTopic),
new(ImageTopicRelation),
new(FileChunk),
new(BlockChain),
new(RecommendOrg),


+ 10
- 0
modules/convert/convert.go View File

@@ -403,6 +403,16 @@ func ToTopicResponse(topic *models.Topic) *api.TopicResponse {
}
}

func ToImageTopicResponse(topic *models.ImageTopic) *api.ImageTopicResponse {
return &api.ImageTopicResponse{
ID: topic.ID,
Name: topic.Name,
ImageCount: topic.ImageCount,
Created: topic.CreatedUnix.AsTime(),
Updated: topic.UpdatedUnix.AsTime(),
}
}

// ToOAuth2Application convert from models.OAuth2Application to api.OAuth2Application
func ToOAuth2Application(app *models.OAuth2Application) *api.OAuth2Application {
return &api.OAuth2Application{


+ 8
- 0
modules/structs/repo_topic.go View File

@@ -17,6 +17,14 @@ type TopicResponse struct {
Updated time.Time `json:"updated"`
}

type ImageTopicResponse struct {
ID int64 `json:"id"`
Name string `json:"topic_name"`
ImageCount int `json:"image_count"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
}

// TopicName a list of repo topic names
type TopicName struct {
TopicNames []string `json:"topics"`


+ 3
- 0
options/locale/locale_en-US.ini View File

@@ -914,6 +914,9 @@ model_download=Model Download
submit_image=Submit Image
image_exist=Image name has been used, please use a new one.
image_commit_fail=Failed to submit image, please try again later.
image_not_exist=Image does not exits.
image_edit_fail=Failed to edit image, please try again later.
image_delete_fail=Failed to delete image, please try again later.
download=Download
score=Score



+ 3
- 0
options/locale/locale_zh-CN.ini View File

@@ -919,6 +919,9 @@ model_download=结果下载
submit_image=提交镜像
image_exist=镜像名称已被使用,请修改镜像名称。
image_commit_fail=提交镜像失败,请稍后再试。
image_not_exist=镜像不存在。
image_edit_fail=编辑镜像失败,请稍后再试。
image_delete_fail=删除镜像失败,请稍后再试。
download=模型下载
score=评分



+ 3
- 0
routers/api/v1/api.go View File

@@ -997,6 +997,9 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/topics", func() {
m.Get("/search", repo.TopicSearch)
})
m.Group("/image/topics", func() {
m.Get("/search", repo.ImageTopicSearch)
})
m.Group("/from_wechat", func() {
m.Get("/event", authentication.ValidEventSource)
m.Post("/event", authentication.AcceptWechatEvent)


+ 60
- 0
routers/api/v1/repo/topic.go View File

@@ -300,3 +300,63 @@ func TopicSearch(ctx *context.APIContext) {
"topics": topicResponses,
})
}

func ImageTopicSearch(ctx *context.APIContext) {
// swagger:operation GET /image/topics/search image topicSearch
// ---
// summary: search topics via keyword
// produces:
// - application/json
// parameters:
// - name: q
// in: query
// description: keywords to search
// required: true
// type: string
// - name: page
// in: query
// description: page number of results to return (1-based)
// type: integer
// - name: limit
// in: query
// description: page size of results, maximum page size is 50
// type: integer
// responses:
// "200":
// "$ref": "#/responses/TopicListResponse"
// "403":
// "$ref": "#/responses/forbidden"

if ctx.User == nil {
ctx.Error(http.StatusForbidden, "UserIsNil", "Only owners could change the topics.")
return
}

kw := ctx.Query("q")

listOptions := utils.GetListOptions(ctx)
if listOptions.Page < 1 {
listOptions.Page = 1
}
if listOptions.PageSize < 1 {
listOptions.PageSize = 10
}

topics, err := models.FindImageTopics(&models.FindImageTopicOptions{
Keyword: kw,
ListOptions: listOptions,
})
if err != nil {
log.Error("SearchImageTopics failed: %v", err)
ctx.InternalServerError(err)
return
}

topicResponses := make([]*api.ImageTopicResponse, len(topics))
for i, topic := range topics {
topicResponses[i] = convert.ToImageTopicResponse(topic)
}
ctx.JSON(http.StatusOK, map[string]interface{}{
"topics": topicResponses,
})
}

+ 146
- 28
routers/repo/cloudbrain.go View File

@@ -461,12 +461,12 @@ func CloudBrainImageEdit(ctx *context.Context) {
var ID = ctx.Params(":id")
id, err := strconv.ParseInt(ID, 10, 64)
if err != nil {
log.Error("GetCloudbrainByID failed:%v", err.Error())
log.Error("GetImageByID failed:%v", err.Error())
ctx.NotFound(ctx.Req.URL.RequestURI(), nil)

image, err := models.GetImageByID(id)
if err != nil {
log.Error("GetCloudbrainByID failed:%v", err.Error())
log.Error("GetImageByID failed:%v", err.Error())
ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
}
ctx.Data["Image"] = image
@@ -476,29 +476,62 @@ func CloudBrainImageEdit(ctx *context.Context) {

func CloudBrainImageEditPost(ctx *context.Context, form auth.EditImageCloudBrainForm) {

}
validTopics, errMessage := checkTopics(form.Topics)
if errMessage != "" {
ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
return
}
image, err := models.GetImageByID(form.ID)
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))

func CloudBrainImageDelete(ctx *context.Context) {
}

}
image.IsPrivate = form.IsPrivate
image.Description = form.Description

func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
err = models.WithTx(func(ctx models.DBContext) error {
if err := models.UpdateLocalImage(image); err != nil {
return err
}
if err := models.SaveImageTopics(image.ID, validTopics...); err != nil {
return err
}
return nil

var topics = make([]string, 0)
var topicsStr = strings.TrimSpace(form.Topics)
if len(topicsStr) > 0 {
topics = strings.Split(topicsStr, ",")
})

if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))

} else {
ctx.JSON(http.StatusOK, models.BaseOKMessage)
}

validTopics, invalidTopics := models.SanitizeAndValidateTopics(topics)
}

if len(validTopics) > 25 {
ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.topic.count_prompt")))
func CloudBrainImageDelete(ctx *context.Context) {
var ID = ctx.Params(":id")
id, err := strconv.ParseInt(ID, 10, 64)
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))
return
}

if len(invalidTopics) > 0 {
ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.topic.format_prompt")))
err = models.DeleteLocalImage(id)
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_delete_fail")))
} else {
ctx.JSON(http.StatusOK, models.BaseOKMessage)
}

}

func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {

validTopics, errMessage := checkTopics(form.Topics)
if errMessage != "" {
ctx.JSON(http.StatusOK, ctx.Tr(errMessage))
return
}

@@ -529,6 +562,27 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain
ctx.JSON(200, models.BaseOKMessage)
}

func checkTopics(Topics string) ([]string, string) {
var topics = make([]string, 0)
var topicsStr = strings.TrimSpace(Topics)
if len(topicsStr) > 0 {
topics = strings.Split(topicsStr, ",")
}

validTopics, invalidTopics := models.SanitizeAndValidateTopics(topics)

if len(validTopics) > 25 {
return nil, "repo.topic.count_prompt"

}

if len(invalidTopics) > 0 {
return nil, "repo.topic.count_prompt"

}
return validTopics, ""
}

func CloudBrainStop(ctx *context.Context) {
var ID = ctx.Params(":id")
var resultCode = "0"
@@ -743,34 +797,98 @@ func CloudBrainShowModels(ctx *context.Context) {

func GetPublicImages(ctx *context.Context) {

getImages(ctx, cloudbrain.Public)
opts := models.SearchImageOptions{
IncludePrivateOnly: true,
UID: -1,
Keyword: ctx.Query("q"),
Topics: ctx.Query("topic"),
IncludeOfficialOnly: ctx.QueryBool("recommend"),
SearchOrderBy: "type desc, num_stars desc",
}

getImages(ctx, &opts)

}

func GetCustomImages(ctx *context.Context) {
var uid int64 = -1
if ctx.IsSigned {
uid = ctx.User.ID
}
opts := models.SearchImageOptions{
UID: uid,
IncludeOwnerOnly: true,
Keyword: ctx.Query("q"),
Topics: ctx.Query("topic"),
SearchOrderBy: "id desc",
}
getImages(ctx, &opts)

}
func GetStarImages(ctx *context.Context) {

getImages(ctx, cloudbrain.Custom)
var uid int64 = -1
if ctx.IsSigned {
uid = ctx.User.ID
}
opts := models.SearchImageOptions{
UID: uid,
IncludeStarByMe: true,
Keyword: ctx.Query("q"),
Topics: ctx.Query("topic"),
SearchOrderBy: "id desc",
}
getImages(ctx, &opts)

}

func getImages(ctx *context.Context, imageType string) {
log.Info("Get images begin")
func GetAllImages(ctx *context.Context) {

opts := models.SearchImageOptions{
UID: -1,
Keyword: ctx.Query("q"),
Topics: ctx.Query("topic"),
SearchOrderBy: "id desc",
}

if ctx.Query("private") != "" {
if ctx.QueryBool("private") {
opts.IncludePrivateOnly = true
} else {
opts.IncludePublicOnly = true
}
}
getImages(ctx, &opts)

}

func getImages(ctx *context.Context, opts *models.SearchImageOptions) {
page := ctx.QueryInt("page")
size := ctx.QueryInt("size")
name := ctx.Query("name")
getImagesResult, err := cloudbrain.GetImagesPageable(page, size, imageType, name)
if page <= 0 {
page = 1
}

pageSize := ctx.QueryInt("pageSize")
if pageSize <= 0 {
pageSize = 15
}
opts.ListOptions = models.ListOptions{
Page: page,
PageSize: pageSize,
}
imageList, total, err := models.SearchImage(opts)
if err != nil {
log.Error("Can not get images:%v", err)
ctx.JSON(http.StatusOK, models.GetImagesPayload{
Count: 0,
TotalPages: 0,
ImageInfo: []*models.ImageInfo{},
ctx.JSON(http.StatusOK, models.ImagesPageResult{
Count: 0,
Images: []*models.Image{},
})
} else {
ctx.JSON(http.StatusOK, getImagesResult.Payload)
ctx.JSON(http.StatusOK, models.ImagesPageResult{
Count: total,
Images: imageList,
})
}
log.Info("Get images end")
}

func GetModelDirs(jobName string, parentDir string) (string, error) {


+ 2
- 1
routers/routes/routes.go View File

@@ -330,7 +330,7 @@ func RegisterRoutes(m *macaron.Macaron) {
})
m.Get("/images/public", repo.GetPublicImages)
m.Get("/images/custom", repo.GetCustomImages)
m.Get("/images/star", repo.GetCustomImages)
m.Get("/images/star", repo.GetStarImages)

m.Get("/repos", routers.ExploreRepos)
m.Get("/datasets", routers.ExploreDatasets)
@@ -527,6 +527,7 @@ func RegisterRoutes(m *macaron.Macaron) {
})
m.Group("/images", func() {
m.Get("", admin.Images)
m.Get("/data", repo.GetAllImages)
})

m.Group("/^:configType(hooks|system-hooks)$", func() {


Loading…
Cancel
Save