Browse Source

update

tags/v1.21.12.1^2
zhoupzh 3 years ago
parent
commit
958b056d82
8 changed files with 166 additions and 23 deletions
  1. +1
    -0
      models/ai_model_manage.go
  2. +49
    -2
      modules/cloudbrain/cloudbrain.go
  3. +7
    -1
      modules/setting/setting.go
  4. +1
    -1
      options/locale/locale_zh-CN.ini
  5. +35
    -0
      routers/home.go
  6. +4
    -0
      routers/private/hook.go
  7. +50
    -4
      routers/repo/ai_model_manage.go
  8. +19
    -15
      routers/routes/routes.go

+ 1
- 0
models/ai_model_manage.go View File

@@ -35,6 +35,7 @@ type AiModelManage struct {
TrainTaskInfo string `xorm:"text NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
IsCanOper bool
}

type AiModelQueryOptions struct {


+ 49
- 2
modules/cloudbrain/cloudbrain.go View File

@@ -1,9 +1,10 @@
package cloudbrain

import (
"code.gitea.io/gitea/modules/setting"
"errors"

"code.gitea.io/gitea/modules/setting"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
@@ -16,7 +17,7 @@ const (
ModelMountPath = "/model"
BenchMarkMountPath = "/benchmark"
Snn4imagenetMountPath = "/snn4imagenet"
BrainScoreMountPath = "/brainscore"
BrainScoreMountPath = "/brainscore"
TaskInfoName = "/taskInfo"

SubTaskName = "task1"
@@ -28,6 +29,52 @@ var (
ResourceSpecs *models.ResourceSpecs
)

func isAdminOrOwnerOrJobCreater(ctx *context.Context, jobId string) bool {

job, err := models.GetCloudbrainByJobID(jobId)

if err != nil {
return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin()
} else {
return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID
}

}

func isAdminOrJobCreater(ctx *context.Context, jobId string) bool {

job, err := models.GetCloudbrainByJobID(jobId)

if err != nil {
return ctx.IsUserSiteAdmin()
} else {
return ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID
}

}

func AdminOrOwnerOrJobCreaterRight(ctx *context.Context) {

var jobID = ctx.Params(":jobid")

if !isAdminOrOwnerOrJobCreater(ctx, jobID) {

ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
}

}

func AdminOrJobCreaterRight(ctx *context.Context) {

var jobID = ctx.Params(":jobid")

if !isAdminOrJobCreater(ctx, jobID) {

ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
}

}

func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, resourceSpecId int) error {
dataActualPath := setting.Attachment.Minio.RealPath +
setting.Attachment.Minio.Bucket + "/" +


+ 7
- 1
modules/setting/setting.go View File

@@ -433,6 +433,9 @@ var (
AuthUser string
AuthPassword string

//home page
RecommentRepoAddr string

//labelsystem config
LabelTaskName string
LabelDatasetDeleteQueue string
@@ -549,7 +552,7 @@ var (
RecordBeginTime string
IgnoreMirrorRepo bool
}{}
Warn_Notify_Mails []string
)

@@ -1224,6 +1227,9 @@ func NewContext() {
LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue")
DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue")

sec = Cfg.Section("homepage")
RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/")

sec = Cfg.Section("cloudbrain")
CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X")
CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC")


+ 1
- 1
options/locale/locale_zh-CN.ini View File

@@ -783,7 +783,7 @@ datasets.desc=数据集功能
cloudbrain_helper=使用GPU/NPU资源,开启Notebook、模型训练任务等

model_manager = 模型管理
model_noright=无权限操作

debug=调试
stop=停止


+ 35
- 0
routers/home.go View File

@@ -7,6 +7,8 @@ package routers

import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"strings"

@@ -511,3 +513,36 @@ func NotFound(ctx *context.Context) {
ctx.Data["Title"] = "Page Not Found"
ctx.NotFound("home.NotFound", nil)
}

func RecommendOrgFromPromote(ctx *context.Context) {
url := setting.RecommentRepoAddr + "organizations"
recommendFromPromote(ctx, url)
}

func recommendFromPromote(ctx *context.Context, url string) {
resp, err := http.Get(url)
if err != nil {
log.Info("Get organizations url error=" + err.Error())
ctx.ServerError("QueryTrainJobList:", err)
return
}
bytes, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
log.Info("Get organizations url error=" + err.Error())
ctx.ServerError("QueryTrainJobList:", err)
return
}

allLineStr := string(bytes)
lines := strings.Split(allLineStr, "\n")
for i, line := range lines {
log.Info("i=" + fmt.Sprint(i) + " line=" + line)
}
ctx.JSON(http.StatusOK, lines)
}

func RecommendRepoFromPromote(ctx *context.Context) {
url := setting.RecommentRepoAddr + "projects"
recommendFromPromote(ctx, url)
}

+ 4
- 0
routers/private/hook.go View File

@@ -164,6 +164,7 @@ func isErrUnverifiedCommit(err error) bool {

// HookPreReceive checks whether a individual commit is acceptable
func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) {
log.Info("Git pre start..................................")
ownerName := ctx.Params(":owner")
repoName := ctx.Params(":repo")
repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName)
@@ -370,6 +371,9 @@ func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) {

// HookPostReceive updates services and users
func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) {

log.Info("Git post start..................................")

ownerName := ctx.Params(":owner")
repoName := ctx.Params(":repo")



+ 50
- 4
routers/repo/ai_model_manage.go View File

@@ -118,6 +118,11 @@ func SaveModel(ctx *context.Context) {
label := ctx.Query("Label")
description := ctx.Query("Description")

if !ctx.Repo.CanWrite(models.UnitTypeCloudBrain) {
ctx.ServerError("No right.", errors.New(ctx.Tr("repo.model_noright")))
return
}

if JobId == "" || VersionName == "" {
ctx.Error(500, fmt.Sprintf("JobId or VersionName is null."))
return
@@ -164,7 +169,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir
func DeleteModel(ctx *context.Context) {
log.Info("delete model start.")
id := ctx.Query("ID")
err := DeleteModelByID(id)
err := deleteModelByID(ctx, id)
if err != nil {
ctx.JSON(500, err.Error())
} else {
@@ -173,11 +178,22 @@ func DeleteModel(ctx *context.Context) {
})
}
}
func isCanDeleteOrDownload(ctx *context.Context, model *models.AiModelManage) bool {
if ctx.User.IsAdmin || ctx.User.ID == model.UserId {
return true
}
if ctx.Repo.IsOwner() {
return true
}
return false
}

func DeleteModelByID(id string) error {
func deleteModelByID(ctx *context.Context, id string) error {
log.Info("delete model start. id=" + id)

model, err := models.QueryModelById(id)
if !isCanDeleteOrDownload(ctx, model) {
return errors.New(ctx.Tr("repo.model_noright"))
}
if err == nil {
log.Info("bucket=" + setting.Bucket + " path=" + model.Path)
if strings.HasPrefix(model.Path, setting.Bucket+"/"+Model_prefix) {
@@ -224,6 +240,11 @@ func DownloadMultiModelFile(ctx *context.Context) {
ctx.ServerError("no such model:", err)
return
}
if !isCanDeleteOrDownload(ctx, task) {
ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright")))
return
}

path := Model_prefix + models.AttachmentRelativePath(id) + "/"

allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path)
@@ -381,9 +402,30 @@ func ShowModelTemplate(ctx *context.Context) {
ctx.HTML(200, tplModelManageIndex)
}

func isQueryRight(ctx *context.Context) bool {
if ctx.Repo.Repository.IsPrivate {
if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
return true
}
return false
} else {
return true
}
}

func isOper(ctx *context.Context, modelUserId int64) bool {
if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId {
return true
}
return false
}

func ShowModelPageInfo(ctx *context.Context) {
log.Info("ShowModelInfo start.")

if !isQueryRight(ctx) {
ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright")))
return
}
page := ctx.QueryInt("page")
if page <= 0 {
page = 1
@@ -404,6 +446,10 @@ func ShowModelPageInfo(ctx *context.Context) {
return
}

for _, model := range modelResult {
model.IsCanOper = isOper(ctx, model.UserId)
}

mapInterface := make(map[string]interface{})
mapInterface["data"] = modelResult
mapInterface["count"] = count


+ 19
- 15
routers/routes/routes.go View File

@@ -12,6 +12,8 @@ import (
"text/template"
"time"

"code.gitea.io/gitea/modules/cloudbrain"

"code.gitea.io/gitea/routers/operation"
"code.gitea.io/gitea/routers/private"

@@ -313,6 +315,8 @@ func RegisterRoutes(m *macaron.Macaron) {
})
m.Get("/", routers.Home)
m.Get("/dashboard", routers.Dashboard)
m.Get("/recommend/org", routers.RecommendOrgFromPromote)
m.Get("/recommend/repo", routers.RecommendRepoFromPromote)
m.Group("/explore", func() {
m.Get("", func(ctx *context.Context) {
ctx.Redirect(setting.AppSubURL + "/explore/repos")
@@ -957,15 +961,15 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex)
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow)
m.Get("/debug", reqRepoCloudBrainReader, repo.CloudBrainDebug)
m.Post("/commit_image", reqRepoCloudBrainWriter, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage)
m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop)
m.Post("/del", reqRepoCloudBrainWriter, repo.CloudBrainDel)
m.Get("/debug", reqRepoCloudBrainWriter, repo.CloudBrainDebug)
m.Post("/commit_image", cloudbrain.AdminOrOwnerOrJobCreaterRight, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage)
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainStop)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDel)
m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate)
m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
m.Get("/download_model", reqRepoCloudBrainReader, repo.CloudBrainDownloadModel)
m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDownloadModel)
})
m.Get("/create", reqRepoCloudBrainReader, repo.CloudBrainNew)
m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate)
}, context.RepoRef())
m.Group("/modelmanage", func() {
@@ -991,9 +995,9 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex)
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.NotebookShow)
m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug)
m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop)
m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel)
m.Get("/debug", reqRepoCloudBrainWriter, repo.NotebookDebug)
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookStop)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookDel)
})
m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate)
@@ -1003,13 +1007,13 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex)
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow)
m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop)
m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel)
m.Get("/model_download", reqRepoCloudBrainReader, repo.ModelDownload)
m.Get("/create_version", reqRepoCloudBrainReader, repo.TrainJobNewVersion)
m.Post("/create_version", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobStop)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobDel)
m.Get("/model_download", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.ModelDownload)
m.Get("/create_version", cloudbrain.AdminOrJobCreaterRight, repo.TrainJobNewVersion)
m.Post("/create_version", cloudbrain.AdminOrJobCreaterRight, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
})
m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew)
m.Get("/create", reqRepoCloudBrainWriter, repo.TrainJobNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate)

m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList)


Loading…
Cancel
Save