@@ -35,6 +35,7 @@ type AiModelManage struct { | |||
TrainTaskInfo string `xorm:"text NULL"` | |||
CreatedUnix timeutil.TimeStamp `xorm:"created"` | |||
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` | |||
IsCanOper bool | |||
} | |||
type AiModelQueryOptions struct { | |||
@@ -1,9 +1,10 @@ | |||
package cloudbrain | |||
import ( | |||
"code.gitea.io/gitea/modules/setting" | |||
"errors" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/context" | |||
"code.gitea.io/gitea/modules/log" | |||
@@ -16,7 +17,7 @@ const ( | |||
ModelMountPath = "/model" | |||
BenchMarkMountPath = "/benchmark" | |||
Snn4imagenetMountPath = "/snn4imagenet" | |||
BrainScoreMountPath = "/brainscore" | |||
BrainScoreMountPath = "/brainscore" | |||
TaskInfoName = "/taskInfo" | |||
SubTaskName = "task1" | |||
@@ -28,6 +29,52 @@ var ( | |||
ResourceSpecs *models.ResourceSpecs | |||
) | |||
func isAdminOrOwnerOrJobCreater(ctx *context.Context, jobId string) bool { | |||
job, err := models.GetCloudbrainByJobID(jobId) | |||
if err != nil { | |||
return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() | |||
} else { | |||
return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID | |||
} | |||
} | |||
func isAdminOrJobCreater(ctx *context.Context, jobId string) bool { | |||
job, err := models.GetCloudbrainByJobID(jobId) | |||
if err != nil { | |||
return ctx.IsUserSiteAdmin() | |||
} else { | |||
return ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID | |||
} | |||
} | |||
func AdminOrOwnerOrJobCreaterRight(ctx *context.Context) { | |||
var jobID = ctx.Params(":jobid") | |||
if !isAdminOrOwnerOrJobCreater(ctx, jobID) { | |||
ctx.NotFound(ctx.Req.URL.RequestURI(), nil) | |||
} | |||
} | |||
func AdminOrJobCreaterRight(ctx *context.Context) { | |||
var jobID = ctx.Params(":jobid") | |||
if !isAdminOrJobCreater(ctx, jobID) { | |||
ctx.NotFound(ctx.Req.URL.RequestURI(), nil) | |||
} | |||
} | |||
func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, resourceSpecId int) error { | |||
dataActualPath := setting.Attachment.Minio.RealPath + | |||
setting.Attachment.Minio.Bucket + "/" + | |||
@@ -433,6 +433,9 @@ var ( | |||
AuthUser string | |||
AuthPassword string | |||
//home page | |||
RecommentRepoAddr string | |||
//labelsystem config | |||
LabelTaskName string | |||
LabelDatasetDeleteQueue string | |||
@@ -549,7 +552,7 @@ var ( | |||
RecordBeginTime string | |||
IgnoreMirrorRepo bool | |||
}{} | |||
Warn_Notify_Mails []string | |||
) | |||
@@ -1224,6 +1227,9 @@ func NewContext() { | |||
LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") | |||
DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") | |||
sec = Cfg.Section("homepage") | |||
RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") | |||
sec = Cfg.Section("cloudbrain") | |||
CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") | |||
CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") | |||
@@ -783,7 +783,7 @@ datasets.desc=数据集功能 | |||
cloudbrain_helper=使用GPU/NPU资源,开启Notebook、模型训练任务等 | |||
model_manager = 模型管理 | |||
model_noright=无权限操作 | |||
debug=调试 | |||
stop=停止 | |||
@@ -7,6 +7,8 @@ package routers | |||
import ( | |||
"bytes" | |||
"fmt" | |||
"io/ioutil" | |||
"net/http" | |||
"strings" | |||
@@ -511,3 +513,36 @@ func NotFound(ctx *context.Context) { | |||
ctx.Data["Title"] = "Page Not Found" | |||
ctx.NotFound("home.NotFound", nil) | |||
} | |||
func RecommendOrgFromPromote(ctx *context.Context) { | |||
url := setting.RecommentRepoAddr + "organizations" | |||
recommendFromPromote(ctx, url) | |||
} | |||
func recommendFromPromote(ctx *context.Context, url string) { | |||
resp, err := http.Get(url) | |||
if err != nil { | |||
log.Info("Get organizations url error=" + err.Error()) | |||
ctx.ServerError("QueryTrainJobList:", err) | |||
return | |||
} | |||
bytes, err := ioutil.ReadAll(resp.Body) | |||
resp.Body.Close() | |||
if err != nil { | |||
log.Info("Get organizations url error=" + err.Error()) | |||
ctx.ServerError("QueryTrainJobList:", err) | |||
return | |||
} | |||
allLineStr := string(bytes) | |||
lines := strings.Split(allLineStr, "\n") | |||
for i, line := range lines { | |||
log.Info("i=" + fmt.Sprint(i) + " line=" + line) | |||
} | |||
ctx.JSON(http.StatusOK, lines) | |||
} | |||
func RecommendRepoFromPromote(ctx *context.Context) { | |||
url := setting.RecommentRepoAddr + "projects" | |||
recommendFromPromote(ctx, url) | |||
} |
@@ -164,6 +164,7 @@ func isErrUnverifiedCommit(err error) bool { | |||
// HookPreReceive checks whether a individual commit is acceptable | |||
func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { | |||
log.Info("Git pre start..................................") | |||
ownerName := ctx.Params(":owner") | |||
repoName := ctx.Params(":repo") | |||
repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) | |||
@@ -370,6 +371,9 @@ func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { | |||
// HookPostReceive updates services and users | |||
func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { | |||
log.Info("Git post start..................................") | |||
ownerName := ctx.Params(":owner") | |||
repoName := ctx.Params(":repo") | |||
@@ -118,6 +118,11 @@ func SaveModel(ctx *context.Context) { | |||
label := ctx.Query("Label") | |||
description := ctx.Query("Description") | |||
if !ctx.Repo.CanWrite(models.UnitTypeCloudBrain) { | |||
ctx.ServerError("No right.", errors.New(ctx.Tr("repo.model_noright"))) | |||
return | |||
} | |||
if JobId == "" || VersionName == "" { | |||
ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) | |||
return | |||
@@ -164,7 +169,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir | |||
func DeleteModel(ctx *context.Context) { | |||
log.Info("delete model start.") | |||
id := ctx.Query("ID") | |||
err := DeleteModelByID(id) | |||
err := deleteModelByID(ctx, id) | |||
if err != nil { | |||
ctx.JSON(500, err.Error()) | |||
} else { | |||
@@ -173,11 +178,22 @@ func DeleteModel(ctx *context.Context) { | |||
}) | |||
} | |||
} | |||
func isCanDeleteOrDownload(ctx *context.Context, model *models.AiModelManage) bool { | |||
if ctx.User.IsAdmin || ctx.User.ID == model.UserId { | |||
return true | |||
} | |||
if ctx.Repo.IsOwner() { | |||
return true | |||
} | |||
return false | |||
} | |||
func DeleteModelByID(id string) error { | |||
func deleteModelByID(ctx *context.Context, id string) error { | |||
log.Info("delete model start. id=" + id) | |||
model, err := models.QueryModelById(id) | |||
if !isCanDeleteOrDownload(ctx, model) { | |||
return errors.New(ctx.Tr("repo.model_noright")) | |||
} | |||
if err == nil { | |||
log.Info("bucket=" + setting.Bucket + " path=" + model.Path) | |||
if strings.HasPrefix(model.Path, setting.Bucket+"/"+Model_prefix) { | |||
@@ -224,6 +240,11 @@ func DownloadMultiModelFile(ctx *context.Context) { | |||
ctx.ServerError("no such model:", err) | |||
return | |||
} | |||
if !isCanDeleteOrDownload(ctx, task) { | |||
ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright"))) | |||
return | |||
} | |||
path := Model_prefix + models.AttachmentRelativePath(id) + "/" | |||
allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) | |||
@@ -381,9 +402,30 @@ func ShowModelTemplate(ctx *context.Context) { | |||
ctx.HTML(200, tplModelManageIndex) | |||
} | |||
func isQueryRight(ctx *context.Context) bool { | |||
if ctx.Repo.Repository.IsPrivate { | |||
if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.Repo.CanRead(models.UnitTypeCloudBrain) { | |||
return true | |||
} | |||
return false | |||
} else { | |||
return true | |||
} | |||
} | |||
func isOper(ctx *context.Context, modelUserId int64) bool { | |||
if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { | |||
return true | |||
} | |||
return false | |||
} | |||
func ShowModelPageInfo(ctx *context.Context) { | |||
log.Info("ShowModelInfo start.") | |||
if !isQueryRight(ctx) { | |||
ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright"))) | |||
return | |||
} | |||
page := ctx.QueryInt("page") | |||
if page <= 0 { | |||
page = 1 | |||
@@ -404,6 +446,10 @@ func ShowModelPageInfo(ctx *context.Context) { | |||
return | |||
} | |||
for _, model := range modelResult { | |||
model.IsCanOper = isOper(ctx, model.UserId) | |||
} | |||
mapInterface := make(map[string]interface{}) | |||
mapInterface["data"] = modelResult | |||
mapInterface["count"] = count | |||
@@ -12,6 +12,8 @@ import ( | |||
"text/template" | |||
"time" | |||
"code.gitea.io/gitea/modules/cloudbrain" | |||
"code.gitea.io/gitea/routers/operation" | |||
"code.gitea.io/gitea/routers/private" | |||
@@ -313,6 +315,8 @@ func RegisterRoutes(m *macaron.Macaron) { | |||
}) | |||
m.Get("/", routers.Home) | |||
m.Get("/dashboard", routers.Dashboard) | |||
m.Get("/recommend/org", routers.RecommendOrgFromPromote) | |||
m.Get("/recommend/repo", routers.RecommendRepoFromPromote) | |||
m.Group("/explore", func() { | |||
m.Get("", func(ctx *context.Context) { | |||
ctx.Redirect(setting.AppSubURL + "/explore/repos") | |||
@@ -957,15 +961,15 @@ func RegisterRoutes(m *macaron.Macaron) { | |||
m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) | |||
m.Group("/:jobid", func() { | |||
m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) | |||
m.Get("/debug", reqRepoCloudBrainReader, repo.CloudBrainDebug) | |||
m.Post("/commit_image", reqRepoCloudBrainWriter, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) | |||
m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop) | |||
m.Post("/del", reqRepoCloudBrainWriter, repo.CloudBrainDel) | |||
m.Get("/debug", reqRepoCloudBrainWriter, repo.CloudBrainDebug) | |||
m.Post("/commit_image", cloudbrain.AdminOrOwnerOrJobCreaterRight, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) | |||
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainStop) | |||
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDel) | |||
m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) | |||
m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) | |||
m.Get("/download_model", reqRepoCloudBrainReader, repo.CloudBrainDownloadModel) | |||
m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDownloadModel) | |||
}) | |||
m.Get("/create", reqRepoCloudBrainReader, repo.CloudBrainNew) | |||
m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew) | |||
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) | |||
}, context.RepoRef()) | |||
m.Group("/modelmanage", func() { | |||
@@ -991,9 +995,9 @@ func RegisterRoutes(m *macaron.Macaron) { | |||
m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex) | |||
m.Group("/:jobid", func() { | |||
m.Get("", reqRepoCloudBrainReader, repo.NotebookShow) | |||
m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug) | |||
m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop) | |||
m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel) | |||
m.Get("/debug", reqRepoCloudBrainWriter, repo.NotebookDebug) | |||
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookStop) | |||
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookDel) | |||
}) | |||
m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew) | |||
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate) | |||
@@ -1003,13 +1007,13 @@ func RegisterRoutes(m *macaron.Macaron) { | |||
m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex) | |||
m.Group("/:jobid", func() { | |||
m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow) | |||
m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) | |||
m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) | |||
m.Get("/model_download", reqRepoCloudBrainReader, repo.ModelDownload) | |||
m.Get("/create_version", reqRepoCloudBrainReader, repo.TrainJobNewVersion) | |||
m.Post("/create_version", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) | |||
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobStop) | |||
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobDel) | |||
m.Get("/model_download", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.ModelDownload) | |||
m.Get("/create_version", cloudbrain.AdminOrJobCreaterRight, repo.TrainJobNewVersion) | |||
m.Post("/create_version", cloudbrain.AdminOrJobCreaterRight, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) | |||
}) | |||
m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) | |||
m.Get("/create", reqRepoCloudBrainWriter, repo.TrainJobNew) | |||
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) | |||
m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList) | |||