diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index fb8280b9e..fbcb5f72f 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -5,12 +5,22 @@ import ( "gitea.com/macaron/macaron" ) -type CreateModelArtsForm struct { +type CreateModelArtsNotebookForm struct { JobName string `form:"job_name" binding:"Required"` Attachment string `form:"attachment" binding:"Required"` Description string `form:"description"` } -func (f *CreateModelArtsForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { +func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { + return validate(errs, ctx.Data, f, ctx.Locale) +} + +type CreateModelArtsTrainJobForm struct { + JobName string `form:"job_name" binding:"Required"` + Attachment string `form:"attachment" binding:"Required"` + Description string `form:"description"` +} + +func (f *CreateModelArtsTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { return validate(errs, ctx.Data, f, ctx.Locale) } diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 2b58325a9..d272d036e 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -19,6 +19,7 @@ const ( NotebookEnv = "Python3" NotebookType = "Ascend" FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 96GiB （modelarts.kat1.xlarge）" + CodeLocalPath = "/code/" ) func GenerateTask(ctx *context.Context, jobName, uuid, description string) error { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 9926c154e..0899f3f34 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -461,7 +461,7 @@ var ( Bucket string Location string BasePath string - //RealPath string + CodePathPrefix string //modelarts config ModelArtsHost string @@ -1169,6 +1169,7 @@ func NewContext() { Bucket = sec.Key("BUCKET").MustString("testopendata") Location = sec.Key("LOCATION").MustString("cn-south-222") BasePath = sec.Key("BASE_PATH").MustString("attachment/") + CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/") sec = Cfg.Section("modelarts") ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80") diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 4e364fe9c..6a55507ce 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -143,3 +143,10 @@ func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { return output.SignedUrl, nil } + +func ObsUploadFileFolder(fileFolder string) error { + input := &obs.PutFileInput{} + input.Key = "" + input.Bucket = setting.Bucket + return nil +} diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index bed57d29a..0db29cfc4 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1,9 +1,14 @@ package repo import ( + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/obs" + "code.gitea.io/gitea/modules/storage" "errors" "github.com/unknwon/com" + "io" + "os" "strconv" "strings" "time" @@ -89,12 +94,11 @@ func NotebookNew(ctx *context.Context) { ctx.HTML(200, tplModelArtsNotebookNew) } -func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsForm) { +func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) { ctx.Data["PageIsCloudBrain"] = true jobName := form.JobName uuid := form.Attachment description := form.Description - //repo := ctx.Repo.Repository err := modelarts.GenerateTask(ctx, jobName, uuid, description) if err != nil { @@ -246,3 +250,152 @@ func NotebookDel(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook") } +func TrainJobIndex(ctx *context.Context) { + MustEnableModelArts(ctx) + repo := ctx.Repo.Repository + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + + tasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repo.ID, + Type: models.TypeCloudBrainTrainJob, + }) + if err != nil { + ctx.ServerError("Cloudbrain", err) + return + } + + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) + pager.SetDefaultParams(ctx) + ctx.Data["Page"] = pager + + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["Tasks"] = tasks + ctx.HTML(200, tplModelArtsNotebookIndex) +} + +func TrainJobNew(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + t := time.Now() + var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["job_name"] = jobName + + attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) + if err != nil { + ctx.ServerError("GetAllUserAttachments failed:", err) + return + } + + ctx.Data["attachments"] = attachs + ctx.Data["dataset_path"] = modelarts.DataSetMountPath + ctx.Data["env"] = modelarts.NotebookEnv + ctx.Data["notebook_type"] = modelarts.NotebookType + ctx.Data["flavor"] = modelarts.FlavorInfo + ctx.HTML(200, tplModelArtsNotebookNew) +} + +func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) { + ctx.Data["PageIsCloudBrain"] = true + jobName := form.JobName + /* + uuid := form.Attachment + description := form.Description + + */ + repo := ctx.Repo.Repository + codePath := setting.JobPath + jobName + modelarts.CodeLocalPath + + if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil { + log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) + ctx.RenderWithErr("Failed to clone repository", tplModelArtsNotebookNew, &form) + return + } + + //todo: upload code (send to file_server todo this work?) + if err := uploadCodeToObs(codePath, jobName, ""); err != nil { + log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) + ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsNotebookNew, &form) + return + } + + /* + err := modelarts.GenerateTask(ctx, jobName, uuid, description) + if err != nil { + ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) + return + } + + */ + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook") +} + +// readDir reads the directory named by dirname and returns +// a list of directory entries sorted by filename. +func readDir(dirname string) ([]os.FileInfo, error) { + f, err := os.Open(dirname) + if err != nil { + return nil, err + } + + list, err := f.Readdir(100) + f.Close() + if err != nil { + if err == io.EOF { + return nil, nil + } + return nil, err + } + + //sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() }) + return list, nil +} + +func uploadCodeToObs(codePath, jobName, parentDir string) error { + log.Info(codePath) + files, err := readDir(codePath) + if err != nil { + log.Error("readDir(%s) failed: %s", codePath, err.Error()) + return err + } + + for _, file := range files { + if file.IsDir() { + input := &obs.PutObjectInput{} + input.Bucket = setting.Bucket + input.Key = codePath + file.Name() + "/" + log.Info(input.Key) + _, err = storage.ObsCli.PutObject(input) + if err != nil { + log.Error("PutObject(%s) failed: %s", input.Key, err.Error()) + return err + } + + if err = uploadCodeToObs(codePath + file.Name() + "/", jobName, parentDir + file.Name() + "/"); err != nil { + log.Error("uploadCodeToObs(%s) failed: %s", file.Name(), err.Error()) + return err + } + } else { + input := &obs.PutFileInput{} + input.Bucket = setting.Bucket + input.Key = setting.CodePathPrefix + jobName + "/" + parentDir + file.Name() + log.Info(input.Key) + input.SourceFile = codePath + file.Name() + log.Info(input.SourceFile) + _, err = storage.ObsCli.PutFile(input) + if err != nil { + log.Error("PutFile(%s) failed: %s", input.SourceFile, err.Error()) + return err + } + } + } + + return nil +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 8bccc4ef7..f14fac7f2 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -926,19 +926,19 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel) }) m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew) - m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.NotebookCreate) + m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate) }) m.Group("/train-job", func() { - m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex) + m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex) m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.NotebookShow) m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug) m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop) m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel) }) - m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew) - m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.NotebookCreate) + m.Get("/create", reqRepoCloudBrainWriter, repo.TrainJobNew) + m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) }) }, context.RepoRef())