uploadCodeToObs

modules/auth/modelarts.go

@@ -5,12 +5,22 @@ import (
 	"gitea.com/macaron/macaron"
 )
 
-type CreateModelArtsForm struct {
+type CreateModelArtsNotebookForm struct {
 	JobName    string `form:"job_name" binding:"Required"`
 	Attachment string `form:"attachment" binding:"Required"`
 	Description string `form:"description"`
 }
 
-func (f *CreateModelArtsForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
+func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
+	return validate(errs, ctx.Data, f, ctx.Locale)
+}
+
+type CreateModelArtsTrainJobForm struct {
+	JobName    string `form:"job_name" binding:"Required"`
+	Attachment string `form:"attachment" binding:"Required"`
+	Description string `form:"description"`
+}
+
+func (f *CreateModelArtsTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
 	return validate(errs, ctx.Data, f, ctx.Locale)
 }

modules/modelarts/modelarts.go

@@ -19,6 +19,7 @@ const (
 	NotebookEnv = "Python3"
 	NotebookType = "Ascend"
 	FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 96GiB （modelarts.kat1.xlarge）"
+	CodeLocalPath   = "/code/"
 )
 
 func GenerateTask(ctx *context.Context, jobName, uuid, description string) error {

modules/setting/setting.go

@@ -461,7 +461,7 @@ var (
 	Bucket          string
 	Location        string
 	BasePath        string
-	//RealPath        string
+	CodePathPrefix        string
 
 	//modelarts config
 	ModelArtsHost	string
@@ -1169,6 +1169,7 @@ func NewContext() {
 	Bucket = sec.Key("BUCKET").MustString("testopendata")
 	Location = sec.Key("LOCATION").MustString("cn-south-222")
 	BasePath = sec.Key("BASE_PATH").MustString("attachment/")
+	CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/")
 
 	sec = Cfg.Section("modelarts")
 	ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80")

modules/storage/obs.go

@@ -143,3 +143,10 @@ func ObsGetPreSignedUrl(uuid, fileName string) (string, error) {
 
 	return output.SignedUrl, nil
 }
+
+func ObsUploadFileFolder(fileFolder string) error {
+	input := &obs.PutFileInput{}
+	input.Key = ""
+	input.Bucket = setting.Bucket
+	return nil
+}

routers/repo/modelarts.go

@@ -1,9 +1,14 @@
 package repo
 
 import (
+	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/modelarts"
+	"code.gitea.io/gitea/modules/obs"
+	"code.gitea.io/gitea/modules/storage"
 	"errors"
 	"github.com/unknwon/com"
+	"io"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -89,12 +94,11 @@ func NotebookNew(ctx *context.Context) {
 	ctx.HTML(200, tplModelArtsNotebookNew)
 }
 
-func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsForm) {
+func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) {
 	ctx.Data["PageIsCloudBrain"] = true
 	jobName := form.JobName
 	uuid := form.Attachment
 	description := form.Description
-	//repo := ctx.Repo.Repository
 
 	err := modelarts.GenerateTask(ctx, jobName, uuid, description)
 	if err != nil {
@@ -246,3 +250,152 @@ func NotebookDel(ctx *context.Context) {
 	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook")
 }
 
+func TrainJobIndex(ctx *context.Context) {
+	MustEnableModelArts(ctx)
+	repo := ctx.Repo.Repository
+	page := ctx.QueryInt("page")
+	if page <= 0 {
+		page = 1
+	}
+
+	tasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
+		ListOptions: models.ListOptions{
+			Page:     page,
+			PageSize: setting.UI.IssuePagingNum,
+		},
+		RepoID: repo.ID,
+		Type:	models.TypeCloudBrainTrainJob,
+	})
+	if err != nil {
+		ctx.ServerError("Cloudbrain", err)
+		return
+	}
+
+	pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
+	pager.SetDefaultParams(ctx)
+	ctx.Data["Page"] = pager
+
+	ctx.Data["PageIsCloudBrain"] = true
+	ctx.Data["Tasks"] = tasks
+	ctx.HTML(200, tplModelArtsNotebookIndex)
+}
+
+func TrainJobNew(ctx *context.Context) {
+	ctx.Data["PageIsCloudBrain"] = true
+
+	t := time.Now()
+	var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
+	ctx.Data["job_name"] = jobName
+
+	attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID)
+	if err != nil {
+		ctx.ServerError("GetAllUserAttachments failed:", err)
+		return
+	}
+
+	ctx.Data["attachments"] = attachs
+	ctx.Data["dataset_path"] = modelarts.DataSetMountPath
+	ctx.Data["env"] = modelarts.NotebookEnv
+	ctx.Data["notebook_type"] = modelarts.NotebookType
+	ctx.Data["flavor"] = modelarts.FlavorInfo
+	ctx.HTML(200, tplModelArtsNotebookNew)
+}
+
+func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) {
+	ctx.Data["PageIsCloudBrain"] = true
+	jobName := form.JobName
+	/*
+	uuid := form.Attachment
+	description := form.Description
+
+	 */
+	repo := ctx.Repo.Repository
+	codePath := setting.JobPath + jobName + modelarts.CodeLocalPath
+
+	if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil {
+		log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
+		ctx.RenderWithErr("Failed to clone repository", tplModelArtsNotebookNew, &form)
+		return
+	}
+
+	//todo: upload code (send to file_server todo this work?)
+	if err := uploadCodeToObs(codePath, jobName, ""); err != nil {
+		log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err)
+		ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsNotebookNew, &form)
+		return
+	}
+
+	/*
+	err := modelarts.GenerateTask(ctx, jobName, uuid, description)
+	if err != nil {
+		ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form)
+		return
+	}
+
+	 */
+
+	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook")
+}
+
+// readDir reads the directory named by dirname and returns
+// a list of directory entries sorted by filename.
+func readDir(dirname string) ([]os.FileInfo, error) {
+	f, err := os.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+
+	list, err := f.Readdir(100)
+	f.Close()
+	if err != nil {
+		if err == io.EOF {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	//sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() })
+	return list, nil
+}
+
+func uploadCodeToObs(codePath, jobName, parentDir string) error {
+	log.Info(codePath)
+	files, err := readDir(codePath)
+	if err != nil {
+		log.Error("readDir(%s) failed: %s", codePath, err.Error())
+		return err
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			input := &obs.PutObjectInput{}
+			input.Bucket = setting.Bucket
+			input.Key = codePath + file.Name() + "/"
+			log.Info(input.Key)
+			_, err = storage.ObsCli.PutObject(input)
+			if err != nil {
+				log.Error("PutObject(%s) failed: %s", input.Key, err.Error())
+				return err
+			}
+
+			if err = uploadCodeToObs(codePath + file.Name() + "/", jobName, parentDir + file.Name() + "/"); err != nil {
+				log.Error("uploadCodeToObs(%s) failed: %s", file.Name(), err.Error())
+				return err
+			}
+		} else {
+			input := &obs.PutFileInput{}
+			input.Bucket = setting.Bucket
+			input.Key = setting.CodePathPrefix + jobName + "/" + parentDir + file.Name()
+			log.Info(input.Key)
+			input.SourceFile = codePath + file.Name()
+			log.Info(input.SourceFile)
+			_, err = storage.ObsCli.PutFile(input)
+			if err != nil {
+				log.Error("PutFile(%s) failed: %s", input.SourceFile, err.Error())
+				return err
+			}
+		}
+	}
+
+	return nil
+}

routers/routes/routes.go

@@ -926,19 +926,19 @@ func RegisterRoutes(m *macaron.Macaron) {
 					m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel)
 				})
 				m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew)
-				m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.NotebookCreate)
+				m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate)
 			})
 
 			m.Group("/train-job", func() {
-				m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex)
+				m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex)
 				m.Group("/:jobid", func() {
 					m.Get("", reqRepoCloudBrainReader, repo.NotebookShow)
 					m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug)
 					m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop)
 					m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel)
 				})
-				m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew)
-				m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.NotebookCreate)
+				m.Get("/create", reqRepoCloudBrainWriter, repo.TrainJobNew)
+				m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate)
 			})
 		}, context.RepoRef())