|
|
|
@@ -4,6 +4,9 @@ import ( |
|
|
|
"fmt" |
|
|
|
"net/http" |
|
|
|
"path" |
|
|
|
"strings" |
|
|
|
|
|
|
|
"code.gitea.io/gitea/modules/notebook" |
|
|
|
|
|
|
|
"code.gitea.io/gitea/modules/modelarts" |
|
|
|
"code.gitea.io/gitea/modules/modelarts_cd" |
|
|
|
@@ -29,6 +32,9 @@ import ( |
|
|
|
) |
|
|
|
|
|
|
|
const NoteBookExtension = ".ipynb" |
|
|
|
const CPUType = 0 |
|
|
|
const GPUType = 1 |
|
|
|
const NPUType = 2 |
|
|
|
|
|
|
|
func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { |
|
|
|
|
|
|
|
@@ -66,7 +72,7 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp |
|
|
|
} |
|
|
|
|
|
|
|
//create repo if not exist |
|
|
|
repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) |
|
|
|
repo, _ := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) |
|
|
|
if repo == nil { |
|
|
|
repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{ |
|
|
|
Name: setting.FileNoteBook.ProjectName, |
|
|
|
@@ -80,17 +86,200 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp |
|
|
|
AutoInit: true, |
|
|
|
DefaultBranch: "master", |
|
|
|
}) |
|
|
|
if err != nil { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) |
|
|
|
return |
|
|
|
} |
|
|
|
} else { |
|
|
|
|
|
|
|
noteBook, _ := models.GetWaitOrRunFileNotebookByRepo(repo.ID, getCloudbrainType(option.Type)) |
|
|
|
if noteBook != nil { |
|
|
|
|
|
|
|
if isRepoConfilcts(option, noteBook) { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_repo_conflict"))) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
if isNotebookSpecMath(option, noteBook) { |
|
|
|
|
|
|
|
err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName) |
|
|
|
if err != nil { |
|
|
|
log.Error("download code failed", err) |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) |
|
|
|
return |
|
|
|
} |
|
|
|
noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName) |
|
|
|
noteBook.BranchName += ";" + option.BranchName |
|
|
|
err := models.UpdateJob(noteBook) |
|
|
|
if err != nil { |
|
|
|
log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
ctx.JSON(http.StatusOK, models.BaseMessageApi{ |
|
|
|
Code: 0, |
|
|
|
Message: noteBook.JobID, |
|
|
|
}) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if option.Type <= GPUType { |
|
|
|
cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) |
|
|
|
} else { |
|
|
|
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOption) { |
|
|
|
if ctx.Written() { |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
if path.Ext(option.File) != NoteBookExtension { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong"))) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
isNotebookFileExist, _ := isNoteBookFileExist(ctx, option) |
|
|
|
if !isNotebookFileExist { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
task, err := models.GetCloudbrainByJobID(option.JobId) |
|
|
|
if err != nil { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) |
|
|
|
log.Error("job not found:"+option.JobId, err) |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Job id may not be right. can not find job.")) |
|
|
|
return |
|
|
|
} |
|
|
|
if option.Type <= 1 { |
|
|
|
cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) |
|
|
|
if task.BootFile == "" || task.Status != string(models.ModelArtsRunning) { |
|
|
|
log.Warn("Boot file is empty or status is running. ") |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Boot file is empty or status is running.")) |
|
|
|
return |
|
|
|
} |
|
|
|
if !isRepoFileMatch(option, task) { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("can not math repo file.")) |
|
|
|
return |
|
|
|
} |
|
|
|
debugBaseUrl, token, err := getBaseUrlAndToken(task) |
|
|
|
if err != nil { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName), task, token) { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseOKMessageApi) |
|
|
|
} else { |
|
|
|
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("upload failed.")) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func getBaseUrlAndToken(task *models.Cloudbrain) (string, string, error) { |
|
|
|
var debugBaseUrl string |
|
|
|
var token string |
|
|
|
if task.Type == models.TypeCloudBrainOne { |
|
|
|
debugBaseUrl = setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName + "/lab" |
|
|
|
|
|
|
|
} else { |
|
|
|
var result *models.GetNotebook2Result |
|
|
|
var err error |
|
|
|
if task.Type == models.TypeCloudBrainTwo { |
|
|
|
result, err = modelarts.GetNotebook2(task.JobID) |
|
|
|
} else if task.Type == models.TypeCDCenter { |
|
|
|
result, err = modelarts_cd.GetNotebook(task.JobID) |
|
|
|
} |
|
|
|
if err != nil || result == nil || result.Status != string(models.ModelArtsRunning) || result.Url == "" { |
|
|
|
log.Error("notebook job not found:"+task.JobID, err) |
|
|
|
return "", "", fmt.Errorf("can not get job or job is invalid.") |
|
|
|
} |
|
|
|
|
|
|
|
debugBaseUrl = result.Url |
|
|
|
token = result.Token |
|
|
|
|
|
|
|
} |
|
|
|
return debugBaseUrl, token, nil |
|
|
|
} |
|
|
|
|
|
|
|
func uploadNotebookFileIfCannotBroswer(debugBaseUrl string, bootFile string, task *models.Cloudbrain, token string) bool { |
|
|
|
c := ¬ebook.NotebookContent{ |
|
|
|
Url: debugBaseUrl, |
|
|
|
Path: bootFile, |
|
|
|
PathType: "file", |
|
|
|
Token: token, |
|
|
|
} |
|
|
|
if c.IsNotebookFileCanBrowser() { |
|
|
|
return true |
|
|
|
} else { |
|
|
|
c.SetCookiesAndCsrf() |
|
|
|
c.UploadNoteBookFile(task) |
|
|
|
return c.IsNotebookFileCanBrowser() |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func isNotebookSpecMath(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { |
|
|
|
if option.Type == NPUType || option.Type == CPUType { |
|
|
|
return true |
|
|
|
} |
|
|
|
spec, err := models.GetCloudbrainSpecByID(book.ID) |
|
|
|
if err != nil { |
|
|
|
log.Warn("can not get spec ", err) |
|
|
|
return false |
|
|
|
} |
|
|
|
return spec.AccCardsNum > 0 |
|
|
|
} |
|
|
|
|
|
|
|
func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { |
|
|
|
bootFiles := strings.Split(book.BootFile, ";") |
|
|
|
branches := strings.Split(book.BranchName, ";") |
|
|
|
|
|
|
|
for i, bootFile := range bootFiles { |
|
|
|
splits := strings.Split(bootFile, "/") |
|
|
|
if len(splits) >= 3 { |
|
|
|
if splits[0] == option.OwnerName && splits[1] == option.ProjectName && branches[i] != option.BranchName { |
|
|
|
return true |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
return false |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { |
|
|
|
bootFiles := strings.Split(book.BootFile, ";") |
|
|
|
branches := strings.Split(book.BranchName, ";") |
|
|
|
|
|
|
|
for i, bootFile := range bootFiles { |
|
|
|
if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName) == bootFile { |
|
|
|
return true |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
return false |
|
|
|
|
|
|
|
} |
|
|
|
func UploadNotebookFiles(task *models.Cloudbrain) { |
|
|
|
if task.Status == string(models.JobRunning) && task.BootFile != "" { |
|
|
|
|
|
|
|
debugBaseUrl, token, err := getBaseUrlAndToken(task) |
|
|
|
if err != nil { |
|
|
|
log.Error("can not get base url:", err) |
|
|
|
return |
|
|
|
} |
|
|
|
bootFiles := strings.Split(task.BootFile, ";") |
|
|
|
|
|
|
|
for _, bootFile := range bootFiles { |
|
|
|
uploadNotebookFileIfCannotBroswer(debugBaseUrl, bootFile, task, token) |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { |
|
|
|
@@ -131,17 +320,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot |
|
|
|
} else { |
|
|
|
if count >= 1 { |
|
|
|
log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) |
|
|
|
ctx.JSON(http.StatusOK,models.BaseMessageApi{ |
|
|
|
Code: 2, |
|
|
|
ctx.JSON(http.StatusOK, models.BaseMessageApi{ |
|
|
|
Code: 2, |
|
|
|
Message: ctx.Tr("repo.cloudbrain.morethanonejob"), |
|
|
|
}) |
|
|
|
return |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName) |
|
|
|
if errStr != "" { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) |
|
|
|
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) |
|
|
|
if err != nil { |
|
|
|
log.Error("download code failed", err) |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) |
|
|
|
return |
|
|
|
} |
|
|
|
command := cloudbrain.GetCloudbrainDebugCommand() |
|
|
|
@@ -185,7 +375,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot |
|
|
|
JobType: jobType, |
|
|
|
Description: getDescription(option), |
|
|
|
BranchName: option.BranchName, |
|
|
|
BootFile: option.File, |
|
|
|
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName), |
|
|
|
Params: "{\"parameter\":[]}", |
|
|
|
CommitID: "", |
|
|
|
BenchmarkTypeID: 0, |
|
|
|
@@ -206,8 +396,15 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func getCodePath(jobName string) string { |
|
|
|
return setting.JobPath + jobName + cloudbrain.CodeMountPath |
|
|
|
func getCloudbrainType(optionType int) int { |
|
|
|
if optionType < 1 { |
|
|
|
return models.TypeCloudBrainOne |
|
|
|
} |
|
|
|
return models.TypeCloudBrainTwo |
|
|
|
} |
|
|
|
|
|
|
|
func getCodePath(jobName string, repo *models.Repository) string { |
|
|
|
return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name |
|
|
|
} |
|
|
|
|
|
|
|
func getDescription(option api.CreateFileNotebookJobOption) string { |
|
|
|
@@ -237,8 +434,8 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote |
|
|
|
} else { |
|
|
|
if count >= 1 { |
|
|
|
log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) |
|
|
|
ctx.JSON(http.StatusOK,models.BaseMessageApi{ |
|
|
|
Code: 2, |
|
|
|
ctx.JSON(http.StatusOK, models.BaseMessageApi{ |
|
|
|
Code: 2, |
|
|
|
Message: ctx.Tr("repo.cloudbrain.morethanonejob"), |
|
|
|
}) |
|
|
|
return |
|
|
|
@@ -260,7 +457,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName) |
|
|
|
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) |
|
|
|
if err != nil { |
|
|
|
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) |
|
|
|
return |
|
|
|
@@ -299,6 +496,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote |
|
|
|
Spec: spec, |
|
|
|
BootFile: "", |
|
|
|
AutoStopDurationMs: modelarts.AutoStopDurationMs / 4, |
|
|
|
BranchName: option.BranchName, |
|
|
|
} |
|
|
|
|
|
|
|
if setting.ModelartsCD.Enabled { |
|
|
|
@@ -347,17 +545,8 @@ func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobO |
|
|
|
return true, nil |
|
|
|
} |
|
|
|
|
|
|
|
func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string { |
|
|
|
err := downloadCode(repo, codePath, branchName) |
|
|
|
if err != nil { |
|
|
|
return "cloudbrain.load_code_failed" |
|
|
|
} |
|
|
|
|
|
|
|
err = uploadOneFileToMinio(codePath, filePath, jobName, cloudbrain.CodeMountPath+"/") |
|
|
|
if err != nil { |
|
|
|
return "cloudbrain.load_code_failed" |
|
|
|
} |
|
|
|
return "" |
|
|
|
func getBootFile(filePath string, ownerName string, projectName string) string { |
|
|
|
return ownerName + "/" + projectName + "/" + filePath |
|
|
|
} |
|
|
|
|
|
|
|
func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) { |
|
|
|
|