|
|
@@ -151,12 +151,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm |
|
|
imageId := form.ImageId |
|
|
imageId := form.ImageId |
|
|
repo := ctx.Repo.Repository |
|
|
repo := ctx.Repo.Repository |
|
|
|
|
|
|
|
|
typeCloudbrain := models.TypeCloudBrainTwo |
|
|
|
|
|
if setting.ModelartsCD.Enabled { |
|
|
|
|
|
typeCloudbrain = models.TypeCDCenter |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID, typeCloudbrain) |
|
|
|
|
|
|
|
|
count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) |
|
|
log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) |
|
|
notebookNewDataPrepare(ctx) |
|
|
notebookNewDataPrepare(ctx) |
|
|
@@ -381,7 +376,7 @@ func NotebookRestart(ctx *context.Context) { |
|
|
break |
|
|
break |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID, task.Type) |
|
|
|
|
|
|
|
|
count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) |
|
|
if err != nil { |
|
|
if err != nil { |
|
|
log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) |
|
|
log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) |
|
|
errorMsg = "system error" |
|
|
errorMsg = "system error" |
|
|
@@ -713,7 +708,7 @@ func trainJobNewDataPrepare(ctx *context.Context) error { |
|
|
ctx.Data["WaitCount"] = waitCount |
|
|
ctx.Data["WaitCount"] = waitCount |
|
|
|
|
|
|
|
|
setMultiNodeIfConfigureMatch(ctx) |
|
|
setMultiNodeIfConfigureMatch(ctx) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return nil |
|
|
return nil |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
@@ -1079,8 +1074,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) |
|
|
VersionCount := modelarts.VersionCountOne |
|
|
VersionCount := modelarts.VersionCountOne |
|
|
EngineName := form.EngineName |
|
|
EngineName := form.EngineName |
|
|
|
|
|
|
|
|
errStr:=checkMultiNode(ctx.User.ID,form.WorkServerNumber) |
|
|
|
|
|
if errStr!=""{ |
|
|
|
|
|
|
|
|
errStr := checkMultiNode(ctx.User.ID, form.WorkServerNumber) |
|
|
|
|
|
if errStr != "" { |
|
|
trainJobErrorNewDataPrepare(ctx, form) |
|
|
trainJobErrorNewDataPrepare(ctx, form) |
|
|
ctx.RenderWithErr(ctx.Tr(errStr), tplModelArtsTrainJobNew, &form) |
|
|
ctx.RenderWithErr(ctx.Tr(errStr), tplModelArtsTrainJobNew, &form) |
|
|
return |
|
|
return |
|
|
@@ -1320,31 +1315,31 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) |
|
|
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") |
|
|
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
func checkMultiNode(userId int64, serverNum int) string{ |
|
|
|
|
|
if serverNum==1{ |
|
|
|
|
|
|
|
|
func checkMultiNode(userId int64, serverNum int) string { |
|
|
|
|
|
if serverNum == 1 { |
|
|
return "" |
|
|
return "" |
|
|
} |
|
|
} |
|
|
modelarts.InitMultiNode() |
|
|
modelarts.InitMultiNode() |
|
|
var isServerNumValid=false |
|
|
|
|
|
|
|
|
var isServerNumValid = false |
|
|
if modelarts.MultiNodeConfig != nil { |
|
|
if modelarts.MultiNodeConfig != nil { |
|
|
for _, info := range modelarts.MultiNodeConfig.Info { |
|
|
for _, info := range modelarts.MultiNodeConfig.Info { |
|
|
if isInOrg, _ := models.IsOrganizationMemberByOrgName(info.Org, userId); isInOrg { |
|
|
if isInOrg, _ := models.IsOrganizationMemberByOrgName(info.Org, userId); isInOrg { |
|
|
if isInNodes(info.Node,serverNum){ |
|
|
|
|
|
isServerNumValid=true |
|
|
|
|
|
|
|
|
if isInNodes(info.Node, serverNum) { |
|
|
|
|
|
isServerNumValid = true |
|
|
break |
|
|
break |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
if isServerNumValid{ |
|
|
|
|
|
|
|
|
if isServerNumValid { |
|
|
return "" |
|
|
return "" |
|
|
}else{ |
|
|
|
|
|
|
|
|
} else { |
|
|
return "repo.modelarts.no_node_right" |
|
|
return "repo.modelarts.no_node_right" |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
func checkInferenceJobMultiNode(userId int64, serverNum int) string{ |
|
|
|
|
|
if serverNum==1{ |
|
|
|
|
|
|
|
|
func checkInferenceJobMultiNode(userId int64, serverNum int) string { |
|
|
|
|
|
if serverNum == 1 { |
|
|
return "" |
|
|
return "" |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
@@ -1353,8 +1348,8 @@ func checkInferenceJobMultiNode(userId int64, serverNum int) string{ |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
func isInNodes(nodes []int, num int) bool { |
|
|
func isInNodes(nodes []int, num int) bool { |
|
|
for _, node:=range nodes{ |
|
|
|
|
|
if node==num{ |
|
|
|
|
|
|
|
|
for _, node := range nodes { |
|
|
|
|
|
if node == num { |
|
|
return true |
|
|
return true |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
@@ -1396,8 +1391,8 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ |
|
|
ctx.Data["PageIsTrainJob"] = true |
|
|
ctx.Data["PageIsTrainJob"] = true |
|
|
var jobID = ctx.Params(":jobid") |
|
|
var jobID = ctx.Params(":jobid") |
|
|
|
|
|
|
|
|
errStr:=checkMultiNode(ctx.User.ID,form.WorkServerNumber) |
|
|
|
|
|
if errStr!=""{ |
|
|
|
|
|
|
|
|
errStr := checkMultiNode(ctx.User.ID, form.WorkServerNumber) |
|
|
|
|
|
if errStr != "" { |
|
|
versionErrorDataPrepare(ctx, form) |
|
|
versionErrorDataPrepare(ctx, form) |
|
|
ctx.RenderWithErr(ctx.Tr(errStr), tplModelArtsTrainJobVersionNew, &form) |
|
|
ctx.RenderWithErr(ctx.Tr(errStr), tplModelArtsTrainJobVersionNew, &form) |
|
|
return |
|
|
return |
|
|
@@ -1738,7 +1733,7 @@ func paramCheckCreateTrainJob(form auth.CreateModelArtsTrainJobForm) error { |
|
|
log.Error("the boot file(%s) must be a python file", strings.TrimSpace(form.BootFile)) |
|
|
log.Error("the boot file(%s) must be a python file", strings.TrimSpace(form.BootFile)) |
|
|
return errors.New("启动文件必须是python文件") |
|
|
return errors.New("启动文件必须是python文件") |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if form.BranchName == "" { |
|
|
if form.BranchName == "" { |
|
|
log.Error("the branch must not be null!", form.BranchName) |
|
|
log.Error("the branch must not be null!", form.BranchName) |
|
|
return errors.New("代码分支不能为空!") |
|
|
return errors.New("代码分支不能为空!") |
|
|
@@ -2037,8 +2032,8 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference |
|
|
ckptUrl := "/" + form.TrainUrl + form.CkptName |
|
|
ckptUrl := "/" + form.TrainUrl + form.CkptName |
|
|
log.Info("ckpt url:" + ckptUrl) |
|
|
log.Info("ckpt url:" + ckptUrl) |
|
|
|
|
|
|
|
|
errStr:=checkInferenceJobMultiNode(ctx.User.ID,form.WorkServerNumber) |
|
|
|
|
|
if errStr!=""{ |
|
|
|
|
|
|
|
|
errStr := checkInferenceJobMultiNode(ctx.User.ID, form.WorkServerNumber) |
|
|
|
|
|
if errStr != "" { |
|
|
inferenceJobErrorNewDataPrepare(ctx, form) |
|
|
inferenceJobErrorNewDataPrepare(ctx, form) |
|
|
ctx.RenderWithErr(ctx.Tr(errStr), tplModelArtsInferenceJobNew, &form) |
|
|
ctx.RenderWithErr(ctx.Tr(errStr), tplModelArtsInferenceJobNew, &form) |
|
|
return |
|
|
return |
|
|
|