Browse Source

Merge pull request 'trainjob' (#628) from liuzx_trainjob into V20211101

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/628
Reviewed-by: lewis <747342561@qq.com>
tags/v1.21.12.1
lewis 3 years ago
parent
commit
91fd792787
36 changed files with 4804 additions and 92 deletions
  1. +1
    -0
      custom/conf/app.ini.sample
  2. +1
    -1
      models/attachment.go
  3. +295
    -17
      models/cloudbrain.go
  4. +4
    -1
      models/file_chunk.go
  5. +10
    -9
      models/repo.go
  6. +29
    -0
      modules/auth/modelarts.go
  7. +2
    -1
      modules/cloudbrain/cloudbrain.go
  8. +180
    -4
      modules/modelarts/modelarts.go
  9. +485
    -4
      modules/modelarts/resty.go
  10. +26
    -9
      modules/setting/setting.go
  11. +74
    -0
      modules/storage/obs.go
  12. +47
    -0
      options/locale/locale_en-US.ini
  13. +1
    -0
      options/locale/locale_zh-CN.ini
  14. +28
    -0
      public/self/test.js
  15. +9
    -1
      routers/api/v1/api.go
  16. +80
    -1
      routers/api/v1/repo/modelarts.go
  17. +1
    -1
      routers/repo/attachment.go
  18. +77
    -10
      routers/repo/cloudbrain.go
  19. +846
    -3
      routers/repo/modelarts.go
  20. +35
    -8
      routers/routes/routes.go
  21. +25
    -1
      templates/repo/cloudbrain/index.tmpl
  22. +6
    -4
      templates/repo/header.tmpl
  23. +21
    -16
      templates/repo/modelarts/index.tmpl
  24. +43
    -0
      templates/repo/modelarts/navbar.tmpl
  25. +498
    -0
      templates/repo/modelarts/notebook/index.tmpl
  26. +240
    -0
      templates/repo/modelarts/notebook/new.tmpl
  27. +122
    -0
      templates/repo/modelarts/notebook/show.tmpl
  28. +239
    -0
      templates/repo/modelarts/trainjob/edit_para.tmpl
  29. +534
    -0
      templates/repo/modelarts/trainjob/index.tmpl
  30. +47
    -0
      templates/repo/modelarts/trainjob/list_model.tmpl
  31. +440
    -0
      templates/repo/modelarts/trainjob/new.tmpl
  32. +154
    -0
      templates/repo/modelarts/trainjob/para_manage.tmpl
  33. +200
    -0
      templates/repo/modelarts/trainjob/show.tmpl
  34. +1
    -0
      vendor/modules.txt
  35. +3
    -1
      web_src/less/openi.less
  36. +0
    -0
      web_src/less/themes/theme-arc-green.less

+ 1
- 0
custom/conf/app.ini.sample View File

@@ -1096,6 +1096,7 @@ LOCATION = cn-south-222
BASE_PATH = attachment/

[modelarts]
ORGANIZATION = modelarts
ENDPOINT = https://modelarts.cn-south-222.ai.pcl.cn
PROJECT_ID = edfccf24aace4e17a56da6bcbb55a5aa
PROJECT_NAME = cn-south-222_test


+ 1
- 1
models/attachment.go View File

@@ -429,7 +429,7 @@ func GetAllUserAttachments(userID int64) ([]*AttachmentUsername, error) {
func getModelArtsUserAttachments(e Engine, userID int64) ([]*AttachmentUsername, error) {
attachments := make([]*AttachmentUsername, 0, 10)
if err := e.Table("attachment").Join("LEFT", "`user`", "attachment.uploader_id "+
"= `user`.id").Where("attachment.type = ? and (uploader_id= ? or is_private = ?)", TypeCloudBrainTwo, userID, false).Find(&attachments); err != nil {
"= `user`.id").Where("attachment.type = ? and (uploader_id= ? or is_private = ?)", TypeCloudBrainNotebook, userID, false).Find(&attachments); err != nil {
return nil, err
}
return attachments, nil


+ 295
- 17
models/cloudbrain.go View File

@@ -5,6 +5,7 @@ import (
"fmt"
"strings"
"time"

"xorm.io/builder"
"xorm.io/xorm"

@@ -47,22 +48,28 @@ const (
)

type Cloudbrain struct {
ID int64 `xorm:"pk autoincr"`
JobID string `xorm:"INDEX NOT NULL"`
JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"`
JobName string `xorm:"INDEX"`
Status string `xorm:"INDEX"`
UserID int64 `xorm:"INDEX"`
RepoID int64 `xorm:"INDEX"`
SubTaskName string `xorm:"INDEX"`
ContainerID string
ContainerIp string
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
DeletedAt time.Time `xorm:"deleted"`
CanDebug bool `xorm:"-"`
CanDel bool `xorm:"-"`
Type int `xorm:"INDEX DEFAULT 0"`
ID int64 `xorm:"pk autoincr"`
JobID string `xorm:"INDEX NOT NULL"`
JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"`
JobName string `xorm:"INDEX"`
Status string `xorm:"INDEX"`
UserID int64 `xorm:"INDEX"`
RepoID int64 `xorm:"INDEX"`
SubTaskName string `xorm:"INDEX"`
ContainerID string
ContainerIp string
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
Duration int `xorm:"INDEX duration"`
TrainJobDuration string
DeletedAt time.Time `xorm:"deleted"`
CanDebug bool `xorm:"-"`
CanDel bool `xorm:"-"`
Type int `xorm:"INDEX DEFAULT 0"`

VersionID int64 `xorm:"INDEX DEFAULT 0"`
VersionName string
Uuid string

User *User `xorm:"-"`
Repo *Repository `xorm:"-"`
@@ -555,6 +562,260 @@ type NotebookDelResult struct {
InstanceID string `json:"instance_id"`
}

type CreateTrainJobParams struct {
JobName string `json:"job_name"`
Description string `json:"job_desc"`
Config Config `json:"config"`
WorkspaceID string `json:"workspace_id"`
}

type Config struct {
WorkServerNum int `json:"worker_server_num"`
AppUrl string `json:"app_url"` //训练作业的代码目录
BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件,需要在代码目录下
Parameter []Parameter `json:"parameter"`
DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL
//DatasetID string `json:"dataset_id"`
//DataVersionID string `json:"dataset_version_id"`
//DataSource []DataSource `json:"data_source"`
//SpecID int64 `json:"spec_id"`
EngineID int64 `json:"engine_id"`
//ModelID int64 `json:"model_id"`
TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL
LogUrl string `json:"log_url"`
//UserImageUrl string `json:"user_image_url"`
//UserCommand string `json:"user_command"`
CreateVersion bool `json:"create_version"`
//Volumes []Volumes `json:"volumes"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
}

type CreateConfigParams struct {
ConfigName string `json:"config_name"`
Description string `json:"config_desc"`
WorkServerNum int `json:"worker_server_num"`
AppUrl string `json:"app_url"` //训练作业的代码目录
BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件,需要在代码目录下
Parameter []Parameter `json:"parameter"`
DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL
//DatasetID string `json:"dataset_id"`
//DataVersionID string `json:"dataset_version_id"`
//DataSource []DataSource `json:"data_source"`
//SpecID int64 `json:"spec_id"`
EngineID int64 `json:"engine_id"`
//ModelID int64 `json:"model_id"`
TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL
LogUrl string `json:"log_url"`
//UserImageUrl string `json:"user_image_url"`
//UserCommand string `json:"user_command"`
//CreateVersion bool `json:"create_version"`
//Volumes []Volumes `json:"volumes"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
}

type Parameter struct {
Label string `json:"label"`
Value string `json:"value"`
}

type Parameters struct {
Parameter []Parameter `json:"parameter"`
}

type DataSource struct {
DatasetID string `json:"dataset_id"`
DatasetVersion string `json:"dataset_version"`
Type string `json:"type"`
DataUrl string `json:"data_url"`
}

type Volumes struct {
Nfs Nfs `json:"nfs"`
HostPath HostPath `json:"host_path"`
}

type Nfs struct {
ID string `json:"id"`
SourcePath string `json:"src_path"`
DestPath string `json:"dest_path"`
ReadOnly bool `json:"read_only"`
}

type HostPath struct {
SourcePath string `json:"src_path"`
DestPath string `json:"dest_path"`
ReadOnly bool `json:"read_only"`
}

type Flavor struct {
Code string `json:"code"`
}

type CreateTrainJobResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
JobName string `json:"job_name"`
JobID int64 `json:"job_id"`
Status int `json:"status"`
CreateTime int64 `json:"create_time"`
VersionID int64 `json:"version_id"`
ResourceID string `json:"resource_id"`
VersionName string `json:"version_name"`
}

type CreateTrainJobConfigResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
}

type GetResourceSpecsResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
SpecTotalCount int `json:"spec_total_count"`
Specs []Specs `json:"specs"`
}

type Specs struct {
Core string `json:"core"`
Cpu string `json:"cpu"`
IsNoResource bool `json:"no_resource"`
GpuType string `json:"gpu_type"`
SpecID int64 `json:"spec_id"`
GpuNum int `json:"gpu_num"`
SpecCode string `json:"spec_code"`
Storage string `json:"storage"`
MaxNum int `json:"max_num"`
UnitNum int `json:"unit_num"`
InterfaceType int `json:"interface_type"`
}

type GetConfigListResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
ConfigTotalCount int `json:"config_total_count"`
ParaConfigs []ParaConfig `json:"configs"`
}

type ParaConfig struct {
ConfigName string `json:"config_name"`
ConfigDesc string `json:"config_desc"`
CreateTime int64 `json:"create_time"`
EngineType int `json:"engine_type"`
EngineName string `json:"engine_name"`
EngineId int64 `json:"engine_id"`
EngineVersion string `json:"engine_version"`
UserImageUrl string `json:"user_image_url"`
UserCommand string `json:"user_command"`
Result GetConfigResult
}

type GetConfigResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
ConfigName string `json:"config_name"`
Description string `json:"config_desc"`
WorkServerNum int `json:"worker_server_num"`
AppUrl string `json:"app_url"` //训练作业的代码目录
BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件,需要在代码目录下
Parameter []Parameter `json:"parameter"`
DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL
//DatasetID string `json:"dataset_id"`
//DataVersionID string `json:"dataset_version_id"`
//DataSource []DataSource `json:"data_source"`
//SpecID int64 `json:"spec_id"`
EngineID int64 `json:"engine_id"`
//ModelID int64 `json:"model_id"`
TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL
LogUrl string `json:"log_url"`
//UserImageUrl string `json:"user_image_url"`
//UserCommand string `json:"user_command"`
//CreateVersion bool `json:"create_version"`
//Volumes []Volumes `json:"volumes"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
}

type ErrorResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_message"`
IsSuccess bool `json:"is_success"`
}

type GetTrainJobResult struct {
IsSuccess bool `json:"is_success"`
JobName string `json:"job_name"`
JobID int64 `json:"job_id"`
Description string `json:"job_desc"`
IntStatus int `json:"status"`
Status string
LongCreateTime int64 `json:"create_time"`
CreateTime string
Duration int64 `json:"duration"` //训练作业的运行时间,单位为毫秒
TrainJobDuration string //训练作业的运行时间,格式为hh:mm:ss
VersionID int64 `json:"version_id"`
ResourceID string `json:"resource_id"`
VersionName string `json:"version_name"`
PreVersionID int64 `json:"pre_version_id"`
WorkServerNum int `json:"worker_server_num"`
AppUrl string `json:"app_url"` //训练作业的代码目录
BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件,需要在代码目录下
Parameter []Parameter `json:"parameter"`
DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL
//DatasetID string `json:"dataset_id"`
//DataVersionID string `json:"dataset_version_id"`
//DataSource []DataSource `json:"data_source"`
//SpecID int64 `json:"spec_id"`
EngineID int64 `json:"engine_id"`
EngineName string `json:"engine_name"`
EngineVersion string `json:"engine_version"`
//ModelID int64 `json:"model_id"`
TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL
LogUrl string `json:"log_url"`
//UserImageUrl string `json:"user_image_url"`
//UserCommand string `json:"user_command"`
//Volumes []Volumes `json:"volumes"`
Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"`
PoolName string `json:"pool_name"`
NasMountPath string `json:"nas_mount_path"`
NasShareAddr string `json:"nas_share_addr"`
DatasetName string
}

type GetTrainJobLogResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
Content string `json:"content"`
Lines int `json:"lines"`
StartLine string `json:"start_line"`
EndLine string `json:"end_line"`
}

type GetTrainJobLogFileNamesResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
LogFileList []string `json:"log_file_list"`
}

type TrainJobResult struct {
ErrorCode string `json:"error_code"`
ErrorMsg string `json:"error_msg"`
IsSuccess bool `json:"is_success"`
}

type LogFile struct {
Name string
}

func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) {
sess := x.NewSession()
defer sess.Close()
@@ -672,6 +933,12 @@ func SetCloudbrainStatusByJobID(jobID string, status CloudbrainStatus) (err erro
return
}

func SetTrainJobStatusByJobID(jobID string, status string, duration int, trainjobduration string) (err error) {
cb := &Cloudbrain{JobID: jobID, Status: string(status), Duration: duration, TrainJobDuration: trainjobduration}
_, err = x.Cols("status", "duration", "train_job_duration").Where("cloudbrain.job_id=?", jobID).Update(cb)
return
}

func UpdateJob(job *Cloudbrain) error {
return updateJob(x, job)
}
@@ -683,6 +950,17 @@ func updateJob(e Engine, job *Cloudbrain) error {
return err
}

// func UpdateTrainJob(job *CloudbrainInfo) error {
// return updateTrainJob(x, job)
// }

// func updateTrainJob(e Engine, job *CloudbrainInfo) error {
// var sess *xorm.Session
// sess = e.Where("job_id = ?", job.Cloudbrain.JobID)
// _, err := sess.Cols("status", "container_id", "container_ip").Update(job)
// return err
// }

func DeleteJob(job *Cloudbrain) error {
return deleteJob(x, job)
}
@@ -698,7 +976,7 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) {
}

func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool {
if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)){
if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) {
return false
}
repo, err := GetRepositoryByID(job.RepoID)


+ 4
- 1
models/file_chunk.go View File

@@ -14,7 +14,10 @@ const (
)

const (
TypeCloudBrainOne = 0
TypeCloudBrainOne = 0
TypeCloudBrainNotebook = 1
TypeCloudBrainTrainJob = 2

TypeCloudBrainTwo = 1
)



+ 10
- 9
models/repo.go View File

@@ -6,13 +6,14 @@
package models

import (
"code.gitea.io/gitea/modules/blockchain"
"context"
"crypto/md5"
"errors"
"fmt"
"html/template"

"code.gitea.io/gitea/modules/blockchain"

// Needed for jpeg support
_ "image/jpeg"
"image/png"
@@ -171,11 +172,11 @@ type Repository struct {
NumOpenIssues int `xorm:"-"`
NumPulls int
NumClosedPulls int
NumOpenPulls int `xorm:"-"`
NumMilestones int `xorm:"NOT NULL DEFAULT 0"`
NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"`
NumOpenMilestones int `xorm:"-"`
NumCommit int64 `xorm:"NOT NULL DEFAULT 0"`
NumOpenPulls int `xorm:"-"`
NumMilestones int `xorm:"NOT NULL DEFAULT 0"`
NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"`
NumOpenMilestones int `xorm:"-"`
NumCommit int64 `xorm:"NOT NULL DEFAULT 0"`

IsPrivate bool `xorm:"INDEX"`
IsEmpty bool `xorm:"INDEX"`
@@ -215,8 +216,8 @@ type Repository struct {
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`

Hot int64 `xorm:"-"`
Active int64 `xorm:"-"`
Hot int64 `xorm:"-"`
Active int64 `xorm:"-"`
}

// SanitizedOriginalURL returns a sanitized OriginalURL
@@ -2473,7 +2474,7 @@ func (repo *Repository) IncreaseCloneCnt() {
}

func UpdateRepositoryCommitNum(repo *Repository) error {
if _,err := x.Exec("UPDATE `repository` SET num_commit = ? where id = ?", repo.NumCommit, repo.ID); err != nil {
if _, err := x.Exec("UPDATE `repository` SET num_commit = ? where id = ?", repo.NumCommit, repo.ID); err != nil {
return err
}



+ 29
- 0
modules/auth/modelarts.go View File

@@ -14,3 +14,32 @@ type CreateModelArtsForm struct {
func (f *CreateModelArtsForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
return validate(errs, ctx.Data, f, ctx.Locale)
}

type CreateModelArtsNotebookForm struct {
JobName string `form:"job_name" binding:"Required"`
Attachment string `form:"attachment"`
Description string `form:"description"`
}

func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
return validate(errs, ctx.Data, f, ctx.Locale)
}

type CreateModelArtsTrainJobForm struct {
JobName string `form:"job_name" binding:"Required"`
Attachment string `form:"attachment" binding:"Required"`
BootFile string `form:"boot_file" binding:"Required"`
WorkServerNumber int `form:"work_server_number" binding:"Required"`
EngineID int `form:"engine_id" binding:"Required"`
PoolID string `form:"pool_id" binding:"Required"`
Flavor string `form:"flavor" binding:"Required"`
Params string `form:"run_para_list" binding:"Required"`
Description string `form:"description"`
IsSaveParam string `form:"is_save_para"`
ParameterTemplateName string `form:"parameter_template_name"`
PrameterDescription string `form:"parameter_description"`
}

func (f *CreateModelArtsTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
return validate(errs, ctx.Data, f, ctx.Locale)
}

+ 2
- 1
modules/cloudbrain/cloudbrain.go View File

@@ -131,7 +131,8 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath,
JobName: jobName,
SubTaskName: SubTaskName,
JobType: jobType,
Type: models.TypeCloudBrainOne,
Type: models.TypeCloudBrainOne,
Uuid: uuid,
})

if err != nil {


+ 180
- 4
modules/modelarts/modelarts.go View File

@@ -1,22 +1,53 @@
package modelarts

import (
"encoding/json"
"path"
"strconv"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"encoding/json"
"path"
)

const (
//notebook
storageTypeOBS = "obs"
autoStopDuration = 4 * 60 * 60

DataSetMountPath = "/home/ma-user/work"
NotebookEnv = "Python3"
NotebookType = "Ascend"
FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 96GiB (modelarts.kat1.xlarge)"

//train-job
// ResourcePools = "{\"resource_pool\":[{\"id\":\"pool1328035d\", \"value\":\"专属资源池\"}]}"
// Engines = "{\"engine\":[{\"id\":1, \"value\":\"Ascend-Powered-Engine\"}]}"
// EngineVersions = "{\"version\":[{\"id\":118,\"value\":\"MindSpore-1.0.0-c75-python3.7-euleros2.8-aarch64\"}," +
// "{\"id\":119,\"value\":\"MindSpore-1.1.1-c76-python3.7-euleros2.8-aarch64\"}," +
// "{\"id\":120,\"value\":\"MindSpore-1.1.1-c76-tr5-python3.7-euleros2.8-aarch64\"}," +
// "{\"id\":117,\"value\":\"TF-1.15-c75-python3.7-euleros2.8-aarch64\"}" +
// "]}"
// TrainJobFlavorInfo = "{\"flavor\":[{\"code\":\"modelarts.bm.910.arm.public.2\",\"value\":\"Ascend : 2 * Ascend 910 CPU:48 核 512GiB\"}," +
// "{\"code\":\"modelarts.bm.910.arm.public.8\",\"value\":\"Ascend : 8 * Ascend 910 CPU:192 核 2048GiB\"}," +
// "{\"code\":\"modelarts.bm.910.arm.public.4\",\"value\":\"Ascend : 4 * Ascend 910 CPU:96 核 1024GiB\"}," +
// "{\"code\":\"modelarts.bm.910.arm.public.1\",\"value\":\"Ascend : 1 * Ascend 910 CPU:24 核 256GiB\"}" +
// "]}"
CodePath = "/code/"
OutputPath = "/output/"
LogPath = "/log/"
JobPath = "/job/"
OrderDesc = "desc" //向下查询
OrderAsc = "asc" //向上查询
Lines = 20
TrainUrl = "train_url"
DataUrl = "data_url"
PerPage = 10

SortByCreateTime = "create_time"
ConfigTypeCustom = "custom"
)

var (
@@ -24,6 +55,50 @@ var (
FlavorInfos *models.FlavorInfos
)

type GenerateTrainJobReq struct {
JobName string
Uuid string
Description string
CodeObsPath string
BootFile string
DataUrl string
TrainUrl string
FlavorCode string
LogUrl string
PoolID string
WorkServerNumber int
EngineID int64
Parameters []models.Parameter
}

type VersionInfo struct {
Version []struct {
ID int `json:"id"`
Value string `json:"value"`
} `json:"version"`
}

type Flavor struct {
Info []struct {
Code string `json:"code"`
Value string `json:"value"`
} `json:"flavor"`
}

type Engine struct {
Info []struct {
ID int `json:"id"`
Value string `json:"value"`
} `json:"engine"`
}

type ResourcePool struct {
Info []struct {
ID string `json:"id"`
Value string `json:"value"`
} `json:"resource_pool"`
}

func GenerateTask(ctx *context.Context, jobName, uuid, description string) error {
var dataActualPath string
if uuid != "" {
@@ -76,20 +151,121 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description string) error
log.Error("CreateJob failed: %v", err.Error())
return err
}

err = models.CreateCloudbrain(&models.Cloudbrain{

Status: string(models.JobWaiting),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: jobResult.ID,
JobName: jobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeCloudBrainTwo,
Type: models.TypeCloudBrainNotebook,
Uuid: uuid,
})

if err != nil {
return err
}

return nil
}

func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) error {
jobResult, err := createTrainJob(models.CreateTrainJobParams{
JobName: req.JobName,
Description: req.Description,
Config: models.Config{
WorkServerNum: req.WorkServerNumber,
AppUrl: req.CodeObsPath,
BootFileUrl: req.BootFile,
DataUrl: req.DataUrl,
EngineID: req.EngineID,
TrainUrl: req.TrainUrl,
LogUrl: req.LogUrl,
PoolID: req.PoolID,
CreateVersion: true,
Flavor: models.Flavor{
Code: req.FlavorCode,
},
Parameter: req.Parameters,
},
})
if err != nil {
log.Error("CreateJob failed: %v", err.Error())
return err
}

err = models.CreateCloudbrain(&models.Cloudbrain{
Status: TransTrainJobStatus(jobResult.Status),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: strconv.FormatInt(jobResult.JobID, 10),
JobName: req.JobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeCloudBrainTrainJob,
VersionID: jobResult.VersionID,
VersionName: jobResult.VersionName,
Uuid: req.Uuid,
})

if err != nil {
log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error())
return err
}

return nil
}

func TransTrainJobStatus(status int) string {
switch status {
case 0:
return "UNKNOWN"
case 1:
return "CREATING"
case 2:
return "IMAGE_CREATING"
case 3:
return "IMAGE_FAILED"
case 4:
return "SUBMIT_TRYING"
case 5:
return "SUBMIT_FAILED"
case 6:
return "DELETE_FAILED"
case 7:
return "WAITING"
case 8:
return "RUNNING"
case 9:
return "STOPPED"
case 10:
return "COMPLETED"
case 11:
return "FAILED"
case 12:
return "STOPPED"
case 13:
return "CANCELED"
case 14:
return "LOST"
case 15:
return "SCALING"
case 16:
return "SUBMIT_MODEL_FAILED"
case 17:
return "DEPLOY_SERVICE_FAILED"
case 18:
return "CHECK_INIT"
case 19:
return "CHECK_RUNNING"
case 20:
return "CHECK_RUNNING_COMPLETED"
case 21:
return "CHECK_FAILED"

default:
return strconv.Itoa(status)
}

return ""
}

+ 485
- 4
modules/modelarts/resty.go View File

@@ -1,13 +1,14 @@
package modelarts

import (
"code.gitea.io/gitea/modules/log"
"crypto/tls"
"encoding/json"
"fmt"
"net/http"
"strconv"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"github.com/go-resty/resty/v2"
)
@@ -23,6 +24,9 @@ const (

urlGetToken = "/v3/auth/tokens"
urlNotebook = "/demanager/instances"
urlTrainJob = "/training-jobs"
urlResourceSpecs = "/job/resource-specs"
urlTrainJobConfig = "/training-job-configs"
errorCodeExceedLimit = "ModelArts.0118"
)

@@ -104,7 +108,7 @@ sendjob:
Post(HOST + "/v1/" + setting.ProjectID + urlNotebook)

if err != nil {
return nil, fmt.Errorf("resty create job: %s", err)
return nil, fmt.Errorf("resty create notebook: %s", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
@@ -121,11 +125,11 @@ sendjob:
}

if len(response.ErrorCode) != 0 {
log.Error("CreateJob failed(%s): %s", response.ErrorCode, response.ErrorMsg)
log.Error("createNotebook failed(%s): %s", response.ErrorCode, response.ErrorMsg)
if response.ErrorCode == errorCodeExceedLimit {
response.ErrorMsg = "所选规格使用数量已超过最大配额限制。"
}
return &result, fmt.Errorf("CreateJob failed(%s): %s", response.ErrorCode, response.ErrorMsg)
return &result, fmt.Errorf("createNotebook failed(%s): %s", response.ErrorCode, response.ErrorMsg)
}

return &result, nil
@@ -210,6 +214,45 @@ sendjob:
return &result, nil
}

func DelNotebook(jobID string) (*models.NotebookDelResult, error) {
checkSetting()
client := getRestyClient()
var result models.NotebookDelResult

retry := 0

sendjob:
res, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetResult(&result).
Delete(HOST + "/v1/" + setting.ProjectID + urlNotebook + "/" + jobID)

if err != nil {
return &result, fmt.Errorf("resty DelJob: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

var response models.NotebookResult
err = json.Unmarshal(res.Body(), &response)
if err != nil {
log.Error("json.Unmarshal failed: %s", err.Error())
return &result, fmt.Errorf("son.Unmarshal failed: %s", err.Error())
}

if len(response.ErrorCode) != 0 {
log.Error("DelJob failed(%s): %s", response.ErrorCode, response.ErrorMsg)
return &result, fmt.Errorf("DelJob failed(%s): %s", response.ErrorCode, response.ErrorMsg)
}

return &result, nil
}

func DelJob(jobID string) (*models.NotebookDelResult, error) {
checkSetting()
client := getRestyClient()
@@ -287,3 +330,441 @@ sendjob:

return &result, nil
}

func createTrainJob(createJobParams models.CreateTrainJobParams) (*models.CreateTrainJobResult, error) {
checkSetting()
client := getRestyClient()
var result models.CreateTrainJobResult

retry := 0

sendjob:
res, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetBody(createJobParams).
SetResult(&result).
Post(HOST + "/v1/" + setting.ProjectID + urlTrainJob)

if err != nil {
return nil, fmt.Errorf("resty create train-job: %s", err)
}

req, _ := json.Marshal(createJobParams)
log.Info("%s", req)

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("createTrainJob failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("createTrainJob failed(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetResourceSpecs() (*models.GetResourceSpecsResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetResourceSpecsResult

retry := 0

sendjob:
res, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlResourceSpecs)

if err != nil {
return nil, fmt.Errorf("resty GetResourceSpecs: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetResourceSpecs failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("GetResourceSpecs failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetResourceSpecs failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("GetResourceSpecs failed(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func CreateTrainJobConfig(req models.CreateConfigParams) (*models.CreateTrainJobConfigResult, error) {
checkSetting()
client := getRestyClient()
var result models.CreateTrainJobConfigResult

retry := 0

sendjob:
res, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetBody(req).
SetResult(&result).
Post(HOST + "/v1/" + setting.ProjectID + urlTrainJobConfig)

if err != nil {
return nil, fmt.Errorf("resty CreateTrainJobConfig: %s", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

//temp, _ := json.Marshal(req)
//log.Info("%s", temp)

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("CreateTrainJobConfig failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("CreateTrainJobConfig failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("CreateTrainJobConfig failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("CreateTrainJobConfig failed(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetConfigList(perPage, page int, sortBy, order, searchContent, configType string) (*models.GetConfigListResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetConfigListResult

retry := 0

sendjob:
res, err := client.R().
SetQueryParams(map[string]string{
"per_page": strconv.Itoa(perPage),
"page": strconv.Itoa(page),
"sortBy": sortBy,
"order": order,
"search_content": searchContent,
"config_type": configType,
}).
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJobConfig)

if err != nil {
return nil, fmt.Errorf("resty GetConfigList: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetConfigList failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("获取参数配置列表失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetConfigList failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("获取参数配置列表失败(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetParaConfig(configName, configType string) (models.GetConfigResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetConfigResult

retry := 0

sendjob:
res, err := client.R().
SetQueryParams(map[string]string{
"config_type": configType,
}).
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJobConfig + "/" + configName)

if err != nil {
return result, fmt.Errorf("resty GetParaConfig: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetParaConfig failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return result, fmt.Errorf("获取参数配置详情失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetParaConfig failed(%s): %s", result.ErrorCode, result.ErrorMsg)
return result, fmt.Errorf("获取参数配置详情失败(%s): %s", result.ErrorCode, result.ErrorMsg)
}

return result, nil
}

func GetTrainJob(jobID, versionID string) (*models.GetTrainJobResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetTrainJobResult

retry := 0

sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID)

if err != nil {
return nil, fmt.Errorf("resty GetTrainJob: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("获取作业详情失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetTrainJob(%s) failed", jobID)
return &result, fmt.Errorf("获取作业详情失败")
}

return &result, nil
}

func GetTrainJobLog(jobID, versionID, baseLine, logFile, order string, lines int) (*models.GetTrainJobLogResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetTrainJobLogResult

retry := 0

sendjob:
res, err := client.R().
SetQueryParams(map[string]string{
"base_line": baseLine,
"lines": strconv.Itoa(lines),
"log_file": logFile,
"order": order,
}).
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/aom-log")

if err != nil {
return nil, fmt.Errorf("resty GetTrainJobLog: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetTrainJobLog failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("获取作业日志失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetTrainJobLog(%s) failed", jobID)
return &result, fmt.Errorf("获取作业日志失败:%s", result.ErrorMsg)
}

return &result, nil
}

func GetTrainJobLogFileNames(jobID, versionID string) (*models.GetTrainJobLogFileNamesResult, error) {
checkSetting()
client := getRestyClient()
var result models.GetTrainJobLogFileNamesResult

retry := 0

sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/log/file-names")

if err != nil {
return nil, fmt.Errorf("resty GetTrainJobLogFileNames: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("GetTrainJobLogFileNames failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("GetTrainJobLogFileNames failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("GetTrainJobLogFileNames(%s) failed", jobID)
return &result, fmt.Errorf("获取作业日志文件失败:%s", result.ErrorMsg)
}

return &result, nil
}

func DelTrainJob(jobID string) (*models.TrainJobResult, error) {
checkSetting()
client := getRestyClient()
var result models.TrainJobResult

retry := 0

sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Delete(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID)

if err != nil {
return &result, fmt.Errorf("resty DelTrainJob: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("DelTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("删除训练作业失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("DelTrainJob(%s) failed", jobID)
return &result, fmt.Errorf("删除训练作业失败:%s", result.ErrorMsg)
}

return &result, nil
}

func StopTrainJob(jobID, versionID string) (*models.TrainJobResult, error) {
checkSetting()
client := getRestyClient()
var result models.TrainJobResult

retry := 0

sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Post(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/stop")

if err != nil {
return &result, fmt.Errorf("resty StopTrainJob: %v", err)
}

if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
var temp models.ErrorResult
if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
}
log.Error("StopTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
return &result, fmt.Errorf("停止训练作业失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
}

if !result.IsSuccess {
log.Error("StopTrainJob(%s) failed", jobID)
return &result, fmt.Errorf("停止训练作业失败:%s", result.ErrorMsg)
}

return &result, nil
}

+ 26
- 9
modules/setting/setting.go View File

@@ -467,13 +467,16 @@ var (
CommitValidDate string

//obs config
Endpoint string
AccessKeyID string
SecretAccessKey string
Bucket string
Location string
BasePath string
UserBasePath string
Endpoint string
AccessKeyID string
SecretAccessKey string
Bucket string
Location string
BasePath string
OutPutPath string
TrainJobModelPath string
CodePathPrefix string
UserBasePath string

//modelarts config
ModelArtsHost string
@@ -483,10 +486,16 @@ var (
ModelArtsUsername string
ModelArtsPassword string
ModelArtsDomain string
AllowedOrg string
ProfileID string
PoolInfos string
Flavor string
FlavorInfos string
//train-job
ResourcePools string
Engines string
EngineVersions string
FlavorInfos string
TrainJobFLAVORINFOS string

//elk config
ElkUrl string
@@ -1246,6 +1255,9 @@ func NewContext() {
Bucket = sec.Key("BUCKET").MustString("testopendata")
Location = sec.Key("LOCATION").MustString("cn-south-222")
BasePath = sec.Key("BASE_PATH").MustString("attachment/")
TrainJobModelPath = sec.Key("TrainJobModel_Path").MustString("job/")
OutPutPath = sec.Key("Output_Path").MustString("output/")
CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/")
UserBasePath = sec.Key("BASE_PATH_USER").MustString("users/")
PROXYURL = sec.Key("PROXY_URL").MustString("")

@@ -1257,10 +1269,15 @@ func NewContext() {
ModelArtsUsername = sec.Key("USERNAME").MustString("")
ModelArtsPassword = sec.Key("PASSWORD").MustString("")
ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222")
AllowedOrg = sec.Key("ORGANIZATION").MustString("")
ProfileID = sec.Key("PROFILE_ID").MustString("")
PoolInfos = sec.Key("POOL_INFOS").MustString("")
Flavor = sec.Key("FLAVOR").MustString("")
Flavor = sec.Key("FLAVOR").MustString("modelarts.bm.910.arm.public.2")
ResourcePools = sec.Key("Resource_Pools").MustString("")
Engines = sec.Key("Engines").MustString("")
EngineVersions = sec.Key("Engine_Versions").MustString("")
FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("")
TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("")

sec = Cfg.Section("elk")
ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch")


+ 74
- 0
modules/storage/obs.go View File

@@ -134,6 +134,7 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) {
input := &obs.GetObjectInput{}
input.Bucket = setting.Bucket
input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
// input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")
output, err := ObsCli.GetObject(input)
if err == nil {
log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n",
@@ -148,6 +149,56 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) {
}
}

func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) {
input := &obs.GetObjectInput{}
input.Bucket = setting.Bucket
input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, JobName, setting.OutPutPath, fileName), "/")
// input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")
output, err := ObsCli.GetObject(input)
if err == nil {
log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n",
output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified)
return output.Body, nil
} else if obsError, ok := err.(obs.ObsError); ok {
fmt.Printf("Code:%s\n", obsError.Code)
fmt.Printf("Message:%s\n", obsError.Message)
return nil, obsError
} else {
return nil, err
}
}

func GetObsListObject(jobName string) ([]string, error) {
// jobName = "liuzx202110271830856"
input := &obs.ListObjectsInput{}
input.Bucket = setting.Bucket
input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath), "/")
log.Info("input.Prefix:", input.Prefix)
output, err := ObsCli.ListObjects(input)
log.Info("output.Prefix:", output)
ModelListArr := make([]string, 0)
if err == nil {
fmt.Printf("RequestId:%s\n", output.RequestId)
for index, val := range output.Contents {
fmt.Printf("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d\n",
index, val.Owner.ID, val.ETag, val.Key, val.LastModified, val.Size)
str1 := strings.Split(val.Key, "/")
ModelList := str1[len(str1)-1]
ModelListArr = append(ModelListArr, ModelList)
log.Info("ModelListArr.Prefix:", ModelListArr)
}
return ModelListArr, err
} else {
if obsError, ok := err.(obs.ObsError); ok {
fmt.Println(obsError.Code)
fmt.Println(obsError.Message)
} else {
fmt.Println(err)
}
return nil, err
}
}

func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) {

input := &obs.CreateSignedUrlInput{}
@@ -171,6 +222,29 @@ func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, file
return output.SignedUrl, nil
}

func GetObsCreateSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) {

input := &obs.CreateSignedUrlInput{}
input.Bucket = setting.Bucket
input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
input.Expires = 60 * 60
input.Method = obs.HttpMethodPut

input.QueryParams = map[string]string{
"partNumber": com.ToStr(partNumber, 10),
"uploadId": uploadId,
//"partSize": com.ToStr(partSize,10),
}

output, err := ObsCli.CreateSignedUrl(input)
if err != nil {
log.Error("CreateSignedUrl failed:", err.Error())
return "", err
}

return output.SignedUrl, nil
}

func ObsGetPreSignedUrl(uuid, fileName string) (string, error) {
input := &obs.CreateSignedUrlInput{}
input.Method = obs.HttpMethodGet


+ 47
- 0
options/locale/locale_en-US.ini View File

@@ -774,6 +774,53 @@ cloudbrain_creator=Creator
cloudbrain_task = Task Name
cloudbrain_operate = Operate
cloudbrain_status_createtime = Status/Createtime
cloudbrain_status_runtime = Running Time


modelarts.notebook=Debug Job
modelarts.train_job=Create Job
modelarts.train_job.new=New Job
modelarts.train_job.basic_info=Basic Info
modelarts.train_job.job_status=Job Status
modelarts.train_job.job_name=Job Name
modelarts.train_job.version=Job Version
modelarts.train_job.start_time=Start Time
modelarts.train_job.dura_time=Duration Time
modelarts.train_job.description=Description
modelarts.train_job.parameter_setting=Parameter setting
modelarts.train_job.parameter_setting_info=Parameter Info
modelarts.train_job.fast_parameter_setting=fast_parameter_setting
modelarts.train_job.fast_parameter_setting_config=fast_parameter_setting_config
modelarts.train_job.fast_parameter_setting_config_link=fast_parameter_setting_config_link
modelarts.train_job.frames=frames
modelarts.train_job.algorithm_origin=算法来源
modelarts.train_job.AI_driver=AI Engine
modelarts.train_job.start_file=启动文件
modelarts.train_job.boot_file_helper=启动文件是您程序执行的入口文件,必须是以.py结尾的文件。
modelarts.train_job.dataset=Dataset
modelarts.train_job.run_parameter=Run Parameter
modelarts.train_job.add_run_parameter=Add Run Parameter
modelarts.train_job.parameter_name=Parameter Name
modelarts.train_job.parameter_value=Parameter Value
modelarts.train_job.resource_setting=resource_setting
modelarts.train_job.resource_setting_info=resource_setting_info
modelarts.train_job.resource_pool=resource_pool
modelarts.train_job.resource_type=resource_type
modelarts.train_job.standard=Standard
modelarts.train_job.NAS_address=NAS地址
modelarts.train_job.NAS_mount_path=NAS挂载路径
modelarts.train_job.query_whether_save_parameter=query_whether_save_parameter
modelarts.train_job.save_helper=save_helper
modelarts.train_job.common_frame=common_frame
modelarts.train_job.amount_of_compute_node=Amount of Compute Node
modelarts.train_job.job_parameter_name=job_parameter_name
modelarts.train_job.parameter_description=parameter_description
modelarts.log=Log
modelarts.version_manage=Version Manage
modelarts.back=Back
modelarts.train_job_para_admin=train_job_para_admin
modelarts.train_job_para.edit=train_job_para.edit
modelarts.train_job_para.connfirm=train_job_para.connfirm

template.items = Template Items
template.git_content = Git Content (Default Branch)


+ 1
- 0
options/locale/locale_zh-CN.ini View File

@@ -776,6 +776,7 @@ cloudbrain_creator=创建者
cloudbrain_task=任务名称
cloudbrain_operate=操作
cloudbrain_status_createtime=状态/创建时间
cloudbrain_status_runtime = 运行时长
cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-,不能以_结尾,最长36个字符。

modelarts.notebook=调试作业


+ 28
- 0
public/self/test.js View File

@@ -0,0 +1,28 @@

function displayDir(uuid){
console.log('uuid 1=' + uuid);
var html="<tr>\
<th></th>\
<th id=\"dataset_head\"></th>\
<th>数据集名称</th>\
<th>数据集类型</th>\
<th>数据集描述</th>\
<th>数据集创建者</th>\
</tr>";
for (var i=0;i<1;i++){
var row = "<tr>\
<td><input type=\"checkbox\"/></td>\
<td id=\"dataset_id\">"+uuid+"</td>\
<td>" + uuid +"</td>\
<td>测试</td>\
<td>测试</td>\
<td>测试</td>\
</tr>";
html=html+row;
}
document.getElementById('dataset-files-table').innerHTML=html;
console.log('uuid 2=' + uuid);
}

+ 9
- 1
routers/api/v1/api.go View File

@@ -853,7 +853,15 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/:jobid", repo.GetCloudbrainTask)
}, reqRepoReader(models.UnitTypeCloudBrain))
m.Group("/modelarts", func() {
m.Get("/:jobid", repo.GetModelArtsTask)
m.Group("/notebook", func() {
m.Get("/:jobid", repo.GetModelArtsNotebook)
})
m.Group("/train-job", func() {
m.Group("/:jobid", func() {
m.Get("", repo.GetModelArtsTrainJob)
m.Get("/log", repo.TrainJobGetLog)
})
})
}, reqRepoReader(models.UnitTypeCloudBrain))
}, repoAssignment())
})


+ 80
- 1
routers/api/v1/repo/modelarts.go View File

@@ -11,9 +11,10 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/modelarts"
"net/http"
"strconv"
)

func GetModelArtsTask(ctx *context.APIContext) {
func GetModelArtsNotebook(ctx *context.APIContext) {
var (
err error
)
@@ -43,3 +44,81 @@ func GetModelArtsTask(ctx *context.APIContext) {
})

}

func GetModelArtsTrainJob(ctx *context.APIContext) {
var (
err error
)

jobID := ctx.Params(":jobid")
repoID := ctx.Repo.Repository.ID
job, err := models.GetRepoCloudBrainByJobID(repoID, jobID)
if err != nil {
ctx.NotFound(err)
return
}
result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(job.VersionID, 10))
if err != nil {
ctx.NotFound(err)
return
}

job.Status = modelarts.TransTrainJobStatus(result.IntStatus)
err = models.UpdateJob(job)
if err != nil {
log.Error("UpdateJob failed:", err)
}

ctx.JSON(http.StatusOK, map[string]interface{}{
"JobID": jobID,
"JobStatus": job.Status,
})

}

func TrainJobGetLog(ctx *context.APIContext) {
var (
err error
)

log.Info("test")

var jobID = ctx.Params(":jobid")
var logFileName = ctx.Query("file_name")
var baseLine = ctx.Query("base_line")
var order = ctx.Query("order")

if order != modelarts.OrderDesc && order != modelarts.OrderAsc {
log.Error("order(%s) check failed", order)
ctx.JSON(http.StatusBadRequest, map[string]interface{}{
"err_msg": "order check failed",
})
return
}

task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
"err_msg": "GetCloudbrainByJobID failed",
})
return
}

result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines)
if err != nil {
log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error())
ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
"err_msg": "GetTrainJobLog failed",
})
return
}

ctx.JSON(http.StatusOK, map[string]interface{}{
"JobID": jobID,
"StartLine": result.StartLine,
"EndLine": result.EndLine,
"Content": result.Content,
"Lines": result.Lines,
})
}

+ 1
- 1
routers/repo/attachment.go View File

@@ -1014,7 +1014,7 @@ func queryDatasets(ctx *context.Context, attachs []*models.AttachmentUsername) {
}

func checkTypeCloudBrain(typeCloudBrain int) error {
if typeCloudBrain != models.TypeCloudBrainOne && typeCloudBrain != models.TypeCloudBrainTwo {
if typeCloudBrain != models.TypeCloudBrainOne && typeCloudBrain != models.TypeCloudBrainNotebook {
log.Error("type error:", typeCloudBrain)
return errors.New("type error")
}


+ 77
- 10
routers/repo/cloudbrain.go View File

@@ -100,15 +100,15 @@ func cutString(str string, lens int) string {
func jobNamePrefixValid(s string) string {
lowStr := strings.ToLower(s)
re := regexp.MustCompile(`[^a-z0-9_\\-]+`)
removeSpecial := re.ReplaceAllString(lowStr, "")
re = regexp.MustCompile(`^[_\\-]+`)
return re.ReplaceAllString(removeSpecial, "")

}

func cloudBrainNewDataPrepare(ctx *context.Context) error{
func cloudBrainNewDataPrepare(ctx *context.Context) error {
ctx.Data["PageIsCloudBrain"] = true
t := time.Now()
var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
@@ -202,7 +202,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
gpuQueue := setting.JobType
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
resourceSpecId := form.ResourceSpecId
if !jobNamePattern.MatchString(jobName) {
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form)
return
@@ -474,7 +474,7 @@ func CloudBrainDel(ctx *context.Context) {
return
}

if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed){
if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) {
log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
return
@@ -580,10 +580,77 @@ func CloudBrainDownloadModel(ctx *context.Context) {
ctx.ServerError("PresignedGetURL", err)
return
}

http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
}

// func TrainJobloadModel(ctx *context.Context) {
// parentDir := ctx.Query("parentDir")
// fileName := ctx.Query("fileName")
// jobName := ctx.Query("jobName")
// filePath := "jobs/" + jobName + "/model/" + parentDir
// url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
// if err != nil {
// log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
// ctx.ServerError("PresignedGetURL", err)
// return
// }

// http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
// }

func TrainJobListModel(ctx *context.Context) {
ctx.Data["PageIsCloudBrain"] = true
jobID := ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("no such job!", ctx.Data["err"])
ctx.ServerError("no such job:", err)
return
}
TrainJobListModel, err := storage.GetObsListObject(task.JobName)
log.Info("TrainJobListModel", TrainJobListModel)
fmt.Println("TrainJobListModel:", TrainJobListModel)
if err != nil {
log.Info("get TrainJobListModel failed:", err)
return
}
ctx.Data["task"] = task
ctx.Data["JobID"] = jobID
ctx.Data["ListModel"] = TrainJobListModel
ctx.HTML(200, tplModelArtsTrainJobListModel)
}

func TrainJobDownloadModel(ctx *context.Context) {
JobName := ctx.Query("JobName")
fileName := ctx.Query("file_name")

// JobName = "liuzx202110271830856"
// fileName = "Untitled.ipynb"

body, err := storage.ObsModelDownload(JobName, fileName)
if err != nil {
log.Info("download error.")
} else {
defer body.Close()
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName)
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
p := make([]byte, 1024)
var readErr error
var readCount int
// 读取对象内容
for {
readCount, readErr = body.Read(p)
if readCount > 0 {
ctx.Resp.Write(p[:readCount])
//fmt.Printf("%s", p[:readCount])
}
if readErr != nil {
break
}
}
}
}

func GetRate(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
job, err := models.GetCloudbrainByJobID(jobID)
@@ -609,9 +676,9 @@ func downloadCode(repo *models.Repository, codePath string) error {
return err
}

configFile, err := os.OpenFile(codePath + "/.git/config", os.O_RDWR, 0666)
configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666)
if err != nil {
log.Error("open file(%s) failed:%v", codePath + "/,git/config", err)
log.Error("open file(%s) failed:%v", codePath+"/,git/config", err)
return err
}

@@ -631,10 +698,10 @@ func downloadCode(repo *models.Repository, codePath string) error {
}
}

if strings.Contains(line, "url") && strings.Contains(line, ".git"){
if strings.Contains(line, "url") && strings.Contains(line, ".git") {
originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n"
if len(line) > len(originUrl) {
originUrl += strings.Repeat( " ", len(line) - len(originUrl))
originUrl += strings.Repeat(" ", len(line)-len(originUrl))
}
bytes := []byte(originUrl)
_, err := configFile.WriteAt(bytes, pos)


+ 846
- 3
routers/repo/modelarts.go View File

@@ -1,14 +1,23 @@
package repo

import (
"code.gitea.io/gitea/modules/modelarts"
"encoding/json"
"errors"
"github.com/unknwon/com"
"fmt"
"io"
"net/http"
"os"
"path"
"strconv"
"strings"
"time"

"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/modelarts"
"code.gitea.io/gitea/modules/obs"
"code.gitea.io/gitea/modules/storage"
"github.com/unknwon/com"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/auth"
"code.gitea.io/gitea/modules/base"
@@ -18,9 +27,19 @@ import (
)

const (
// tplModelArtsNotebookIndex base.TplName = "repo/modelarts/notebook/index"
tplModelArtsNotebookIndex base.TplName = "repo/modelarts/notebook/index"
tplModelArtsNotebookNew base.TplName = "repo/modelarts/notebook/new"
tplModelArtsNotebookShow base.TplName = "repo/modelarts/notebook/show"

tplModelArtsIndex base.TplName = "repo/modelarts/index"
tplModelArtsNew base.TplName = "repo/modelarts/new"
tplModelArtsShow base.TplName = "repo/modelarts/show"

tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index"
tplModelArtsTrainJobNew base.TplName = "repo/modelarts/trainjob/new"
tplModelArtsTrainJobShow base.TplName = "repo/modelarts/trainjob/show"
tplModelArtsTrainJobListModel base.TplName = "repo/modelarts/trainjob/list_model"
)

// MustEnableDataset check if repository enable internal cb
@@ -30,6 +49,7 @@ func MustEnableModelArts(ctx *context.Context) {
return
}
}

func ModelArtsIndex(ctx *context.Context) {
MustEnableModelArts(ctx)
repo := ctx.Repo.Repository
@@ -231,7 +251,7 @@ func ModelArtsDel(ctx *context.Context) {
return
}

if task.Status != string(models.ModelArtsCreateFailed) && task.Status != string(models.ModelArtsStartFailed) && task.Status != string(models.ModelArtsStopped){
if task.Status != string(models.ModelArtsCreateFailed) && task.Status != string(models.ModelArtsStartFailed) && task.Status != string(models.ModelArtsStopped) {
log.Error("the job(%s) has not been stopped", task.JobName)
ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
return
@@ -252,3 +272,826 @@ func ModelArtsDel(ctx *context.Context) {

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts")
}

func NotebookIndex(ctx *context.Context) {
MustEnableModelArts(ctx)
repo := ctx.Repo.Repository
page := ctx.QueryInt("page")
if page <= 0 {
page = 1
}

ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
ListOptions: models.ListOptions{
Page: page,
PageSize: setting.UI.IssuePagingNum,
},
RepoID: repo.ID,
Type: models.TypeCloudBrainNotebook,
})
if err != nil {
ctx.ServerError("Cloudbrain", err)
return
}

for i, task := range ciTasks {
if task.Status == string(models.JobRunning) {
ciTasks[i].CanDebug = true
} else {
ciTasks[i].CanDebug = false
}
}

pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager

ctx.Data["PageIsCloudBrain"] = true
ctx.Data["Tasks"] = ciTasks
ctx.HTML(200, tplModelArtsNotebookIndex)
}

func NotebookNew(ctx *context.Context) {
ctx.Data["PageIsCloudBrain"] = true

t := time.Now()
var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
ctx.Data["job_name"] = jobName

attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID)
if err != nil {
ctx.ServerError("GetAllUserAttachments failed:", err)
return
}

ctx.Data["attachments"] = attachs
ctx.Data["dataset_path"] = modelarts.DataSetMountPath
ctx.Data["env"] = modelarts.NotebookEnv
ctx.Data["notebook_type"] = modelarts.NotebookType
if modelarts.FlavorInfos == nil {
json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos)
}
ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo

ctx.HTML(200, tplModelArtsNotebookNew)
}

func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) {
ctx.Data["PageIsNotebook"] = true
jobName := form.JobName
uuid := form.Attachment
description := form.Description

err := modelarts.GenerateTask(ctx, jobName, uuid, description)
if err != nil {
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook")
}

func NotebookShow(ctx *context.Context) {
ctx.Data["PageIsCloudBrain"] = true

var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
ctx.Data["error"] = err.Error()
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

result, err := modelarts.GetJob(jobID)
if err != nil {
ctx.Data["error"] = err.Error()
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

if result != nil {
task.Status = result.Status
err = models.UpdateJob(task)
if err != nil {
ctx.Data["error"] = err.Error()
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

createTime, _ := com.StrTo(result.CreationTimestamp).Int64()
result.CreateTime = time.Unix(int64(createTime/1000), 0).Format("2006-01-02 15:04:05")
endTime, _ := com.StrTo(result.LatestUpdateTimestamp).Int64()
result.LatestUpdateTime = time.Unix(int64(endTime/1000), 0).Format("2006-01-02 15:04:05")
result.QueuingInfo.BeginTime = time.Unix(int64(result.QueuingInfo.BeginTimestamp/1000), 0).Format("2006-01-02 15:04:05")
result.QueuingInfo.EndTime = time.Unix(int64(result.QueuingInfo.EndTimestamp/1000), 0).Format("2006-01-02 15:04:05")
}

ctx.Data["task"] = task
ctx.Data["jobID"] = jobID
ctx.Data["result"] = result
ctx.HTML(200, tplModelArtsNotebookShow)
}

func NotebookDebug(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
_, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
ctx.ServerError("GetCloudbrainByJobID failed", err)
return
}

result, err := modelarts.GetJob(jobID)
if err != nil {
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

res, err := modelarts.GetJobToken(jobID)
if err != nil {
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return
}

urls := strings.Split(result.Spec.Annotations.Url, "/")
urlPrefix := result.Spec.Annotations.TargetDomain
for i, url := range urls {
if i > 2 {
urlPrefix += "/" + url
}
}

debugUrl := urlPrefix + "?token=" + res.Token
ctx.Redirect(debugUrl)
}

func NotebookStop(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
log.Info(jobID)
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
ctx.ServerError("GetCloudbrainByJobID failed", err)
return
}

if task.Status != string(models.JobRunning) {
log.Error("the job(%s) is not running", task.JobName)
ctx.ServerError("the job is not running", errors.New("the job is not running"))
return
}

param := models.NotebookAction{
Action: models.ActionStop,
}
res, err := modelarts.StopJob(jobID, param)
if err != nil {
log.Error("StopJob(%s) failed:%v", task.JobName, err.Error())
ctx.ServerError("StopJob failed", err)
return
}

task.Status = res.CurrentStatus
err = models.UpdateJob(task)
if err != nil {
ctx.ServerError("UpdateJob failed", err)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook")
}

func NotebookDel(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
ctx.ServerError("GetCloudbrainByJobID failed", err)
return
}

if task.Status != string(models.JobStopped) {
log.Error("the job(%s) has not been stopped", task.JobName)
ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
return
}

_, err = modelarts.DelNotebook(jobID)
if err != nil {
log.Error("DelJob(%s) failed:%v", task.JobName, err.Error())
ctx.ServerError("DelJob failed", err)
return
}

err = models.DeleteJob(task)
if err != nil {
ctx.ServerError("DeleteJob failed", err)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook")
}

func TrainJobIndex(ctx *context.Context) {
MustEnableModelArts(ctx)

can, err := canUserCreateTrainJob(ctx.User.ID)
if err != nil {
ctx.ServerError("canUserCreateTrainJob", err)
return
}

ctx.Data["CanCreate"] = can

repo := ctx.Repo.Repository
page := ctx.QueryInt("page")
if page <= 0 {
page = 1
}

tasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{
ListOptions: models.ListOptions{
Page: page,
PageSize: setting.UI.IssuePagingNum,
},
RepoID: repo.ID,
Type: models.TypeCloudBrainTrainJob,
})
if err != nil {
ctx.ServerError("Cloudbrain", err)
return
}

for i := range tasks {
TrainJobDetail, err := modelarts.GetTrainJob(tasks[i].Cloudbrain.JobID, strconv.FormatInt(tasks[i].Cloudbrain.VersionID, 10))
if TrainJobDetail != nil {
TrainJobDetail.CreateTime = time.Unix(int64(TrainJobDetail.LongCreateTime/1000), 0).Format("2006-01-02 15:04:05")
if TrainJobDetail.Duration != 0 {
TrainJobDetail.TrainJobDuration = addZero(TrainJobDetail.Duration/3600000) + ":" + addZero(TrainJobDetail.Duration%3600000/60000) + ":" + addZero(TrainJobDetail.Duration%60000/1000)

} else {
TrainJobDetail.TrainJobDuration = "00:00:00"
}
}
if err != nil {
log.Error("GetJob(%s) failed:%v", tasks[i].Cloudbrain.JobID, err.Error())
return
}
err = models.SetTrainJobStatusByJobID(tasks[i].Cloudbrain.JobID, modelarts.TransTrainJobStatus(TrainJobDetail.IntStatus), int(TrainJobDetail.Duration), string(TrainJobDetail.TrainJobDuration))
// err = models.UpdateJob(tasks[i].Cloudbrain)
if err != nil {
ctx.ServerError("UpdateJob failed", err)
return
}
}

trainTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
ListOptions: models.ListOptions{
Page: page,
PageSize: setting.UI.IssuePagingNum,
},
RepoID: repo.ID,
Type: models.TypeCloudBrainTrainJob,
})
if err != nil {
ctx.ServerError("Cloudbrain", err)
return
}

pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager

ctx.Data["PageIsCloudBrain"] = true
ctx.Data["Tasks"] = trainTasks
ctx.HTML(200, tplModelArtsTrainJobIndex)
}

func TrainJobNew(ctx *context.Context) {
ctx.Data["PageIsCloudBrain"] = true

can, err := canUserCreateTrainJob(ctx.User.ID)
if err != nil {
ctx.ServerError("canUserCreateTrainJob", err)
return
}

if !can {
log.Error("the user can not create train-job")
ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job"))
return
}

t := time.Now()
var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
ctx.Data["job_name"] = jobName

attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID)
if err != nil {
ctx.ServerError("GetAllUserAttachments failed:", err)
return
}
ctx.Data["attachments"] = attachs

var resourcePools modelarts.ResourcePool
if err = json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil {
ctx.ServerError("json.Unmarshal failed:", err)
return
}
ctx.Data["resource_pools"] = resourcePools.Info

var engines modelarts.Engine
if err = json.Unmarshal([]byte(setting.Engines), &engines); err != nil {
ctx.ServerError("json.Unmarshal failed:", err)
return
}
ctx.Data["engines"] = engines.Info

var versionInfos modelarts.VersionInfo
if err = json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil {
ctx.ServerError("json.Unmarshal failed:", err)
return
}
ctx.Data["engine_versions"] = versionInfos.Version

var flavorInfos modelarts.Flavor
if err = json.Unmarshal([]byte(setting.TrainJobFLAVORINFOS), &flavorInfos); err != nil {
ctx.ServerError("json.Unmarshal failed:", err)
return
}
ctx.Data["flavor_infos"] = flavorInfos.Info

outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
ctx.Data["train_url"] = outputObsPath

configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom)
if err != nil {
ctx.ServerError("getConfigList failed:", err)
return
}

ctx.Data["config_list"] = configList.ParaConfigs

ctx.HTML(200, tplModelArtsTrainJobNew)
}

func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) {
ctx.Data["PageIsTrainJob"] = true
jobName := form.JobName
uuid := form.Attachment
description := form.Description
workServerNumber := form.WorkServerNumber
engineID := form.EngineID
bootFile := form.BootFile
flavorCode := form.Flavor
params := form.Params
poolID := form.PoolID
isSaveParam := form.IsSaveParam
repo := ctx.Repo.Repository
codeLocalPath := setting.JobPath + jobName + modelarts.CodePath
codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath
outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath
logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath
dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + "/"

can, err := canUserCreateTrainJob(ctx.User.ID)
if err != nil {
ctx.ServerError("canUserCreateTrainJob", err)
return
}

if !can {
log.Error("the user can not create train-job")
ctx.RenderWithErr("the user can not create train-job", tplModelArtsTrainJobNew, &form)
return
}

//param check
if err := paramCheckCreateTrainJob(form); err != nil {
log.Error("paramCheckCreateTrainJob failed:(%v)", err)
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form)
return
}

if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{}); err != nil {
log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
ctx.RenderWithErr("Failed to clone repository", tplModelArtsTrainJobNew, &form)
return
}

//todo: upload code (send to file_server todo this work?)
if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath); err != nil {
log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err)
ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobNew, &form)
return
}

if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath); err != nil {
log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err)
ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobNew, &form)
return
}

if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil {
log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err)
ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobNew, &form)
return
}

//todo: del local code?

var parameters models.Parameters
param := make([]models.Parameter, 0)
param = append(param, models.Parameter{
Label: modelarts.TrainUrl,
Value: outputObsPath,
}, models.Parameter{
Label: modelarts.DataUrl,
Value: dataPath,
})
if len(params) != 0 {
err := json.Unmarshal([]byte(params), &parameters)
if err != nil {
log.Error("Failed to Unmarshal params: %s (%v)", params, err)
ctx.RenderWithErr("运行参数错误", tplModelArtsTrainJobNew, &form)
return
}

for _, parameter := range parameters.Parameter {
if parameter.Label != modelarts.TrainUrl && parameter.Label != modelarts.DataUrl {
param = append(param, models.Parameter{
Label: parameter.Label,
Value: parameter.Value,
})
}
}
}

//save param config
if isSaveParam == "on" {
if form.ParameterTemplateName == "" {
log.Error("ParameterTemplateName is empty")
ctx.RenderWithErr("保存作业参数时,作业参数名称不能为空", tplModelArtsTrainJobNew, &form)
return
}

_, err := modelarts.CreateTrainJobConfig(models.CreateConfigParams{
ConfigName: form.ParameterTemplateName,
Description: form.PrameterDescription,
DataUrl: dataPath,
AppUrl: codeObsPath,
BootFileUrl: codeObsPath + bootFile,
TrainUrl: outputObsPath,
Flavor: models.Flavor{
Code: flavorCode,
},
WorkServerNum: workServerNumber,
EngineID: int64(engineID),
LogUrl: logObsPath,
PoolID: poolID,
Parameter: param,
})

if err != nil {
log.Error("Failed to CreateTrainJobConfig: %v", err)
ctx.RenderWithErr("保存作业参数失败:"+err.Error(), tplModelArtsTrainJobNew, &form)
return
}
}

req := &modelarts.GenerateTrainJobReq{
JobName: jobName,
DataUrl: dataPath,
Description: description,
CodeObsPath: codeObsPath,
BootFile: codeObsPath + bootFile,
TrainUrl: outputObsPath,
FlavorCode: flavorCode,
WorkServerNumber: workServerNumber,
EngineID: int64(engineID),
LogUrl: logObsPath,
PoolID: poolID,
Uuid: uuid,
Parameters: param,
}

err = modelarts.GenerateTrainJob(ctx, req)
if err != nil {
log.Error("GenerateTrainJob failed:%v", err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form)
return
}
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
}

// readDir reads the directory named by dirname and returns
// a list of directory entries sorted by filename.
func readDir(dirname string) ([]os.FileInfo, error) {
f, err := os.Open(dirname)
if err != nil {
return nil, err
}

list, err := f.Readdir(100)
f.Close()
if err != nil {
//todo: can not upload empty folder
if err == io.EOF {
return nil, nil
}
return nil, err
}

//sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() })
return list, nil
}

func uploadCodeToObs(codePath, jobName, parentDir string) error {
files, err := readDir(codePath)
if err != nil {
log.Error("readDir(%s) failed: %s", codePath, err.Error())
return err
}

for _, file := range files {
if file.IsDir() {
input := &obs.PutObjectInput{}
input.Bucket = setting.Bucket
input.Key = parentDir + file.Name() + "/"
_, err = storage.ObsCli.PutObject(input)
if err != nil {
log.Error("PutObject(%s) failed: %s", input.Key, err.Error())
return err
}

if err = uploadCodeToObs(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil {
log.Error("uploadCodeToObs(%s) failed: %s", file.Name(), err.Error())
return err
}
} else {
input := &obs.PutFileInput{}
input.Bucket = setting.Bucket
input.Key = setting.CodePathPrefix + jobName + "/code/" + parentDir + file.Name()
input.SourceFile = codePath + file.Name()
_, err = storage.ObsCli.PutFile(input)
if err != nil {
log.Error("PutFile(%s) failed: %s", input.SourceFile, err.Error())
return err
}
}
}

return nil
}

func obsMkdir(dir string) error {
input := &obs.PutObjectInput{}
input.Bucket = setting.Bucket
input.Key = dir
_, err := storage.ObsCli.PutObject(input)
if err != nil {
log.Error("PutObject(%s) failed: %s", input.Key, err.Error())
return err
}

return nil
}

func paramCheckCreateTrainJob(form auth.CreateModelArtsTrainJobForm) error {
if !strings.HasSuffix(form.BootFile, ".py") {
log.Error("the boot file(%s) must be a python file", form.BootFile)
return errors.New("启动文件必须是python文件")
}

if form.WorkServerNumber > 25 || form.WorkServerNumber < 1 {
log.Error("the WorkServerNumber(%d) must be in (1,25)", form.WorkServerNumber)
return errors.New("计算节点数必须在1-25之间")
}

return nil
}

func TrainJobShow(ctx *context.Context) {
ctx.Data["PageIsCloudBrain"] = true

var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

attach, err := models.GetAttachmentByUUID(task.Uuid)
if err != nil {
log.Error("GetAttachmentByUUID(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(task.VersionID, 10))
if err != nil {
log.Error("GetJob(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

if result != nil {
result.CreateTime = time.Unix(int64(result.LongCreateTime/1000), 0).Format("2006-01-02 15:04:05")
if result.Duration != 0 {
result.TrainJobDuration = addZero(result.Duration/3600000) + ":" + addZero(result.Duration%3600000/60000) + ":" + addZero(result.Duration%60000/1000)

} else {
result.TrainJobDuration = "00:00:00"
}
err = models.SetTrainJobStatusByJobID(jobID, modelarts.TransTrainJobStatus(result.IntStatus), int(result.Duration), string(result.TrainJobDuration))
if err != nil {
ctx.ServerError("UpdateJob failed", err)
return
}
result.Status = modelarts.TransTrainJobStatus(result.IntStatus)
result.DatasetName = attach.Name
}

resultLogFile, resultLog, err := trainJobGetLog(jobID)
if err != nil {
log.Error("trainJobGetLog(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

ctx.Data["log_file_name"] = resultLogFile.LogFileList[0]
ctx.Data["log"] = resultLog
ctx.Data["task"] = task
ctx.Data["jobID"] = jobID
ctx.Data["result"] = result
ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow)
}

func addZero(t int64) (m string) {
if t < 10 {
m = "0" + strconv.FormatInt(t, 10)
return m
} else {
return strconv.FormatInt(t, 10)
}
}

func TrainJobGetLog(ctx *context.Context) {
ctx.Data["PageIsTrainJob"] = true

var jobID = ctx.Params(":jobid")
var logFileName = ctx.Query("file_name")
var baseLine = ctx.Query("base_line")
var order = ctx.Query("order")

if order != modelarts.OrderDesc && order != modelarts.OrderAsc {
log.Error("order(%s) check failed", order)
ctx.HTML(http.StatusBadRequest, tplModelArtsTrainJobShow)
return
}

task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines)
if err != nil {
log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

ctx.Data["log"] = result
//ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow)
}

func trainJobGetLog(jobID string) (*models.GetTrainJobLogFileNamesResult, *models.GetTrainJobLogResult, error) {
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
return nil, nil, err
}

resultLogFile, err := modelarts.GetTrainJobLogFileNames(jobID, strconv.FormatInt(task.VersionID, 10))
if err != nil {
log.Error("GetTrainJobLogFileNames(%s) failed:%v", jobID, err.Error())
return nil, nil, err
}

result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), "", resultLogFile.LogFileList[0], modelarts.OrderDesc, modelarts.Lines)
if err != nil {
log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error())
return nil, nil, err
}

return resultLogFile, result, err
}

func TrainJobDel(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

_, err = modelarts.DelTrainJob(jobID)
if err != nil {
log.Error("DelTrainJob(%s) failed:%v", task.JobName, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

err = models.DeleteJob(task)
if err != nil {
ctx.ServerError("DeleteJob failed", err)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
}

func TrainJobStop(ctx *context.Context) {
var jobID = ctx.Params(":jobid")
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

_, err = modelarts.StopTrainJob(jobID, strconv.FormatInt(task.VersionID, 10))
if err != nil {
log.Error("StopTrainJob(%s) failed:%v", task.JobName, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil)
return
}

ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
}

func canUserCreateTrainJob(uid int64) (bool, error) {
org, err := models.GetOrgByName(setting.AllowedOrg)
if err != nil {
log.Error("get allowed org failed: ", setting.AllowedOrg)
return false, err
}

return org.IsOrgMember(uid)
}

func TrainJobGetConfigList(ctx *context.Context) {
ctx.Data["PageIsTrainJob"] = true

var jobID = ctx.Params(":jobid")
var logFileName = ctx.Query("file_name")
var baseLine = ctx.Query("base_line")
var order = ctx.Query("order")

if order != modelarts.OrderDesc && order != modelarts.OrderAsc {
log.Error("order(%s) check failed", order)
ctx.HTML(http.StatusBadRequest, tplModelArtsTrainJobShow)
return
}

task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines)
if err != nil {
log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error())
ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil)
return
}

ctx.Data["log"] = result
//ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow)
}

func getConfigList(perPage, page int, sortBy, order, searchContent, configType string) (*models.GetConfigListResult, error) {
var result models.GetConfigListResult

list, err := modelarts.GetConfigList(perPage, page, sortBy, order, searchContent, configType)
if err != nil {
log.Error("GetConfigList failed:", err)
return &result, err
}

for _, config := range list.ParaConfigs {
paraConfig, err := modelarts.GetParaConfig(config.ConfigName, configType)
if err != nil {
log.Error("GetParaConfig failed:", err)
return &result, err
}

config.Result = paraConfig
}

return list, nil
}

+ 35
- 8
routers/routes/routes.go View File

@@ -962,15 +962,42 @@ func RegisterRoutes(m *macaron.Macaron) {
}, context.RepoRef())

m.Group("/modelarts", func() {
m.Get("", reqRepoCloudBrainReader, repo.ModelArtsIndex)
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.ModelArtsShow)
m.Get("/debug", reqRepoCloudBrainReader, repo.ModelArtsDebug)
m.Post("/stop", reqRepoCloudBrainWriter, repo.ModelArtsStop)
m.Post("/del", reqRepoCloudBrainWriter, repo.ModelArtsDel)
// m.Get("", reqRepoCloudBrainReader, repo.ModelArtsIndex)
// m.Group("/:jobid", func() {
// m.Get("", reqRepoCloudBrainReader, repo.ModelArtsShow)
// m.Get("/debug", reqRepoCloudBrainReader, repo.ModelArtsDebug)
// m.Post("/stop", reqRepoCloudBrainWriter, repo.ModelArtsStop)
// m.Post("/del", reqRepoCloudBrainWriter, repo.ModelArtsDel)
// })
// m.Get("/create", reqRepoCloudBrainWriter, repo.ModelArtsNew)
// m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.ModelArtsCreate)

m.Group("/notebook", func() {
m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex)
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.NotebookShow)
m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug)
m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop)
m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel)
})
m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate)
})

m.Group("/train-job", func() {
m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex)
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow)
m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop)
m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel)
m.Get("/log", reqRepoCloudBrainReader, repo.TrainJobGetLog)
m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobListModel)
m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobDownloadModel)
})
m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate)
m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList)
})
m.Get("/create", reqRepoCloudBrainWriter, repo.ModelArtsNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.ModelArtsCreate)
}, context.RepoRef())

m.Group("/blockchain", func() {


+ 25
- 1
templates/repo/cloudbrain/index.tmpl View File

@@ -215,7 +215,7 @@
<div class="ui container">

<!-- 中间云脑和新建任务按钮 -->
<div class="ui three column stack able grid">
<!-- <div class="ui three column stack able grid">
<div class="column">
<h2>{{.i18n.Tr "repo.cloudbrain1"}}</h2>
</div>
@@ -233,6 +233,30 @@
<div class="ui blue mini menu selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/cloudbrain">{{svg "octicon-server" 16}} CPU / GPU</a>
<a class="item" href="{{.RepoLink}}/modelarts">{{svg "octicon-server" 16}} Ascend NPU</a>
</div> -->

<div class="ui two column stackable grid ">

<div class="column">
<div class="ui blue small menu compact selectcloudbrain">
<a class="active item">调试任务</a>
<!-- <a class="item" href="{{.RepoLink}}/modelarts">训练任务</a> -->
</div>
</div>
<div class="column right aligned">
<div class="ui selection dropdown" style="min-width: 10em;min-height:2.6em;border-radius: .28571429rem;margin-right: 1em;padding: .67em 3.2em .7em 1em;">
{{svg "octicon-server" 16}}
<div class="default text" style="color: rgba(0,0,0,.87);"> CPU / GPU</div>
<i class="dropdown icon"></i>
<div class="menu">
<a class="item" href="{{.RepoLink}}/cloudbrain" data-value="11">CPU / GPU</a>
<a class="item" href="{{.RepoLink}}/modelarts/notebook" data-value="22">Ascend NPU</a>
</div>
</div>
<a class="ui green button" href="{{.RepoLink}}/cloudbrain/create">新建调试任务</a>
</div>
</div>

<!-- 中下列表展示区 -->


+ 6
- 4
templates/repo/header.tmpl View File

@@ -176,13 +176,13 @@
<label for="CloudBrain">{{$.i18n.Tr "repo.cloudbrain_platform_selection"}}</label>
<div class="field">
<div class="ui radio checkbox">
<input type="radio" name="CloudBrain" checked tabindex="0" class="hidden" value="0">
<input type="radio" name="CloudBrainSelect" checked tabindex="0" class="hidden" value="0">
<label>{{$.i18n.Tr "repo.cloudbrain1"}}</label>
</div>
</div>
<div class="field">
<div class="ui radio checkbox">
<input type="radio" name="CloudBrain" tabindex="0" class="hidden" value="1">
<input type="radio" name="CloudBrainSelect" tabindex="0" class="hidden" value="1">
<label>{{$.i18n.Tr "repo.cloudbrain2"}}</label>
</div>
</div>
@@ -209,14 +209,16 @@
$('.ui.radio.checkbox').checkbox();

var repolink = $(".cloudbrain_link").text()
console.log(repolink)
$(".ui.positive.right.icon.button").click(function(){
// 声明一个变量来接收以及获取单选框选择的情况
var checked_radio = $("input[type='radio']:checked").val()
var checked_radio = $("input[name='CloudBrainSelect']:checked").val()
console.log(checked_radio)

if(checked_radio=='0'){
window.location.href = repolink+'/cloudbrain'
}else if(checked_radio=='1'){
window.location.href = repolink+'/modelarts'
window.location.href = repolink+'/modelarts/notebook'
}else{
return;
}


+ 21
- 16
templates/repo/modelarts/index.tmpl View File

@@ -202,24 +202,29 @@
<div class="ui container">

<!-- 中间云脑和新建任务按钮 -->
<div class="ui three column stack able grid">
<div class="column">
<h2>{{.i18n.Tr "repo.cloudbrain2"}}</h2>
</div>

<div class="column">
</div>

<div class="column right aligned">
{{if .Permission.CanWrite $.UnitTypeCloudBrain}}
<a class="ui green button" href="{{.RepoLink}}/modelarts/create">{{.i18n.Tr "repo.cloudbrain.new"}}</a> {{end}}
</div>
</div>
<div class="ui two column stackable grid ">

<p>使用鹏城云脑计算资源进行调试,云脑1提供CPU / GPU资源,云脑2提供Ascend NPU资源;调试使用的数据集也需要上传到对应的环境。</p>
<div class="ui blue mini menu selectcloudbrain">
<a class="item" href="{{.RepoLink}}/cloudbrain">{{svg "octicon-server" 16}} CPU / GPU</a>
<a class="active item" href="{{.RepoLink}}/modelarts">{{svg "octicon-server" 16}} Ascend NPU</a>
<div class="column">
<div class="ui blue small menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/modelarts/notebook">调试任务</a>
<a class="item" href="{{.RepoLink}}/modelarts/train-job">训练任务</a>
</div>
</div>
<div class="column right aligned">
<div class="ui selection dropdown" style="min-width: 10em;min-height:2.6em;border-radius: .28571429rem;margin-right: 1em;padding: .67em 3.2em .7em 1em;">
{{svg "octicon-server" 16}}
<div class="default text" style="color: rgba(0,0,0,.87);"> Ascend NPU</div>
<i class="dropdown icon"></i>
<div class="menu">
<a class="item" href="{{.RepoLink}}/cloudbrain" data-value="11">CPU / GPU</a>
<a class="item" href="{{.RepoLink}}/modelarts" data-value="22">Ascend NPU</a>
</div>
</div>
<a class="ui green button" href="{{.RepoLink}}/modelarts/create">新建调试任务</a>
</div>
</div>

<!-- 中下列表展示区 -->


+ 43
- 0
templates/repo/modelarts/navbar.tmpl View File

@@ -0,0 +1,43 @@
<style>
.dis {
margin-bottom: 20px;
}
.disabled {
cursor: pointer;
pointer-events: none;
}

.ui.segment.bottom.attached {
border: none;
}
.ui.secondary.vertical.pointing.menu{
border-right-width: 0px !important;
}

.vertical.menu .item {
border-right-color: white !important;
}

.vertical.menu .activate.item {
font-weight: 700;
}
</style>
<div class="three wide column">
<div class="ui grid">
<div class="sixteen wide column ui secondary sticky pointing tabular vertical menu">
<a class="{{if .PageIsNotebook}}active{{end}} item" href="{{.RepoLink}}/modelarts/notebook">
{{svg "octicon-repo" 16}} {{.i18n.Tr "repo.modelarts.notebook"}}
</a>
<a class="{{if .PageIsTrainJob}}active{{end}} item" href="{{.RepoLink}}/modelarts/train-job">
{{svg "octicon-inbox" 16}} {{.i18n.Tr "repo.modelarts.train_job"}}
</a>
<!--
<a class="{{if .PageIsParaManage}}active{{end}} item" href="{{.RepoLink}}/modelarts/train-job/para-manage">
{{svg "octicon-inbox" 16}} {{.i18n.Tr "repo.modelarts.train_job_para_admin"}}
</a>
-->
</div>
</div>
</div>

+ 498
- 0
templates/repo/modelarts/notebook/index.tmpl View File

@@ -0,0 +1,498 @@
<!-- 头部导航栏 -->
{{template "base/head" .}}

<style>
.selectcloudbrain .active.item{
color: #0087f5 !important;
border: 1px solid #0087f5;
margin: -1px;
background: #FFF !important;
}
#deletemodel {
width: 100%;
height: 100%;
}
/* 弹窗 */

#mask {
position: fixed;
top: 0px;
left: 0px;
right: 0px;
bottom: 0px;
filter: alpha(opacity=60);
background-color: #777;
z-index: 1000;
display: none;
opacity: 0.8;
-moz-opacity: 0.5;
padding-top: 100px;
color: #000000
}

#loadingPage {
margin: 200px auto;
width: 50px;
height: 40px;
text-align: center;
font-size: 10px;
display: block;
}

#loadingPage>div {
background-color: green;
height: 100%;
width: 6px;
display: inline-block;
-webkit-animation: sk-stretchdelay 1.2s infinite ease-in-out;
animation: sk-stretchdelay 1.2s infinite ease-in-out;
}

#loadingPage .rect2 {
-webkit-animation-delay: -1.1s;
animation-delay: -1.1s;
}

#loadingPage .rect3 {
-webkit-animation-delay: -1.0s;
animation-delay: -1.0s;
}

#loadingPage .rect4 {
-webkit-animation-delay: -0.9s;
animation-delay: -0.9s;
}

#loadingPage .rect5 {
-webkit-animation-delay: -0.8s;
animation-delay: -0.8s;
}

@-webkit-keyframes sk-stretchdelay {
0%,
40%,
100% {
-webkit-transform: scaleY(0.4)
}
20% {
-webkit-transform: scaleY(1.0)
}
}

@keyframes sk-stretchdelay {
0%,
40%,
100% {
transform: scaleY(0.4);
-webkit-transform: scaleY(0.4);
}
20% {
transform: scaleY(1.0);
-webkit-transform: scaleY(1.0);
}
}
/* 消息框 */

.alert {
display: none;
position: fixed;
width: 100%;
z-index: 1001;
padding: 15px;
border: 1px solid transparent;
border-radius: 4px;
text-align: center;
font-weight: bold;
}

.alert-success {
color: #3c763d;
background-color: #dff0d8;
border-color: #d6e9c6;
}

.alert-info {
color: #31708f;
background-color: #d9edf7;
border-color: #bce8f1;
}

.alert-warning {
color: #8a6d3b;
background-color: #fcf8e3;
border-color: #faebcc;
}

.alert-danger {
color: #a94442;
background-color: #f2dede;
border-color: #ebccd1;
}

.pusher {
width: calc(100% - 260px);
box-sizing: border-box;
}
/* 弹窗 (background) */

#imageModal {
display: none;
position: fixed;
z-index: 1;
left: 0;
top: 0;
width: 100%;
height: 100%;
overflow: auto;
background-color: rgb(0, 0, 0);
background-color: rgba(0, 0, 0, 0.4);
}
/* 弹窗内容 */

.modal-content {
background-color: #fefefe;
margin: 15% auto;
padding: 20px;
border: 1px solid #888;
width: 30%;
}
/* 关闭按钮 */

.close {
color: #aaa;
float: right;
font-size: 28px;
font-weight: bold;
}

.close:hover,
.close:focus {
color: black;
text-decoration: none;
cursor: pointer;
}

.dis {
margin-bottom: 20px;
}

.disabled {
cursor: pointer;
pointer-events: none;
}
</style>

<!-- 弹窗 -->
<div id="mask">
<div id="loadingPage">
<div class="rect1"></div>
<div class="rect2"></div>
<div class="rect3"></div>
<div class="rect4"></div>
<div class="rect5"></div>
</div>
</div>

<!-- 提示框 -->
<div class="alert"></div>

<div class="repository release dataset-list view">
{{template "repo/header" .}}
<!-- 列表容器 -->
<div class="ui container">

<!-- 中间云脑和新建任务按钮 -->

<div class="ui two column stackable grid ">

<div class="column">
<div class="ui blue small menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/modelarts/notebook">调试任务</a>
<a class="item" href="{{.RepoLink}}/modelarts/train-job">训练任务</a>
</div>
</div>
<div class="column right aligned">
<div class="ui selection dropdown" style="min-width: 10em;min-height:2.6em;border-radius: .28571429rem;margin-right: 1em;padding: .67em 3.2em .7em 1em;">
{{svg "octicon-server" 16}}
<div class="default text" style="color: rgba(0,0,0,.87);"> Ascend NPU</div>
<i class="dropdown icon"></i>
<div class="menu">
<a class="item" href="{{.RepoLink}}/cloudbrain" data-value="11">CPU / GPU</a>
<a class="item" href="{{.RepoLink}}/modelarts/notebook" data-value="22">Ascend NPU</a>
</div>
</div>
<a class="ui green button" href="{{.RepoLink}}/modelarts/notebook/create">新建调试任务</a>
</div>
</div>

<!-- 中下列表展示区 -->
<div class="ui grid">
<div class="row">
<div class="ui sixteen wide column">

<!-- 排序区 -->
<!-- <div class="ui sixteen wide column">
<div class="ui two column stackable grid">
<div class="column">
</div>
<div class="column right aligned">
<div class="ui right dropdown type jump item">
<span class="text">
{{.i18n.Tr "repo.issues.filter_sort"}}<i class="dropdown icon"></i>
</span>
</div>
</div>
</div>
</div> -->

<!-- 任务展示 -->
<div class="dataset list">

<!-- 表头 -->
<div class="ui grid stackable" style="background: #f0f0f0;;">
<div class="row">
<div class="five wide column">
<span style="margin:0 6px">{{$.i18n.Tr "repo.cloudbrain_task"}}</span>
</div>
<div class="three wide column">
<span>{{$.i18n.Tr "repo.cloudbrain_status_createtime"}}</span>
</div>
<div class="one wide column">
<span>{{$.i18n.Tr "repo.cloudbrain_creator"}}</span>
</div>
<div class="seven wide column text center">
<span style="margin-left: 10rem;">{{$.i18n.Tr "repo.cloudbrain_operate"}}</span>
</div>

</div>
</div>



{{range .Tasks}}
<div class="ui grid stackable item">
<div class="row">
<!-- 任务名 -->
<div class="five wide column">
<a class="title" href="{{$.Link}}/{{.JobID}}" title="{{.JobName}}" style="font-size: 15px;">
<span class="fitted" style="vertical-align: middle;">{{svg "octicon-tasklist" 16}}</span>
<span class="fitted" style="width: 90%;vertical-align: middle;margin-left: 0.4rem;">{{.JobName}}</span>
</a>
</div>

<div class="three wide column">
<!--任务状态 -->
<!-- <span class="ui compact button job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
{{.Status}}
</span> -->
<span class="job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
<span><i style="vertical-align: middle;" class="{{.Status}}"></i><span style="margin-left: 0.4em;font-size: 12px;">{{.Status}}</span></span>
</span>
<!-- 任务创建时间 -->
<span style="font-size: 12px;margin-left: 0.4rem;" class="">{{TimeSinceUnix .Cloudbrain.CreatedUnix $.Lang}}</span>
</div>

<div class="one wide column">
{{if .User.Name}}
<a href="{{AppSubUrl}}/{{.User.Name}}" title="{{.User.Name}}"><img class="ui avatar image" src="{{.User.RelAvatarLink}}"></a>
{{else}}
<a title="Ghost"><img class="ui avatar image" src="{{AppSubUrl}}/user/avatar/Ghost/-1"></a>
{{end}}
</div>

<div class="seven wide column text right">
<div class="ui compact buttons">
<!-- <a class="ui basic blue button" href="{{$.Link}}/{{.JobID}}">
查看
</a> -->
<a class="ui basic {{if not .CanDebug}}disabled {{else}}blue {{end}}button" href="{{$.Link}}/{{.JobID}}/debug" target="_blank">
调试
</a>
<form id="stopForm-{{.JobID}}" action="{{if ne .Status "RUNNING"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post" style="margin-left:-1px;">
{{$.CsrfTokenHtml}}
<a class="ui basic {{if ne .Status "RUNNING"}}disabled {{else}}blue {{end}}button" onclick="document.getElementById('stopForm-{{.JobID}}').submit();">
停止
</a>
</form>
</div>

<div class="ui compact buttons" style="margin-right:10px;">
<!-- 模型下载 -->
<a class="ui basic blue button" href="{{$.Link}}/{{.JobID}}/models" target="_blank">
模型下载
</a>
<!-- 接收结果 -->
<iframe src="" frameborder="0" name="iframeContent" style="display: none;"></iframe>
<a class="imageBtn ui basic {{if not .CanDebug}}disabled {{else}}blue {{end}}button" value="{{.CanDebug}}">提交镜像</a>
</div>

<!-- 删除任务 -->
<form class="ui compact buttons" id="delForm-{{.JobID}}" action="{{if eq .Status "RUNNING"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
{{$.CsrfTokenHtml}}
<a class="ui compact {{if eq .Status "RUNNING"}}disabled {{else}}red {{end}}button" onclick="assertDelete(this)" style="border-radius: .28571429rem;">
删除
</a>
</form>
</div>



</div>
</div>
{{end}} {{template "base/paginate" .}}
</div>

</div>
</div>
</div>

</div>

</div>
</div>

</div>

<!-- 确认模态框 -->
<div id="deletemodel">
<div class="ui basic modal">
<div class="ui icon header">
<i class="trash icon"></i> 删除任务
</div>

<div class="content">
<p>你确认删除该任务么?此任务一旦删除不可恢复。</p>
</div>
<div class="actions">
<div class="ui red basic inverted cancel button">
<i class="remove icon"></i> 取消操作
</div>
<div class="ui green basic inverted ok button">
<i class="checkmark icon"></i> 确定操作
</div>
</div>
</div>
</div>

</div>
{{template "base/footer" .}}

<script>
// 调试和评分新开窗口
function stop(obj) {
if (obj.style.color != "rgb(204, 204, 204)") {
obj.target = '_blank'
} else {
return
}
}

// 删除时用户确认
function assertDelete(obj) {
if (obj.style.color == "rgb(204, 204, 204)") {
return
} else {
var delId = obj.parentNode.id
flag = 1;
$('.ui.basic.modal')
.modal({
onDeny: function() {
flag = false
},
onApprove: function() {
document.getElementById(delId).submit()
flag = true
},
onHidden: function() {
if (flag == false) {
$('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut();
}
}
})
.modal('show')
}
}

// 加载任务状态
var timeid = window.setInterval(loadJobStatus, 15000);
// $(document).ready(loadJobStatus);
function loadJobStatus() {
$(".job-status").each((index, job) => {
const jobID = job.dataset.jobid;
const repoPath = job.dataset.repopath;
if (job.textContent.trim() == 'STOPPED') {
return
}

$.get(`/api/v1/repos/${repoPath}/modelarts/notebook/${jobID}`, (data) => {
const jobID = data.JobID
const status = data.JobStatus
if (status != job.textContent.trim() || status=='RUNNING') {
//$('#' + jobID).text(status)
//if (status == 'STOPPED') {
window.location.reload()
//}
}
}).fail(function(err) {
console.log(err);
});
});
};

// 获取弹窗
var modal = document.getElementById('imageModal');

// 打开弹窗的按钮对象
var btns = document.getElementsByClassName("imageBtn");

// 获取 <span> 元素,用于关闭弹窗
var spans = document.getElementsByClassName('close');

// 点击按钮打开弹窗
for (i = 0; i < btns.length; i++) {
btns[i].onclick = function() {
modal.style.display = "block";
}
}

// 点击 <span> (x), 关闭弹窗
for (i = 0; i < spans.length; i++) {
spans[i].onclick = function() {
modal.style.display = "none";
}
}

// 在用户点击其他地方时,关闭弹窗
window.onclick = function(event) {
if (event.target == modal) {
modal.style.display = "none";
}
}

// 显示弹窗,弹出相应的信息
function showmask() {
$('#imageModal').css('display', 'none')
$('#mask').css('display', 'block')

$("iframe[name=iframeContent]").on("load", function() {  
var responseText = $("iframe")[0].contentDocument.body.getElementsByTagName("pre")[0].innerHTML; 
var json1 = JSON.parse(responseText)
$('#mask').css('display', 'none')
parent.location.href

if (json1.result_code === "0") {
$('.alert').html('操作成功!').removeClass('alert-danger').addClass('alert-success').show().delay(1500).fadeOut();
} else {
$('.alert').html(json1.error_msg).removeClass('alert-success').addClass('alert-danger').show().delay(5000).fadeOut();
}
})
}
</script>

+ 240
- 0
templates/repo/modelarts/notebook/new.tmpl View File

@@ -0,0 +1,240 @@
{{template "base/head" .}}
<style>
/* 遮罩层css效果图 */
#mask {
position: fixed;
top: 0px;
left: 0px;
right: 0px;
bottom: 0px;
filter: alpha(opacity=60);
background-color: #777;
z-index: 1000;
display: none;
opacity: 0.8;
-moz-opacity: 0.5;
padding-top: 100px;
color: #000000
}
/* 加载圈css效果图 */
#loadingPage {
margin: 200px auto;
width: 50px;
height: 40px;
text-align: center;
font-size: 10px;
display: block;
}
#loadingPage>div {
background-color: green;
height: 100%;
width: 6px;
display: inline-block;
-webkit-animation: sk-stretchdelay 1.2s infinite ease-in-out;
animation: sk-stretchdelay 1.2s infinite ease-in-out;
}
#loadingPage .rect2 {
-webkit-animation-delay: -1.1s;
animation-delay: -1.1s;
}
#loadingPage .rect3 {
-webkit-animation-delay: -1.0s;
animation-delay: -1.0s;
}
#loadingPage .rect4 {
-webkit-animation-delay: -0.9s;
animation-delay: -0.9s;
}
#loadingPage .rect5 {
-webkit-animation-delay: -0.8s;
animation-delay: -0.8s;
}
@-webkit-keyframes sk-stretchdelay {
0%,
40%,
100% {
-webkit-transform: scaleY(0.4)
}
20% {
-webkit-transform: scaleY(1.0)
}
}
@keyframes sk-stretchdelay {
0%,
40%,
100% {
transform: scaleY(0.4);
-webkit-transform: scaleY(0.4);
}
20% {
transform: scaleY(1.0);
-webkit-transform: scaleY(1.0);
}
}
.inline.required.field.cloudbrain_benchmark {
display: none;
}
</style>

<div id="mask">
<div id="loadingPage">
<div class="rect1"></div>
<div class="rect2"></div>
<div class="rect3"></div>
<div class="rect4"></div>
<div class="rect5"></div>
</div>
</div>
<div class="repository">
{{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
{{template "base/alert" .}}
<div class="ui positive message" id="messageInfo">
<p></p>
</div>
<form class="ui form" id="form_id" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<h3 class="ui top attached header">
{{.i18n.Tr "repo.cloudbrain.new"}}
</h3>
<div class="ui attached segment">
<!-- <br> -->
<div class="inline required field">
<label>任务名称</label>
<input name="job_name" id="cloudbrain_job_name" placeholder="任务名称" value="{{.job_name}}" tabindex="3" autofocus required maxlength="255">
</div>

<div class="inline field">
<label>数据集</label>
<input type="text" list="cloudbrain_dataset" placeholder="选择数据集" name="" id="answerInput" autofocus maxlength="36">
<datalist id="cloudbrain_dataset" class="ui search" style='width:385px' name="attachment">
{{range .attachments}}
<option name="attachment" data-value="{{.UUID}}">{{.Attachment.Name}}</option>
{{end}}
</datalist>
<input type="hidden" name="attachment" id="answerInput-hidden">
</div>

<div class="inline required field">
<label>工作环境</label>
<input name="de" id="cloudbrain_de" value="{{.env}}" tabindex="3" disabled autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline required field">
<label>类型</label>
<input name="job_type" id="cloudbrain_job_type" value="{{.notebook_type}}" tabindex="3" disabled autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline required field">
<label>规格</label>
<select id="cloudbrain_flavor" class="ui search dropdown" placeholder="选择规格" style='width:385px' name="flavor">
{{range .flavors}}
<option name="flavor" value="{{.Value}}">{{.Value}}</option>

{{end}}
</select>
</div>
<div class="inline required field">
<label>数据集存放路径</label>
<input name="dataset_path" id="cloudbrain_dataset_path" value="{{.dataset_path}}" tabindex="3" disabled autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline field">
<label>描述</label>
<input name="description" id="cloudbrain_description" tabindex="3" autofocus maxlength="255">
</div>
<div class="inline field">
<label></label>
<button class="ui green button">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
</div>
</form>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
// 取消创建跳转
let url_href = window.location.pathname.split('create')[0]
$(".ui.button").attr('href',url_href)

// 判断必填选项是否填写正确
let form = document.getElementById('form_id');

$('#messageInfo').css('display','none')

form.onsubmit = function(e){
let value_task = $("input[name='job_name']").val()
let re = /^[a-z0-9][a-z0-9-_]{1,36}$/
let flag = re.test(value_task)
if(!flag){
$('#messageInfo').css('display','block')
let str = '只能以小写字母或数字开头且只包含小写字母、数字、_和-、最长36个字符。'
$('#messageInfo p').text(str)
return false
}
let min_value_task = value_task.toLowerCase()
$("input[name='job_name']").attr("value",min_value_task)
document.getElementById("mask").style.display = "block"
}
// 点击按钮后遮罩层显示
// function showmask() {
// document.getElementById("mask").style.display = "block"
// }

// 页面加载完毕后遮罩层隐藏
document.onreadystatechange = function() {
if (document.readyState === "complete") {
document.getElementById("mask").style.display = "none"
}
}

$('select.dropdown')
.dropdown();

$(function() {
$("#cloudbrain_job_type").change(function() {
if ($(this).val() == 'BENCHMARK') {
$(".cloudbrain_benchmark").show();
} else {
$(".cloudbrain_benchmark").hide();
}
})
})
document.querySelector('input[list]').addEventListener('input',function(e){
var input = e.target,
list = input.getAttribute('list'),
options = document.querySelectorAll('#'+list+' option'),
hiddenInput = document.getElementById(input.getAttribute('id')+'-hidden'),
inputValue = input.value;
hiddenInput.value = inputValue;
for (let i=0;i<options.length;i++){
var option = options[i]
if(option.innerText===inputValue){
hiddenInput.value = option.getAttribute('data-value');
break
}
}


})
</script>

+ 122
- 0
templates/repo/modelarts/notebook/show.tmpl View File

@@ -0,0 +1,122 @@
{{template "base/head" .}}
<div class="repository">
{{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
{{template "base/alert" .}}

<h4 class="ui header" id="vertical-segment">
<a href="javascript:window.history.back();"><i class="arrow left icon"></i>返回</a>
</h4>
<div>
<div class="ui yellow segment">
{{with .task}}
<p>任务名称: {{.JobName}}</p>
{{end}}
</div>
<div class="ui green segment">
<p>任务结果:</p>
{{with .result}}
<table class="ui celled striped table">
<tbody>
<tr>
<td class="four wide"> 状态 </td>
<td> {{.Status}} </td>
</tr>
<tr>
<td> 开始时间 </td>
<td>{{.CreateTime}}</td>
</tr>
<tr>
<td> 最后更新时间 </td>
<td>{{.LatestUpdateTime}}</td>
</tr>
</tbody>
</table>
{{end}}
</div>
<div class="ui blue segment">
{{with .result}}
<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> 配置信息 </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> 开发环境类型 </td>
<td>{{.Profile.DeType}}</td>
</tr>
<tr>
<td> 硬件类型 </td>
<td>{{.Profile.FlavorType}}</td>
</tr>
</tbody>
</table>

<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> 机器规格详情 </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> 机器规格 </td>
<td> {{.Flavor}} </td>
</tr>
<tr>
<td> 规格名称 </td>
<td>{{.FlavorDetails.Name}}</td>
</tr>
<tr>
<td> 规格销售状态 </td>
<td>{{.FlavorDetails.Status}}</td>
</tr>
<tr>
<td> 排队个数 </td>
<td>{{.FlavorDetails.QueuingNum}}</td>
</tr>
<tr>
<td> 排到队的剩余时间(秒) </td>
<td>{{.FlavorDetails.QueueLeftTime}}</td>
</tr>
<tr>
<td> 自动停止时间(秒) </td>
<td>{{.FlavorDetails.Duration}}</td>
</tr>
</tbody>
</table>

<table class="ui celled striped table" {{if eq .QueuingInfo.RemainTime 0}}hidden{{end}}>
<thead>
<tr> <th colspan="2"> 排队信息 </th> </tr>
</thead>
<tbody>
<tr>
<td> 实例状态 </td>
<td>{{.QueuingInfo.Status}}</td>
</tr>
<tr>
<td> 实例排队的开始时间 </td>
<td>{{.QueuingInfo.BeginTime}}</td>
</tr>
<tr>
<td> 排到队的剩余时间(秒) </td>
<td>{{.QueuingInfo.RemainTime}}</td>
</tr>
<tr>
<td> 实例排队的预计停止时间 </td>
<td>{{.QueuingInfo.EndTime}}</td>
</tr>
<tr>
<td> 实例在队列中的排位 </td>
<td>{{.QueuingInfo.Rank}}</td>
</tr>
</tbody>
</table>
{{end}}
</div>
</div>

</div>
</div>
</div>
{{template "base/footer" .}}

+ 239
- 0
templates/repo/modelarts/trainjob/edit_para.tmpl View File

@@ -0,0 +1,239 @@
{{template "base/head" .}}
<div class="ui page dimmer">
<div class="ui text loader">{{.i18n.Tr "loading"}}</div>
</div>

<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
{{template "base/alert" .}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new"}}
</h4>
<div class="ui attached segment">
<!-- equal width -->
<form class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="update">
<h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}</h4>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input name="job_name" id="trainjob_job_name" placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.job_name}}" tabindex="3" autofocus required maxlength="255" readonly="">
</div>
<div class="field">
<label for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
<textarea id="description" name="description" rows="2"></textarea>
</div>
<h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}</h4>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.algorithm_origin"}}</label>
<div class="ui top attached tabular menu">
<a class="item active" data-tab="frame">{{svg "octicon-repo" 16}}{{.i18n.Tr "repo.modelarts.train_job.frames"}}</a>
</div>
<div class="ui bottom attached tab active segment" data-tab="frame">
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</label>
<div class="two fields">
<div class="field">
<select class="ui search dropdown" id="trainjob_engines" style='width:385px'>
{{range .engines}}
<option value="{{.Value}}">{{.Value}}</option>
{{end}}
</select>
</div>
<div class="field">
<select class="ui search dropdown" id="trainjob_engine_versions" style='width:385px' name="engine_id">
{{range .engine_versions}}
<option name="engine_id" value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>
</div>
</div>
</div>
<div class="inline required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
<input name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255">
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i>
</span>
</div>
</div>
</div>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.dataset"}}</label>
<select class="ui search dropdown" id="trainjob_datasets" style='width:385px' name="attachment">
{{range .attachments}}
<option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>
{{end}}
</select>
</div>
<div class="inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
<span id="add_run_para"><i class="plus circle icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>
<input id="store_run_para" type="hidden" name="run_para_list">
<div class="dynamic field">
{{range .para}}
<div class="two fields">
<div class="field">
<input type="text" name="shipping_first-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}>
</div>
<div class="field">
<input type="text" name="shipping_last-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>
</div>
<span>
<i class="trash icon">
</i>
</span>
</div>
{{end}}
</div>
</div>

<h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.resource_setting"}}</h4>
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.resource_pool"}}</label>
<select class="ui search dropdown" id="trainjob_resource_pool" style='width:385px' name="pool_id">
{{range .resource_pools}}
<option value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>
</div>

<div class="required grouped fields">
<label for="resource_type">{{.i18n.Tr "repo.modelarts.train_job.resource_type"}}</label>
{{range .benchmark_categories}}
<div class="field">
<div class="ui grid">
<div class="four wide column">
<div class="ui radio checkbox">
<input type="radio" name="resource_type" checked="" tabindex="0" class="hidden">
</div>
</div>
<div class="four wide column">train-private-1</div>
<div class="four wide column">{{svg "octicon-verified" 16}} 运行中</div>
<div class="four wide column"> CPU:192 核 2048GiB</div>
</div>
</div>
{{end}}
</div>

<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label>
<select class="ui search dropdown" id="trainjob-flavor" style='width:385px' name="flavor">
{{range .flavor_infos}}
<option name="flavor" value="{{.Code}}">{{.Value}}</option>
{{end}}
</select>
</div>
<div class="inline required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</label>
<input name="work_server_number" id="trainjob_work_server_num" tabindex="3" autofocus required maxlength="255">
</div>
<div class="inline field">
<button class="ui green button">
{{.i18n.Tr "repo.modelarts.train_job_para.connfirm"}}
</button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
</form>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
$('select.dropdown')
.dropdown();

$('.menu .item')
.tab();

// 参数增加、删除、修改、保存
function Add_parameter(){
value = '<div class="two fields">' +
'<div class="field">' +
'<input type="text" name="shipping_first-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}> ' +
'</div> ' +
'<div class="field"> ' +
'<input type="text" name="shipping_last-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>' +
'</div>'+
'<span>' +
'<i class="trash icon">' +
'</i>' +
'</span>' +
'</div>'
$(".dynamic.field").append(value)
}

$('#add_run_para').click(function(){
Add_parameter()
});

$(".dynamic.field").on("click",".trash.icon", function() {
var index = $(this).parent().parent().index()
$(this).parent().parent().remove()
});

$('.question.circle.icon').hover(function(){
$(this).popup('show')
});

$('.ui.deny.button').click(function(){
$('.ui.parameter.modal')
.modal('hide');
})

function validate(){
$('.ui.form')
.form({
on: 'blur',
inline:true,
fields: {
boot_file: {
identifier : 'boot_file',
rules: [
{
type: 'regExp[/.+\.py$/g]',
prompt : '启动文件必须为.py结尾'
}
]
},
work_server_number: {
identifier : 'work_server_number',
rules: [
{
type : 'integer[1..25]',
prompt : '计算节点需要在1-25之间,请您键入正确的值'
}
]
}
},
onSuccess: function(){
$('.ui.page.dimmer').dimmer('show')
},
onFailure: function(e){
return false;
}
})
}

function send_run_para(){
var run_parameters = []
var msg = {}
$(".dynamic.field .two.fields").each(function(){
var para_name = $(this).find('input[name=shipping_first-name]').val()
var para_value = $(this).find('input[name=shipping_last-name]').val()
run_parameters.push({"label": para_name, "value": para_value})
})
msg["parameter"] = run_parameters
msg = JSON.stringify(msg)
$('#store_run_para').val(msg)
}

$('.ui.green.button').click(function(e) {
send_run_para()
validate()
})

</script>

+ 534
- 0
templates/repo/modelarts/trainjob/index.tmpl View File

@@ -0,0 +1,534 @@
<!-- 头部导航栏 -->
{{template "base/head" .}}

<style>
.selectcloudbrain .active.item{
color: #0087f5 !important;
border: 1px solid #0087f5;
margin: -1px;
background: #FFF !important;
}
#deletemodel {
width: 100%;
height: 100%;
}
/* 弹窗 */

#mask {
position: fixed;
top: 0px;
left: 0px;
right: 0px;
bottom: 0px;
filter: alpha(opacity=60);
background-color: #777;
z-index: 1000;
display: none;
opacity: 0.8;
-moz-opacity: 0.5;
padding-top: 100px;
color: #000000
}

#loadingPage {
margin: 200px auto;
width: 50px;
height: 40px;
text-align: center;
font-size: 10px;
display: block;
}

#loadingPage>div {
background-color: green;
height: 100%;
width: 6px;
display: inline-block;
-webkit-animation: sk-stretchdelay 1.2s infinite ease-in-out;
animation: sk-stretchdelay 1.2s infinite ease-in-out;
}

#loadingPage .rect2 {
-webkit-animation-delay: -1.1s;
animation-delay: -1.1s;
}

#loadingPage .rect3 {
-webkit-animation-delay: -1.0s;
animation-delay: -1.0s;
}

#loadingPage .rect4 {
-webkit-animation-delay: -0.9s;
animation-delay: -0.9s;
}

#loadingPage .rect5 {
-webkit-animation-delay: -0.8s;
animation-delay: -0.8s;
}

@-webkit-keyframes sk-stretchdelay {
0%,
40%,
100% {
-webkit-transform: scaleY(0.4)
}
20% {
-webkit-transform: scaleY(1.0)
}
}

@keyframes sk-stretchdelay {
0%,
40%,
100% {
transform: scaleY(0.4);
-webkit-transform: scaleY(0.4);
}
20% {
transform: scaleY(1.0);
-webkit-transform: scaleY(1.0);
}
}
/* 消息框 */

.alert {
display: none;
position: fixed;
width: 100%;
z-index: 1001;
padding: 15px;
border: 1px solid transparent;
border-radius: 4px;
text-align: center;
font-weight: bold;
}

.alert-success {
color: #3c763d;
background-color: #dff0d8;
border-color: #d6e9c6;
}

.alert-info {
color: #31708f;
background-color: #d9edf7;
border-color: #bce8f1;
}

.alert-warning {
color: #8a6d3b;
background-color: #fcf8e3;
border-color: #faebcc;
}

.alert-danger {
color: #a94442;
background-color: #f2dede;
border-color: #ebccd1;
}

.pusher {
width: calc(100% - 260px);
box-sizing: border-box;
}
/* 弹窗 (background) */

#imageModal {
display: none;
position: fixed;
z-index: 1;
left: 0;
top: 0;
width: 100%;
height: 100%;
overflow: auto;
background-color: rgb(0, 0, 0);
background-color: rgba(0, 0, 0, 0.4);
}
/* 弹窗内容 */

.modal-content {
background-color: #fefefe;
margin: 15% auto;
padding: 20px;
border: 1px solid #888;
width: 30%;
}
/* 关闭按钮 */

.close {
color: #aaa;
float: right;
font-size: 28px;
font-weight: bold;
}

.close:hover,
.close:focus {
color: black;
text-decoration: none;
cursor: pointer;
}

.dis {
margin-bottom: 20px;
}

.disabled {
cursor: pointer;
pointer-events: none;
}
</style>

<!-- 弹窗 -->
<div id="mask">
<div id="loadingPage">
<div class="rect1"></div>
<div class="rect2"></div>
<div class="rect3"></div>
<div class="rect4"></div>
<div class="rect5"></div>
</div>
</div>

<!-- 提示框 -->
<div class="alert"></div>

<div class="repository release dataset-list view">
{{template "repo/header" .}}
<!-- 列表容器 -->
<div class="ui container">

<!-- 中间云脑和新建任务按钮 -->
<!-- <div class="ui three column stack able grid">
<div class="column">
<h2>{{.i18n.Tr "repo.cloudbrain2"}}</h2>
</div>

<div class="column">
</div>

<div class="column right aligned">
{{if .Permission.CanWrite $.UnitTypeCloudBrain}}
<a class="ui green button" href="{{.RepoLink}}/modelarts/create">{{.i18n.Tr "repo.cloudbrain.new"}}</a> {{end}}
</div>

<div class="column right aligned">
{{if .Permission.CanWrite $.UnitTypeCloudBrain}}
<a class="ui blue button" href="{{.RepoLink}}/modelarts/train-job/create">{{.i18n.Tr "repo.cloudbrain.new"}}</a> {{end}}
</div>
</div>

<p>使用鹏城云脑计算资源进行调试,云脑1提供CPU / GPU资源,云脑2提供Ascend NPU资源;调试使用的数据集也需要上传到对应的环境。</p>
<div class="ui blue mini menu selectcloudbrain">
<a class="item" href="{{.RepoLink}}/cloudbrain">{{svg "octicon-server" 16}} CPU / GPU</a>
<a class="active item" href="{{.RepoLink}}/modelarts">{{svg "octicon-server" 16}} Ascend NPU</a>
</div> -->

<div class="ui two column stackable grid ">

<div class="column">
<div class="ui blue small menu compact selectcloudbrain">
<a class="item" href="{{.RepoLink}}/modelarts/notebook">调试任务</a>
<a class="active item" href="{{.RepoLink}}/modelarts/train-job">训练任务</a>
</div>
</div>
<div class="column right aligned">
<div class="ui selection dropdown" style="min-width: 10em;min-height:2.6em;border-radius: .28571429rem;margin-right: 1em;padding: .67em 3.2em .7em 1em;">
{{svg "octicon-server" 16}}
<div class="default text" style="color: rgba(0,0,0,.87);"> Ascend NPU</div>
<i class="dropdown icon"></i>
<div class="menu">
<a class="item" href="{{.RepoLink}}/cloudbrain" data-value="11">CPU / GPU</a>
<a class="item" href="{{.RepoLink}}/modelarts/notebook" data-value="22">Ascend NPU</a>
</div>
</div>
<a class="ui green button" href="{{.RepoLink}}/modelarts/train-job/create">新建训练任务</a>
</div>
</div>

<!-- 中下列表展示区 -->
<div class="ui grid">
<div class="row">
<div class="ui sixteen wide column">

<!-- 排序区 -->
<!-- <div class="ui sixteen wide column">
<div class="ui two column stackable grid">
<div class="column">
</div>
<div class="column right aligned">
<div class="ui right dropdown type jump item">
<span class="text">
{{.i18n.Tr "repo.issues.filter_sort"}}<i class="dropdown icon"></i>
</span>
</div>
</div>
</div>
</div> -->

<!-- 任务展示 -->
<div class="dataset list">

<!-- 表头 -->
<div class="ui grid stackable" style="background: #f0f0f0;;">
<div class="row">
<div class="five wide column">
<span style="margin:0 6px">{{$.i18n.Tr "repo.cloudbrain_task"}}</span>
</div>
<div class="three wide column">
<span>{{$.i18n.Tr "repo.cloudbrain_status_createtime"}}</span>
</div>
<div class="three wide column">
<span>{{$.i18n.Tr "repo.cloudbrain_status_runtime"}}</span>
</div>
<div class="one wide column">
<span>{{$.i18n.Tr "repo.cloudbrain_creator"}}</span>
</div>
<div class="four wide column text center">
<span style="margin-left: 10rem;">{{$.i18n.Tr "repo.cloudbrain_operate"}}</span>
</div>

</div>
</div>



{{range .Tasks}}
<div class="ui grid stackable item">
<div class="row">
<!-- 任务名 -->
<div class="five wide column">
<a class="title" href="{{$.Link}}/{{.JobID}}" title="{{.JobName}}" style="font-size: 15px;">
<span class="fitted" style="vertical-align: middle;">{{svg "octicon-tasklist" 16}}</span>
<span class="fitted" style="width: 90%;vertical-align: middle;margin-left: 0.4rem;">{{.JobName}}</span>
</a>
</div>

<div class="three wide column">
<!--任务状态 -->
<!-- <span class="ui compact button job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
{{.Status}}
</span> -->
<span class="job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
<span><i style="vertical-align: middle;" class="{{.Status}}"></i><span style="margin-left: 0.4em;font-size: 12px;">{{.Status}}</span></span>
</span>
<!-- 任务创建时间 -->
<span style="font-size: 12px;margin-left: 0.4rem;" class="">{{TimeSinceUnix .Cloudbrain.CreatedUnix $.Lang}}</span>
</div>

<div class="three wide column">
<!--任务状态 -->
<!-- <span class="ui compact button job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
{{.Status}}
</span> -->
<span>{{.TrainJobDuration}}</span>
<!-- 任务创建时间 -->
<!-- <span style="font-size: 12px;margin-left: 0.4rem;" class="">{{TimeSinceUnix .Cloudbrain.CreatedUnix $.Lang}}</span> -->
</div>

<div class="one wide column">
{{if .User.Name}}
<a href="{{AppSubUrl}}/{{.User.Name}}" title="{{.User.Name}}"><img class="ui avatar image" src="{{.User.RelAvatarLink}}"></a>
{{else}}
<a title="Ghost"><img class="ui avatar image" src="{{AppSubUrl}}/user/avatar/Ghost/-1"></a>
{{end}}
</div>

<div class="four wide column text right">
<div class="ui compact buttons">
<!-- <a class="ui basic blue button" href="{{$.Link}}/{{.JobID}}">
查看
</a>
<a class="ui basic {{if not .CanDebug}}disabled {{else}}blue {{end}}button" href="{{$.Link}}/{{.JobID}}/debug" target="_blank">
调试
</a> -->
<form id="stopForm-{{.JobID}}" action="{{if ne .Status "RUNNING"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post" style="margin-left:-1px;">
{{$.CsrfTokenHtml}}
<a class="ui basic {{if ne .Status "RUNNING"}}disabled {{else}}blue {{end}}button" onclick="document.getElementById('stopForm-{{.JobID}}').submit();">
停止
</a>
</form>
</div>
<div class="ui compact buttons" style="margin-right:10px;">
<!-- 模型下载 -->
<a class="ui basic blue button" href="{{$.Link}}/{{.JobID}}/models" target="_blank">
模型下载
</a>
<!-- 接收结果 -->
<!-- <iframe src="" frameborder="0" name="iframeContent" style="display: none;"></iframe>
<a class="imageBtn ui basic {{if not .CanDebug}}disabled {{else}}blue {{end}}button" value="{{.CanDebug}}">提交镜像</a> -->
</div>
<!-- 删除任务 -->
<form class="ui compact buttons" id="delForm-{{.JobID}}" action="{{if eq .Status "RUNNING"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/del{{end}}" method="post">
{{$.CsrfTokenHtml}}
<a class="ui compact {{if eq .Status "RUNNING" }}disabled {{else}}red {{end}}button" onclick="assertDelete(this)" style="border-radius: .28571429rem;">
删除
</a>
</form>
</div>



</div>
</div>
{{end}} {{template "base/paginate" .}}
</div>

</div>
</div>
</div>

</div>

</div>
</div>

</div>

<!-- 确认模态框 -->
<div id="deletemodel">
<div class="ui basic modal">
<div class="ui icon header">
<i class="trash icon"></i> 删除任务
</div>

<div class="content">
<p>你确认删除该任务么?此任务一旦删除不可恢复。</p>
</div>
<div class="actions">
<div class="ui red basic inverted cancel button">
<i class="remove icon"></i> 取消操作
</div>
<div class="ui green basic inverted ok button">
<i class="checkmark icon"></i> 确定操作
</div>
</div>
</div>
</div>

</div>
{{template "base/footer" .}}

<script>
// 调试和评分新开窗口
function stop(obj) {
if (obj.style.color != "rgb(204, 204, 204)") {
obj.target = '_blank'
} else {
return
}
}

// 删除时用户确认
function assertDelete(obj) {
if (obj.style.color == "rgb(204, 204, 204)") {
return
} else {
var delId = obj.parentNode.id
flag = 1;
$('.ui.basic.modal')
.modal({
onDeny: function() {
flag = false
},
onApprove: function() {
document.getElementById(delId).submit()
flag = true
},
onHidden: function() {
if (flag == false) {
$('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut();
}
}
})
.modal('show')
}
}

// 加载任务状态
var timeid = window.setInterval(loadJobStatus, 15000);
// $(document).ready(loadJobStatus);
function loadJobStatus() {
$(".job-status").each((index, job) => {
const jobID = job.dataset.jobid;
const repoPath = job.dataset.repopath;
if (job.textContent.trim() == 'STOPPED') {
return
}

$.get(`/api/v1/repos/${repoPath}/modelarts/train-job/${jobID}`, (data) => {
const jobID = data.JobID
const status = data.JobStatus
if (status != job.textContent.trim() || status=='RUNNING') {
//$('#' + jobID).text(status)
//if (status == 'STOPPED') {
window.location.reload()
//}
}
}).fail(function(err) {
console.log(err);
});
});
};

// 获取弹窗
var modal = document.getElementById('imageModal');

// 打开弹窗的按钮对象
var btns = document.getElementsByClassName("imageBtn");

// 获取 <span> 元素,用于关闭弹窗
var spans = document.getElementsByClassName('close');

// 点击按钮打开弹窗
for (i = 0; i < btns.length; i++) {
btns[i].onclick = function() {
modal.style.display = "block";
}
}

// 点击 <span> (x), 关闭弹窗
for (i = 0; i < spans.length; i++) {
spans[i].onclick = function() {
modal.style.display = "none";
}
}

// 在用户点击其他地方时,关闭弹窗
window.onclick = function(event) {
if (event.target == modal) {
modal.style.display = "none";
}
}

// 显示弹窗,弹出相应的信息
function showmask() {
$('#imageModal').css('display', 'none')
$('#mask').css('display', 'block')

$("iframe[name=iframeContent]").on("load", function() {  
var responseText = $("iframe")[0].contentDocument.body.getElementsByTagName("pre")[0].innerHTML; 
var json1 = JSON.parse(responseText)
$('#mask').css('display', 'none')
parent.location.href

if (json1.result_code === "0") {
$('.alert').html('操作成功!').removeClass('alert-danger').addClass('alert-success').show().delay(1500).fadeOut();
} else {
$('.alert').html(json1.error_msg).removeClass('alert-success').addClass('alert-danger').show().delay(5000).fadeOut();
}
})
}
</script>

+ 47
- 0
templates/repo/modelarts/trainjob/list_model.tmpl View File

@@ -0,0 +1,47 @@
{{template "base/head" .}}
<div class="repository">
{{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
{{template "base/alert" .}}
<h4 class="ui top attached header">
<div class="ui two column grid">
<div class="column">
{{$.i18n.Tr "repo.modelarts.version_manage"}}
</div>
<div class="column right aligned">
<a href="javascript:window.history.back();">{{svg "octicon-reply" 16}}{{$.i18n.Tr "repo.modelarts.back"}}</a>
</div>
</div>
</h4>
<div class="ui attached segment">
<div class="ui style accordion">
<div class="title active">
<i class="dropdown icon"></i>
{{$.i18n.Tr "repo.modelarts.train_job.version"}}
</div>
<div class="content active">
<div class="ui list">
{{.ListModel}}
<div class="item">
<i class="file icon"></i>
<div class="content">
<div class="header">semantic.json</div>
<div class="description">Contains build settings for gulp</div>
</div>
</div>
</div>
</div>
</div>

</div>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
</script>

+ 440
- 0
templates/repo/modelarts/trainjob/new.tmpl View File

@@ -0,0 +1,440 @@
{{template "base/head" .}}
<style>

.unite{
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
}

.title{
font-size: 16px !important;
padding-left: 3rem !important;
}
.min_title{
font-size: 14px !important;
padding-left: 6rem !important;
margin-bottom: 2rem !important;

}
.width{
width:100% !important;
}
.width80{
width: 80.7% !important;
margin-left: 10px;
}
.width85{
width: 85% !important;
margin-left: 4.5rem !important;
}
.width81{
margin-left: 1.5rem;
width: 81% !important;
}

.add{font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 0px 5px 5px 0px;
line-height: 21px;
text-align: center;
color: #C2C7CC;
}
.min{
font-size: 18px;
padding: 0.5rem;
border: 1px solid rgba(187, 187, 187, 100);
border-radius: 5px 0px 0px 5px;
line-height: 21px;
text-align: center;
color: #C2C7CC;"
}

</style>
<div class="ui page dimmer">
<div class="ui text loader">{{.i18n.Tr "loading"}}</div>
</div>

<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
{{template "base/alert" .}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new"}}
</h4>
<div class="ui attached segment">
<!-- equal width -->
<form class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="update">
<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required unite min_title inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input style="width: 80%;" name="job_name" id="trainjob_job_name" placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.job_name}}" tabindex="3" autofocus required maxlength="255">
</div>
<!--<div class="inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.version"}}</label>
<span>第一版本</span>
</div>
-->
<div class="unite min_title inline field">
<label for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}&nbsp;&nbsp;</label>
<textarea style="width: 80%;" id="description" name="description" rows="2"></textarea>
</div>
<!-- <h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}</h4>
<div class="inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.fast_parameter_setting"}}</label>
<span>
{{.i18n.Tr "repo.modelarts.train_job.fast_parameter_setting_config"}}
<a class="item active parameter_config">{{.i18n.Tr "repo.modelarts.train_job.fast_parameter_setting_config_link"}}</a>
</span>
</div> -->
<div class="ui divider"></div>

<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>
<div class="required unite min_title inline fields" style="width: 90%;">
<label>{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</label>
<div class="field" style="flex: 1.5;">
<select class="ui search dropdown width" id="trainjob_engines" >
{{range .engines}}
<option value="{{.Value}}">{{.Value}}</option>
{{end}}
</select>

</div>
<div class="field" style="flex: 2;">
<select class="ui search dropdown width" id="trainjob_engine_versions" style='width: 100%;' name="engine_id">
{{range .engine_versions}}
<option name="engine_id" value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>
</div>

</div>
<!-- <div class="required inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.algorithm_origin"}}</label>
<div class="ui top attached tabular menu">
<a class="item active" data-tab="frame">{{svg "octicon-repo" 16}}{{.i18n.Tr "repo.modelarts.train_job.frames"}}</a>
</div>
<div class="ui bottom attached tab active segment" data-tab="frame">
<div class="required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</label>
<div class="two fields">
<div class="field">
<select class="ui search dropdown" id="trainjob_engines" style='width:385px'>
{{range .engines}}
<option value="{{.Value}}">{{.Value}}</option>
{{end}}
</select>
</div>
<div class="field">
<select class="ui search dropdown" id="trainjob_engine_versions" style='width:385px' name="engine_id">
{{range .engine_versions}}
<option name="engine_id" value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>
</div>
</div>
</div>
<div class="inline required field">
<label>{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
<input name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255">
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i>
</span>
</div>
</div>
</div> -->
<div class="inline unite min_title field">
<label>{{.i18n.Tr "repo.modelarts.train_job.start_file"}}</label>
<input style="width: 33.5%;" name="boot_file" id="trainjob_boot_file" value="" tabindex="3" autofocus required maxlength="255" placeholder="{{.i18n.Tr "repo.modelarts.train_job.boot_file_place"}}">
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i>
</span>
</div>
<div class="required unite min_title inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.dataset"}}</label>
<select class="ui search dropdown width80" id="trainjob_datasets" name="attachment">
{{range .attachments}}
<option name="attachment" value="{{.UUID}}">{{.Attachment.Name}}</option>
{{end}}
</select>
</div>
<div class="inline unite min_title field">
<label>{{.i18n.Tr "repo.modelarts.train_job.run_parameter"}}</label>
<!-- <i class="plus square outline icon"></i> -->
<span id="add_run_para" style="margin-left: 0.5rem;cursor:pointer;color: rgba(3, 102, 214, 100);font-size: 14px;line-height: 26px;font-family: SourceHanSansSC-medium;"><i class="plus square outline icon"></i>{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}}</span>
<input id="store_run_para" type="hidden" name="run_para_list">
<div class="dynamic field" style="margin-top: 1rem;"></div>
<!-- <div class="dynamic field">
</div> -->
</div>

<!-- <h4 class="ui dividing header">{{.i18n.Tr "repo.modelarts.train_job.resource_setting"}}</h4> -->
<div class="required field " style="display: none;">
<label>{{.i18n.Tr "repo.modelarts.train_job.resource_pool"}}</label>
<select class="ui search dropdown" id="trainjob_resource_pool" style='width:385px' name="pool_id">
{{range .resource_pools}}
<option value="{{.ID}}">{{.Value}}</option>
{{end}}
</select>
</div>

<div class="required grouped fields" style="display: none;">
<label for="resource_type">{{.i18n.Tr "repo.modelarts.train_job.resource_type"}}</label>
<div class="field">
<div class="ui grid">
<div class="column">
<div class="ui radio checkbox">
<input type="radio" name="resource_type" checked="" tabindex="0">
</div>
</div>
<div class="three wide column">train-private-1</div>
<div class="three wide column">{{svg "octicon-verified" 16}} 运行中</div>
<div class="three wide column"> CPU:192 核 2048GiB</div>
</div>
</div>
</div>

<div class="required unite min_title inline field">
<label>{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label>
<select class="ui search dropdown width81" id="trainjob-flavor" style='width:385px' name="flavor">
{{range .flavor_infos}}
<option name="flavor" value="{{.Code}}">{{.Value}}</option>
{{end}}
</select>
</div>
<div class="inline required unite min_title field">
<label>{{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</label>
<div class="ui labeled input">
<span class="min"><i class="minus icon"></i></span>
<input style="border-radius: 0;" name="work_server_number" id="trainjob_work_server_num" tabindex="3" autofocus required maxlength="255" value="1">
<span class="add"><i class="plus icon"></i></span>
</div>
<!-- <input name="work_server_number" id="trainjob_work_server_num" tabindex="3" autofocus required maxlength="255"> -->
</div>
<!--
<div class="inline field">
<div class="ui save checkbox">
<input name="is_save_para" type="checkbox">
<label>{{.i18n.Tr "repo.modelarts.train_job.query_whether_save_parameter"}}
<span>
<i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.save_helper"}} data-position="right center" data-variation="mini"></i>
</span>
</label>
</div>
</div>
<div class="disabled field" id="save_para">
<div class="field">
<label>{{.i18n.Tr "repo.modelarts.train_job.job_parameter_name"}}</label>
<input name="parameter_template_name" id="parameter_template_name" tabindex="3" autofocus maxlength="255">
</div>
<div class="field">
<label for="parameter_description">{{.i18n.Tr "repo.modelarts.train_job.parameter_description"}}</label>
<textarea id="parameter_description" name="parameter_description" rows="2"></textarea>
</div>
</div>
-->
<div class="inline unite min_title field">
<button class="ui create_train_job green button">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
<!-- 模态框 -->
</form>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
let url_href = window.location.pathname.split('create')[0]
$(".ui.button").attr('href',url_href)
$('select.dropdown')
.dropdown();

$('.menu .item')
.tab();

let sever_num = $('#trainjob_work_server_num')
$('.add').click(function(){
sever_num.val(parseInt(sever_num.val())+1)
})
$('.min').click(function(){
sever_num.val(parseInt(sever_num.val())-1)
if(sever_num.val()<=0){
sever_num.val(parseInt(sever_num.val())+1)
}
})

// 参数增加、删除、修改、保存
function Add_parameter(i){
value = '<div class="two fields width85" id= "para'+ i +'">' +
'<div class="field">' +
'<input type="text" name="shipping_first-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_name"}}> ' +
'</div> ' +
'<div class="field"> ' +
'<input type="text" name="shipping_last-name" placeholder={{.i18n.Tr "repo.modelarts.train_job.parameter_value"}}>' +
'</div>'+
'<span>' +
'<i class="trash icon">' +
'</i>' +
'</span>' +
'</div>'
$(".dynamic.field").append(value)
}

$('#add_run_para').click(function(){
var len = $(".dynamic.field .two.fields").length
Add_parameter(len)
});

$(".dynamic.field").on("click",".trash.icon", function() {
var index = $(this).parent().parent().index()
$(this).parent().parent().remove()
var len = $(".dynamic.field .two.fields").length
$(".dynamic.field .two.fields").each(function(){
var cur_index = $(this).index()
$(this).attr('id', 'para' + cur_index)
})
});

$('.ui.parameter.green.button').click(function(){
var parameters = [];
$('table tr').each(function() {
$(this).find('td:eq(1)').each(function(){
parameters.push($(this).text());
})
$(this).find('input').each(function(){
parameters.push($(this).text())
})
});
console.log(parameters)
$('.ui.parameter.modal')
.modal('hide');
for(var i = 2; i < parameters.length; i++){
switch(i) {
// 数据集uuid待完成
// case (2):
// console.log(1)
// break;
// $("#trainjob_datasets").val(parameters[i]);
// console.log($("#trainjob_datasets").val())
case (3):
$("input[name='boot_file']").val(parameters[i]);
break;
case (4):
var para = parameters[i].split(" ")
for(var j = 0; j < para.length; j++){
var para_name = para[j].split('=')[0]
var para_value = para[j].split('=')[1]
var len = $(".dynamic.field .two.fields").length
Add_parameter(len)
var pid = 'para' + len
$(".dynamic.field"+ " #" + pid + "").find("input[name=shipping_first-name]").val(para_name)
$(".dynamic.field"+ " #" + pid + "").find("input[name=shipping_last-name]").val(para_value)
}
break;
// 数据集pool_id待完成
// case (5):
// $("select[name='pool_id']").val(parameters[i]);
// break;
case (6):
$("input[name='work_server_number']").val(parameters[i]);
break;
}
}
})

$('.ui.save.checkbox').click(function(){
$(this).checkbox({
onChange: function(){
if ($('.ui.save.checkbox').checkbox('is checked')){
$('#save_para').removeClass("disabled")
}else{
$('#save_para').addClass("disabled")
}
}
});
})

$('.question.circle.icon').hover(function(){
$(this).popup('show')
});

$(".item.active.parameter_config").click(function(){
$('.ui.parameter.modal')
.modal('setting', 'closable', false)
.modal('show');
})

$('.ui.deny.button').click(function(){
$('.ui.parameter.modal')
.modal('hide');
})

function validate(){
$('.ui.form')
.form({
on: 'blur',
inline:true,
fields: {
boot_file: {
identifier : 'boot_file',
rules: [
{
type: 'regExp[/.+\.py$/g]',
prompt : '启动文件必须为.py结尾'
}
]
},
work_server_number: {
identifier : 'work_server_number',
rules: [
{
type : 'integer[1..25]',
prompt : '计算节点需要在1-25之间,请您键入正确的值'
}
]
}
},
onSuccess: function(){
$('.ui.page.dimmer').dimmer('show')
},
onFailure: function(e){
return false;
}
})
}

function send_run_para(){
var run_parameters = []
var msg = {}
$(".dynamic.field .two.fields").each(function(){
var para_name = $(this).find('input[name=shipping_first-name]').val()
var para_value = $(this).find('input[name=shipping_last-name]').val()
run_parameters.push({"label": para_name, "value": para_value})
})
msg["parameter"] = run_parameters
msg = JSON.stringify(msg)
$('#store_run_para').val(msg)
}

$('.ui.create_train_job.green.button').click(function(e) {
send_run_para()
validate()
})
</script>

+ 154
- 0
templates/repo/modelarts/trainjob/para_manage.tmpl View File

@@ -0,0 +1,154 @@
{{template "base/head" .}}
<div class="modelarts">
<div class="repository release modelarts train_job view container">
{{template "repo/header" .}}
<div class="ui container">
<div class="ui grid">
{{template "repo/modelarts/navbar" .}}
<!-- 右侧 -->
<div class="ui thirteen wide column">
<div class="ui column stackable grid">
<div class="column">
<h2>{{.i18n.Tr "repo.modelarts.train_job_para_admin"}}</h2>
</div>
</div>
<div class="ui divider"></div>
<div class="ui grid">
<div class="row">
<div class="ui sixteen wide column">
<div class="ui sixteen wide column">
<div class="ui two column stackable grid">
<div class="column">
</div>
<div class="column right aligned">
<div class="ui right dropdown type jump item">
<span class="text">
{{.i18n.Tr "repo.issues.filter_sort"}}<i class="dropdown icon"></i>
</span>
</div>
</div>
</div>
</div>
<!-- 任务展示 -->
<div class="dataset list">
{{range .Tasks}}
<div class="ui grid stackable item">
<div class="row">
<!-- 任务名 -->
<div class="five wide column">
<a class="title" href="{{$.Link}}/{{.JobID}}">
<span class="fitted">{{svg "octicon-tasklist" 16}}</span>
<span class="fitted">{{.JobName}}</span>
</a>
</div>
<!-- 引擎类型-->
<div class="four wide column job-status" id="{{.JobID}}" data-repopath="{{$.RepoRelPath}}" data-jobid="{{.JobID}}">
{{.Status}}
</div>

<!-- 创建时间 -->
<div class="three wide column">
<span class="ui text center">{{svg "octicon-clock" 16}} {{TimeSinceUnix .CreatedUnix $.Lang}}</span>
</div>

<!-- 编辑 -->
<div class="two wide column">
<a class="title" href="{{$.Link}}/{{.JobID}}/edit">
<span class="fitted">编辑</span>
</a>
</div>
<!-- 删除 -->
<div class="two wide column">
<div class="ui text center clipboard">
<form id="delForm-{{.JobID}}" action="{{$.Link}}/{{.JobID}}/del" method="post">
{{$.CsrfTokenHtml}}
<a class="fitted" onclick="assertDelete(this)" style="font-size:16px; font-weight:bold">删除</a>
</form>
</div>
</div>
</div>
</div>
{{end}} {{template "base/paginate" .}}
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>

<!-- 确认模态框 -->
<div id="deletemodel">
<div class="ui basic modal">
<div class="ui icon header">
<i class="trash icon"></i> 删除任务
</div>

<div class="content">
<p>你确认删除该任务么?此任务一旦删除不可恢复。</p>
</div>
<div class="actions">
<div class="ui red basic inverted cancel button">
<i class="remove icon"></i> 取消操作
</div>
<div class="ui green basic inverted ok button">
<i class="checkmark icon"></i> 确定操作
</div>
</div>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
// 删除时用户确认
function assertDelete(obj) {
if (obj.style.color == "rgb(204, 204, 204)") {
return
} else {
var delId = obj.parentNode.id
flag = 1;
$('.ui.basic.modal')
.modal({
onDeny: function() {
flag = false
},
onApprove: function() {
document.getElementById(delId).submit()
flag = true
},
onHidden: function() {
if (flag == false) {
$('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut();
}
}
})
.modal('show')
}
}

// 加载任务状态
$(document).ready(function() {
$(".job-status").each((index, job) => {
const jobID = job.dataset.jobid;
const repoPath = job.dataset.repopath;
if (job.textContent.trim() == 'STOPPED') {
return
}

$.get(`/api/v1/repos/${repoPath}/modelarts/train-job/${jobID}`, (data) => {
const jobID = data.JobID
const status = data.JobStatus
$('#' + jobID).text(status)
// console.log(data)
}).fail(function(err) {
console.log(err);
});
});
});
</script>

+ 200
- 0
templates/repo/modelarts/trainjob/show.tmpl View File

@@ -0,0 +1,200 @@
{{template "base/head" .}}
<div class="repository">
{{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
{{template "base/alert" .}}
<h4 class="ui top attached header">
<div class="ui two column grid">
<div class="column">
{{$.i18n.Tr "repo.modelarts.version_manage"}}
</div>
<div class="column right aligned">
<a href="javascript:window.history.back();">{{svg "octicon-reply" 16}}{{$.i18n.Tr "repo.modelarts.back"}}</a>
</div>
</div>
</h4>

<div class="ui attached segment">
<div class="ui style accordion">
<div class="title active">
<i class="dropdown icon"></i>
{{$.i18n.Tr "repo.modelarts.train_job.version"}}
</div>
<div class="content active">
<div class="ui container">
<div class="ui top attached tabular menu">
<a class="item active" data-tab="configs">配置信息</a>
<a class="item logs" data-tab="logs">{{$.i18n.Tr "repo.modelarts.log"}}</a>
<!-- <a class="item" data-tab="resources">资源占用情况</a> -->
</div>
<div class="ui bottom attached tab segment active" data-tab="configs">
<div>
<div class="ui yellow segment">
<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> {{.i18n.Tr "repo.modelarts.train_job.basic_info"}} </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.job_name"}} </td>
<td>{{.result.JobName}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.job_status"}} </td>
<td>{{.result.Status}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.version"}} </td>
<td>{{.result.VersionName}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.start_time"}} </td>
<td>{{.result.CreateTime}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.dura_time"}} </td>
<td>{{.result.TrainJobDuration}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.description"}} </td>
<td>{{.result.Description}}</td>
</tr>
</tbody>
</table>
</div>
<div class="ui green segment">
<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> {{.i18n.Tr "repo.modelarts.train_job.parameter_setting_info"}} </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.AI_driver"}} </td>
<td>{{.result.EngineName}} | {{.result.EngineVersion}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.start_file"}}</td>
<td>{{.result.BootFileUrl}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.dataset"}} </td>
<td>{{.result.DatasetName}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.run_parameter"}} </td>
<td>{{.result.Parameter}}</td>
</tr>
</tbody>
</table>
</div>
<div class="ui blue segment">
<table class="ui celled striped table">
<thead>
<tr> <th colspan="2"> {{.i18n.Tr "repo.modelarts.train_job.resource_setting_info"}} </th> </tr>
</thead>
<tbody>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.resource_pool"}} </td>
<td>{{.result.PoolName}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.amount_of_compute_node"}}</td>
<td>{{.result.WorkServerNum}}</td>
</tr>
<tr>
<td class="four wide"> {{.i18n.Tr "repo.modelarts.train_job.NAS_mount_path"}} </td>
<td>{{.result.NasMountPath}}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="ui bottom attached tab segment" data-tab="logs">
<div class="ui message" style="display: none;">
<div class="header">
</div>
</div>
<div class="ui top attached segment" style="background: #f0f0f0;">
<div class="center aligned">
<label>{{$.i18n.Tr "repo.modelarts.log"}}:</label>
<span class="fitted file_name">{{.log_file_name}}</span>
<input type="hidden" name="file_name" value={{.log_file_name}}>
<input type="hidden" name="start_line" value={{.log.StartLine}}>
<input type="hidden" name="end_line" value={{.log.EndLine}}>
</div>
</div>
<div class="ui attached segment log" style="height: 300px !important; overflow: auto;">
<pre>{{.log.Content}}</pre>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
{{template "base/footer" .}}

<script>
$('.menu .item').tab()
$('.ui.style.accordion').accordion();

var userName
var repoPath
var jobID
$(document).ready(function(){
var url = window.location.href;
var urlArr = url.split('/')
userName = urlArr.slice(-5)[0]
repoPath = urlArr.slice(-4)[0]
jobID = urlArr.slice(-1)[0]
})
$(".log").scroll(function () {
var scrollTop = $(this)[0].scrollTop; // 滚动距离
var scrollHeight = $(this)[0].scrollHeight; // 文档高度
var divHeight = $(this).height(); // 可视区高度
var file_name = $('input[name=file_name]').val()

if(parseInt(scrollTop) + divHeight + 29 == scrollHeight){
var end_line = $('input[name=end_line]').val()
$.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/log?file_name=${file_name}&base_line=${end_line}&order=desc`, (data) => {
if (data.lines == 0){
$('.header').text('您已翻阅至日志底部')
$('.message').css('display', 'block')
setTimeout(function(){
$('.message').css('display', 'none')
}, 1000)
}else{
$('input[name=end_line]').val(data.EndLine)
$('.log').append('<pre>' + data.Content)
}
}).fail(function(err) {
console.log(err);
});
}
if(scrollTop == 0){
var start_line = $('input[name=start_line]').val()
$.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/log?file_name=${file_name}&base_line=${start_line}&order=asc`, (data) => {
if (data.lines == 0){
$('.header').text('您已翻阅至日志顶部')
$('.message').css('display', 'block')
setTimeout(function(){
$('.message').css('display', 'none')
}, 1000)
}else{
$('input[name=start_line]').val(data.StartLine) //如果变动就改变所对应的值
$(".log").prepend('<pre>' + data.Content)
}
}).fail(function(err) {
console.log(err);
});
}
})
</script>

+ 1
- 0
vendor/modules.txt View File

@@ -444,6 +444,7 @@ github.com/golang/protobuf/ptypes/timestamp
# github.com/golang/snappy v0.0.1
github.com/golang/snappy
# github.com/gomodule/redigo v2.0.0+incompatible
## explicit
github.com/gomodule/redigo/internal
github.com/gomodule/redigo/redis
# github.com/google/go-github/v24 v24.0.1


+ 3
- 1
web_src/less/openi.less View File

@@ -224,7 +224,7 @@ footer .column{margin-bottom:0!important; padding-bottom:0!important;}
// icon cloudbrain
.i-round{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;}
.i-bg-organ{background-position: -496px -52px;}
.STOPPED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -459px -52px;}
.STOPPED, .KILLED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -459px -52px;}
.RUNNING{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -478px -52px;}
.i-bg-orange{background-position: -495px -51px;}
.FAILED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -532px -52px;}
@@ -233,6 +233,8 @@ footer .column{margin-bottom:0!important; padding-bottom:0!important;}
.icon-bind{background-position: -550px -52px;}
.icon-unbind{background-position: -568px -52px;}
.CREATING, .STOPPING, .DELETING, .STARTING, .WAITING{display:inline-block;background-image:url('/img/loading.gif');background-repeat:no-repeat;width:16px;height:16px;background-size:16px 16px;margin-right:5px;}

.COMPLETED{display:inline-block;width:18px;height:18px;background:url("/img/icons.svg");background-position: -496px -52px;background-position: -441px -52px;}
.text_over{
overflow: hidden;
text-overflow: ellipsis;


+ 0
- 0
web_src/less/themes/theme-arc-green.less View File


Loading…
Cancel
Save