Browse Source

Merge remote-tracking branch 'origin/V20221228' into fix-3319

tags/v1.22.12.2^2
liuzx 2 years ago
parent
commit
a9cd02268d
100 changed files with 5673 additions and 3009 deletions
  1. +4
    -0
      models/action.go
  2. +31
    -0
      models/ai_model_manage.go
  3. +11
    -0
      models/attachment.go
  4. +130
    -14
      models/cloudbrain.go
  5. +64
    -1
      models/cloudbrain_static.go
  6. +20
    -0
      models/list_options.go
  7. +229
    -4
      models/repo.go
  8. +33
    -21
      models/repo_list.go
  9. +30
    -1
      models/repo_statistic.go
  10. +31
    -0
      models/repo_tag.go
  11. +2
    -0
      models/task_config.go
  12. +14
    -0
      models/topic.go
  13. +40
    -1
      models/user.go
  14. +180
    -49
      models/user_business_analysis.go
  15. +9
    -0
      models/user_business_struct.go
  16. +21
    -0
      modules/auth/grampus.go
  17. +13
    -7
      modules/auth/modelarts.go
  18. +68
    -31
      modules/cloudbrain/cloudbrain.go
  19. +7
    -5
      modules/convert/cloudbrain.go
  20. +3
    -2
      modules/cron/tasks_basic.go
  21. +178
    -9
      modules/grampus/grampus.go
  22. +112
    -4
      modules/grampus/resty.go
  23. +47
    -213
      modules/modelarts/modelarts.go
  24. +3
    -1
      modules/modelarts/resty.go
  25. +35
    -27
      modules/modelarts_cd/modelarts.go
  26. +198
    -0
      modules/notebook/contentManager.go
  27. +9
    -0
      modules/redis/redis_key/repo_redis_key.go
  28. +47
    -17
      modules/setting/setting.go
  29. +28
    -27
      modules/structs/cloudbrain.go
  30. +23
    -0
      modules/structs/pipeline.go
  31. +17
    -2
      modules/templates/helper.go
  32. +22
    -3
      options/locale/locale_en-US.ini
  33. +26
    -5
      options/locale/locale_zh-CN.ini
  34. +34
    -65
      package-lock.json
  35. +2
    -1
      package.json
  36. +13
    -14
      public/home/home.js
  37. +1
    -1
      routers/admin/cloudbrains.go
  38. +34
    -0
      routers/admin/resources.go
  39. +17
    -1
      routers/api/v1/api.go
  40. +15
    -0
      routers/api/v1/notify/pipeline.go
  41. +85
    -9
      routers/api/v1/repo/cloudbrain.go
  42. +38
    -62
      routers/api/v1/repo/cloudbrain_dashboard.go
  43. +1
    -1
      routers/api/v1/repo/images.go
  44. +14
    -0
      routers/api/v1/repo/mlops.go
  45. +1
    -1
      routers/api/v1/repo/modelarts.go
  46. +51
    -0
      routers/api/v1/repo/modelmanage.go
  47. +8
    -6
      routers/api/v1/repo/repo_dashbord.go
  48. +14
    -2
      routers/api/v1/repo/topic.go
  49. +20
    -0
      routers/api/v1/user/repo.go
  50. +106
    -0
      routers/home.go
  51. +3
    -0
      routers/private/internal.go
  52. +36
    -5
      routers/repo/ai_model_convert.go
  53. +59
    -27
      routers/repo/ai_model_manage.go
  54. +4
    -1
      routers/repo/aisafety.go
  55. +3
    -0
      routers/repo/attachment.go
  56. +225
    -91
      routers/repo/cloudbrain.go
  57. +73
    -113
      routers/repo/cloudbrain_statistic.go
  58. +1
    -0
      routers/repo/dataset.go
  59. +668
    -81
      routers/repo/grampus.go
  60. +170
    -38
      routers/repo/modelarts.go
  61. +11
    -2
      routers/repo/repo_statistic.go
  62. +12
    -0
      routers/repo/user_data_analysis.go
  63. +33
    -15
      routers/routes/routes.go
  64. +1
    -1
      routers/user/home.go
  65. +14
    -14
      services/cloudbrain/clear.go
  66. +12
    -2
      services/cloudbrain/cloudbrainTask/count.go
  67. +252
    -29
      services/cloudbrain/cloudbrainTask/notebook.go
  68. +78
    -0
      services/cloudbrain/cloudbrainTask/sync_status.go
  69. +88
    -0
      services/repository/contributor.go
  70. +51
    -6
      services/repository/repository.go
  71. +315
    -0
      services/repository/square.go
  72. +1
    -1
      services/socketwrap/clientManager.go
  73. +19
    -19
      templates/admin/cloudbrain/list.tmpl
  74. +5
    -5
      templates/base/head_navbar.tmpl
  75. +5
    -5
      templates/base/head_navbar_fluid.tmpl
  76. +5
    -5
      templates/base/head_navbar_home.tmpl
  77. +4
    -4
      templates/base/head_navbar_pro.tmpl
  78. +2
    -2
      templates/custom/select_model.tmpl
  79. +2
    -2
      templates/custom/task_wait_count.tmpl
  80. +2
    -2
      templates/explore/navbar.tmpl
  81. +8
    -0
      templates/explore/repos/search.tmpl
  82. +16
    -0
      templates/explore/repos/square.tmpl
  83. +123
    -257
      templates/repo/cloudbrain/benchmark/new.tmpl
  84. +31
    -218
      templates/repo/cloudbrain/benchmark/show.tmpl
  85. +7
    -11
      templates/repo/cloudbrain/inference/new.tmpl
  86. +12
    -199
      templates/repo/cloudbrain/inference/show.tmpl
  87. +142
    -191
      templates/repo/cloudbrain/new.tmpl
  88. +71
    -255
      templates/repo/cloudbrain/show.tmpl
  89. +2
    -16
      templates/repo/cloudbrain/trainjob/new.tmpl
  90. +24
    -198
      templates/repo/cloudbrain/trainjob/show.tmpl
  91. +20
    -16
      templates/repo/debugjob/index.tmpl
  92. +162
    -0
      templates/repo/grampus/notebook/gpu/new.tmpl
  93. +164
    -0
      templates/repo/grampus/notebook/npu/new.tmpl
  94. +439
    -0
      templates/repo/grampus/notebook/show.tmpl
  95. +2
    -8
      templates/repo/grampus/trainjob/gpu/new.tmpl
  96. +2
    -15
      templates/repo/grampus/trainjob/npu/new.tmpl
  97. +23
    -193
      templates/repo/grampus/trainjob/show.tmpl
  98. +6
    -11
      templates/repo/modelarts/inferencejob/new.tmpl
  99. +12
    -159
      templates/repo/modelarts/inferencejob/show.tmpl
  100. +114
    -185
      templates/repo/modelarts/notebook/new.tmpl

+ 4
- 0
models/action.go View File

@@ -65,6 +65,8 @@ const (
ActionCreateImage //36 ActionCreateImage //36
ActionImageRecommend //37 ActionImageRecommend //37
ActionChangeUserAvatar //38 ActionChangeUserAvatar //38
ActionCreateGrampusNPUDebugTask //39
ActionCreateGrampusGPUDebugTask //40
) )


// Action represents user operation type and other information to // Action represents user operation type and other information to
@@ -375,6 +377,8 @@ func (a *Action) IsCloudbrainAction() bool {
ActionCreateInferenceTask, ActionCreateInferenceTask,
ActionCreateBenchMarkTask, ActionCreateBenchMarkTask,
ActionCreateGPUTrainTask, ActionCreateGPUTrainTask,
ActionCreateGrampusGPUDebugTask,
ActionCreateGrampusNPUDebugTask,
ActionCreateGrampusNPUTrainTask, ActionCreateGrampusNPUTrainTask,
ActionCreateGrampusGPUTrainTask: ActionCreateGrampusGPUTrainTask:
return true return true


+ 31
- 0
models/ai_model_manage.go View File

@@ -221,6 +221,19 @@ func SaveModelToDb(model *AiModelManage) error {
return nil return nil
} }


func QueryModelConvertByName(name string, repoId int64) ([]*AiModelConvert, error) {
sess := x.NewSession()
defer sess.Close()
sess.Select("*").Table(new(AiModelConvert)).
Where("name='" + name + "' and repo_id=" + fmt.Sprint(repoId)).OrderBy("created_unix desc")
aiModelManageConvertList := make([]*AiModelConvert, 0)
err := sess.Find(&aiModelManageConvertList)
if err == nil {
return aiModelManageConvertList, nil
}
return nil, err
}

func QueryModelConvertById(id string) (*AiModelConvert, error) { func QueryModelConvertById(id string) (*AiModelConvert, error) {
sess := x.NewSession() sess := x.NewSession()
defer sess.Close() defer sess.Close()
@@ -390,6 +403,18 @@ func QueryModelByName(name string, repoId int64) []*AiModelManage {
return aiModelManageList return aiModelManageList
} }


func QueryModelByPath(path string) (*AiModelManage, error) {
modelManage := new(AiModelManage)
has, err := x.Where("path=?", path).Get(modelManage)
if err != nil {
return nil, err
}
if !has {
return nil, ErrNotExist{}
}
return modelManage, nil
}

func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) {
sess := x.NewSession() sess := x.NewSession()
defer sess.Close() defer sess.Close()
@@ -460,6 +485,12 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) {
return aiModelManages, count, nil return aiModelManages, count, nil
} }


func QueryModelConvertCountByRepoID(repoId int64) int64 {
convert := new(AiModelConvert)
total, _ := x.Where("repo_id =?", repoId).Count(convert)
return total
}

func QueryModelConvertByRepoID(repoId int64) ([]*AiModelConvert, error) { func QueryModelConvertByRepoID(repoId int64) ([]*AiModelConvert, error) {
sess := x.NewSession() sess := x.NewSession()
defer sess.Close() defer sess.Close()


+ 11
- 0
models/attachment.go View File

@@ -259,6 +259,17 @@ func GetAttachmentsByCommentID(commentID int64) ([]*Attachment, error) {
return getAttachmentsByCommentID(x, commentID) return getAttachmentsByCommentID(x, commentID)
} }


func GetAttachmentByDatasetIdFileName(fileName string, datasetId int64) (*Attachment, error) {
attach := &Attachment{DatasetID: datasetId, Name: fileName}
has, err := x.Get(attach)
if err != nil {
return nil, err
} else if !has {
return nil, err
}
return attach, nil
}

func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) { func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) {
attachments := make([]*Attachment, 0, 10) attachments := make([]*Attachment, 0, 10)
return attachments, e.Where("comment_id=?", commentID).Find(&attachments) return attachments, e.Where("comment_id=?", commentID).Find(&attachments)


+ 130
- 14
models/cloudbrain.go View File

@@ -60,6 +60,7 @@ const (
JobTypeModelSafety JobType = "MODELSAFETY" JobTypeModelSafety JobType = "MODELSAFETY"
JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeSnn4imagenet JobType = "SNN4IMAGENET"
JobTypeBrainScore JobType = "BRAINSCORE" JobTypeBrainScore JobType = "BRAINSCORE"
JobTypeSnn4Ecoset JobType = "SNN4ECOSET"
JobTypeTrain JobType = "TRAIN" JobTypeTrain JobType = "TRAIN"
JobTypeInference JobType = "INFERENCE" JobTypeInference JobType = "INFERENCE"


@@ -114,6 +115,7 @@ const (
GrampusStatusFailed = "FAILED" GrampusStatusFailed = "FAILED"
GrampusStatusSucceeded = "SUCCEEDED" GrampusStatusSucceeded = "SUCCEEDED"
GrampusStatusStopped = "STOPPED" GrampusStatusStopped = "STOPPED"
GrampusStatusStopping = "STOPPING"
GrampusStatusUnknown = "UNKNOWN" GrampusStatusUnknown = "UNKNOWN"
GrampusStatusWaiting = "WAITING" GrampusStatusWaiting = "WAITING"


@@ -181,7 +183,7 @@ type Cloudbrain struct {
BranchName string //分支名称 BranchName string //分支名称
Parameters string //传给modelarts的param参数 Parameters string //传给modelarts的param参数
BootFile string //启动文件 BootFile string //启动文件
DataUrl string //数据集的obs路径
DataUrl string `xorm:"varchar(3500)"` //数据集的obs路径
LogUrl string //日志输出的obs路径 LogUrl string //日志输出的obs路径
PreVersionId int64 //父版本的版本id PreVersionId int64 //父版本的版本id
FlavorCode string //modelarts上的规格id FlavorCode string //modelarts上的规格id
@@ -298,6 +300,12 @@ func (task *Cloudbrain) IsUserHasRight(user *User) bool {
} }
return user.IsAdmin || user.ID == task.UserID return user.IsAdmin || user.ID == task.UserID
} }
func (task *Cloudbrain) IsGPUTask() bool {
return task.ComputeResource == GPUResource
}
func (task *Cloudbrain) IsNPUTask() bool {
return task.ComputeResource == NPUResource
}


func ConvertDurationToStr(duration int64) string { func ConvertDurationToStr(duration int64) string {
if duration <= 0 { if duration <= 0 {
@@ -328,6 +336,9 @@ func IsModelArtsDebugJobTerminal(status string) bool {
func IsCloudBrainOneDebugJobTerminal(status string) bool { func IsCloudBrainOneDebugJobTerminal(status string) bool {
return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded)
} }
func IsModelBenchMarkJobType(jobType string) bool {
return jobType == string(JobTypeSnn4imagenet) || jobType == string(JobTypeBrainScore) || jobType == string(JobTypeSnn4Ecoset)
}


func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) { func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) {
isActivated := result.JobStatus.CreatedTime > 0 isActivated := result.JobStatus.CreatedTime > 0
@@ -1064,6 +1075,9 @@ type UserImageConfig struct {
CreateVersion bool `json:"create_version"` CreateVersion bool `json:"create_version"`
Flavor Flavor `json:"flavor"` Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"` PoolID string `json:"pool_id"`
ShareAddr string `json:"nas_share_addr"`
MountPath string `json:"nas_mount_path"`
NasType string `json:"nas_type"`
} }


type CreateTrainJobParams struct { type CreateTrainJobParams struct {
@@ -1087,13 +1101,18 @@ type Config struct {
CreateVersion bool `json:"create_version"` CreateVersion bool `json:"create_version"`
Flavor Flavor `json:"flavor"` Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"` PoolID string `json:"pool_id"`
ShareAddr string `json:"nas_share_addr"`
MountPath string `json:"nas_mount_path"`
NasType string `json:"nas_type"`
} }

type CreateInferenceJobParams struct { type CreateInferenceJobParams struct {
JobName string `json:"job_name"` JobName string `json:"job_name"`
Description string `json:"job_desc"` Description string `json:"job_desc"`
InfConfig InfConfig `json:"config"` InfConfig InfConfig `json:"config"`
WorkspaceID string `json:"workspace_id"` WorkspaceID string `json:"workspace_id"`
} }

type CreateInfUserImageParams struct { type CreateInfUserImageParams struct {
JobName string `json:"job_name"` JobName string `json:"job_name"`
Description string `json:"job_desc"` Description string `json:"job_desc"`
@@ -1151,6 +1170,9 @@ type TrainJobVersionConfig struct {
Flavor Flavor `json:"flavor"` Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"` PoolID string `json:"pool_id"`
PreVersionId int64 `json:"pre_version_id"` PreVersionId int64 `json:"pre_version_id"`
ShareAddr string `json:"nas_share_addr"`
MountPath string `json:"nas_mount_path"`
NasType string `json:"nas_type"`
} }


type TrainJobVersionUserImageConfig struct { type TrainJobVersionUserImageConfig struct {
@@ -1166,6 +1188,9 @@ type TrainJobVersionUserImageConfig struct {
PreVersionId int64 `json:"pre_version_id"` PreVersionId int64 `json:"pre_version_id"`
UserImageUrl string `json:"user_image_url"` UserImageUrl string `json:"user_image_url"`
UserCommand string `json:"user_command"` UserCommand string `json:"user_command"`
ShareAddr string `json:"nas_share_addr"`
MountPath string `json:"nas_mount_path"`
NasType string `json:"nas_type"`
} }


type CreateConfigParams struct { type CreateConfigParams struct {
@@ -1181,6 +1206,7 @@ type CreateConfigParams struct {
LogUrl string `json:"log_url"` LogUrl string `json:"log_url"`
Flavor Flavor `json:"flavor"` Flavor Flavor `json:"flavor"`
PoolID string `json:"pool_id"` PoolID string `json:"pool_id"`
Volumes []Volumes `json:"volumes"`
} }


type Parameter struct { type Parameter struct {
@@ -1203,6 +1229,13 @@ type DatasetDownload struct {
IsDelete bool `json:"is_delete"` IsDelete bool `json:"is_delete"`
} }


type ModelDownload struct {
Name string `json:"name"`
DownloadLink string `json:"download_link"`
RepositoryLink string `json:"repository_link"`
IsDelete bool `json:"is_delete"`
}

type DataSource struct { type DataSource struct {
DatasetID string `json:"dataset_id"` DatasetID string `json:"dataset_id"`
DatasetVersion string `json:"dataset_version"` DatasetVersion string `json:"dataset_version"`
@@ -1446,6 +1479,20 @@ type GrampusJobInfo struct {
UserID string `json:"userId"` UserID string `json:"userId"`
Tasks []GrampusTasks `json:"tasks"` Tasks []GrampusTasks `json:"tasks"`
} }

type GrampusNotebookInfo struct {
StartedAt int64 `json:"startedAt"`
RunSec int64 `json:"runSec"`
CompletedAt int64 `json:"completedAt"`
CreatedAt int64 `json:"createdAt"`
UpdatedAt int64 `json:"updatedAt"`
Desc string `json:"desc"`
JobID string `json:"id"`
Name string `json:"name"`
Status string `json:"status"`
UserID string `json:"userId"`
Tasks []GrampusNotebookTask `json:"tasks"`
}
type Center struct { type Center struct {
ID string `json:"id"` ID string `json:"id"`
Name string `json:"name"` Name string `json:"name"`
@@ -1522,9 +1569,22 @@ type GetGrampusJobResponse struct {
JobInfo GrampusJobInfo `json:"otJob"` JobInfo GrampusJobInfo `json:"otJob"`
} }


type GrampusNotebookResponse struct {
GrampusResult
JobInfo GrampusNotebookInfo `json:"otJob"`
}

type GrampusNotebookRestartResponse struct {
GrampusResult
NewId string `json:"newId"`
Status string `json:"status"`
}

type GrampusStopJobResponse struct { type GrampusStopJobResponse struct {
GrampusResult GrampusResult
StoppedAt int64 `json:"stoppedAt"`
StoppedAt int64 `json:"stoppedAt"`
ID string `json:"id"`
Status string `json:"status"`
} }


type GrampusTasks struct { type GrampusTasks struct {
@@ -1541,12 +1601,32 @@ type GrampusTasks struct {
Code GrampusDataset `json:"code"` Code GrampusDataset `json:"code"`
BootFile string `json:"bootFile"` BootFile string `json:"bootFile"`
} }
type GrampusNotebookTask struct {
AutoStopDuration int `json:"autoStopDuration"`
Name string `json:"name"`
Capacity int `json:"capacity"`
CenterID []string `json:"centerID"`
CenterName []string `json:"centerName"`
Code GrampusDataset `json:"code"`
Datasets []GrampusDataset `json:"datasets"`
CodeUrl string `json:"codeUrl"`
DataUrl string `json:"dataUrl"`
ImageId string `json:"imageId"`
ImageUrl string `json:"imageUrl"`
ResourceSpecId string `json:"resourceSpecId"`
Token string `json:"token"`
Url string `json:"url"`
Status string `json:"status"`
Command string `json:"command"`
}


type GrampusDataset struct { type GrampusDataset struct {
Name string `json:"name"`
Bucket string `json:"bucket"`
EndPoint string `json:"endPoint"`
ObjectKey string `json:"objectKey"`
Name string `json:"name"`
Bucket string `json:"bucket"`
EndPoint string `json:"endPoint"`
ObjectKey string `json:"objectKey"`
ContainerPath string `json:"containerPath"`
ReadOnly bool `json:"readOnly"`
} }


type CreateGrampusJobRequest struct { type CreateGrampusJobRequest struct {
@@ -1554,6 +1634,11 @@ type CreateGrampusJobRequest struct {
Tasks []GrampusTasks `json:"tasks"` Tasks []GrampusTasks `json:"tasks"`
} }


type CreateGrampusNotebookRequest struct {
Name string `json:"name"`
Tasks []GrampusNotebookTask `json:"tasks"`
}

type GetTrainJobMetricStatisticResult struct { type GetTrainJobMetricStatisticResult struct {
TrainJobResult TrainJobResult
Interval int `json:"interval"` //查询的时间间隔,单位为分钟 Interval int `json:"interval"` //查询的时间间隔,单位为分钟
@@ -1745,7 +1830,7 @@ func QueryModelTrainJobVersionList(jobId string) ([]*Cloudbrain, int, error) {
return cloudbrains, int(len(cloudbrains)), nil return cloudbrains, int(len(cloudbrains)), nil
} }


func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) {
func QueryModelTrainJobList(repoId int64) ([]*Cloudbrain, int, error) {
sess := x.NewSession() sess := x.NewSession()
defer sess.Close() defer sess.Close()
var cond = builder.NewCond() var cond = builder.NewCond()
@@ -1762,14 +1847,14 @@ func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) {
// builder.In("type", 0, 1), // builder.In("type", 0, 1),
// ) // )


cloudbrains := make([]*CloudbrainInfo, 0)
cloudbrains := make([]*Cloudbrain, 0)
if err := sess.Select("job_id,display_job_name").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC"). if err := sess.Select("job_id,display_job_name").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC").
Find(&cloudbrains); err != nil { Find(&cloudbrains); err != nil {
return nil, 0, fmt.Errorf("Find: %v", err) return nil, 0, fmt.Errorf("Find: %v", err)
} }


keys := make(map[string]string) keys := make(map[string]string)
uniqueElements := make([]*CloudbrainInfo, 0)
uniqueElements := make([]*Cloudbrain, 0)
for _, entry := range cloudbrains { for _, entry := range cloudbrains {
if _, value := keys[entry.JobID]; !value { if _, value := keys[entry.JobID]; !value {
keys[entry.JobID] = entry.DisplayJobName keys[entry.JobID] = entry.DisplayJobName
@@ -1865,6 +1950,7 @@ func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) {
session.Commit() session.Commit()


go IncreaseDatasetUseCount(cloudbrain.Uuid) go IncreaseDatasetUseCount(cloudbrain.Uuid)
go OperateRepoAITaskNum(cloudbrain.RepoID, 1)
return nil return nil
} }


@@ -2020,13 +2106,42 @@ func DeleteJob(job *Cloudbrain) error {


func deleteJob(e Engine, job *Cloudbrain) error { func deleteJob(e Engine, job *Cloudbrain) error {
_, err := e.ID(job.ID).Delete(job) _, err := e.ID(job.ID).Delete(job)
if err == nil {
go updateAITaskNumWhenDeleteJob(job)
}
return err return err
} }


func updateAITaskNumWhenDeleteJob(job *Cloudbrain) {
repoId := job.RepoID
if repoId == 0 {
t := &Cloudbrain{}
_, tempErr := x.ID(job.ID).Unscoped().Get(t)
if tempErr != nil {
log.Error("updateAITaskNumWhenDeleteJob error.%v", tempErr)
return
}
repoId = t.RepoID
}

if repoId > 0 {
go OperateRepoAITaskNum(repoId, -1)
}
}

func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { func GetCloudbrainByName(jobName string) (*Cloudbrain, error) {
cb := &Cloudbrain{JobName: jobName} cb := &Cloudbrain{JobName: jobName}
return getRepoCloudBrain(cb) return getRepoCloudBrain(cb)
} }
func GetWaitOrRunFileNotebookByRepo(repoId int64, cloudbrainType int) (*Cloudbrain, error) {
cloudBrain := new(Cloudbrain)
has, err := x.In("status", JobWaiting, JobRunning, ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting,
ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsDeleting, ModelArtsRestarting).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain)
if has {
return cloudBrain, err
}
return nil, err
}


func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool {
if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) { if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) {
@@ -2060,7 +2175,7 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) {
Find(&cloudbrains) Find(&cloudbrains)
} }


func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
func GetGPUStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
cloudbrains := make([]*Cloudbrain, 0, 10) cloudbrains := make([]*Cloudbrain, 0, 10)
endTimeBefore := time.Now().Unix() - int64(days)*24*3600 endTimeBefore := time.Now().Unix() - int64(days)*24*3600
missEndTimeBefore := endTimeBefore - 24*3600 missEndTimeBefore := endTimeBefore - 24*3600
@@ -2069,7 +2184,7 @@ func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbra
JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted,
ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed,
ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed).
Where("(((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false and type=0 and job_type != 'DEBUG'", missEndTimeBefore, endTimeBefore).
Where("(((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false and (type=0 or (type =2 and compute_resource='CPU/GPU')) and job_type != 'DEBUG'", missEndTimeBefore, endTimeBefore).
Limit(limit). Limit(limit).
Find(&cloudbrains) Find(&cloudbrains)
} }
@@ -2077,14 +2192,14 @@ func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbra
/** /**
本方法考虑了再次调试的情况,多次调试取最后一次的任务的结束时间 本方法考虑了再次调试的情况,多次调试取最后一次的任务的结束时间
*/ */
func GetCloudBrainOneStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
func GetGPUStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
cloudbrains := make([]*Cloudbrain, 0, 10) cloudbrains := make([]*Cloudbrain, 0, 10)
endTimeBefore := time.Now().Unix() - int64(days)*24*3600 endTimeBefore := time.Now().Unix() - int64(days)*24*3600
missEndTimeBefore := endTimeBefore - 24*3600 missEndTimeBefore := endTimeBefore - 24*3600
sql := `SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name) sql := `SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name)
id, job_name, job_id,status,end_time,updated_unix,cleared id, job_name, job_id,status,end_time,updated_unix,cleared
FROM cloudbrain FROM cloudbrain
where type=0 and job_type='DEBUG'
where (type=0 or (type =2 and compute_resource='CPU/GPU')) and job_type='DEBUG'
ORDER BY job_name, updated_unix DESC) a ORDER BY job_name, updated_unix DESC) a
where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false` where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false`


@@ -2225,7 +2340,6 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) {
} }


go IncreaseDatasetUseCount(new.Uuid) go IncreaseDatasetUseCount(new.Uuid)

return nil return nil
} }
func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) {
@@ -2538,6 +2652,7 @@ type DatasetInfo struct {
DataLocalPath string DataLocalPath string
Name string Name string
FullName string FullName string
Size int
} }


func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) { func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) {
@@ -2593,6 +2708,7 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn
DataLocalPath: dataLocalPath, DataLocalPath: dataLocalPath,
Name: fileName, Name: fileName,
FullName: attach.Name, FullName: attach.Name,
Size: int(attach.Size),
} }
if i == 0 { if i == 0 {
datasetNames = attach.Name datasetNames = attach.Name


+ 64
- 1
models/cloudbrain_static.go View File

@@ -92,6 +92,17 @@ type HourTimeStatistic struct {
HourTimeTotalDuration map[string]int `json:"hourTimeTotalDuration"` HourTimeTotalDuration map[string]int `json:"hourTimeTotalDuration"`
HourTimeUsageRate map[string]float64 `json:"hourTimeUsageRate"` HourTimeUsageRate map[string]float64 `json:"hourTimeUsageRate"`
} }
type CloudbrainTypeDuration []struct {
Type int `xorm:"type"`
DurationSum int `xorm:"durationSum"`
CardDurationSum int `xorm:"cardDurationSum"`
Count int `xorm:"count"`
}
type CloudbrainAllDuration struct {
DurationSum int `xorm:"durationSum"`
CardDurationSum int `xorm:"cardDurationSum"`
Count int `xorm:"count"`
}


func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(distinct user_id) FROM " + countSql := "SELECT count(distinct user_id) FROM " +
@@ -303,7 +314,7 @@ func GetCloudbrainByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, err
builder.And(builder.Gte{"cloudbrain.start_time": beginTime}, builder.Lte{"cloudbrain.start_time": endTime}, builder.Gt{"cloudbrain.start_time": 0}), builder.And(builder.Gte{"cloudbrain.start_time": beginTime}, builder.Lte{"cloudbrain.start_time": endTime}, builder.Gt{"cloudbrain.start_time": 0}),
) )
cond = cond.Or( cond = cond.Or(
builder.And(builder.Eq{"cloudbrain.status": string(JobRunning)}),
builder.And(builder.Eq{"cloudbrain.status": string(JobRunning)}, builder.Lte{"cloudbrain.start_time": beginTime}),
) )
sess.OrderBy("cloudbrain.id ASC") sess.OrderBy("cloudbrain.id ASC")
cloudbrains := make([]*CloudbrainInfo, 0, 10) cloudbrains := make([]*CloudbrainInfo, 0, 10)
@@ -425,3 +436,55 @@ func DeleteCloudbrainDurationStatistic(beginTime timeutil.TimeStamp, endTime tim
} }
return nil return nil
} }

func GetCloudbrainTypeCardDuration() (CloudbrainTypeDuration, error) {
query := `
SELECT
cloudbrain.type,
SUM(cloudbrain.duration) as durationSum,
SUM(
COALESCE(cloudbrain.duration *
CASE
WHEN cloudbrain.work_server_number = 0 THEN 1
ELSE COALESCE(cloudbrain.work_server_number, 1)
END *
COALESCE(cloudbrain_spec.acc_cards_num, 1), 0)
) as cardDurationSum,
COUNT(*) as count
FROM cloudbrain
LEFT JOIN cloudbrain_spec
ON cloudbrain.id = cloudbrain_spec.cloudbrain_id
GROUP BY cloudbrain.type
`
// 执行查询
var results CloudbrainTypeDuration
if err := x.SQL(query).Find(&results); err != nil {
panic(err)
}
return results, nil
}

func GetCloudbrainAllCardDuration() (CloudbrainAllDuration, error) {
query := `
SELECT
SUM(cloudbrain.duration) as durationSum,
SUM(
COALESCE(cloudbrain.duration *
CASE
WHEN cloudbrain.work_server_number = 0 THEN 1
ELSE COALESCE(cloudbrain.work_server_number, 1)
END *
COALESCE(cloudbrain_spec.acc_cards_num, 1), 0)
) as cardDurationSum,
COUNT(*) as count
FROM cloudbrain
LEFT JOIN cloudbrain_spec
ON cloudbrain.id = cloudbrain_spec.cloudbrain_id
`
// 执行查询
var result CloudbrainAllDuration
if _, err := x.SQL(query).Get(&result); err != nil {
panic(err)
}
return result, nil
}

+ 20
- 0
models/list_options.go View File

@@ -10,6 +10,26 @@ import (
"xorm.io/xorm" "xorm.io/xorm"
) )


type AvailablePageSize int

const (
PageSize15 AvailablePageSize = 15
PageSize30 AvailablePageSize = 30
PageSize50 AvailablePageSize = 50
)

func (s AvailablePageSize) IsLegal() bool {
switch s {
case PageSize30, PageSize50, PageSize15:
return true
}
return false
}

func (s AvailablePageSize) Int() int {
return int(s)
}

// ListOptions options to paginate results // ListOptions options to paginate results
type ListOptions struct { type ListOptions struct {
PageSize int PageSize int


+ 229
- 4
models/repo.go View File

@@ -231,10 +231,43 @@ type Repository struct {
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`


Hot int64 `xorm:"-"`
Active int64 `xorm:"-"`
Alias string `xorm:"INDEX"`
LowerAlias string `xorm:"INDEX"`
Hot int64 `xorm:"-"`
Active int64 `xorm:"-"`
Alias string `xorm:"INDEX"`
LowerAlias string `xorm:"INDEX"`
AiTaskCnt int64 `xorm:"NOT NULL DEFAULT 0"`
ModelCnt int64 `xorm:"NOT NULL DEFAULT 0"`
DatasetCnt int64 `xorm:"NOT NULL DEFAULT 0"`
LastMonthVisits int64 `xorm:"NOT NULL DEFAULT 0"`
LastFourMonthCommits int64 `xorm:"NOT NULL DEFAULT 0"`
}

// Repository4Card format for front display
type Repository4Card struct {
ID int64
OwnerID int64
OwnerName string
LowerName string
Name string
Alias string
NumWatches int
NumStars int
NumForks int
Description string
Topics []string
AiTaskCnt int64
ModelCnt int64
DatasetCnt int64
CreatedUnix timeutil.TimeStamp
UpdatedUnix timeutil.TimeStamp
PrimaryLanguage *LanguageStat
RelAvatarLink string
Contributors []*ContributorInfo
IsPrivate bool
IsFork bool
IsMirror bool
IsOwnerPrivate bool
IsArchived bool
} }


type RepositoryShow struct { type RepositoryShow struct {
@@ -243,6 +276,47 @@ type RepositoryShow struct {
Alias string Alias string
} }


func (repo *Repository) ToCardFormat() *Repository4Card {
link := repo.RelAvatarLink()
var isOwnerPrivate bool
if repo.Owner != nil && repo.Owner.Visibility.IsPrivate() {
isOwnerPrivate = true
}
result := &Repository4Card{
ID: repo.ID,
OwnerID: repo.OwnerID,
OwnerName: repo.OwnerName,
LowerName: repo.LowerName,
Name: repo.Name,
NumWatches: repo.NumWatches,
NumStars: repo.NumStars,
NumForks: repo.NumForks,
Description: repo.Description,
Topics: repo.Topics,
AiTaskCnt: repo.AiTaskCnt,
ModelCnt: repo.ModelCnt,
DatasetCnt: repo.DatasetCnt,
CreatedUnix: repo.CreatedUnix,
UpdatedUnix: repo.UpdatedUnix,
PrimaryLanguage: repo.PrimaryLanguage,
RelAvatarLink: link,
Alias: repo.Alias,
IsPrivate: repo.IsPrivate,
IsFork: repo.IsFork,
IsMirror: repo.IsMirror,
IsOwnerPrivate: isOwnerPrivate,
IsArchived: repo.IsArchived,
}
return result
}

type ContributorInfo struct {
RelAvatarLink string
UserName string
Email string
CommitCnt int
}

// SanitizedOriginalURL returns a sanitized OriginalURL // SanitizedOriginalURL returns a sanitized OriginalURL
func (repo *Repository) SanitizedOriginalURL() string { func (repo *Repository) SanitizedOriginalURL() string {
if repo.OriginalURL == "" { if repo.OriginalURL == "" {
@@ -2379,6 +2453,75 @@ func CheckRepoStats(ctx context.Context) error {
} }
} }
// ***** END: Repository.NumForks ***** // ***** END: Repository.NumForks *****

// ***** START: Repository.DatasetCnt *****
desc = "repository count 'dataset_cnt'"
results, err = x.Query("SELECT repository.id FROM `repository` WHERE repository.dataset_cnt!=(select count(1) from attachment inner join dataset on attachment.dataset_id = dataset.id where dataset.repo_id = repository.id)")
if err != nil {
log.Error("Select %s: %v", desc, err)
} else {
for _, result := range results {
id := com.StrTo(result["id"]).MustInt64()
select {
case <-ctx.Done():
log.Warn("CheckRepoStats: Cancelled")
return ErrCancelledf("during %s for repo ID %d", desc, id)
default:
}
log.Trace("Updating %s: %d", desc, id)
err = ResetRepoDatasetNum(id)
if err != nil {
log.Error("Update %s[%d]: %v", desc, id, err)
}
}
}
// ***** END: Repository.DatasetCnt *****

// ***** START: Repository.ModelCnt *****
desc = "repository count 'model_cnt'"
results, err = x.Query("SELECT repository.id FROM `repository` WHERE repository.model_cnt!=(select count(1) from ai_model_manage where repository.id = ai_model_manage.repo_id and ai_model_manage.size > 0 )")
if err != nil {
log.Error("Select %s: %v", desc, err)
} else {
for _, result := range results {
id := com.StrTo(result["id"]).MustInt64()
select {
case <-ctx.Done():
log.Warn("CheckRepoStats: Cancelled")
return ErrCancelledf("during %s for repo ID %d", desc, id)
default:
}
log.Trace("Updating %s: %d", desc, id)
err = ResetRepoModelNum(id)
if err != nil {
log.Error("Update %s[%d]: %v", desc, id, err)
}
}
}
// ***** END: Repository.ModelCnt *****

// ***** START: Repository.AiTaskCnt *****
desc = "repository count 'ai_task_cnt'"
results, err = x.Query("SELECT repository.id FROM `repository` WHERE repository.ai_task_cnt!=(select count(1) from cloudbrain where repository.id = cloudbrain.repo_id and (cloudbrain.deleted_at is null or cloudbrain.deleted_at = '0001-01-01 00:00:00') )")
if err != nil {
log.Error("Select %s: %v", desc, err)
} else {
for _, result := range results {
id := com.StrTo(result["id"]).MustInt64()
select {
case <-ctx.Done():
log.Warn("CheckRepoStats: Cancelled")
return ErrCancelledf("during %s for repo ID %d", desc, id)
default:
}
log.Trace("Updating %s: %d", desc, id)
err = ResetRepoAITaskNum(id)
if err != nil {
log.Error("Update %s[%d]: %v", desc, id, err)
}
}
}
// ***** END: Repository.AiTaskCnt *****
return nil return nil
} }


@@ -2775,3 +2918,85 @@ func ReadLatestFileInRepo(userName, repoName, refName, treePath string) (*RepoFi
} }
return &RepoFile{CommitId: commitId, Content: d}, nil return &RepoFile{CommitId: commitId, Content: d}, nil
} }

func ResetRepoAITaskNum(repoId int64) error {
n, err := x.Where("repo_id = ? ", repoId).Count(&Cloudbrain{})
if err != nil {
return err
}
r := Repository{
AiTaskCnt: n,
}
_, err = x.Cols("ai_task_cnt").Where("id = ?", repoId).Update(&r)
return err
}

func ResetRepoDatasetNum(repoId int64) error {
n, err := x.Table("attachment").Join("inner", "dataset", "attachment.dataset_id = dataset.id").Where("dataset.repo_id = ?", repoId).Count()
if err != nil {
return err
}
r := Repository{
DatasetCnt: n,
}
_, err = x.Cols("dataset_cnt").Where("id = ?", repoId).Update(&r)
return err
}

func ResetRepoModelNum(repoId int64) error {
_, err := x.Exec("update repository set model_cnt = (select count(1) from ai_model_manage where ai_model_manage.repo_id = ? and size > 0) where id = ?", repoId, repoId)
return err
}

func operateRepoCol(repoId int64, colName string, amount int64, engines ...*xorm.Engine) error {
var err error

if amount == 0 {
return nil
}
var ee *xorm.Engine
if len(engines) == 0 {
ee = x
} else {
ee = engines[0]
}
if amount > 0 {
_, err = ee.Exec(fmt.Sprintf("update repository set %s = %s + ? where id = ?", colName, colName), amount, repoId)
} else {
_, err = ee.Exec(fmt.Sprintf("update repository set %s = %s - ? where id = ?", colName, colName), -1*amount, repoId)
}

return err
}

func OperateRepoDatasetNum(repoId int64, amount int64, engines ...*xorm.Engine) error {
return operateRepoCol(repoId, "dataset_cnt", amount, engines...)
}

func OperateRepoModelNum(repoId int64, amount int64, engines ...*xorm.Engine) error {
return operateRepoCol(repoId, "model_cnt", amount, engines...)
}

func OperateRepoAITaskNum(repoId int64, amount int64, engines ...*xorm.Engine) error {
return operateRepoCol(repoId, "ai_task_cnt", amount, engines...)
}

func UpdateRepositoryLastFourMonthCommits(repoID int64, amount int64) error {
_, err := x.Exec("update repository set last_four_month_commits = ? where id = ?", amount, repoID)
return err
}
func UpdateRepositoryLastMonthVisits(repoID int64, amount int64) error {
_, err := x.Exec("update repository set last_month_visits = ? where id = ?", amount, repoID)
return err
}

func SyncStatDataToRepo(repo *Repository) {
//Save the visit number of repository in the last month
if lv, err := SumLastMonthNumVisits(repo.ID); err == nil {
UpdateRepositoryLastMonthVisits(repo.ID, lv)
}
//Save the commits number of repository in the last four month
if lc, err := SumLastFourMonthNumCommits(repo.ID); err == nil {
UpdateRepositoryLastFourMonthCommits(repo.ID, lc)
}
}

+ 33
- 21
models/repo_list.go View File

@@ -201,29 +201,41 @@ func (s SearchOrderBy) String() string {
return string(s) return string(s)
} }


type FindReposResponse struct {
Repos []*Repository4Card
Page int
PageSize int
Total int64
}

// Strings for sorting result // Strings for sorting result
const ( const (
SearchOrderByAlphabetically SearchOrderBy = "name ASC"
SearchOrderByAlphabeticallyReverse SearchOrderBy = "name DESC"
SearchOrderByLeastUpdated SearchOrderBy = "updated_unix ASC"
SearchOrderByRecentUpdated SearchOrderBy = "updated_unix DESC"
SearchOrderByOldest SearchOrderBy = "created_unix ASC"
SearchOrderByNewest SearchOrderBy = "created_unix DESC"
SearchOrderBySize SearchOrderBy = "size ASC"
SearchOrderBySizeReverse SearchOrderBy = "size DESC"
SearchOrderByID SearchOrderBy = "id ASC"
SearchOrderByIDReverse SearchOrderBy = "id DESC"
SearchOrderByStars SearchOrderBy = "num_stars ASC"
SearchOrderByStarsReverse SearchOrderBy = "num_stars DESC"
SearchOrderByForks SearchOrderBy = "num_forks ASC"
SearchOrderByForksReverse SearchOrderBy = "num_forks DESC"
SearchOrderByDownloadTimes SearchOrderBy = "download_times DESC"
SearchOrderByUseCount SearchOrderBy = "use_count ASC"
SearchOrderByUseCountReverse SearchOrderBy = "use_count DESC"
SearchOrderByHot SearchOrderBy = "(num_watches + num_stars + num_forks + clone_cnt) DESC"
SearchOrderByActive SearchOrderBy = "(num_issues + num_pulls + num_commit) DESC"
SearchOrderByWatches SearchOrderBy = "num_watches DESC"
SearchOrderByDefault SearchOrderBy = "recommend desc,num_stars DESC,updated_unix DESC"
SearchOrderByAlphabetically SearchOrderBy = "name ASC"
SearchOrderByAlphabeticallyReverse SearchOrderBy = "name DESC"
SearchOrderByLeastUpdated SearchOrderBy = "updated_unix ASC"
SearchOrderByRecentUpdated SearchOrderBy = "updated_unix DESC"
SearchOrderByOldest SearchOrderBy = "created_unix ASC"
SearchOrderByNewest SearchOrderBy = "created_unix DESC"
SearchOrderBySize SearchOrderBy = "size ASC"
SearchOrderBySizeReverse SearchOrderBy = "size DESC"
SearchOrderByID SearchOrderBy = "id ASC"
SearchOrderByIDReverse SearchOrderBy = "id DESC"
SearchOrderByStars SearchOrderBy = "num_stars ASC"
SearchOrderByStarsReverse SearchOrderBy = "num_stars DESC"
SearchOrderByForks SearchOrderBy = "num_forks ASC"
SearchOrderByForksReverse SearchOrderBy = "num_forks DESC"
SearchOrderByDownloadTimes SearchOrderBy = "download_times DESC"
SearchOrderByUseCount SearchOrderBy = "use_count ASC"
SearchOrderByUseCountReverse SearchOrderBy = "use_count DESC"
SearchOrderByHot SearchOrderBy = "(num_watches + num_stars + num_forks + clone_cnt) DESC"
SearchOrderByActive SearchOrderBy = "(num_issues + num_pulls + num_commit) DESC"
SearchOrderByWatches SearchOrderBy = "num_watches DESC"
SearchOrderByDefault SearchOrderBy = "recommend desc,num_stars DESC,updated_unix DESC"
SearchOrderByAiTaskCntReverse SearchOrderBy = "ai_task_cnt desc"
SearchOrderByModelCntReverse SearchOrderBy = "model_cnt desc"
SearchOrderByDatasetCntReverse SearchOrderBy = "dataset_cnt desc"
SearchOrderByLastMonthVisitsReverse SearchOrderBy = "last_month_visits desc"
SearchOrderByLastFourMonthCommitsReverse SearchOrderBy = "last_four_month_commits desc"
) )


// SearchRepositoryCondition creates a query condition according search repository options // SearchRepositoryCondition creates a query condition according search repository options


+ 30
- 1
models/repo_statistic.go View File

@@ -36,7 +36,7 @@ type RepoStatistic struct {
NumDevMonths int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` NumDevMonths int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`
RepoSize int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` RepoSize int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`
DatasetSize int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` DatasetSize int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`
NumModels int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`
NumModels int64 `xorm:"NOT NULL DEFAULT 0" json:"model"`
NumWikiViews int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` NumWikiViews int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`
NumCommits int64 `xorm:"NOT NULL DEFAULT 0" json:"commit"` NumCommits int64 `xorm:"NOT NULL DEFAULT 0" json:"commit"`
NumCommitsAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` NumCommitsAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`
@@ -55,6 +55,15 @@ type RepoStatistic struct {
NumIssuesGrowth int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` NumIssuesGrowth int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`
NumCommentsGrowth int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` NumCommentsGrowth int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`


NumDatasetFile int64 `xorm:"NOT NULL DEFAULT 0" json:"datasetFiles"`
NumCloudbrain int64 `xorm:"NOT NULL DEFAULT 0" json:"cloudbrains"`
NumModelConvert int64 `xorm:"NOT NULL DEFAULT 0" json:"modelConverts"`

NumDatasetFileAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`
NumCloudbrainAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"-"`
NumModelConvertAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"- "`
NumModelsAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"- "`

Impact float64 `xorm:"NOT NULL DEFAULT 0" json:"impact"` Impact float64 `xorm:"NOT NULL DEFAULT 0" json:"impact"`
Completeness float64 `xorm:"NOT NULL DEFAULT 0" json:"completeness"` Completeness float64 `xorm:"NOT NULL DEFAULT 0" json:"completeness"`
Liveness float64 `xorm:"NOT NULL DEFAULT 0" json:"liveness"` Liveness float64 `xorm:"NOT NULL DEFAULT 0" json:"liveness"`
@@ -200,3 +209,23 @@ func UpdateRepoStatVisits(repoStat *RepoStatistic) error {
_, err := xStatistic.Exec(sql, repoStat.NumVisits, repoStat.RepoID, repoStat.Date) _, err := xStatistic.Exec(sql, repoStat.NumVisits, repoStat.RepoID, repoStat.Date)
return err return err
} }

func SumRepoStatColumn(begin, end time.Time, repoId int64, columnName string) (int64, error) {
res, err := xStatistic.Where("created_unix <= ? and created_unix >= ? and repo_id = ? ", end.Unix(), begin.Unix(), repoId).Sum(&RepoStatistic{}, columnName)
if err != nil {
return 0, err
}
return int64(res), nil
}

func SumLastMonthNumVisits(repoId int64) (int64, error) {
end := time.Now()
begin := end.AddDate(0, 0, -30)
return SumRepoStatColumn(begin, end, repoId, "num_visits")
}

func SumLastFourMonthNumCommits(repoId int64) (int64, error) {
end := time.Now()
begin := end.AddDate(0, 0, -120)
return SumRepoStatColumn(begin, end, repoId, "num_commits_added")
}

+ 31
- 0
models/repo_tag.go View File

@@ -4,6 +4,7 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"fmt" "fmt"
"xorm.io/builder"
) )


type OfficialTag struct { type OfficialTag struct {
@@ -166,3 +167,33 @@ func GetAllOfficialTags() ([]OfficialTag, error) {
} }
return o, nil return o, nil
} }

type FindSelectedReposOpts struct {
ListOptions
OrgId int64
OnlyPublic bool
}

func GetSelectedRepos(opts FindSelectedReposOpts) ([]*Repository, error) {
if opts.Page < 1 {
opts.Page = 1
}
var cond = builder.NewCond()
cond = cond.And(builder.Eq{"official_tag.code": "selected"})
if opts.OrgId > 0 {
cond = cond.And(builder.Eq{"official_tag_repos.org_id": opts.OrgId})
}
if opts.OnlyPublic {
cond = cond.And(builder.Eq{"repository.is_private": false})
}
t := make([]*Repository, 0)
err := x.Join("inner", "official_tag_repos", "repository.id = official_tag_repos.repo_id").
Join("inner", "official_tag", "official_tag.id = official_tag_repos.tag_id").
Where(cond).OrderBy("repository.updated_unix desc").Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).Find(&t)

if err != nil {
return nil, err
}

return t, nil
}

+ 2
- 0
models/task_config.go View File

@@ -36,6 +36,8 @@ func GetTaskTypeFromAction(a ActionType) TaskType {
ActionCreateInferenceTask, ActionCreateInferenceTask,
ActionCreateBenchMarkTask, ActionCreateBenchMarkTask,
ActionCreateGPUTrainTask, ActionCreateGPUTrainTask,
ActionCreateGrampusGPUDebugTask,
ActionCreateGrampusNPUDebugTask,
ActionCreateGrampusNPUTrainTask, ActionCreateGrampusNPUTrainTask,
ActionCreateGrampusGPUTrainTask: ActionCreateGrampusGPUTrainTask:
return TaskCreateCloudbrainTask return TaskCreateCloudbrainTask


+ 14
- 0
models/topic.go View File

@@ -9,6 +9,7 @@ import (
"regexp" "regexp"
"strings" "strings"
"unicode/utf8" "unicode/utf8"
"xorm.io/xorm"


"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"


@@ -337,3 +338,16 @@ func GetOrgTopics(orgId int64) ([]Topic, error) {


return result, nil return result, nil
} }

func UpdateRepoTopics(repoID int64, topicNames []string, sess ...*xorm.Engine) error {
e := x
if len(sess) > 0 {
e = sess[0]
}
if _, err := e.ID(repoID).Cols("topics").Update(&Repository{
Topics: topicNames,
}); err != nil {
return err
}
return nil
}

+ 40
- 1
models/user.go View File

@@ -16,6 +16,7 @@ import (
"fmt" "fmt"
_ "image/jpeg" // Needed for jpeg support _ "image/jpeg" // Needed for jpeg support
"image/png" "image/png"
"math/rand"
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
@@ -198,6 +199,40 @@ type SearchOrganizationsOptions struct {
All bool All bool
} }


type User4Front struct {
ID int64
LowerName string `xorm:"UNIQUE NOT NULL"`
Name string `xorm:"UNIQUE NOT NULL"`
FullName string
Email string `xorm:"NOT NULL"`
Language string `xorm:"VARCHAR(5)"`
Description string
RelAvatarLink string
NumMembers int
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}

func (u *User) ToFrontFormat() *User4Front {
uf := &User4Front{
ID: u.ID,
LowerName: u.LowerName,
Name: u.Name,
FullName: u.FullName,
Email: u.Email,
Language: u.Language,
Description: u.Description,
CreatedUnix: u.CreatedUnix,
UpdatedUnix: u.UpdatedUnix,
NumMembers: u.NumMembers,
}
if !u.KeepEmailPrivate {
uf.Email = u.Email
}
uf.RelAvatarLink = u.RelAvatarLink()
return uf
}

// GenerateRandomAvatar generates a random avatar for user. // GenerateRandomAvatar generates a random avatar for user.
func (u *User) IsBindWechat() bool { func (u *User) IsBindWechat() bool {
return u.WechatOpenId != "" return u.WechatOpenId != ""
@@ -461,7 +496,11 @@ func (u *User) RealSizedAvatarLink(size int) string {
// may either be a sub-URL to this site, or a full URL to an external avatar // may either be a sub-URL to this site, or a full URL to an external avatar
// service. // service.
func (u *User) RelAvatarLink() string { func (u *User) RelAvatarLink() string {
return u.SizedRelAvatarLink(base.DefaultAvatarSize)
append := ""
if u.UseCustomAvatar {
append = "?" + fmt.Sprint(rand.Intn(100))
}
return u.SizedRelAvatarLink(base.DefaultAvatarSize) + append
} }


// AvatarLink returns user avatar absolute link. // AvatarLink returns user avatar absolute link.


+ 180
- 49
models/user_business_analysis.go View File

@@ -355,6 +355,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi
OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix)
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix)
AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)


CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix)
@@ -427,6 +428,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi
dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap)
dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap)


dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset)
dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset)
@@ -546,6 +548,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus
resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum
resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
resultMap[userRecord.ID].ModelConvertCount += userRecord.ModelConvertCount
resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
@@ -583,7 +586,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
startTime := currentTimeNow.AddDate(0, 0, -1) startTime := currentTimeNow.AddDate(0, 0, -1)


CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5)
CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5)
IssueCountMap := queryCreateIssue(start_unix, end_unix) IssueCountMap := queryCreateIssue(start_unix, end_unix)


CommentCountMap := queryComment(start_unix, end_unix) CommentCountMap := queryComment(start_unix, end_unix)
@@ -599,29 +602,25 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
//log.Info("CommitCodeSizeMapJson=" + string(CommitCodeSizeMapJson)) //log.Info("CommitCodeSizeMapJson=" + string(CommitCodeSizeMapJson))
} }
//CommitCodeSizeMap := queryCommitCodeSize(StartTimeNextDay.Unix(), EndTimeNextDay.Unix()) //CommitCodeSizeMap := queryCommitCodeSize(StartTimeNextDay.Unix(), EndTimeNextDay.Unix())
CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix)
CommitDatasetSizeMap, CommitDatasetNumMap, _ := queryDatasetSize(start_unix, end_unix)
SolveIssueCountMap := querySolveIssue(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix)
CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix)
CreateRepoCountMap, _, _ := queryUserCreateRepo(start_unix, end_unix)
LoginCountMap := queryLoginCount(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix)


OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix)
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix)
AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)


CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
RecommendDataset, CreatedDataset := queryRecommedDataSet(start_unix, end_unix)
RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix)
CollectImage, CollectedImage := queryImageStars(start_unix, end_unix) CollectImage, CollectedImage := queryImageStars(start_unix, end_unix)
RecommendImage := queryRecommedImage(start_unix, end_unix) RecommendImage := queryRecommedImage(start_unix, end_unix)


InvitationMap := queryUserInvitationCount(start_unix, end_unix) InvitationMap := queryUserInvitationCount(start_unix, end_unix)


DataDate := currentTimeNow.Format("2006-01-02") + " 00:01" DataDate := currentTimeNow.Format("2006-01-02") + " 00:01"
bonusMap := make(map[string]map[string]int)
if isUserYearData(tableName) {
bonusMap = getBonusMap()
log.Info("truncate all data from table:user_summary_current_year ")
statictisSess.Exec("TRUNCATE TABLE user_summary_current_year")
}

cond := "type != 1 and is_active=true" cond := "type != 1 and is_active=true"
count, err := sess.Where(cond).Count(new(User)) count, err := sess.Where(cond).Count(new(User))
if err != nil { if err != nil {
@@ -687,6 +686,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap) dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap)
dateRecordAll.ModelConvertCount = getMapValue(dateRecordAll.ID, AiModelConvertMap)
dateRecordAll.CollectDataset = getMapValue(dateRecordAll.ID, CollectDataset) dateRecordAll.CollectDataset = getMapValue(dateRecordAll.ID, CollectDataset)
dateRecordAll.CollectedDataset = getMapValue(dateRecordAll.ID, CollectedDataset) dateRecordAll.CollectedDataset = getMapValue(dateRecordAll.ID, CollectedDataset)
dateRecordAll.RecommendDataset = getMapValue(dateRecordAll.ID, RecommendDataset) dateRecordAll.RecommendDataset = getMapValue(dateRecordAll.ID, RecommendDataset)
@@ -719,37 +719,6 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
userMetrics["TotalHasActivityUser"] = getMapKeyStringValue("TotalHasActivityUser", userMetrics) + 1 userMetrics["TotalHasActivityUser"] = getMapKeyStringValue("TotalHasActivityUser", userMetrics) + 1
} }
} }
if isUserYearData(tableName) {
//年度数据
subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC())
mostActiveDay := ""
if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok {
mostActiveDay = getMostActiveJson(userInfo)
}
scoreMap := make(map[string]float64)
repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap)
dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset)
scoreMap["datasetscore"] = datasetscore
codeInfo, codescore := getCodeInfo(dateRecordAll)
scoreMap["codescore"] = codescore
cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap)
playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap)
re := &UserSummaryCurrentYear{
ID: dateRecordAll.ID,
Name: dateRecordAll.Name,
Email: dateRecordAll.Email,
Phone: dateRecordAll.Phone,
RegistDate: dateRecordAll.RegistDate,
DateCount: int(subTime.Hours()) / 24,
MostActiveDay: mostActiveDay,
RepoInfo: repoInfo,
DataSetInfo: dataSetInfo,
CodeInfo: codeInfo,
CloudBrainInfo: cloudBrainInfo,
PlayARoll: playARoll,
}
statictisSess.Insert(re)
}
} }
if len(dateRecordBatch) > 0 { if len(dateRecordBatch) > 0 {
err := insertTable(dateRecordBatch, tableName, statictisSess) err := insertTable(dateRecordBatch, tableName, statictisSess)
@@ -779,14 +748,136 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount)) log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount))
} }


func RefreshUserYearTable(pageStartTime time.Time, pageEndTime time.Time) {
sess := x.NewSession()
defer sess.Close()
log.Info("RefreshUserYearTable start....")
statictisSess := xStatistic.NewSession()
defer statictisSess.Close()

log.Info("UserYear StartTime:" + pageStartTime.Format("2006-01-02 15:04:05"))
log.Info("UserYear EndTime time:" + pageEndTime.Format("2006-01-02 15:04:05"))

start_unix := pageStartTime.Unix()
end_unix := pageEndTime.Unix()

CodeMergeCountMap := queryPullRequest(start_unix, end_unix)
CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5)
IssueCountMap := queryCreateIssue(start_unix, end_unix)

CommentCountMap := queryComment(start_unix, end_unix)

CommitCodeSizeMap, err := GetAllUserKPIStats(pageStartTime, pageEndTime)
if err != nil {
log.Info("query commit code errr.")
} else {
log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap)))
}
CommitDatasetSizeMap, CommitDatasetNumMap, dataSetDownloadMap := queryDatasetSize(start_unix, end_unix)
SolveIssueCountMap := querySolveIssue(start_unix, end_unix)
CreateRepoCountMap, DetailInfoMap, MostDownloadMap := queryUserCreateRepo(start_unix, end_unix)

CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)

_, CollectedDataset := queryDatasetStars(start_unix, end_unix)
_, CreatedDataset := queryRecommedDataSet(start_unix, end_unix)

bonusMap := getBonusMap()
log.Info("truncate all data from table:user_summary_current_year ")
statictisSess.Exec("TRUNCATE TABLE user_summary_current_year")

cond := "type != 1 and is_active=true"
count, err := sess.Where(cond).Count(new(User))
if err != nil {
log.Info("query user error. return.")
return
}
var indexTotal int64
indexTotal = 0
for {
sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
userList := make([]*User, 0)
sess.Find(&userList)
for _, userRecord := range userList {
var dateRecordAll UserBusinessAnalysisAll
dateRecordAll.ID = userRecord.ID
dateRecordAll.Email = userRecord.Email
dateRecordAll.Phone = userRecord.PhoneNumber
dateRecordAll.RegistDate = userRecord.CreatedUnix
dateRecordAll.Name = userRecord.Name

dateRecordAll.CodeMergeCount = getMapValue(dateRecordAll.ID, CodeMergeCountMap)
dateRecordAll.CommitCount = getMapValue(dateRecordAll.ID, CommitCountMap)
dateRecordAll.IssueCount = getMapValue(dateRecordAll.ID, IssueCountMap)
dateRecordAll.CommentCount = getMapValue(dateRecordAll.ID, CommentCountMap)

if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok {
dateRecordAll.CommitCodeSize = 0
} else {
dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines)
}
//dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap)
dateRecordAll.CommitDatasetSize = getMapValue(dateRecordAll.ID, CommitDatasetSizeMap)
dateRecordAll.CommitDatasetNum = getMapValue(dateRecordAll.ID, CommitDatasetNumMap)
dateRecordAll.SolveIssueCount = getMapValue(dateRecordAll.ID, SolveIssueCountMap)
dateRecordAll.CreateRepoCount = getMapValue(dateRecordAll.ID, CreateRepoCountMap)

dateRecordAll.CloudBrainTaskNum = getMapValue(dateRecordAll.ID, CloudBrainTaskMap)
dateRecordAll.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuDebugJob", CloudBrainTaskItemMap)
dateRecordAll.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuDebugJob", CloudBrainTaskItemMap)
dateRecordAll.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuTrainJob", CloudBrainTaskItemMap)
dateRecordAll.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuTrainJob", CloudBrainTaskItemMap)
dateRecordAll.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap)
dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecordAll.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)

//年度数据
subTime := time.Now().UTC().Sub(dateRecordAll.RegistDate.AsTime().UTC())
mostActiveDay := ""
if userInfo, ok := mostActiveMap[dateRecordAll.ID]; ok {
mostActiveDay = getMostActiveJson(userInfo)
}
scoreMap := make(map[string]float64)
repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap)
dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset)
scoreMap["datasetscore"] = datasetscore
codeInfo, codescore := getCodeInfo(dateRecordAll)
scoreMap["codescore"] = codescore
cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap)
playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap)
re := &UserSummaryCurrentYear{
ID: dateRecordAll.ID,
Name: dateRecordAll.Name,
Email: dateRecordAll.Email,
Phone: dateRecordAll.Phone,
RegistDate: dateRecordAll.RegistDate,
DateCount: int(subTime.Hours()) / 24,
MostActiveDay: mostActiveDay,
RepoInfo: repoInfo,
DataSetInfo: dataSetInfo,
CodeInfo: codeInfo,
CloudBrainInfo: cloudBrainInfo,
PlayARoll: playARoll,
}
statictisSess.Insert(re)
}
indexTotal += PAGE_SIZE
if indexTotal >= count {
break
}
}
log.Info("update user year data finished. ")
}

func isUserYearData(tableName string) bool { func isUserYearData(tableName string) bool {
if tableName == "user_business_analysis_current_year" { if tableName == "user_business_analysis_current_year" {
currentTimeNow := time.Now() currentTimeNow := time.Now()
if currentTimeNow.Year() >= 2023 { if currentTimeNow.Year() >= 2023 {
return false return false
} }
return true
} }
return true
return false
} }


func getBonusMap() map[string]map[string]int { func getBonusMap() map[string]map[string]int {
@@ -811,6 +902,7 @@ func getBonusMap() map[string]map[string]int {
record, ok := bonusMap[userName] record, ok := bonusMap[userName]
if !ok { if !ok {
record = make(map[string]int) record = make(map[string]int)
bonusMap[userName] = record
} }
record["times"] = getMapKeyStringValue("times", record) + getIntValue(aLine[3]) record["times"] = getMapKeyStringValue("times", record) + getIntValue(aLine[3])
record["total_bonus"] = getMapKeyStringValue("total_bonus", record) + getIntValue(aLine[4]) record["total_bonus"] = getMapKeyStringValue("total_bonus", record) + getIntValue(aLine[4])
@@ -996,7 +1088,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static


insertBatchSql := "INSERT INTO public." + tableName + insertBatchSql := "INSERT INTO public." + tableName +
"(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " +
"commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num) " +
"commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location,focus_other_user,collect_dataset,collected_dataset,recommend_dataset,collect_image,collected_image,recommend_image,user_index_primitive,phone,invitation_user_num,model_convert_count) " +
"VALUES" "VALUES"


for i, record := range dateRecords { for i, record := range dateRecords {
@@ -1005,7 +1097,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static
", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) + ", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) +
", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) + ", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) +
", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "'," + ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "'," +
fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + ")"
fmt.Sprint(record.FocusOtherUser) + "," + fmt.Sprint(record.CollectDataset) + "," + fmt.Sprint(record.CollectedDataset) + "," + fmt.Sprint(record.RecommendDataset) + "," + fmt.Sprint(record.CollectImage) + "," + fmt.Sprint(record.CollectedImage) + "," + fmt.Sprint(record.RecommendImage) + "," + fmt.Sprint(record.UserIndexPrimitive) + ",'" + record.Phone + "'" + "," + fmt.Sprint(record.InvitationUserNum) + "," + fmt.Sprint(record.ModelConvertCount) + ")"
if i < (len(dateRecords) - 1) { if i < (len(dateRecords) - 1) {
insertBatchSql += "," insertBatchSql += ","
} }
@@ -1096,6 +1188,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix)
CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix) AiModelManageMap := queryUserModel(start_unix, end_unix)
AiModelConvertMap := queryUserModelConvert(start_unix, end_unix)


CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix) CollectDataset, CollectedDataset := queryDatasetStars(start_unix, end_unix)
RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix) RecommendDataset, _ := queryRecommedDataSet(start_unix, end_unix)
@@ -1177,7 +1270,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap) dateRecord.CloudBrainRunTime = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_CloudBrainRunTime", CloudBrainTaskItemMap)
dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap)
dateRecord.ModelConvertCount = getMapValue(dateRecord.ID, AiModelConvertMap)
dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset) dateRecord.CollectDataset = getMapValue(dateRecord.ID, CollectDataset)
dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset) dateRecord.CollectedDataset = getMapValue(dateRecord.ID, CollectedDataset)
dateRecord.RecommendDataset = getMapValue(dateRecord.ID, RecommendDataset) dateRecord.RecommendDataset = getMapValue(dateRecord.ID, RecommendDataset)
@@ -1366,6 +1459,7 @@ func getUserIndexFromAnalysisAll(dateRecord UserBusinessAnalysisAll, ParaWeight
result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05)
result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3)
result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2)
result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2)
result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1)


result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1) result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1)
@@ -1391,6 +1485,7 @@ func getUserActivateAll(dateRecord UserBusinessAnalysisAll) int {
result += dateRecord.CreateRepoCount result += dateRecord.CreateRepoCount
result += dateRecord.CloudBrainTaskNum result += dateRecord.CloudBrainTaskNum
result += dateRecord.CommitModelCount result += dateRecord.CommitModelCount
result += dateRecord.ModelConvertCount
result += dateRecord.CommitDatasetNum result += dateRecord.CommitDatasetNum
result += dateRecord.FocusOtherUser result += dateRecord.FocusOtherUser
result += dateRecord.CollectDataset result += dateRecord.CollectDataset
@@ -1412,6 +1507,7 @@ func getUserActivate(dateRecord UserBusinessAnalysis) int {
result += dateRecord.CreateRepoCount result += dateRecord.CreateRepoCount
result += dateRecord.CloudBrainTaskNum result += dateRecord.CloudBrainTaskNum
result += dateRecord.CommitModelCount result += dateRecord.CommitModelCount
result += dateRecord.ModelConvertCount
result += dateRecord.CommitDatasetNum result += dateRecord.CommitDatasetNum
result += dateRecord.FocusOtherUser result += dateRecord.FocusOtherUser
result += dateRecord.CollectDataset result += dateRecord.CollectDataset
@@ -1448,6 +1544,7 @@ func getUserIndex(dateRecord UserBusinessAnalysis, ParaWeight map[string]float64
result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05) result += float64(dateRecord.CreateRepoCount) * getParaWeightValue("CreateRepoCount", ParaWeight, 0.05)
result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3) result += float64(dateRecord.CloudBrainTaskNum) * getParaWeightValue("CloudBrainTaskNum", ParaWeight, 0.3)
result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2) result += float64(dateRecord.CommitModelCount) * getParaWeightValue("CommitModelCount", ParaWeight, 0.2)
result += float64(dateRecord.ModelConvertCount) * getParaWeightValue("ModelConvertCount", ParaWeight, 0.2)
result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1) result += dateRecord.OpenIIndex * getParaWeightValue("OpenIIndex", ParaWeight, 0.1)


result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1) result += float64(dateRecord.CollectDataset) * getParaWeightValue("CollectDataset", ParaWeight, 0.1)
@@ -1492,10 +1589,6 @@ func getInt(str string) int {
return int(re) return int(re)
} }


func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) {
CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false)
}

func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { func querySolveIssue(start_unix int64, end_unix int64) map[int64]int {
sess := x.NewSession() sess := x.NewSession()
defer sess.Close() defer sess.Close()
@@ -2276,6 +2369,38 @@ func queryUserModel(start_unix int64, end_unix int64) map[int64]int {
return resultMap return resultMap
} }


func queryUserModelConvert(start_unix int64, end_unix int64) map[int64]int {
sess := x.NewSession()
defer sess.Close()
resultMap := make(map[int64]int)
cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)
count, err := sess.Where(cond).Count(new(AiModelConvert))
if err != nil {
log.Info("query AiModelConvert error. return.")
return resultMap
}
var indexTotal int64
indexTotal = 0
for {
sess.Select("id,user_id").Table("ai_model_convert").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
aiModelList := make([]*AiModelConvert, 0)
sess.Find(&aiModelList)
log.Info("query AiModelConvert size=" + fmt.Sprint(len(aiModelList)))
for _, aiModelRecord := range aiModelList {
if _, ok := resultMap[aiModelRecord.UserId]; !ok {
resultMap[aiModelRecord.UserId] = 1
} else {
resultMap[aiModelRecord.UserId] += 1
}
}
indexTotal += PAGE_SIZE
if indexTotal >= count {
break
}
}
return resultMap
}

func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) { func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) {
sess := x.NewSession() sess := x.NewSession()
defer sess.Close() defer sess.Close()
@@ -2441,3 +2566,9 @@ func GetContentFromPromote(url string) (string, error) {
allLineStr := string(bytes) allLineStr := string(bytes)
return allLineStr, nil return allLineStr, nil
} }

func QueryLast30DaysHighestIndexUsers(size int) ([]int64, error) {
userIds := make([]int64, 0)
err := xStatistic.Table("user_business_analysis_last30_day").Cols("id").OrderBy("user_index desc").Limit(size).Find(&userIds)
return userIds, err
}

+ 9
- 0
models/user_business_struct.go View File

@@ -89,6 +89,7 @@ type UserBusinessAnalysisCurrentYear struct {


Phone string `xorm:"NULL"` Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
} }


type UserBusinessAnalysisLast30Day struct { type UserBusinessAnalysisLast30Day struct {
@@ -157,6 +158,7 @@ type UserBusinessAnalysisLast30Day struct {


Phone string `xorm:"NULL"` Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
} }


type UserBusinessAnalysisLastMonth struct { type UserBusinessAnalysisLastMonth struct {
@@ -225,6 +227,7 @@ type UserBusinessAnalysisLastMonth struct {


Phone string `xorm:"NULL"` Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
} }


type UserBusinessAnalysisCurrentMonth struct { type UserBusinessAnalysisCurrentMonth struct {
@@ -293,6 +296,7 @@ type UserBusinessAnalysisCurrentMonth struct {


Phone string `xorm:"NULL"` Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
} }


type UserBusinessAnalysisCurrentWeek struct { type UserBusinessAnalysisCurrentWeek struct {
@@ -362,6 +366,7 @@ type UserBusinessAnalysisCurrentWeek struct {


Phone string `xorm:"NULL"` Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
} }


type UserBusinessAnalysisYesterday struct { type UserBusinessAnalysisYesterday struct {
@@ -431,6 +436,7 @@ type UserBusinessAnalysisYesterday struct {


Phone string `xorm:"NULL"` Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
} }


type UserBusinessAnalysisLastWeek struct { type UserBusinessAnalysisLastWeek struct {
@@ -500,6 +506,7 @@ type UserBusinessAnalysisLastWeek struct {


Phone string `xorm:"NULL"` Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
} }


type UserAnalysisPara struct { type UserAnalysisPara struct {
@@ -616,6 +623,7 @@ type UserBusinessAnalysisAll struct {


Phone string `xorm:"NULL"` Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
} }


type UserBusinessAnalysis struct { type UserBusinessAnalysis struct {
@@ -704,4 +712,5 @@ type UserBusinessAnalysis struct {


Phone string `xorm:"NULL"` Phone string `xorm:"NULL"`
InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"` InvitationUserNum int `xorm:"NOT NULL DEFAULT 0"`
ModelConvertCount int `xorm:"NOT NULL DEFAULT 0"`
} }

+ 21
- 0
modules/auth/grampus.go View File

@@ -29,3 +29,24 @@ type CreateGrampusTrainJobForm struct {
func (f *CreateGrampusTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { func (f *CreateGrampusTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
return validate(errs, ctx.Data, f, ctx.Locale) return validate(errs, ctx.Data, f, ctx.Locale)
} }

type CreateGrampusNotebookForm struct {
Type int `form:"type"`
DisplayJobName string `form:"display_job_name" binding:"Required"`
Attachment string `form:"attachment"`
ImageID string `form:"image_id" binding:"Required"`
Description string `form:"description"`
BranchName string `form:"branch_name" binding:"Required"`
Image string `form:"image" binding:"Required"`
DatasetName string `form:"dataset_name"`
ModelName string `form:"model_name"`
ModelVersion string `form:"model_version"`
CkptName string `form:"ckpt_name"`
LabelName string `form:"label_names"`
PreTrainModelUrl string `form:"pre_train_model_url"`
SpecId int64 `form:"spec_id" binding:"Required"`
}

func (f *CreateGrampusNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
return validate(errs, ctx.Data, f, ctx.Locale)
}

+ 13
- 7
modules/auth/modelarts.go View File

@@ -16,13 +16,19 @@ func (f *CreateModelArtsForm) Validate(ctx *macaron.Context, errs binding.Errors
} }


type CreateModelArtsNotebookForm struct { type CreateModelArtsNotebookForm struct {
DisplayJobName string `form:"display_job_name" binding:"Required"`
JobName string `form:"job_name" binding:"Required"`
Attachment string `form:"attachment"`
Description string `form:"description"`
Flavor string `form:"flavor" binding:"Required"`
ImageId string `form:"image_id" binding:"Required"`
SpecId int64 `form:"spec_id" binding:"Required"`
DisplayJobName string `form:"display_job_name" binding:"Required"`
JobName string `form:"job_name" binding:"Required"`
Attachment string `form:"attachment"`
Description string `form:"description"`
Flavor string `form:"flavor" binding:"Required"`
ImageId string `form:"image_id" binding:"Required"`
ModelName string `form:"model_name"`
ModelVersion string `form:"model_version"`
CkptName string `form:"ckpt_name"`
LabelName string `form:"label_names"`
PreTrainModelUrl string `form:"pre_train_model_url"`
SpecId int64 `form:"spec_id" binding:"Required"`
DatasetName string `form:"dataset_name"`
} }


func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {


+ 68
- 31
modules/cloudbrain/cloudbrain.go View File

@@ -5,6 +5,7 @@ import (
"errors" "errors"
"os" "os"
"strconv" "strconv"
"strings"


"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"


@@ -31,10 +32,10 @@ const (
Snn4imagenetMountPath = "/snn4imagenet" Snn4imagenetMountPath = "/snn4imagenet"
BrainScoreMountPath = "/brainscore" BrainScoreMountPath = "/brainscore"
TaskInfoName = "/taskInfo" TaskInfoName = "/taskInfo"
Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt`
BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt`
SubTaskName = "task1"
Snn4imagenetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt`
BrainScoreCommand = `bash /benchmark/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel/%s' -d '%s' >/model/benchmark-log.txt`
Snn4EcosetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --datapath '/dataset' --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt`
SubTaskName = "task1"


Success = "S000" Success = "S000"


@@ -145,7 +146,7 @@ func isAdminOrImageCreater(ctx *context.Context, image *models.Image, err error)
func AdminOrOwnerOrJobCreaterRight(ctx *context.Context) { func AdminOrOwnerOrJobCreaterRight(ctx *context.Context) {


var id = ctx.Params(":id") var id = ctx.Params(":id")
job, err := GetCloudBrainByIdOrJobId(id)
job, err := GetCloudBrainByIdOrJobId(id, "id")
if err != nil { if err != nil {
log.Error("GetCloudbrainByID failed:%v", err.Error()) log.Error("GetCloudbrainByID failed:%v", err.Error())
ctx.NotFound(ctx.Req.URL.RequestURI(), nil) ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
@@ -161,7 +162,7 @@ func AdminOrOwnerOrJobCreaterRight(ctx *context.Context) {
func AdminOrJobCreaterRight(ctx *context.Context) { func AdminOrJobCreaterRight(ctx *context.Context) {


var id = ctx.Params(":id") var id = ctx.Params(":id")
job, err := GetCloudBrainByIdOrJobId(id)
job, err := GetCloudBrainByIdOrJobId(id, "id")
if err != nil { if err != nil {
log.Error("GetCloudbrainByID failed:%v", err.Error()) log.Error("GetCloudbrainByID failed:%v", err.Error())
ctx.NotFound(ctx.Req.URL.RequestURI(), nil) ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
@@ -177,7 +178,7 @@ func AdminOrJobCreaterRight(ctx *context.Context) {
func AdminOrOwnerOrJobCreaterRightForTrain(ctx *context.Context) { func AdminOrOwnerOrJobCreaterRightForTrain(ctx *context.Context) {


var jobID = ctx.Params(":jobid") var jobID = ctx.Params(":jobid")
job, err := GetCloudBrainByIdOrJobId(jobID)
job, err := GetCloudBrainByIdOrJobId(jobID, "jobid")
if err != nil { if err != nil {
log.Error("GetCloudbrainByJobID failed:%v", err.Error()) log.Error("GetCloudbrainByJobID failed:%v", err.Error())
ctx.NotFound(ctx.Req.URL.RequestURI(), nil) ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
@@ -193,7 +194,7 @@ func AdminOrOwnerOrJobCreaterRightForTrain(ctx *context.Context) {
func AdminOrJobCreaterRightForTrain(ctx *context.Context) { func AdminOrJobCreaterRightForTrain(ctx *context.Context) {


var jobID = ctx.Params(":jobid") var jobID = ctx.Params(":jobid")
job, err := GetCloudBrainByIdOrJobId(jobID)
job, err := GetCloudBrainByIdOrJobId(jobID, "jobid")
if err != nil { if err != nil {
log.Error("GetCloudbrainByJobID failed:%v", err.Error()) log.Error("GetCloudbrainByJobID failed:%v", err.Error())
ctx.NotFound(ctx.Req.URL.RequestURI(), nil) ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
@@ -256,20 +257,6 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) {
ReadOnly: true, ReadOnly: true,
}, },
}, },
{
HostPath: models.StHostPath{
Path: req.Snn4ImageNetPath,
MountPath: Snn4imagenetMountPath,
ReadOnly: true,
},
},
{
HostPath: models.StHostPath{
Path: req.BrainScorePath,
MountPath: BrainScoreMountPath,
ReadOnly: true,
},
},
{ {
HostPath: models.StHostPath{ HostPath: models.StHostPath{
Path: req.ResultPath, Path: req.ResultPath,
@@ -405,7 +392,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) {
} }


func IsBenchmarkJob(jobType string) bool { func IsBenchmarkJob(jobType string) bool {
return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType
return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType || string(models.JobTypeSnn4Ecoset) == jobType
} }


func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 {
@@ -490,6 +477,21 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e
} }
} }


if task.PreTrainModelUrl != "" { //预训练
_, err := models.QueryModelByPath(task.PreTrainModelUrl)
if err != nil {
log.Warn("The model may be deleted", err)
} else {
volumes = append(volumes, models.Volume{
HostPath: models.StHostPath{
Path: setting.Attachment.Minio.RealPath + task.PreTrainModelUrl,
MountPath: PretrainModelMountPath,
ReadOnly: true,
},
})
}
}

createTime := timeutil.TimeStampNow() createTime := timeutil.TimeStampNow()
jobResult, err := CreateJob(jobName, models.CreateJobParams{ jobResult, err := CreateJob(jobName, models.CreateJobParams{
JobName: jobName, JobName: jobName,
@@ -540,10 +542,16 @@ func RestartTask(ctx *context.Context, task *models.Cloudbrain, newID *string) e
GpuQueue: task.GpuQueue, GpuQueue: task.GpuQueue,
ResourceSpecId: task.ResourceSpecId, ResourceSpecId: task.ResourceSpecId,
ComputeResource: task.ComputeResource, ComputeResource: task.ComputeResource,
CreatedUnix: createTime,
UpdatedUnix: createTime,
BranchName: task.BranchName,
Spec: spec,

CreatedUnix: createTime,
UpdatedUnix: createTime,
BranchName: task.BranchName,
Spec: spec,
ModelName: task.ModelName,
ModelVersion: task.ModelVersion,
LabelName: task.LabelName,
PreTrainModelUrl: task.PreTrainModelUrl,
CkptName: task.CkptName,
} }


err = models.RestartCloudbrain(task, newTask) err = models.RestartCloudbrain(task, newTask)
@@ -653,18 +661,47 @@ func IsElementExist(s []string, str string) bool {
return false return false
} }


func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain,error) {

func GetCloudBrainByIdOrJobId(id string, initialQuery string) (*models.Cloudbrain, error) {
_, err := strconv.ParseInt(id, 10, 64) _, err := strconv.ParseInt(id, 10, 64)
var job *models.Cloudbrain var job *models.Cloudbrain
if err != nil { if err != nil {


job, err = models.GetCloudbrainByJobID(id) job, err = models.GetCloudbrainByJobID(id)
} else { } else {
job, err = models.GetCloudbrainByID(id)
if err!=nil{

if strings.EqualFold(initialQuery, "id") {
job, err = models.GetCloudbrainByID(id)
if err != nil {
job, err = models.GetCloudbrainByJobID(id)
}
} else {
job, err = models.GetCloudbrainByJobID(id) job, err = models.GetCloudbrainByJobID(id)
if err != nil {
job, err = models.GetCloudbrainByID(id)
}
} }


} }
return job,err
return job, err
}

type GenerateModelArtsNotebookReq struct {
JobName string
DisplayJobName string
Uuid string
Description string

BootFile string

ImageId string
AutoStopDurationMs int64
BranchName string

Spec *models.Specification
ModelName string
LabelName string
CkptName string
ModelVersion string
PreTrainModelUrl string
} }

+ 7
- 5
modules/convert/cloudbrain.go View File

@@ -28,14 +28,13 @@ func ToCloudBrain(task *models.Cloudbrain) *api.Cloudbrain {
BootFile: task.BootFile, BootFile: task.BootFile,
Description: task.Description, Description: task.Description,
ModelName: task.ModelName, ModelName: task.ModelName,
ModelVersion: task.ModelVersion,
CkptName: task.CkptName,
VersionName: task.VersionName,
ModelVersion: task.ModelVersion,
CkptName: task.CkptName,


StartTime: int64(task.StartTime), StartTime: int64(task.StartTime),
EndTime: int64(task.EndTime), EndTime: int64(task.EndTime),

Spec: ToSpecification(task.Spec),
Spec: ToSpecification(task.Spec),
} }
} }
func ToAttachment(attachment *models.Attachment) *api.AttachmentShow { func ToAttachment(attachment *models.Attachment) *api.AttachmentShow {
@@ -89,6 +88,9 @@ func ToDataset(dataset *models.Dataset) *api.Dataset {
} }


func ToSpecification(s *models.Specification) *api.SpecificationShow { func ToSpecification(s *models.Specification) *api.SpecificationShow {
if s == nil {
return nil
}
return &api.SpecificationShow{ return &api.SpecificationShow{
ID: s.ID, ID: s.ID,
AccCardsNum: s.AccCardsNum, AccCardsNum: s.AccCardsNum,


+ 3
- 2
modules/cron/tasks_basic.go View File

@@ -5,10 +5,11 @@
package cron package cron


import ( import (
"code.gitea.io/gitea/modules/setting"
"context" "context"
"time" "time"


"code.gitea.io/gitea/modules/setting"

"code.gitea.io/gitea/modules/urfs_client/urchin" "code.gitea.io/gitea/modules/urfs_client/urchin"
cloudbrainService "code.gitea.io/gitea/services/cloudbrain" cloudbrainService "code.gitea.io/gitea/services/cloudbrain"


@@ -296,7 +297,7 @@ func registerHandleCloudbrainDurationStatistic() {
RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{
Enabled: true, Enabled: true,
RunAtStart: false, RunAtStart: false,
Schedule: "1 0 * * * ?",
Schedule: "1 1 * * * ?",
}, func(ctx context.Context, _ *models.User, _ Config) error { }, func(ctx context.Context, _ *models.User, _ Config) error {
repo.CloudbrainDurationStatisticHour() repo.CloudbrainDurationStatisticHour()
return nil return nil


+ 178
- 9
modules/grampus/grampus.go View File

@@ -1,7 +1,8 @@
package grampus package grampus


import ( import (
"encoding/json"
"fmt"
"strconv"
"strings" "strings"


"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
@@ -26,8 +27,10 @@ const (


CodeArchiveName = "master.zip" CodeArchiveName = "master.zip"


BucketRemote = "grampus"
RemoteModelPath = "/output/" + models.ModelSuffix
BucketRemote = "grampus"
RemoteModelPath = "/output/" + models.ModelSuffix
autoStopDurationMs = 4 * 60 * 60 * 1000
CommandGpuDebug = "mkdir -p /dataset;%s! [ -x \"$(command -v jupyter)\" ] && pip install jupyterlab==3 -i https://pypi.tuna.tsinghua.edu.cn/simple;jupyter lab --ServerApp.shutdown_no_activity_timeout=%s --TerminalManager.cull_inactive_timeout=%s --TerminalManager.cull_interval=%s --MappingKernelManager.cull_idle_timeout=%s --MappingKernelManager.cull_interval=%s --MappingKernelManager.cull_connected=True --MappingKernelManager.cull_busy=True --no-browser --ip=0.0.0.0 --allow-root --notebook-dir='/code' --port=$OCTOPUS_NOTEBOOK_PORT --LabApp.token='' --LabApp.allow_origin='*' --LabApp.base_url=$OCTOPUS_NOTEBOOK_BASE_URL;"
) )


var ( var (
@@ -37,7 +40,7 @@ var (


SpecialPools *models.SpecialPools SpecialPools *models.SpecialPools


CommandPrepareScriptGpu = ";mkdir -p output;mkdir -p code;mkdir -p dataset;mkdir -p pretrainmodel;echo \"start loading script\";wget -q https://openi.pcl.ac.cn/OpenIOSSG/%s/archive/master.zip;" +
CommandPrepareScriptGpu = ";mkdir -p output;mkdir -p code;mkdir -p dataset;mkdir -p pretrainmodel;echo \"start loading script\";wget -q https://git.openi.org.cn/OpenIOSSG/%s/archive/master.zip;" +
"echo \"finish loading script\";unzip -q master.zip;cd %s;chmod 777 downloader_for_obs uploader_for_npu downloader_for_minio uploader_for_gpu;" "echo \"finish loading script\";unzip -q master.zip;cd %s;chmod 777 downloader_for_obs uploader_for_npu downloader_for_minio uploader_for_gpu;"
) )


@@ -81,6 +84,32 @@ type GenerateTrainJobReq struct {
CodeName string CodeName string
} }


type GenerateNotebookJobReq struct {
JobName string
Command string
ImageUrl string
ImageId string
DisplayJobName string
Uuid string
Description string
CodeStoragePath string
CommitID string
BranchName string
ComputeResource string
ProcessType string
DatasetNames string
DatasetInfos map[string]models.DatasetInfo
ModelName string
LabelName string
CkptName string
ModelVersion string
PreTrainModelPath string
PreTrainModelUrl string
Spec *models.Specification
CodeName string
ModelPath string //参考启智GPU调试, 挂载/model目录用户的模型可以输出到这个目录
}

func getEndPoint() string { func getEndPoint() string {
index := strings.Index(setting.Endpoint, "//") index := strings.Index(setting.Endpoint, "//")
endpoint := setting.Endpoint[index+2:] endpoint := setting.Endpoint[index+2:]
@@ -101,6 +130,151 @@ func getDatasetGrampus(datasetInfos map[string]models.DatasetInfo) []models.Gram
} }
return datasetGrampus return datasetGrampus
} }
func getDatasetGPUGrampus(datasetInfos map[string]models.DatasetInfo) ([]models.GrampusDataset, string) {
var datasetGrampus []models.GrampusDataset
var command = ""
for uuid, datasetInfo := range datasetInfos {
datasetGrampus = append(datasetGrampus, models.GrampusDataset{
Name: datasetInfo.FullName,
Bucket: setting.Attachment.Minio.Bucket,
EndPoint: setting.Attachment.Minio.Endpoint,
ObjectKey: datasetInfo.DataLocalPath,
ReadOnly: true,
ContainerPath: "/dataset1/" + datasetInfo.Name,
})

command += "cp /dataset1/'" + datasetInfo.Name + "'/" + uuid + " /dataset/'" + datasetInfo.FullName + "';"

}
return datasetGrampus, command
}

func GenerateNotebookJob(ctx *context.Context, req *GenerateNotebookJobReq) (jobId string, err error) {
createTime := timeutil.TimeStampNow()

var datasetGrampus []models.GrampusDataset
var codeGrampus models.GrampusDataset
var cpCommand string
imageUrl := req.ImageUrl
if ProcessorTypeNPU == req.ProcessType {
datasetGrampus = getDatasetGrampus(req.DatasetInfos)
if len(req.ModelName) != 0 {
datasetGrampus = append(datasetGrampus, models.GrampusDataset{
Name: req.ModelName,
Bucket: setting.Bucket,
EndPoint: getEndPoint(),
ReadOnly: true,
ObjectKey: req.PreTrainModelPath,
})
}

codeGrampus = models.GrampusDataset{
Name: req.CodeName,
Bucket: setting.Bucket,
EndPoint: getEndPoint(),
ObjectKey: req.CodeStoragePath + cloudbrain.DefaultBranchName + ".zip",
ReadOnly: false,
}
imageUrl = ""
req.Command = ""
} else {
datasetGrampus, cpCommand = getDatasetGPUGrampus(req.DatasetInfos)
if len(req.ModelName) != 0 {
datasetGrampus = append(datasetGrampus, models.GrampusDataset{
Name: req.ModelName,
Bucket: setting.Attachment.Minio.Bucket,
EndPoint: setting.Attachment.Minio.Endpoint,
ObjectKey: req.PreTrainModelPath,
ReadOnly: true,
ContainerPath: cloudbrain.PretrainModelMountPath,
})
}
codeGrampus = models.GrampusDataset{
Name: req.CodeName,
Bucket: setting.Attachment.Minio.Bucket,
EndPoint: setting.Attachment.Minio.Endpoint,
ObjectKey: req.CodeStoragePath + cloudbrain.DefaultBranchName + ".zip",
ReadOnly: false,
ContainerPath: cloudbrain.CodeMountPath,
}
req.Command = fmt.Sprintf(CommandGpuDebug, cpCommand, setting.CullIdleTimeout, setting.CullIdleTimeout, setting.CullInterval, setting.CullIdleTimeout, setting.CullInterval)
log.Info("debug command:" + req.Command)

}

jobResult, err := createNotebookJob(models.CreateGrampusNotebookRequest{
Name: req.JobName,
Tasks: []models.GrampusNotebookTask{
{
Name: req.JobName,
ResourceSpecId: req.Spec.SourceSpecId,
ImageId: req.ImageId,
ImageUrl: imageUrl,
Datasets: datasetGrampus,
Code: codeGrampus,
AutoStopDuration: autoStopDurationMs,
Capacity: setting.Capacity,
Command: req.Command,
},
},
})
if err != nil {
log.Error("createNotebookJob failed: %v", err.Error())
return "", err
}

jobID := jobResult.JobInfo.JobID
err = models.CreateCloudbrain(&models.Cloudbrain{
Status: TransTrainJobStatus(jobResult.JobInfo.Status),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: jobID,
JobName: req.JobName,
DisplayJobName: req.DisplayJobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeC2Net,
Uuid: req.Uuid,
DatasetName: req.DatasetNames,
CommitID: req.CommitID,
IsLatestVersion: "1",
ComputeResource: req.ComputeResource,
ImageID: req.ImageId,
BranchName: req.BranchName,
Description: req.Description,
WorkServerNumber: 1,
EngineName: req.ImageUrl,
CreatedUnix: createTime,
UpdatedUnix: createTime,
Spec: req.Spec,
ModelName: req.ModelName,
ModelVersion: req.ModelVersion,
LabelName: req.LabelName,
PreTrainModelUrl: req.PreTrainModelUrl,
CkptName: req.CkptName,
})

if err != nil {
log.Error("CreateCloudbrain(%s) failed:%v", req.DisplayJobName, err.Error())
return "", err
}

var actionType models.ActionType
if req.ComputeResource == models.NPUResource {
actionType = models.ActionCreateGrampusNPUDebugTask
} else if req.ComputeResource == models.GPUResource {
actionType = models.ActionCreateGrampusGPUDebugTask
}
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil {
log.Error("GetCloudbrainByJobID failed: %v", err.Error())
return "", err
}

stringId := strconv.FormatInt(task.ID, 10)
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, req.DisplayJobName, actionType)

return jobID, nil
}


func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId string, err error) { func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId string, err error) {
createTime := timeutil.TimeStampNow() createTime := timeutil.TimeStampNow()
@@ -269,11 +443,6 @@ func TransTrainJobStatus(status string) string {


return strings.ToUpper(status) return strings.ToUpper(status)
} }
func InitSpecialPool() {
if SpecialPools == nil && setting.Grampus.SpecialPools != "" {
json.Unmarshal([]byte(setting.Grampus.SpecialPools), &SpecialPools)
}
}


func GetNpuModelRemoteObsUrl(jobName string) string { func GetNpuModelRemoteObsUrl(jobName string) string {
return "s3:///" + BucketRemote + "/" + GetNpuModelObjectKey(jobName) return "s3:///" + BucketRemote + "/" + GetNpuModelObjectKey(jobName)


+ 112
- 4
modules/grampus/resty.go View File

@@ -26,6 +26,7 @@ const (
urlGetResourceSpecs = urlOpenApiV1 + "resourcespec" urlGetResourceSpecs = urlOpenApiV1 + "resourcespec"
urlGetAiCenter = urlOpenApiV1 + "sharescreen/aicenter" urlGetAiCenter = urlOpenApiV1 + "sharescreen/aicenter"
urlGetImages = urlOpenApiV1 + "image" urlGetImages = urlOpenApiV1 + "image"
urlNotebookJob = urlOpenApiV1 + "notebook"


errorIllegalToken = 1005 errorIllegalToken = 1005
) )
@@ -87,6 +88,39 @@ func getToken() error {
return nil return nil
} }


func createNotebookJob(req models.CreateGrampusNotebookRequest) (*models.GrampusNotebookResponse, error) {
checkSetting()
client := getRestyClient()
var result models.GrampusNotebookResponse

retry := 0

sendjob:
_, err := client.R().
SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN).
SetBody(req).
SetResult(&result).
Post(HOST + urlNotebookJob)

if err != nil {
return nil, fmt.Errorf("resty CreateNotebookJob: %s", err)
}

if result.ErrorCode == errorIllegalToken && retry < 1 {
retry++
_ = getToken()
goto sendjob
}

if result.ErrorCode != 0 {
log.Error("CreateNotebookJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
return &result, fmt.Errorf("CreateNotebookJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func createJob(req models.CreateGrampusJobRequest) (*models.CreateGrampusJobResponse, error) { func createJob(req models.CreateGrampusJobRequest) (*models.CreateGrampusJobResponse, error) {
checkSetting() checkSetting()
client := getRestyClient() client := getRestyClient()
@@ -120,6 +154,38 @@ sendjob:
return &result, nil return &result, nil
} }


func GetNotebookJob(jobID string) (*models.GrampusNotebookResponse, error) {
checkSetting()
client := getRestyClient()
var result models.GrampusNotebookResponse

retry := 0

sendjob:
_, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + urlNotebookJob + "/" + jobID)

if err != nil {
return nil, fmt.Errorf("resty GetNotebookJob: %v", err)
}

if result.ErrorCode == errorIllegalToken && retry < 1 {
retry++
log.Info("retry get token")
_ = getToken()
goto sendjob
}

if result.ErrorCode != 0 {
log.Error("GetNotebookJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
return nil, fmt.Errorf("GetNotebookJob failed(%d): %s", result.ErrorCode, result.ErrorMsg)
}

return &result, nil
}

func GetJob(jobID string) (*models.GetGrampusJobResponse, error) { func GetJob(jobID string) (*models.GetGrampusJobResponse, error) {
checkSetting() checkSetting()
client := getRestyClient() client := getRestyClient()
@@ -184,18 +250,23 @@ sendjob:
return &result, nil return &result, nil
} }


func GetImages(processorType string) (*models.GetGrampusImagesResult, error) {
func GetImages(processorType string, jobType string) (*models.GetGrampusImagesResult, error) {
checkSetting() checkSetting()
client := getRestyClient() client := getRestyClient()
var result models.GetGrampusImagesResult var result models.GetGrampusImagesResult


retry := 0 retry := 0


queryType := "TrainJob"
if jobType == string(models.JobTypeDebug) {
queryType = "Notebook"
}

sendjob: sendjob:
_, err := client.R(). _, err := client.R().
SetAuthToken(TOKEN). SetAuthToken(TOKEN).
SetResult(&result). SetResult(&result).
Get(HOST + urlGetImages + "?processorType=" + processorType)
Get(HOST + urlGetImages + "?processorType=" + processorType + "&trainType=" + queryType)


if err != nil { if err != nil {
return nil, fmt.Errorf("resty GetImages: %v", err) return nil, fmt.Errorf("resty GetImages: %v", err)
@@ -271,19 +342,26 @@ func GetGrampusMetrics(jobID string) (models.GetTrainJobMetricStatisticResult, e
return result, nil return result, nil
} }


func StopJob(jobID string) (*models.GrampusStopJobResponse, error) {
func StopJob(jobID string, jobType ...string) (*models.GrampusStopJobResponse, error) {
checkSetting() checkSetting()
client := getRestyClient() client := getRestyClient()
var result models.GrampusStopJobResponse var result models.GrampusStopJobResponse


retry := 0 retry := 0


url := urlTrainJob
if len(jobType) > 0 {
if jobType[0] == string(models.JobTypeDebug) {
url = urlNotebookJob
}
}

sendjob: sendjob:
_, err := client.R(). _, err := client.R().
//SetHeader("Content-Type", "application/json"). //SetHeader("Content-Type", "application/json").
SetAuthToken(TOKEN). SetAuthToken(TOKEN).
SetResult(&result). SetResult(&result).
Post(HOST + urlTrainJob + "/" + jobID + "/stop")
Post(HOST + url + "/" + jobID + "/stop")


if err != nil { if err != nil {
return &result, fmt.Errorf("resty StopTrainJob: %v", err) return &result, fmt.Errorf("resty StopTrainJob: %v", err)
@@ -335,3 +413,33 @@ sendjob:


return &result, nil return &result, nil
} }

func RestartNotebookJob(jobID string) (*models.GrampusNotebookRestartResponse, error) {
checkSetting()
client := getRestyClient()
var restartResponse *models.GrampusNotebookRestartResponse
retry := 0

sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&restartResponse).
Post(HOST + urlNotebookJob + "/" + jobID + "/start")

if err != nil {
return nil, fmt.Errorf("resty grampus restart note book job: %v", err)
}
if restartResponse.ErrorCode == errorIllegalToken && retry < 1 {
retry++
log.Info("retry get token")
_ = getToken()
goto sendjob
}

if res.StatusCode() != http.StatusOK {
log.Error("resty grampus restart note book job failed(%s): %v", res.String(), err.Error())
return nil, fmt.Errorf("resty grampus restart note book job failed: %v", err)
}

return restartResponse, nil
}

+ 47
- 213
modules/modelarts/modelarts.go View File

@@ -1,13 +1,9 @@
package modelarts package modelarts


import ( import (
"encoding/base64"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/ioutil"
"net/http"
"path"
"strconv" "strconv"
"strings" "strings"


@@ -20,34 +16,16 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
) )


const ( const (
//notebook //notebook

storageTypeOBS = "obs" storageTypeOBS = "obs"
autoStopDuration = 4 * 60 * 60 autoStopDuration = 4 * 60 * 60
AutoStopDurationMs = 4 * 60 * 60 * 1000 AutoStopDurationMs = 4 * 60 * 60 * 1000
MORDELART_USER_IMAGE_ENGINE_ID = -1
DataSetMountPath = "/home/ma-user/work"
NotebookEnv = "Python3"
NotebookType = "Ascend"
FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 96GiB (modelarts.kat1.xlarge)"

//train-job
// ResourcePools = "{\"resource_pool\":[{\"id\":\"pool1328035d\", \"value\":\"专属资源池\"}]}"
// Engines = "{\"engine\":[{\"id\":1, \"value\":\"Ascend-Powered-Engine\"}]}"
// EngineVersions = "{\"version\":[{\"id\":118,\"value\":\"MindSpore-1.0.0-c75-python3.7-euleros2.8-aarch64\"}," +
// "{\"id\":119,\"value\":\"MindSpore-1.1.1-c76-python3.7-euleros2.8-aarch64\"}," +
// "{\"id\":120,\"value\":\"MindSpore-1.1.1-c76-tr5-python3.7-euleros2.8-aarch64\"}," +
// "{\"id\":117,\"value\":\"TF-1.15-c75-python3.7-euleros2.8-aarch64\"}" +
// "]}"
// TrainJobFlavorInfo = "{\"flavor\":[{\"code\":\"modelarts.bm.910.arm.public.2\",\"value\":\"Ascend : 2 * Ascend 910 CPU:48 核 512GiB\"}," +
// "{\"code\":\"modelarts.bm.910.arm.public.8\",\"value\":\"Ascend : 8 * Ascend 910 CPU:192 核 2048GiB\"}," +
// "{\"code\":\"modelarts.bm.910.arm.public.4\",\"value\":\"Ascend : 4 * Ascend 910 CPU:96 核 1024GiB\"}," +
// "{\"code\":\"modelarts.bm.910.arm.public.1\",\"value\":\"Ascend : 1 * Ascend 910 CPU:24 核 256GiB\"}" +
// "]}"

CodePath = "/code/" CodePath = "/code/"
OutputPath = "/output/" OutputPath = "/output/"
ResultPath = "/result/" ResultPath = "/result/"
@@ -190,14 +168,6 @@ type OrgMultiNode struct {
Node []int `json:"node"` Node []int `json:"node"`
} }


// type Parameter struct {
// Label string `json:"label"`
// Value string `json:"value"`
// }

// type Parameters struct {
// Parameter []Parameter `json:"parameter"`
// }


type Parameters struct { type Parameters struct {
Parameter []struct { Parameter []struct {
@@ -206,98 +176,23 @@ type Parameters struct {
} `json:"parameter"` } `json:"parameter"`
} }


func GenerateTask(ctx *context.Context, jobName, uuid, description, flavor string) error {
var dataActualPath string
if uuid != "" {
dataActualPath = setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + "/"
} else {
userPath := setting.UserBasePath + ctx.User.Name + "/"
isExist, err := storage.ObsHasObject(userPath)
if err != nil {
log.Error("ObsHasObject failed:%v", err.Error(), ctx.Data["MsgID"])
return err
}

if !isExist {
if err = storage.ObsCreateObject(userPath); err != nil {
log.Error("ObsCreateObject failed:%v", err.Error(), ctx.Data["MsgID"])
return err
}
}

dataActualPath = setting.Bucket + "/" + userPath
}

if poolInfos == nil {
json.Unmarshal([]byte(setting.PoolInfos), &poolInfos)
}
createTime := timeutil.TimeStampNow()
jobResult, err := CreateJob(models.CreateNotebookParams{
JobName: jobName,
Description: description,
ProfileID: setting.ProfileID,
Flavor: flavor,
Pool: models.Pool{
ID: poolInfos.PoolInfo[0].PoolId,
Name: poolInfos.PoolInfo[0].PoolName,
Type: poolInfos.PoolInfo[0].PoolType,
},
Spec: models.Spec{
Storage: models.Storage{
Type: storageTypeOBS,
Location: models.Location{
Path: dataActualPath,
},
},
AutoStop: models.AutoStop{
Enable: true,
Duration: autoStopDuration,
},
},
})
if err != nil {
log.Error("CreateJob failed: %v", err.Error())
return err
}
err = models.CreateCloudbrain(&models.Cloudbrain{

Status: string(models.JobWaiting),
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: jobResult.ID,
JobName: jobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeCloudBrainTwo,
Uuid: uuid,
ComputeResource: models.NPUResource,
CreatedUnix: createTime,
UpdatedUnix: createTime,
})

if err != nil {
return err
}
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobResult.ID, jobName, models.ActionCreateDebugNPUTask)
return nil
}

func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification, bootFile string,autoStopDurationInMs int64) (string, error) {
func GenerateNotebook2(ctx *context.Context, req cloudbrain.GenerateModelArtsNotebookReq) (string, error) {
if poolInfos == nil { if poolInfos == nil {
json.Unmarshal([]byte(setting.PoolInfos), &poolInfos) json.Unmarshal([]byte(setting.PoolInfos), &poolInfos)
} }


imageName, err := GetNotebookImageName(imageId)
imageName, err := GetNotebookImageName(req.ImageId)
if err != nil { if err != nil {
log.Error("GetNotebookImageName failed: %v", err.Error()) log.Error("GetNotebookImageName failed: %v", err.Error())
return "", err return "", err
} }
createTime := timeutil.TimeStampNow() createTime := timeutil.TimeStampNow()
jobResult, err := createNotebook2(models.CreateNotebook2Params{ jobResult, err := createNotebook2(models.CreateNotebook2Params{
JobName: jobName,
Description: description,
Flavor: spec.SourceSpecId,
Duration: autoStopDurationInMs,
ImageID: imageId,
JobName: req.JobName,
Description: req.Description,
Flavor: req.Spec.SourceSpecId,
Duration: req.AutoStopDurationMs,
ImageID: req.ImageId,
PoolID: poolInfos.PoolInfo[0].PoolId, PoolID: poolInfos.PoolInfo[0].PoolId,
Feature: models.NotebookFeature, Feature: models.NotebookFeature,
Volume: models.VolumeReq{ Volume: models.VolumeReq{
@@ -310,13 +205,13 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc
if err != nil { if err != nil {
log.Error("createNotebook2 failed: %v", err.Error()) log.Error("createNotebook2 failed: %v", err.Error())
if strings.HasPrefix(err.Error(), UnknownErrorPrefix) { if strings.HasPrefix(err.Error(), UnknownErrorPrefix) {
log.Info("(%s)unknown error, set temp status", displayJobName)
log.Info("(%s)unknown error, set temp status", req.DisplayJobName)
errTemp := models.InsertCloudbrainTemp(&models.CloudbrainTemp{ errTemp := models.InsertCloudbrainTemp(&models.CloudbrainTemp{
JobID: models.TempJobId, JobID: models.TempJobId,
VersionID: models.TempVersionId, VersionID: models.TempVersionId,
Status: models.TempJobStatus, Status: models.TempJobStatus,
Type: models.TypeCloudBrainTwo, Type: models.TypeCloudBrainTwo,
JobName: jobName,
JobName: req.JobName,
JobType: string(models.JobTypeDebug), JobType: string(models.JobTypeDebug),
}) })
if errTemp != nil { if errTemp != nil {
@@ -327,23 +222,29 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc
return "", err return "", err
} }
task := &models.Cloudbrain{ task := &models.Cloudbrain{
Status: jobResult.Status,
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: jobResult.ID,
JobName: jobName,
FlavorCode: spec.SourceSpecId,
DisplayJobName: displayJobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeCloudBrainTwo,
Uuid: uuid,
ComputeResource: models.NPUResource,
Image: imageName,
BootFile: bootFile,
Description: description,
CreatedUnix: createTime,
UpdatedUnix: createTime,
Spec: spec,
Status: jobResult.Status,
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: jobResult.ID,
JobName: req.JobName,
FlavorCode: req.Spec.SourceSpecId,
DisplayJobName: req.DisplayJobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeCloudBrainTwo,
Uuid: req.Uuid,
ComputeResource: models.NPUResource,
Image: imageName,
BootFile: req.BootFile,
BranchName: req.BranchName,
Description: req.Description,
CreatedUnix: createTime,
UpdatedUnix: createTime,
Spec: req.Spec,
ModelName: req.ModelName,
ModelVersion: req.ModelVersion,
LabelName: req.LabelName,
PreTrainModelUrl: req.PreTrainModelUrl,
CkptName: req.CkptName,
} }


err = models.CreateCloudbrain(task) err = models.CreateCloudbrain(task)
@@ -352,7 +253,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc
} }


stringId := strconv.FormatInt(task.ID, 10) stringId := strconv.FormatInt(task.ID, 10)
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateDebugNPUTask)
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, req.DisplayJobName, models.ActionCreateDebugNPUTask)
return jobResult.ID, nil return jobResult.ID, nil
} }


@@ -379,6 +280,9 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str
Parameter: req.Parameters, Parameter: req.Parameters,
UserImageUrl: req.UserImageUrl, UserImageUrl: req.UserImageUrl,
UserCommand: req.UserCommand, UserCommand: req.UserCommand,
ShareAddr: setting.ModelArtsShareAddr,
MountPath: setting.ModelArtsMountPath,
NasType: setting.ModelArtsNasType,
}, },
}) })
} else { } else {
@@ -399,6 +303,9 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId str
Code: req.Spec.SourceSpecId, Code: req.Spec.SourceSpecId,
}, },
Parameter: req.Parameters, Parameter: req.Parameters,
ShareAddr: setting.ModelArtsShareAddr,
MountPath: setting.ModelArtsMountPath,
NasType: setting.ModelArtsNasType,
}, },
}) })
} }
@@ -517,6 +424,9 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job
PreVersionId: req.PreVersionId, PreVersionId: req.PreVersionId,
UserImageUrl: req.UserImageUrl, UserImageUrl: req.UserImageUrl,
UserCommand: req.UserCommand, UserCommand: req.UserCommand,
ShareAddr: setting.ModelArtsShareAddr,
MountPath: setting.ModelArtsMountPath,
NasType: setting.ModelArtsNasType,
}, },
}, jobId) }, jobId)
} else { } else {
@@ -536,6 +446,9 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job
}, },
Parameter: req.Parameters, Parameter: req.Parameters,
PreVersionId: req.PreVersionId, PreVersionId: req.PreVersionId,
ShareAddr: setting.ModelArtsShareAddr,
MountPath: setting.ModelArtsMountPath,
NasType: setting.ModelArtsNasType,
}, },
}, jobId) }, jobId)
} }
@@ -914,10 +827,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error {
task.FlavorCode = result.Flavor task.FlavorCode = result.Flavor
} }


if oldStatus != task.Status && task.Status == string(models.ModelArtsRunning) && task.BootFile != "" {
uploadNoteBookFile(task, result)

}
err = models.UpdateJob(task) err = models.UpdateJob(task)
if err != nil { if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
@@ -928,81 +837,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error {
return nil return nil
} }


func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) {
jupyterUrl := result.Url + "?token=" + result.Token

cookies, xsrf := getCookiesAndCsrf(jupyterUrl)
if xsrf == "" {
log.Error("browser jupyterUrl failed:%v", task.DisplayJobName)
} else {

codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath
fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile)
if err != nil {
log.Error("read jupyter file failed:%v", task.DisplayJobName, err)
}

base64Content := base64.StdEncoding.EncodeToString(fileContents)
client := getRestyClient()
uploadUrl := getJupyterBaseUrl(result.Url) + "api/contents/" + path.Base(task.BootFile)
res, err := client.R().
SetCookies(cookies).
SetHeader("X-XSRFToken", xsrf).
SetBody(map[string]interface{}{
"type": "file",
"format": "base64",
"name": path.Base(task.BootFile),
"path": path.Base(task.BootFile),
"content": base64Content}).
Put(uploadUrl)
if err != nil {
log.Error("upload jupyter file failed:%v", task.DisplayJobName, err)
} else if res.StatusCode() != http.StatusCreated {
log.Error("upload jupyter file failed:%v", task.DisplayJobName, err)
}

}

}

func getJupyterBaseUrl(url string) string {
jupyterUrlLength := len(url)
baseUrl := url[0 : jupyterUrlLength-len(path.Base(url))]
return baseUrl
}

func getCookiesAndCsrf(jupyterUrl string) ([]*http.Cookie, string) {
log.Info("jupyter url:"+jupyterUrl)
var cookies []*http.Cookie
const retryTimes = 10
for i := 0; i < retryTimes; i++ {
res, err := http.Get(jupyterUrl)
if err != nil {
log.Error("browser jupyterUrl failed.",err)
if i==retryTimes-1{
return cookies, ""
}

} else {
cookies = res.Cookies()
xsrf := ""
for _, cookie := range cookies {
if cookie.Name == "_xsrf" {
xsrf = cookie.Value
break
}

}
if xsrf != "" {
return cookies, xsrf
}

}
}
return cookies, ""

}

func SyncTempStatusJob() { func SyncTempStatusJob() {
jobs, err := models.GetCloudBrainTempJobs() jobs, err := models.GetCloudBrainTempJobs()
if err != nil { if err != nil {


+ 3
- 1
modules/modelarts/resty.go View File

@@ -497,7 +497,7 @@ sendjob:
} }


req, _ := json.Marshal(createJobParams) req, _ := json.Marshal(createJobParams)
log.Info("%s", req)
log.Info("postapi json: %s", req)


if res.StatusCode() == http.StatusUnauthorized && retry < 1 { if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
retry++ retry++
@@ -543,6 +543,8 @@ func createTrainJob(createJobParams models.CreateTrainJobParams) (*models.Create
var result models.CreateTrainJobResult var result models.CreateTrainJobResult


retry := 0 retry := 0
req, _ := json.Marshal(createJobParams)
log.Info("postapi json: %s", req)


sendjob: sendjob:
res, err := client.R(). res, err := client.R().


+ 35
- 27
modules/modelarts_cd/modelarts.go View File

@@ -5,6 +5,8 @@ import (
"strconv" "strconv"
"strings" "strings"


"code.gitea.io/gitea/modules/cloudbrain"

"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
@@ -88,19 +90,19 @@ type Parameters struct {
} `json:"parameter"` } `json:"parameter"`
} }


func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification, bootFile string,autoStopDurationInMs int64) (string, error) {
imageName, err := GetNotebookImageName(imageId)
func GenerateNotebook(ctx *context.Context, req cloudbrain.GenerateModelArtsNotebookReq) (string, error) {
imageName, err := GetNotebookImageName(req.ImageId)
if err != nil { if err != nil {
log.Error("GetNotebookImageName failed: %v", err.Error()) log.Error("GetNotebookImageName failed: %v", err.Error())
return "", err return "", err
} }
createTime := timeutil.TimeStampNow() createTime := timeutil.TimeStampNow()
jobResult, err := createNotebook(models.CreateNotebookWithoutPoolParams{ jobResult, err := createNotebook(models.CreateNotebookWithoutPoolParams{
JobName: jobName,
Description: description,
Flavor: spec.SourceSpecId,
Duration: autoStopDurationInMs,
ImageID: imageId,
JobName: req.JobName,
Description: req.Description,
Flavor: req.Spec.SourceSpecId,
Duration: req.AutoStopDurationMs,
ImageID: req.ImageId,
Feature: models.NotebookFeature, Feature: models.NotebookFeature,
Volume: models.VolumeReq{ Volume: models.VolumeReq{
Capacity: setting.Capacity, Capacity: setting.Capacity,
@@ -112,13 +114,13 @@ func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, descr
if err != nil { if err != nil {
log.Error("createNotebook failed: %v", err.Error()) log.Error("createNotebook failed: %v", err.Error())
if strings.HasPrefix(err.Error(), UnknownErrorPrefix) { if strings.HasPrefix(err.Error(), UnknownErrorPrefix) {
log.Info("(%s)unknown error, set temp status", displayJobName)
log.Info("(%s)unknown error, set temp status", req.DisplayJobName)
errTemp := models.InsertCloudbrainTemp(&models.CloudbrainTemp{ errTemp := models.InsertCloudbrainTemp(&models.CloudbrainTemp{
JobID: models.TempJobId, JobID: models.TempJobId,
VersionID: models.TempVersionId, VersionID: models.TempVersionId,
Status: models.TempJobStatus, Status: models.TempJobStatus,
Type: models.TypeCDCenter, Type: models.TypeCDCenter,
JobName: jobName,
JobName: req.JobName,
JobType: string(models.JobTypeDebug), JobType: string(models.JobTypeDebug),
}) })
if errTemp != nil { if errTemp != nil {
@@ -129,23 +131,29 @@ func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, descr
return "", err return "", err
} }
task := &models.Cloudbrain{ task := &models.Cloudbrain{
Status: jobResult.Status,
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: jobResult.ID,
JobName: jobName,
FlavorCode: spec.SourceSpecId,
DisplayJobName: displayJobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeCDCenter,
Uuid: uuid,
ComputeResource: models.NPUResource,
Image: imageName,
Description: description,
CreatedUnix: createTime,
UpdatedUnix: createTime,
Spec: spec,
BootFile: bootFile,
Status: jobResult.Status,
UserID: ctx.User.ID,
RepoID: ctx.Repo.Repository.ID,
JobID: jobResult.ID,
JobName: req.JobName,
FlavorCode: req.Spec.SourceSpecId,
DisplayJobName: req.DisplayJobName,
JobType: string(models.JobTypeDebug),
Type: models.TypeCDCenter,
Uuid: req.Uuid,
ComputeResource: models.NPUResource,
Image: imageName,
Description: req.Description,
CreatedUnix: createTime,
UpdatedUnix: createTime,
Spec: req.Spec,
BootFile: req.BootFile,
BranchName: req.BranchName,
ModelName: req.ModelName,
ModelVersion: req.ModelVersion,
LabelName: req.LabelName,
PreTrainModelUrl: req.PreTrainModelUrl,
CkptName: req.CkptName,
} }


err = models.CreateCloudbrain(task) err = models.CreateCloudbrain(task)
@@ -154,7 +162,7 @@ func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, descr
} }


stringId := strconv.FormatInt(task.ID, 10) stringId := strconv.FormatInt(task.ID, 10)
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateDebugNPUTask)
notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, req.DisplayJobName, models.ActionCreateDebugNPUTask)
return jobResult.ID, nil return jobResult.ID, nil
} }




+ 198
- 0
modules/notebook/contentManager.go View File

@@ -0,0 +1,198 @@
package notebook

import (
"crypto/tls"
"encoding/base64"
"fmt"
"io/ioutil"
"net/http"
"path"
"strings"

"github.com/go-resty/resty/v2"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cloudbrain"
"code.gitea.io/gitea/modules/setting"

"code.gitea.io/gitea/modules/log"
)

var restyClient *resty.Client

type NotebookApiResponse struct {
Name string `json:"name"`
Path string `json:"path"`
}

type NotebookContent struct {
Url string
Path string
Cookies []*http.Cookie
Xsrf string
PathType string //file directory
Token string
}

func (c *NotebookContent) IsNotebookFileCanBrowser() bool {
if c.Xsrf == "" {
c.SetCookiesAndCsrf()
}
if c.Xsrf == "" {
log.Warn("xsrf is empty, can not broswer url:" + c.Url)
return false
}
return c.IsNoteBookContentsExist()

}

func (c *NotebookContent) SetCookiesAndCsrf() {
log.Info("jupyter url:" + c.Url)
var cookies []*http.Cookie
const retryTimes = 10
url := c.Url
if c.Token != "" {
url = c.Url + "?token=" + c.Token
}
for i := 0; i < retryTimes; i++ {
res, err := http.Get(url)
if err != nil {
log.Error("browser jupyterUrl failed.", err)
if i == retryTimes-1 {
c.Cookies = cookies
}

} else {
cookies = res.Cookies()
xsrf := ""
for _, cookie := range cookies {
if cookie.Name == "_xsrf" {
xsrf = cookie.Value
if len(cookies) > 1 {
break
}

}

}
if xsrf != "" {
c.Cookies = cookies
c.Xsrf = xsrf
}

}
}
c.Cookies = cookies

}

func (c *NotebookContent) IsNoteBookContentsExist() bool {
client := getRestyClient()
uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + "?type=" + c.PathType
res, err := client.R().
SetCookies(c.Cookies).
SetHeader("X-XSRFToken", c.Xsrf).
Get(uploadUrl)
if err != nil {
log.Warn("browser url error:"+uploadUrl, err)
return false
}
return res.StatusCode() == http.StatusOK
}

func (c *NotebookContent) UploadNoteBookFile(task *models.Cloudbrain) error {

err := c.MakeNoteBookDir()
if err != nil {
return err
}

codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath
fileContents, err := ioutil.ReadFile(codePath + "/" + c.Path)
if err != nil {
log.Error("read jupyter file failed:%v", task.DisplayJobName, err)
}

base64Content := base64.StdEncoding.EncodeToString(fileContents)
client := getRestyClient()
uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path
res, err := client.R().
SetCookies(c.Cookies).
SetHeader("X-XSRFToken", c.Xsrf).
SetBody(map[string]interface{}{
"type": "file",
"format": "base64",
"name": path.Base(c.Path),
"path": c.Path,
"content": base64Content}).
Put(uploadUrl)
if err != nil {
log.Error("upload jupyter file failed:%v", task.DisplayJobName, err)
return err
} else if res.StatusCode() != http.StatusCreated {
log.Error("upload jupyter file failed:%v, status is %s", task.DisplayJobName, res.Status(), err)
return fmt.Errorf("status:", res.StatusCode())
}
return nil
}

/**
if c.Path is a/b/c.txt
makedir a/b
if c.Path is a/b/c
makedir a/b
*/
func (c *NotebookContent) MakeNoteBookDir() error {
filePaths := strings.Split(c.Path, "/")

for i := 0; i < len(filePaths)-1; i++ {
cTemp := &NotebookContent{
Url: c.Url,
Cookies: c.Cookies,
Path: path.Join(filePaths[0 : i+1]...),
PathType: "directory",
Xsrf: c.Xsrf,
}
if !cTemp.IsNoteBookContentsExist() {

createTempDirUrl := getJupyterBaseUrl(cTemp.Url) + "api/contents/" + cTemp.Path
client := getRestyClient()
var jobResult NotebookApiResponse
res, err := client.R().
SetCookies(c.Cookies).
SetHeader("X-XSRFToken", c.Xsrf).
SetBody(map[string]interface{}{
"type": cTemp.PathType,
"path": cTemp.Path,
}).SetResult(&jobResult).
Put(createTempDirUrl)
if err != nil {
return err
}
if res.StatusCode() != http.StatusCreated {
return fmt.Errorf("status code:" + res.Status())
}

}

}
return nil
}

func getJupyterBaseUrl(url string) string {
jupyterUrlLength := len(url)
baseUrl := url
if strings.HasSuffix(url, "lab") {
baseUrl = url[0 : jupyterUrlLength-len(path.Base(url))]
}

return baseUrl
}

func getRestyClient() *resty.Client {
if restyClient == nil {
restyClient = resty.New()
restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}
return restyClient
}

+ 9
- 0
modules/redis/redis_key/repo_redis_key.go View File

@@ -0,0 +1,9 @@
package redis_key

import "fmt"

const REPO_PREFIX = "repo"

func RepoTopNContributors(repoId int64, N int) string {
return KeyJoin(REPO_PREFIX, fmt.Sprint(repoId), fmt.Sprint(N), "top_n_contributor")
}

+ 47
- 17
modules/setting/setting.go View File

@@ -518,7 +518,7 @@ var (
MaxDatasetNum int MaxDatasetNum int
CullIdleTimeout string CullIdleTimeout string
CullInterval string CullInterval string
DebugAttachSize int


//benchmark config //benchmark config
IsBenchmarkEnabled bool IsBenchmarkEnabled bool
@@ -544,6 +544,12 @@ var (
BrainScoreName string BrainScoreName string
BrainScoreServerHost string BrainScoreServerHost string


IsSnn4EcosetEnabled bool
Snn4EcosetOwner string
Snn4EcosetName string
Snn4EcosetServerHost string
Snn4AttachmentName string

//blockchain config //blockchain config
BlockChainHost string BlockChainHost string
CommitValidDate string CommitValidDate string
@@ -585,6 +591,9 @@ var (
TrainJobFLAVORINFOS string TrainJobFLAVORINFOS string
ModelArtsSpecialPools string ModelArtsSpecialPools string
ModelArtsMultiNode string ModelArtsMultiNode string
ModelArtsShareAddr string
ModelArtsMountPath string
ModelArtsNasType string
//kanban //kanban
IsCloudbrainTimingEnabled bool IsCloudbrainTimingEnabled bool


@@ -614,14 +623,14 @@ var (
UsageRateBeginTime string UsageRateBeginTime string
}{} }{}


ClearStrategy= struct {
Enabled bool
ResultSaveDays int
BatchSize int
DebugJobSize int
TrashSaveDays int
Cron string
RunAtStart bool
ClearStrategy = struct {
Enabled bool
ResultSaveDays int
BatchSize int
DebugJobSize int
TrashSaveDays int
Cron string
RunAtStart bool
}{} }{}


C2NetInfos *C2NetSqInfos C2NetInfos *C2NetSqInfos
@@ -677,6 +686,10 @@ var (
CloudbrainStoppedTitle string CloudbrainStoppedTitle string
CloudbrainStoppedRemark string CloudbrainStoppedRemark string


//repo square config
IncubationSourceOrgName string
PaperRepoTopicName string

//nginx proxy //nginx proxy
PROXYURL string PROXYURL string
RadarMap = struct { RadarMap = struct {
@@ -704,6 +717,7 @@ var (


ProjectHealth float64 ProjectHealth float64
ProjectHealthIssueCompleteRatio float64 ProjectHealthIssueCompleteRatio float64
ProjectHealth0IssueCloseRatio float64


TeamHealth float64 TeamHealth float64
TeamHealthContributors float64 TeamHealthContributors float64
@@ -1490,6 +1504,7 @@ func NewContext() {
MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5)
CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900")
CullInterval = sec.Key("CULL_INTERVAL").MustString("60") CullInterval = sec.Key("CULL_INTERVAL").MustString("60")
DebugAttachSize = sec.Key("DEBUG_ATTACH_SIZE").MustInt(20)


sec = Cfg.Section("benchmark") sec = Cfg.Section("benchmark")
IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)
@@ -1515,6 +1530,13 @@ func NewContext() {
BrainScoreName = sec.Key("NAME").MustString("") BrainScoreName = sec.Key("NAME").MustString("")
BrainScoreServerHost = sec.Key("HOST").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("")


sec = Cfg.Section("snn4ecoset")
IsSnn4EcosetEnabled = sec.Key("ENABLED").MustBool(false)
Snn4EcosetOwner = sec.Key("OWNER").MustString("")
Snn4EcosetName = sec.Key("NAME").MustString("")
Snn4EcosetServerHost = sec.Key("HOST").MustString("")
Snn4AttachmentName = sec.Key("DATASET").MustString("")

sec = Cfg.Section("blockchain") sec = Cfg.Section("blockchain")
BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/")
CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15")
@@ -1553,6 +1575,9 @@ func NewContext() {
TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("") TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("")
ModelArtsSpecialPools = sec.Key("SPECIAL_POOL").MustString("") ModelArtsSpecialPools = sec.Key("SPECIAL_POOL").MustString("")
ModelArtsMultiNode = sec.Key("MULTI_NODE").MustString("") ModelArtsMultiNode = sec.Key("MULTI_NODE").MustString("")
ModelArtsShareAddr = sec.Key("ModelArts_Share_Addr").MustString("192.168.0.30:/")
ModelArtsMountPath = sec.Key("ModelArts_Mount_Path").MustString("/cache/sfs")
ModelArtsNasType = sec.Key("ModelArts_Nas_Type").MustString("nfs")


sec = Cfg.Section("elk") sec = Cfg.Section("elk")
ElkUrl = sec.Key("ELKURL").MustString("") ElkUrl = sec.Key("ELKURL").MustString("")
@@ -1585,6 +1610,10 @@ func NewContext() {
CloudbrainStoppedTitle = sec.Key("CLOUDBRAIN_STOPPED_TITLE").MustString("您好,您申请的算力资源已结束使用,任务已完成运行,状态为%s,请您关注运行结果") CloudbrainStoppedTitle = sec.Key("CLOUDBRAIN_STOPPED_TITLE").MustString("您好,您申请的算力资源已结束使用,任务已完成运行,状态为%s,请您关注运行结果")
CloudbrainStoppedRemark = sec.Key("CLOUDBRAIN_STOPPED_REMARK").MustString("感谢您的耐心等待。") CloudbrainStoppedRemark = sec.Key("CLOUDBRAIN_STOPPED_REMARK").MustString("感谢您的耐心等待。")


sec = Cfg.Section("repo-square")
IncubationSourceOrgName = sec.Key("INCUBATION_ORG_NAME").MustString("OpenI")
PaperRepoTopicName = sec.Key("PAPER_REPO_TOPIC_NAME").MustString("openi-paper")

sec = Cfg.Section("point") sec = Cfg.Section("point")
CloudBrainPaySwitch = sec.Key("CLOUDBRAIN_PAY_SWITCH").MustBool(false) CloudBrainPaySwitch = sec.Key("CLOUDBRAIN_PAY_SWITCH").MustBool(false)
CloudBrainPayDelay = sec.Key("CLOUDBRAIN_PAY_DELAY").MustDuration(30 * time.Minute) CloudBrainPayDelay = sec.Key("CLOUDBRAIN_PAY_DELAY").MustDuration(30 * time.Minute)
@@ -1691,16 +1720,16 @@ func getModelartsCDConfig() {
getNotebookFlavorInfos() getNotebookFlavorInfos()
} }


func getClearStrategy(){
func getClearStrategy() {


sec := Cfg.Section("clear_strategy") sec := Cfg.Section("clear_strategy")
ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false)
ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30)
ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500)
ClearStrategy.DebugJobSize=sec.Key("DEBUG_BATCH_SIZE").MustInt(100)
ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90)
ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ?")
ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false)
ClearStrategy.Enabled = sec.Key("ENABLED").MustBool(false)
ClearStrategy.ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt(30)
ClearStrategy.BatchSize = sec.Key("BATCH_SIZE").MustInt(500)
ClearStrategy.DebugJobSize = sec.Key("DEBUG_BATCH_SIZE").MustInt(100)
ClearStrategy.TrashSaveDays = sec.Key("TRASH_SAVE_DAYS").MustInt(90)
ClearStrategy.Cron = sec.Key("CRON").MustString("* 0,30 2-8 * * ?")
ClearStrategy.RunAtStart = sec.Key("RUN_AT_START").MustBool(false)
} }


func getGrampusConfig() { func getGrampusConfig() {
@@ -1767,6 +1796,7 @@ func SetRadarMapConfig() {
RadarMap.LivenessRelease = sec.Key("liveness_release").MustFloat64(0.4) RadarMap.LivenessRelease = sec.Key("liveness_release").MustFloat64(0.4)
RadarMap.ProjectHealth = sec.Key("project_health").MustFloat64(0.1) RadarMap.ProjectHealth = sec.Key("project_health").MustFloat64(0.1)
RadarMap.ProjectHealthIssueCompleteRatio = sec.Key("project_health_issue_complete_ratio").MustFloat64(100) RadarMap.ProjectHealthIssueCompleteRatio = sec.Key("project_health_issue_complete_ratio").MustFloat64(100)
RadarMap.ProjectHealth0IssueCloseRatio = sec.Key("project_health_0_issue_close_ratio").MustFloat64(0.0)
RadarMap.TeamHealth = sec.Key("team_health").MustFloat64(0.1) RadarMap.TeamHealth = sec.Key("team_health").MustFloat64(0.1)
RadarMap.TeamHealthContributors = sec.Key("team_health_contributors").MustFloat64(0.2) RadarMap.TeamHealthContributors = sec.Key("team_health_contributors").MustFloat64(0.2)
RadarMap.TeamHealthKeyContributors = sec.Key("team_health_key_contributors").MustFloat64(0.6) RadarMap.TeamHealthKeyContributors = sec.Key("team_health_key_contributors").MustFloat64(0.6)


+ 28
- 27
modules/structs/cloudbrain.go View File

@@ -47,36 +47,37 @@ type CreateFileNotebookJobOption struct {
BranchName string `json:"branch_name" binding:"Required"` BranchName string `json:"branch_name" binding:"Required"`
OwnerName string `json:"owner_name" binding:"Required"` OwnerName string `json:"owner_name" binding:"Required"`
ProjectName string `json:"project_name" binding:"Required"` ProjectName string `json:"project_name" binding:"Required"`
JobId string `json:"job_id"`
} }


type Cloudbrain struct { type Cloudbrain struct {
ID int64 `json:"id"`
JobID string `json:"job_id"`
JobType string `json:"job_type"`
Type int `json:"type"`
DisplayJobName string `json:"display_job_name"`
Status string `json:"status"`
CreatedUnix int64 `json:"created_unix"`
RepoID int64 `json:"repo_id"`
Duration int64 `json:"duration"` //运行时长 单位秒
TrainJobDuration string `json:"train_job_duration"`
ImageID string `json:"image_id"` //grampus image_id
Image string `json:"image"`
Uuid string `json:"uuid"` //数据集id
DatasetName string `json:"dataset_name"`
ComputeResource string `json:"compute_resource"` //计算资源,例如npu
AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name
BranchName string `json:"branch_name"` //分支名称
Parameters string `json:"parameters"` //传给modelarts的param参数
BootFile string `json:"boot_file"` //启动文件
Description string `json:"description"` //描述
ModelName string `json:"model_name"` //模型名称
ModelVersion string `json:"model_version"` //模型版本
CkptName string `json:"ckpt_name"` //权重文件名称
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
Spec *SpecificationShow `json:"spec"`
ID int64 `json:"id"`
JobID string `json:"job_id"`
JobType string `json:"job_type"`
Type int `json:"type"`
DisplayJobName string `json:"display_job_name"`
Status string `json:"status"`
CreatedUnix int64 `json:"created_unix"`
RepoID int64 `json:"repo_id"`
Duration int64 `json:"duration"` //运行时长 单位秒
TrainJobDuration string `json:"train_job_duration"`
ImageID string `json:"image_id"` //grampus image_id
Image string `json:"image"`
Uuid string `json:"uuid"` //数据集id
DatasetName string `json:"dataset_name"`
ComputeResource string `json:"compute_resource"` //计算资源,例如npu
AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name
BranchName string `json:"branch_name"` //分支名称
Parameters string `json:"parameters"` //传给modelarts的param参数
BootFile string `json:"boot_file"` //启动文件
Description string `json:"description"` //描述
ModelName string `json:"model_name"` //模型名称
ModelVersion string `json:"model_version"` //模型版本
CkptName string `json:"ckpt_name"` //权重文件名称
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
VersionName string `json:"version_name"`
Spec *SpecificationShow `json:"spec"`
} }


type SpecificationShow struct { type SpecificationShow struct {


+ 23
- 0
modules/structs/pipeline.go View File

@@ -0,0 +1,23 @@
package structs

type Pipeline struct {
ID int64 `json:"id"`
Name string `json:"name"`
Status string `json:"status"`
}
type NodeInfo struct {
Name string `json:"name"`
Status string `json:"status"`
Code string `json:"code"`
Message string `json:"message"`
}

type PipelineNotification struct {
Type int `json:"type"`
Username string `json:"username"`
Reponame string `json:"reponame"`
Pipeline Pipeline `json:"pipeline"`
PipelineRunId string `json:"pipeline_run_id"`
Node NodeInfo `json:"node"`
OccurTime int64 `json:"occur_time"`
}

+ 17
- 2
modules/templates/helper.go View File

@@ -47,6 +47,7 @@ const (
REF_TYPE_BRANCH = "branch" REF_TYPE_BRANCH = "branch"
REF_TYPE_TAG = "tag" REF_TYPE_TAG = "tag"
REF_TYPE_PATTERN = "(refs/heads/|refs/tags/)" REF_TYPE_PATTERN = "(refs/heads/|refs/tags/)"
DURATION_STR_ZERO = "00:00:00"
) )


// Used from static.go && dynamic.go // Used from static.go && dynamic.go
@@ -97,11 +98,15 @@ func NewFuncMap() []template.FuncMap {
"AllowedReactions": func() []string { "AllowedReactions": func() []string {
return setting.UI.Reactions return setting.UI.Reactions
}, },
"DebugAttachSize": func() int {
return setting.DebugAttachSize * 1000 * 1000 * 1000
},
"AvatarLink": models.AvatarLink, "AvatarLink": models.AvatarLink,
"Safe": Safe, "Safe": Safe,
"SafeJS": SafeJS, "SafeJS": SafeJS,
"Str2html": Str2html, "Str2html": Str2html,
"subOne": subOne, "subOne": subOne,
"addOne": addOne,
"TimeSince": timeutil.TimeSince, "TimeSince": timeutil.TimeSince,
"TimeSinceUnix": timeutil.TimeSinceUnix, "TimeSinceUnix": timeutil.TimeSinceUnix,
"TimeSinceUnix1": timeutil.TimeSinceUnix1, "TimeSinceUnix1": timeutil.TimeSinceUnix1,
@@ -109,6 +114,7 @@ func NewFuncMap() []template.FuncMap {
"AttachmentStatus": dataset.GetStatusText, "AttachmentStatus": dataset.GetStatusText,
"IsShowDataSetOfCurrentRepo": dataset.IsShowDataSetOfCurrentRepo, "IsShowDataSetOfCurrentRepo": dataset.IsShowDataSetOfCurrentRepo,
"TimeSinceUnixShort": timeutil.TimeSinceUnixShort, "TimeSinceUnixShort": timeutil.TimeSinceUnixShort,
"ConvertDurationToStr": ConvertDurationToStr,
"RawTimeSince": timeutil.RawTimeSince, "RawTimeSince": timeutil.RawTimeSince,
"FileSize": base.FileSize, "FileSize": base.FileSize,
"PrettyNumber": base.PrettyNumber, "PrettyNumber": base.PrettyNumber,
@@ -151,7 +157,7 @@ func NewFuncMap() []template.FuncMap {
"EscapePound": func(str string) string { "EscapePound": func(str string) string {
return strings.NewReplacer("%", "%25", "#", "%23", " ", "%20", "?", "%3F").Replace(str) return strings.NewReplacer("%", "%25", "#", "%23", " ", "%20", "?", "%3F").Replace(str)
}, },
"IpynbBool":func(str string) bool{
"IpynbBool": func(str string) bool {
return strings.Contains(str, ".ipynb") return strings.Contains(str, ".ipynb")
}, },
"nl2br": func(text string) template.HTML { "nl2br": func(text string) template.HTML {
@@ -365,6 +371,7 @@ func NewTextFuncMap() []texttmpl.FuncMap {
"TimeSinceUnix": timeutil.TimeSinceUnix, "TimeSinceUnix": timeutil.TimeSinceUnix,
"TimeSinceUnix1": timeutil.TimeSinceUnix1, "TimeSinceUnix1": timeutil.TimeSinceUnix1,
"TimeSinceUnixShort": timeutil.TimeSinceUnixShort, "TimeSinceUnixShort": timeutil.TimeSinceUnixShort,
"ConvertDurationToStr": ConvertDurationToStr,
"RawTimeSince": timeutil.RawTimeSince, "RawTimeSince": timeutil.RawTimeSince,
"AttachmentResourceType": dataset.GetResourceType, "AttachmentResourceType": dataset.GetResourceType,
"AttachmentStatus": dataset.GetStatusText, "AttachmentStatus": dataset.GetStatusText,
@@ -467,7 +474,9 @@ func Str2html(raw string) template.HTML {
func subOne(length int) int { func subOne(length int) int {
return length - 1 return length - 1
} }

func addOne(length int64) int64 {
return length + 1
}
// Escape escapes a HTML string // Escape escapes a HTML string
func Escape(raw string) string { func Escape(raw string) string {
return html.EscapeString(raw) return html.EscapeString(raw)
@@ -804,3 +813,9 @@ func MB2GB(size int) string {
} }
return s return s
} }
func ConvertDurationToStr(duration int64) string {
if duration <= 0 {
return DURATION_STR_ZERO
}
return util.AddZero(duration/3600) + ":" + util.AddZero(duration%3600/60) + ":" + util.AddZero(duration%60)
}

+ 22
- 3
options/locale/locale_en-US.ini View File

@@ -577,6 +577,7 @@ static.CloudBrainTaskNum=CloudBrain Task Count
static.CloudBrainRunTime=CloudBrain Run Time static.CloudBrainRunTime=CloudBrain Run Time
static.CommitDatasetNum=Commit Dataset Count static.CommitDatasetNum=Commit Dataset Count
static.CommitModelCount=Commit Model Count static.CommitModelCount=Commit Model Count
static.ModelConvertCount=Model Convert Count
static.UserIndex=Normalized user index static.UserIndex=Normalized user index
static.UserIndexPrimitive=User Index static.UserIndexPrimitive=User Index
static.countdate=Count Date static.countdate=Count Date
@@ -852,6 +853,7 @@ description = Description
description_format_err=Description's length can be up to %s characters long. description_format_err=Description's length can be up to %s characters long.
create_dataset = Create Dataset create_dataset = Create Dataset
download_url=Download Url download_url=Download Url
download_model_url=Download Url
download_oper=Operation download_oper=Operation
download_copy=Copy URL download_copy=Copy URL
create_dataset_fail=Failed to create dataset. create_dataset_fail=Failed to create dataset.
@@ -1060,6 +1062,9 @@ model_rename=Duplicate model name, please modify model name.
notebook_file_not_exist=Notebook file does not exist. notebook_file_not_exist=Notebook file does not exist.
notebook_select_wrong=Please select a Notebook(.ipynb) file first. notebook_select_wrong=Please select a Notebook(.ipynb) file first.
notebook_file_no_right=You have no right to access the Notebook(.ipynb) file. notebook_file_no_right=You have no right to access the Notebook(.ipynb) file.
notebook_repo_conflict=The files in different branches of the same repository can not run together.
debug_again_fail=Fail to restart debug task, please try again later.
debug_again_fail_forever=The task was scheduled failed last time, can not restart.


date=Date date=Date
repo_add=Project Increment repo_add=Project Increment
@@ -1077,6 +1082,7 @@ delete=Delete
more=More more=More
gpu_type_all=All gpu_type_all=All
model_download=Model Download model_download=Model Download
all_result_download=All result download
submit_image=Submit Image submit_image=Submit Image
modify_image=Modify Image modify_image=Modify Image
image_exist=Image name has been used, please use a new one. image_exist=Image name has been used, please use a new one.
@@ -1089,8 +1095,8 @@ image_delete_fail=Failed to delete image, please try again later.
image_overwrite=You had submitted the same name image before, are you sure to overwrite the original image? image_overwrite=You had submitted the same name image before, are you sure to overwrite the original image?
download=Download download=Download
score=Score score=Score
wait_count_start = There are currently
wait_count_end = tasks queued
wait_count_start = Your current queue position is
wait_count_end =
file_limit_100 = Display up to 100 files or folders in a single directory file_limit_100 = Display up to 100 files or folders in a single directory
images.name = Image Tag images.name = Image Tag
images.name_placerholder = Please enter the image name images.name_placerholder = Please enter the image name
@@ -1342,9 +1348,12 @@ modelconvert.inputshapeerror=Format input error, please input such as: 1,1,32,32


modelconvert.manage.create_error1=A model transformation task with the same name already exists. modelconvert.manage.create_error1=A model transformation task with the same name already exists.
modelconvert.manage.create_error2=Only one running model transformation task can be created. modelconvert.manage.create_error2=Only one running model transformation task can be created.
modelconvert.manage.model_not_exist=The model does not exist.
modelconvert.manage.model_not_exist=The model in the task does not exist or has been deleted.
modelconvert.manage.no_operate_right=You have no right to do the operation. modelconvert.manage.no_operate_right=You have no right to do the operation.


debug.manage.model_not_exist=The model in the task does not exist or has been deleted, please create a new debug job.
debug.manage.dataset_not_exist=The part of datasets in the task does not exist or has been deleted, please create a new debug job.

grampus.train_job.ai_center = AI Center grampus.train_job.ai_center = AI Center
grampus.dataset_path_rule = The code is storaged in /cache/code;the dataset is storaged in /cache/dataset;and please put your model into /cache/output, then you can download it online。 grampus.dataset_path_rule = The code is storaged in /cache/code;the dataset is storaged in /cache/dataset;and please put your model into /cache/output, then you can download it online。
grampus.gpu_dataset_path_rule = The code is storaged in /tmp/code;the dataset is storaged in /tmp/dataset;and please put your model into /tmp/output, then you can download it online。 grampus.gpu_dataset_path_rule = The code is storaged in /tmp/code;the dataset is storaged in /tmp/dataset;and please put your model into /tmp/output, then you can download it online。
@@ -2750,6 +2759,10 @@ repos.pr=PR
repos.commit=Commit repos.commit=Commit
repos.closedIssues=Closed Issue repos.closedIssues=Closed Issue
repos.contributor=Contributor repos.contributor=Contributor
repos.numDataset=Dataset File
repos.numCloudbrain=Cloudbrain Task
repos.numModel=Model
repos.numModelConvert=Model Convert Task
repos.yes=Yes repos.yes=Yes
repos.no=No repos.no=No


@@ -3120,6 +3133,8 @@ reject_pull_request = `suggested changes for <a href="%s/pulls/%s">%s#%[2]s</a>`
upload_dataset=`upload dataset <a href="%s/datasets">%s</a>` upload_dataset=`upload dataset <a href="%s/datasets">%s</a>`
task_gpudebugjob=`created CPU/GPU type debugging task <a href="%s/cloudbrain/%s">%s</a>` task_gpudebugjob=`created CPU/GPU type debugging task <a href="%s/cloudbrain/%s">%s</a>`
task_npudebugjob=`created NPU type debugging task <a href="%s/modelarts/notebook/%s">%s</a>` task_npudebugjob=`created NPU type debugging task <a href="%s/modelarts/notebook/%s">%s</a>`
task_c2net_gpudebugjob=`created CPU/GPU type debugging task <a href="%s/grampus/notebook/%s">%s</a>`
task_c2net_npudebugjob=`created NPU type debugging task <a href="%s/grampus/notebook/%s">%s</a>`
task_nputrainjob=`created NPU training task <a href="%s/modelarts/train-job/%s">%s</a>` task_nputrainjob=`created NPU training task <a href="%s/modelarts/train-job/%s">%s</a>`
task_inferencejob=`created reasoning task <a href="%s/modelarts/inference-job/%s">%s</a>` task_inferencejob=`created reasoning task <a href="%s/modelarts/inference-job/%s">%s</a>`
task_benchmark=`created profiling task <a href="%s/cloudbrain/benchmark/%s">%s</a>` task_benchmark=`created profiling task <a href="%s/cloudbrain/benchmark/%s">%s</a>`
@@ -3239,6 +3254,7 @@ dataset = Dataset
resource_specification = Resource specification resource_specification = Resource specification
dataset_storage_path = Dataset storage path dataset_storage_path = Dataset storage path
model_storage_path = Model storage path model_storage_path = Model storage path
output_storage_path = Output storage path
code_storage_path = Code storage path code_storage_path = Code storage path
benchmark_path = Benchmark script path benchmark_path = Benchmark script path
snn4imagenet_path = Snn4imagenet script path snn4imagenet_path = Snn4imagenet script path
@@ -3294,8 +3310,11 @@ load_code_failed=Fail to load code, please check if the right branch is selected


error.dataset_select = dataset select error:the count exceed the limit or has same name error.dataset_select = dataset select error:the count exceed the limit or has same name
new_train_gpu_tooltips = The code is storaged in <strong style="color:#010101">%s</strong>, the dataset is storaged in <strong style="color:#010101">%s</strong>, the pre-trained model is storaged in the run parameter <strong style="color:#010101">%s</strong>, and please put your model into <strong style="color:#010101">%s</strong> then you can download it online new_train_gpu_tooltips = The code is storaged in <strong style="color:#010101">%s</strong>, the dataset is storaged in <strong style="color:#010101">%s</strong>, the pre-trained model is storaged in the run parameter <strong style="color:#010101">%s</strong>, and please put your model into <strong style="color:#010101">%s</strong> then you can download it online
new_debug_gpu_tooltips = The code is storaged in <strong style="color:#010101">%s</strong>, the dataset is storaged in <strong style="color:#010101">%s</strong>, the pre-trained model is storaged in the <strong style="color:#010101">%s</strong>, and please put your model into <strong style="color:#010101">%s</strong> then you can download it online
new_debug_gpu_tooltips1 = The code is storaged in <strong style="color:#010101">%s</strong>, the dataset is storaged in <strong style="color:#010101">%s</strong>, the pre-trained model is storaged in the <strong style="color:#010101">%s</strong>.
new_train_npu_tooltips = The code is storaged in <strong style="color:#010101">%s</strong>, the pre-trained model is storaged in the run parameter <strong style="color:#010101">%s</strong>, and please put your model into <strong style="color:#010101">%s</strong> then you can download it online new_train_npu_tooltips = The code is storaged in <strong style="color:#010101">%s</strong>, the pre-trained model is storaged in the run parameter <strong style="color:#010101">%s</strong>, and please put your model into <strong style="color:#010101">%s</strong> then you can download it online
new_infer_gpu_tooltips = The dataset is stored in <strong style="color:#010101">%s</strong>, the model file is stored in <strong style="color:#010101">%s</strong>, please store the inference output in <strong style="color:#010101">%s</strong> for subsequent downloads. new_infer_gpu_tooltips = The dataset is stored in <strong style="color:#010101">%s</strong>, the model file is stored in <strong style="color:#010101">%s</strong>, please store the inference output in <strong style="color:#010101">%s</strong> for subsequent downloads.
code_obs_address = Code OBS address


[points] [points]
points = points points = points


+ 26
- 5
options/locale/locale_zh-CN.ini View File

@@ -581,6 +581,7 @@ static.CloudBrainTaskNum=云脑任务数
static.CloudBrainRunTime=云脑运行时间(小时) static.CloudBrainRunTime=云脑运行时间(小时)
static.CommitDatasetNum=上传(提交)数据集文件数 static.CommitDatasetNum=上传(提交)数据集文件数
static.CommitModelCount=提交模型数 static.CommitModelCount=提交模型数
static.ModelConvertCount=模型转换数
static.UserIndex=归一化用户指数 static.UserIndex=归一化用户指数
static.UserIndexPrimitive=用户指数 static.UserIndexPrimitive=用户指数
static.countdate=系统统计时间 static.countdate=系统统计时间
@@ -863,6 +864,7 @@ reference_dataset_fail=关联数据集失败,请稍后再试。
cancel_reference_dataset_fail=取消关联数据集失败,请稍后再试。 cancel_reference_dataset_fail=取消关联数据集失败,请稍后再试。


download_url=数据集下载地址 download_url=数据集下载地址
download_model_url=模型文件下载地址
download_copy=复制链接 download_copy=复制链接
download_oper=操作 download_oper=操作
show_dataset=数据集 show_dataset=数据集
@@ -1059,6 +1061,9 @@ model_rename=模型名称重复,请修改模型名称
notebook_file_not_exist=Notebook文件不存在。 notebook_file_not_exist=Notebook文件不存在。
notebook_select_wrong=请先选择Notebook(.ipynb)文件。 notebook_select_wrong=请先选择Notebook(.ipynb)文件。
notebook_file_no_right=您没有这个Notebook文件的读权限。 notebook_file_no_right=您没有这个Notebook文件的读权限。
notebook_repo_conflict=同一个仓库的不同分支文件不能同时运行。
debug_again_fail=再次调试失败,请稍后再试。
debug_again_fail_forever=这个任务之前没有调度成功,不能再次调试。


date=日期 date=日期
repo_add=新增项目 repo_add=新增项目
@@ -1076,6 +1081,7 @@ delete=删除
more=更多 more=更多
gpu_type_all=全部 gpu_type_all=全部
model_download=结果下载 model_download=结果下载
all_result_download=全部结果下载
submit_image=提交镜像 submit_image=提交镜像
modify_image=修改镜像 modify_image=修改镜像
image_exist=镜像Tag已被使用,请修改镜像Tag。 image_exist=镜像Tag已被使用,请修改镜像Tag。
@@ -1088,8 +1094,8 @@ image_delete_fail=删除镜像失败,请稍后再试。
image_overwrite=您已经提交过相同名称的镜像,您确定要覆盖原来提交的镜像吗? image_overwrite=您已经提交过相同名称的镜像,您确定要覆盖原来提交的镜像吗?
download=模型下载 download=模型下载
score=评分 score=评分
wait_count_start = 当前有
wait_count_end = 个任务正在排队
wait_count_start = 您当前排队位置是第
wait_count_end =
file_limit_100 = 单目录下最多显示100个文件或文件夹 file_limit_100 = 单目录下最多显示100个文件或文件夹
images.name = 镜像Tag images.name = 镜像Tag
images.name_placerholder = 请输入镜像Tag images.name_placerholder = 请输入镜像Tag
@@ -1356,9 +1362,13 @@ modelconvert.modelfileempty=请选择模型文件。


modelconvert.manage.create_error1=相同的名称模型转换任务已经存在。 modelconvert.manage.create_error1=相同的名称模型转换任务已经存在。
modelconvert.manage.create_error2=只能创建一个正在运行的模型转换任务。 modelconvert.manage.create_error2=只能创建一个正在运行的模型转换任务。
modelconvert.manage.model_not_exist=选择的模型不存在。
modelconvert.manage.model_not_exist=任务中选择的模型不存在或者已被删除
modelconvert.manage.no_operate_right=您没有操作权限。 modelconvert.manage.no_operate_right=您没有操作权限。



debug.manage.model_not_exist=任务中选择的模型不存在或者已被删除,请新建调试任务。
debug.manage.dataset_not_exist=任务中选择的部分数据集不存在或者已被删除,请新建调试任务。

grampus.train_job.ai_center=智算中心 grampus.train_job.ai_center=智算中心
grampus.dataset_path_rule = 训练脚本存储在/cache/code中,数据集存储在/cache/dataset中,训练输出请存储在/cache/output中以供后续下载。 grampus.dataset_path_rule = 训练脚本存储在/cache/code中,数据集存储在/cache/dataset中,训练输出请存储在/cache/output中以供后续下载。
grampus.gpu_dataset_path_rule = 训练脚本存储在/tmp/code中,数据集存储在/tmp/dataset中,训练输出请存储在/tmp/output中以供后续下载。 grampus.gpu_dataset_path_rule = 训练脚本存储在/tmp/code中,数据集存储在/tmp/dataset中,训练输出请存储在/tmp/output中以供后续下载。
@@ -1666,7 +1676,7 @@ issues.action_assignee_no_select=未指派
issues.opened_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s创建 issues.opened_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s创建
pulls.merged_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s 合并 pulls.merged_by=由 <a href="%[2]s">%[3]s</a> 于 %[1]s 合并
pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并 pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并
issues.closed_by=按 <a href="%[2]s">%[3]s</a> 关闭%[1]s
issues.closed_by=由 <a href="%[2]s">%[3]s</a> 创建,被关闭于 %[1]s
issues.opened_by_fake=由 %[2]s 于 %[1]s创建 issues.opened_by_fake=由 %[2]s 于 %[1]s创建
issues.closed_by_fake=通过 %[2]s 关闭 %[1]s issues.closed_by_fake=通过 %[2]s 关闭 %[1]s
issues.previous=上一页 issues.previous=上一页
@@ -2767,6 +2777,11 @@ repos.pr=PR数
repos.commit=Commit数 repos.commit=Commit数
repos.closedIssues=已解决任务数 repos.closedIssues=已解决任务数
repos.contributor=贡献者数 repos.contributor=贡献者数
repos.numDataset=数据集文件数
repos.numCloudbrain=云脑任务数
repos.numModel=模型数
repos.numModelConvert=转换任务数

repos.yes=是 repos.yes=是
repos.no=否 repos.no=否


@@ -3137,6 +3152,8 @@ reject_pull_request=`建议变更 <a href="%s/pulls/%s">%s#%[2]s</a>`
upload_dataset=`上传了数据集文件 <a href="%s/datasets">%s</a>` upload_dataset=`上传了数据集文件 <a href="%s/datasets">%s</a>`
task_gpudebugjob=`创建了CPU/GPU类型调试任务 <a href="%s/cloudbrain/%s">%s</a>` task_gpudebugjob=`创建了CPU/GPU类型调试任务 <a href="%s/cloudbrain/%s">%s</a>`
task_npudebugjob=`创建了NPU类型调试任务 <a href="%s/modelarts/notebook/%s">%s</a>` task_npudebugjob=`创建了NPU类型调试任务 <a href="%s/modelarts/notebook/%s">%s</a>`
task_c2net_gpudebugjob=`创建了CPU/GPU类型调试任务 <a href="%s/grampus/notebook/%s">%s</a>`
task_c2net_npudebugjob=`创建了NPU类型调试任务 <a href="%s/grampus/notebook/%s">%s</a>`
task_nputrainjob=`创建了NPU类型训练任务 <a href="%s/modelarts/train-job/%s">%s</a>` task_nputrainjob=`创建了NPU类型训练任务 <a href="%s/modelarts/train-job/%s">%s</a>`
task_inferencejob=`创建了推理任务 <a href="%s/modelarts/inference-job/%s">%s</a>` task_inferencejob=`创建了推理任务 <a href="%s/modelarts/inference-job/%s">%s</a>`
task_benchmark=`创建了评测任务 <a href="%s/cloudbrain/benchmark/%s">%s</a>` task_benchmark=`创建了评测任务 <a href="%s/cloudbrain/benchmark/%s">%s</a>`
@@ -3257,6 +3274,7 @@ resource_specification = 资源规格
dataset_storage_path = 数据集存放路径 dataset_storage_path = 数据集存放路径
model_storage_path = 模型存放路径 model_storage_path = 模型存放路径
code_storage_path = 代码存放路径 code_storage_path = 代码存放路径
output_storage_path = 输出存放路径
benchmark_path = benchmark脚本存放路径 benchmark_path = benchmark脚本存放路径
snn4imagenet_path = snn4imagenet脚本存放路径 snn4imagenet_path = snn4imagenet脚本存放路径
brainscore_path = brainscore脚本存放路径 brainscore_path = brainscore脚本存放路径
@@ -3311,11 +3329,14 @@ Stopped_failed=任务停止失败,请稍后再试。
Stopped_success_update_status_fail=任务停止成功,状态及运行时间更新失败。 Stopped_success_update_status_fail=任务停止成功,状态及运行时间更新失败。
load_code_failed=代码加载失败,请确认选择了正确的分支。 load_code_failed=代码加载失败,请确认选择了正确的分支。


error.debug_datasetsize = 数据集大小超过限制('%d'GB)
error.dataset_select = 数据集选择错误:数量超过限制或者有同名数据集 error.dataset_select = 数据集选择错误:数量超过限制或者有同名数据集
new_train_gpu_tooltips = 训练脚本存储在 <strong style="color:#010101">%s</strong> 中,数据集存储在 <strong style="color:#010101">%s</strong> 中,预训练模型存放在运行参数 <strong style="color:#010101">%s</strong> 中,训练输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。 new_train_gpu_tooltips = 训练脚本存储在 <strong style="color:#010101">%s</strong> 中,数据集存储在 <strong style="color:#010101">%s</strong> 中,预训练模型存放在运行参数 <strong style="color:#010101">%s</strong> 中,训练输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。
new_debug_gpu_tooltips = 项目代码存储在 <strong style="color:#010101">%s</strong> 中,数据集存储在 <strong style="color:#010101">%s</strong> 中,选择的模型存储在 <strong style="color:#010101">%s</strong> 中,调试输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。
new_debug_gpu_tooltips1 = 项目代码存储在 <strong style="color:#010101">%s</strong> 中,数据集存储在 <strong style="color:#010101">%s</strong> 中,选择的模型存储在 <strong style="color:#010101">%s</strong> 中。
new_train_npu_tooltips = 训练脚本存储在 <strong style="color:#010101">%s</strong> 中,预训练模型存放在运行参数 <strong style="color:#010101">%s</strong> 中,训练输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。 new_train_npu_tooltips = 训练脚本存储在 <strong style="color:#010101">%s</strong> 中,预训练模型存放在运行参数 <strong style="color:#010101">%s</strong> 中,训练输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。
new_infer_gpu_tooltips = 数据集存储在 <strong style="color:#010101">%s</strong> 中,模型文件存储在 <strong style="color:#010101">%s</strong> 中,推理输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。 new_infer_gpu_tooltips = 数据集存储在 <strong style="color:#010101">%s</strong> 中,模型文件存储在 <strong style="color:#010101">%s</strong> 中,推理输出请存储在 <strong style="color:#010101">%s</strong> 中以供后续下载。
code_obs_address = 代码obs地址


[points] [points]
points = 积分 points = 积分


+ 34
- 65
package-lock.json View File

@@ -1982,28 +1982,6 @@
"object.assign": "^4.1.0" "object.assign": "^4.1.0"
} }
}, },
"babel-polyfill": {
"version": "6.26.0",
"resolved": "https://registry.npm.taobao.org/babel-polyfill/download/babel-polyfill-6.26.0.tgz",
"integrity": "sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=",
"requires": {
"babel-runtime": "^6.26.0",
"core-js": "^2.5.0",
"regenerator-runtime": "^0.10.5"
},
"dependencies": {
"core-js": {
"version": "2.6.12",
"resolved": "https://registry.npm.taobao.org/core-js/download/core-js-2.6.12.tgz?cache=0&sync_timestamp=1611040749668&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fcore-js%2Fdownload%2Fcore-js-2.6.12.tgz",
"integrity": "sha1-2TM9+nsGXjR8xWgiGdb2kIWcwuw="
},
"regenerator-runtime": {
"version": "0.10.5",
"resolved": "https://registry.npm.taobao.org/regenerator-runtime/download/regenerator-runtime-0.10.5.tgz",
"integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg="
}
}
},
"babel-runtime": { "babel-runtime": {
"version": "6.26.0", "version": "6.26.0",
"resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
@@ -2194,9 +2172,9 @@
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
}, },
"blueimp-md5": { "blueimp-md5": {
"version": "2.18.0",
"resolved": "https://registry.npm.taobao.org/blueimp-md5/download/blueimp-md5-2.18.0.tgz",
"integrity": "sha1-EVK+EzXwxrORHtnjbbVPPmrFKTU="
"version": "2.19.0",
"resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.19.0.tgz",
"integrity": "sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w=="
}, },
"bn.js": { "bn.js": {
"version": "5.1.1", "version": "5.1.1",
@@ -3536,6 +3514,11 @@
"resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.2.0.tgz", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.2.0.tgz",
"integrity": "sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI=" "integrity": "sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI="
}, },
"dayjs": {
"version": "1.10.7",
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz",
"integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig=="
},
"de-indent": { "de-indent": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz", "resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz",
@@ -4152,25 +4135,29 @@
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
}, },
"esdk-obs-browserjs": { "esdk-obs-browserjs": {
"version": "3.20.7",
"resolved": "https://registry.npm.taobao.org/esdk-obs-browserjs/download/esdk-obs-browserjs-3.20.7.tgz",
"integrity": "sha1-vhziRlKEhW3PgZPl0DyX68bJI0s=",
"version": "3.22.3",
"resolved": "https://registry.npmjs.org/esdk-obs-browserjs/-/esdk-obs-browserjs-3.22.3.tgz",
"integrity": "sha512-MATZXp0FwjPtKG9tpdfURa3koUarR/ev+tbO0oUKgj0GRt0798ZxmfCvYvRpgNst4w1ht4E79ikD4H40UYLgPA==",
"requires": { "requires": {
"axios": "^0.19.0",
"babel-polyfill": "^6.26.0",
"blueimp-md5": "^2.10.0",
"js-base64": "^2.3.2",
"jssha": "^2.3.1",
"urijs": "^1.19.1"
"axios": "^0.26.1",
"blueimp-md5": "^2.18.0",
"js-base64": "^3.7.1",
"jssha": "^3.2.0",
"urijs": "^1.19.7"
}, },
"dependencies": { "dependencies": {
"axios": { "axios": {
"version": "0.19.2",
"resolved": "https://registry.npm.taobao.org/axios/download/axios-0.19.2.tgz?cache=0&sync_timestamp=1608609215811&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Faxios%2Fdownload%2Faxios-0.19.2.tgz",
"integrity": "sha1-PqNsXYgY0NX4qKl6bTa4bNwAyyc=",
"version": "0.26.1",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
"integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
"requires": { "requires": {
"follow-redirects": "1.5.10"
"follow-redirects": "^1.14.8"
} }
},
"js-base64": {
"version": "3.7.3",
"resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.3.tgz",
"integrity": "sha512-PAr6Xg2jvd7MCR6Ld9Jg3BmTcjYsHEBx1VlwEwULb/qowPf5VD9kEMagj23Gm7JRnSvE/Da/57nChZjnvL8v6A=="
} }
} }
}, },
@@ -5382,27 +5369,9 @@
} }
}, },
"follow-redirects": { "follow-redirects": {
"version": "1.5.10",
"resolved": "https://registry.npm.taobao.org/follow-redirects/download/follow-redirects-1.5.10.tgz?cache=0&sync_timestamp=1611606737937&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Ffollow-redirects%2Fdownload%2Ffollow-redirects-1.5.10.tgz",
"integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=",
"requires": {
"debug": "=3.1.0"
},
"dependencies": {
"debug": {
"version": "3.1.0",
"resolved": "https://registry.npm.taobao.org/debug/download/debug-3.1.0.tgz?cache=0&sync_timestamp=1607566533140&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdebug%2Fdownload%2Fdebug-3.1.0.tgz",
"integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}
}
"version": "1.15.2",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
"integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
}, },
"fomantic-ui": { "fomantic-ui": {
"version": "2.8.4", "version": "2.8.4",
@@ -7884,9 +7853,9 @@
} }
}, },
"jssha": { "jssha": {
"version": "2.4.2",
"resolved": "https://registry.npm.taobao.org/jssha/download/jssha-2.4.2.tgz",
"integrity": "sha1-2VCwlWNJKL1rK9odQtqaOnYtZek="
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/jssha/-/jssha-3.3.0.tgz",
"integrity": "sha512-w9OtT4ALL+fbbwG3gw7erAO0jvS5nfvrukGPMWIAoea359B26ALXGpzy4YJSp9yGnpUvuvOw1nSjSoHDfWSr1w=="
}, },
"just-debounce": { "just-debounce": {
"version": "1.0.0", "version": "1.0.0",
@@ -14400,9 +14369,9 @@
} }
}, },
"urijs": { "urijs": {
"version": "1.19.6",
"resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.6.tgz",
"integrity": "sha512-eSXsXZ2jLvGWeLYlQA3Gh36BcjF+0amo92+wHPyN1mdR8Nxf75fuEuYTd9c0a+m/vhCjRK0ESlE9YNLW+E1VEw=="
"version": "1.19.11",
"resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz",
"integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ=="
}, },
"urix": { "urix": {
"version": "0.1.0", "version": "0.1.0",


+ 2
- 1
package.json View File

@@ -17,11 +17,12 @@
"core-js": "3.6.5", "core-js": "3.6.5",
"css-loader": "3.5.3", "css-loader": "3.5.3",
"cssnano": "4.1.10", "cssnano": "4.1.10",
"dayjs": "1.10.7",
"domino": "2.1.5", "domino": "2.1.5",
"dropzone": "5.7.2", "dropzone": "5.7.2",
"echarts": "3.8.5", "echarts": "3.8.5",
"element-ui": "2.15.5", "element-ui": "2.15.5",
"esdk-obs-browserjs": "3.20.7",
"esdk-obs-browserjs": "3.22.3",
"esdk-obs-nodejs": "3.20.11", "esdk-obs-nodejs": "3.20.11",
"fast-glob": "3.2.2", "fast-glob": "3.2.2",
"file-loader": "6.0.0", "file-loader": "6.0.0",


+ 13
- 14
public/home/home.js View File

@@ -247,7 +247,7 @@ document.onreadystatechange = function () {
html += recordPrefix + actionName; html += recordPrefix + actionName;
html += " <a href=\"" + getTaskLink(record) + "\" rel=\"nofollow\">" + record.RefName + "</a>" html += " <a href=\"" + getTaskLink(record) + "\" rel=\"nofollow\">" + record.RefName + "</a>"
} }
else if(record.OpType == "25" || record.OpType == "29"){
else if(record.OpType == "25" || record.OpType == "29" || record.OpType == "39" || record.OpType == "40"){
html += recordPrefix + actionName; html += recordPrefix + actionName;
html += " <a href=\"" + getTaskLink(record) + "\" rel=\"nofollow\">" + record.RefName + "</a>" html += " <a href=\"" + getTaskLink(record) + "\" rel=\"nofollow\">" + record.RefName + "</a>"
} }
@@ -294,7 +294,10 @@ function getTaskLink(record){
re = re + "/cloudbrain/train-job/" + record.Content; re = re + "/cloudbrain/train-job/" + record.Content;
}else if(record.OpType == 32 || record.OpType == 33){ }else if(record.OpType == 32 || record.OpType == 33){
re = re + "/grampus/train-job/" + record.Content; re = re + "/grampus/train-job/" + record.Content;
}else if(record.OpType == 39 || record.OpType == 40){
re = re + "/grampus/notebook/" + record.Content;
} }
re = encodeURI(re); re = encodeURI(re);
return re; return re;
} }
@@ -450,7 +453,9 @@ var actionNameZH={
"33":"创建了CPU/GPU类型训练任务", "33":"创建了CPU/GPU类型训练任务",
"35":"创建的数据集 {dataset} 被设置为推荐数据集", "35":"创建的数据集 {dataset} 被设置为推荐数据集",
"36":"提交了镜像 {image}", "36":"提交了镜像 {image}",
"37":"提交的镜像 {image} 被设置为推荐镜像",
"37": "提交的镜像 {image} 被设置为推荐镜像",
"39":"创建了CPU/GPU类型调试任务",
"40":"创建了NPU类型调试任务",
}; };


var actionNameEN={ var actionNameEN={
@@ -481,7 +486,9 @@ var actionNameEN={
"33":" created CPU/GPU type training task", "33":" created CPU/GPU type training task",
"35":" created dataset {dataset} was set as recommended dataset", "35":" created dataset {dataset} was set as recommended dataset",
"36":"committed image {image}", "36":"committed image {image}",
"37":"committed image {image} was set as recommended image",
"37": "committed image {image} was set as recommended image",
"39":" created CPU/GPU type debugging task ",
"40":" created NPU type debugging task ",
}; };


var repoAndOrgZH={ var repoAndOrgZH={
@@ -622,20 +629,12 @@ function displayRepo(json){
for (var i = 0, iLen = repos.length; i < iLen; i++) { for (var i = 0, iLen = repos.length; i < iLen; i++) {
if (i >= 4) break; if (i >= 4) break;
var repo = repos[i]; var repo = repos[i];
// <i class="ri-star-line"></i>${repo["NumStars"]}<i class="ri-git-branch-line am-ml-10"></i>${repo["NumForks"]}</span> <div class="ui tags nowrap am-mt-10"></div>
html += `<div class="ui fluid card" style="border-radius:6px;"> html += `<div class="ui fluid card" style="border-radius:6px;">
<div class="content">
${repo["Avatar"] ? `<img class="left floated mini ui image" src="${repo["Avatar"]}">` : `<img class="left floated mini ui image" avatar="${repo["OwnerName"]}">`}
<div class="content" style="position:relative;">
${repo["Avatar"] ? `<img style="border-radius:100%;" class="left floated mini ui image" src="${repo["Avatar"]}">` : `<img style="border-radius:100%;" class="left floated mini ui image" avatar="${repo["OwnerName"]}">`}
<a class="header nowrap" style="color:rgb(50, 145, 248);font-size:14px;" href="/${repo["OwnerName"]}/${repo["Name"]}" title="${repo["Alias"]}">${repo["Alias"]}</a> <a class="header nowrap" style="color:rgb(50, 145, 248);font-size:14px;" href="/${repo["OwnerName"]}/${repo["Name"]}" title="${repo["Alias"]}">${repo["Alias"]}</a>
<div class="description nowrap-2" style="rgba(136,136,136,1);;font-size:12px;" title="${repo["Description"]}">${repo["Description"]}</div> <div class="description nowrap-2" style="rgba(136,136,136,1);;font-size:12px;" title="${repo["Description"]}">${repo["Description"]}</div>
`;
// if (repo["Topics"] != null) {
// for(var j = 0; j < repo["Topics"].length; j++){
// var topic = repo["Topics"][j];
// var url = "/explore/repos?q=" + (topic) + "&amp;topic="
// html += `<a class="ui small label topic" href=" ${url}">${topic}</a>`;
// }
// }
<a href="/${repo["OwnerName"]}/${repo["Name"]}" style="height:100%;width:100%;position:absolute;left:0;top:0"></a>`;
html += ` html += `
</div> </div>
</div>`; </div>`;


+ 1
- 1
routers/admin/cloudbrains.go View File

@@ -53,7 +53,7 @@ func CloudBrains(ctx *context.Context) {
var jobTypes []string var jobTypes []string
jobTypeNot := false jobTypeNot := false
if jobType == string(models.JobTypeBenchmark) { if jobType == string(models.JobTypeBenchmark) {
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset))
} else if jobType != "all" && jobType != "" { } else if jobType != "all" && jobType != "" {
jobTypes = append(jobTypes, jobType) jobTypes = append(jobTypes, jobType)
} }


+ 34
- 0
routers/admin/resources.go View File

@@ -307,3 +307,37 @@ func RefreshHistorySpec(ctx *context.Context) {
r["total"] = total r["total"] = total
ctx.JSON(http.StatusOK, response.SuccessWithData(r)) ctx.JSON(http.StatusOK, response.SuccessWithData(r))
} }

func RefreshReposHistoryCnt(ctx *context.Context) {
scope := ctx.Query("scope")
list := ctx.Query("list")

var scopeAll = false
if scope == "all" {
scopeAll = true
}
var ids = make([]int64, 0)
if list != "" {
strs := strings.Split(list, "|")
for _, s := range strs {
i, err := strconv.ParseInt(s, 10, 64)
if err != nil {
ctx.JSON(http.StatusOK, response.ServerError(err.Error()))
return
}
ids = append(ids, i)
}

}

total, success, err := resource.RefreshHistorySpec(scopeAll, ids)
if err != nil {
log.Error("RefreshHistorySpec error. %v", err)
ctx.JSON(http.StatusOK, response.ServerError(err.Error()))
return
}
r := make(map[string]interface{}, 0)
r["success"] = success
r["total"] = total
ctx.JSON(http.StatusOK, response.SuccessWithData(r))
}

+ 17
- 1
routers/api/v1/api.go View File

@@ -544,6 +544,12 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/complete_multipart", repo.CompleteMultipart) m.Post("/complete_multipart", repo.CompleteMultipart)


}, reqToken()) }, reqToken())
m.Group("/pipeline", func() {
m.Post("/notification", bind(api.PipelineNotification{}), notify.PipelineNotify)

}, reqToken())

m.Get("/compute-nodes", reqToken(), user.GetComputeNodes)


// Notifications // Notifications
m.Group("/notifications", func() { m.Group("/notifications", func() {
@@ -741,7 +747,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/file_notebook", func() { m.Group("/file_notebook", func() {
m.Get("", repo.GetFileNoteBookInfo) m.Get("", repo.GetFileNoteBookInfo)
m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook)
m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus)
}) })


m.Group("/repos", func() { m.Group("/repos", func() {
@@ -759,6 +765,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/:username/:reponame", func() { m.Group("/:username/:reponame", func() {
m.Get("/right", reqToken(), repo.GetRight) m.Get("/right", reqToken(), repo.GetRight)
m.Get("/tagger", reqToken(), repo.ListTagger) m.Get("/tagger", reqToken(), repo.ListTagger)
m.Get("/cloudBrainJobId", repo.GetCloudBrainJobId)
m.Combo("").Get(reqAnyRepoReader(), repo.Get). m.Combo("").Get(reqAnyRepoReader(), repo.Get).
Delete(reqToken(), reqOwner(), repo.Delete). Delete(reqToken(), reqOwner(), repo.Delete).
Patch(reqToken(), reqAdmin(), bind(api.EditRepoOption{}), context.RepoRef(), repo.Edit) Patch(reqToken(), reqAdmin(), bind(api.EditRepoOption{}), context.RepoRef(), repo.Edit)
@@ -995,6 +1002,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/detail", reqToken(), reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) m.Get("/detail", reqToken(), reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow)
m.Get("/model_list", repo.CloudBrainModelList) m.Get("/model_list", repo.CloudBrainModelList)
m.Post("/stop_version", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo_ext.CloudBrainStop) m.Post("/stop_version", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo_ext.CloudBrainStop)
m.Put("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GeneralCloudBrainJobStop)
}) })
}) })
m.Group("/inference-job", func() { m.Group("/inference-job", func() {
@@ -1015,12 +1023,17 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Delete("/delete_model", repo.DeleteModel) m.Delete("/delete_model", repo.DeleteModel)
m.Get("/downloadall", repo.DownloadModel) m.Get("/downloadall", repo.DownloadModel)
m.Get("/query_model_byId", repo.QueryModelById) m.Get("/query_model_byId", repo.QueryModelById)
m.Get("/query_model_byName", repo.QueryModelByName)
m.Get("/query_model_for_predict", repo.QueryModelListForPredict) m.Get("/query_model_for_predict", repo.QueryModelListForPredict)
m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict)
m.Get("/query_train_job", repo.QueryTrainJobList)
m.Get("/query_train_job_version", repo.QueryTrainJobVersionList)
m.Get("/query_train_model", repo.QueryTrainModelList) m.Get("/query_train_model", repo.QueryTrainModelList)
m.Post("/create_model_convert", repo.CreateModelConvert) m.Post("/create_model_convert", repo.CreateModelConvert)
m.Post("/convert_stop", repo.StopModelConvert)
m.Get("/show_model_convert_page", repo.ShowModelConvertPage) m.Get("/show_model_convert_page", repo.ShowModelConvertPage)
m.Get("/query_model_convert_byId", repo.QueryModelConvertById) m.Get("/query_model_convert_byId", repo.QueryModelConvertById)
m.Get("/query_model_convert_byName", repo.QueryModelConvertByName)


m.Get("/:id", repo.GetCloudbrainModelConvertTask) m.Get("/:id", repo.GetCloudbrainModelConvertTask)
m.Get("/:id/log", repo.CloudbrainForModelConvertGetLog) m.Get("/:id/log", repo.CloudbrainForModelConvertGetLog)
@@ -1053,6 +1066,9 @@ func RegisterRoutes(m *macaron.Macaron) {
}) })
}, reqRepoReader(models.UnitTypeCloudBrain)) }, reqRepoReader(models.UnitTypeCloudBrain))
m.Group("/grampus", func() { m.Group("/grampus", func() {
m.Group("/notebook", func() {
m.Get("/:id", repo_ext.GetGrampusNotebook)
})
m.Group("/train-job", func() { m.Group("/train-job", func() {
m.Group("/:jobid", func() { m.Group("/:jobid", func() {
m.Get("", repo.GetModelArtsTrainJobVersion) m.Get("", repo.GetModelArtsTrainJobVersion)


+ 15
- 0
routers/api/v1/notify/pipeline.go View File

@@ -0,0 +1,15 @@
package notify

import (
"net/http"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
api "code.gitea.io/gitea/modules/structs"
)

func PipelineNotify(ctx *context.APIContext, form api.PipelineNotification) {

ctx.JSON(http.StatusOK, models.BaseOKMessageApi)

}

+ 85
- 9
routers/api/v1/repo/cloudbrain.go View File

@@ -9,6 +9,7 @@ import (
"bufio" "bufio"
"encoding/json" "encoding/json"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"os" "os"
"path" "path"
@@ -17,6 +18,8 @@ import (
"strings" "strings"
"time" "time"


"code.gitea.io/gitea/modules/grampus"

cloudbrainService "code.gitea.io/gitea/services/cloudbrain" cloudbrainService "code.gitea.io/gitea/services/cloudbrain"


"code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/convert"
@@ -80,9 +83,36 @@ func CloudBrainShow(ctx *context.APIContext) {
ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: convert.ToCloudBrain(task)}) ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: convert.ToCloudBrain(task)})


} }
func GeneralCloudBrainJobStop(ctx *context.APIContext) {
task := ctx.Cloudbrain
if task.IsTerminal() {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("cloudbrain.Already_stopped"))
return
}
var err error

if ctx.Cloudbrain.Type == models.TypeCloudBrainOne {
err = cloudbrain.StopJob(task.JobID)
} else if ctx.Cloudbrain.Type == models.TypeCloudBrainTwo {
_, err = modelarts.StopTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10))
} else {
_, err = grampus.StopJob(task.JobID)
}

if err != nil {
log.Warn("cloud brain stopped failed.", err)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("cloudbrain.Stopped_failed"))
return
}

ctx.JSON(http.StatusOK, models.BaseOKMessageApi)
}
func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) {
cloudbrainTask.FileNotebookCreate(ctx.Context, option) cloudbrainTask.FileNotebookCreate(ctx.Context, option)
} }
func FileNoteBookStatus(ctx *context.APIContext, option api.CreateFileNotebookJobOption) {
cloudbrainTask.FileNotebookStatus(ctx.Context, option)
}


func GetFileNoteBookInfo(ctx *context.APIContext) { func GetFileNoteBookInfo(ctx *context.APIContext) {
//image description spec description waiting count //image description spec description waiting count
@@ -211,7 +241,7 @@ func GetCloudbrainTask(ctx *context.APIContext) {


ID := ctx.Params(":id") ID := ctx.Params(":id")


job, err := cloudbrain.GetCloudBrainByIdOrJobId(ID)
job, err := cloudbrain.GetCloudBrainByIdOrJobId(ID, "id")


if err != nil { if err != nil {
ctx.NotFound(err) ctx.NotFound(err)
@@ -621,6 +651,19 @@ func CloudbrainDownloadLogFile(ctx *context.Context) {
} }
} }


existStr := ""
if job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference) {
if job.Type == models.TypeCloudBrainOne {
result, err := cloudbrain.GetJob(job.JobID)
if err == nil && result != nil {
jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
taskRoles := jobRes.TaskRoles
taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
existStr = taskRes.TaskStatuses[0].ExitDiagnostics
}
}
}

logDir := "/model" logDir := "/model"
if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) { if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) {
logDir = cloudbrain.ResultPath logDir = cloudbrain.ResultPath
@@ -638,17 +681,30 @@ func CloudbrainDownloadLogFile(ctx *context.Context) {
} }
} }
if fileName != "" { if fileName != "" {
prefix := "/" + setting.CBCodePathPrefix + job.JobName + logDir
url, err := storage.Attachments.PresignedGetURL(prefix+"/"+fileName, fileName)
prefix := "/" + setting.CBCodePathPrefix + job.JobName + "/model"
filePath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + prefix + "/" + fileName
// Read the file contents into a byte slice
data, err := ioutil.ReadFile(filePath)
if err != nil { if err != nil {
log.Error("Get minio get SignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
ctx.ServerError("ReadFile", err)
return
}

// Set the appropriate response headers
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName)

// Write the file contents to the response
if _, err := ctx.Resp.Write(data); err != nil {
ctx.ServerError("Write", err)
return
}
if _, err := ctx.Resp.Write([]byte(existStr)); err != nil {
log.Error("Write failed: %v", err.Error(), ctx.Data["msgID"])
return return
} }
log.Info("fileName=" + fileName)
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect)
} else { } else {
log.Info("fileName is null.") log.Info("fileName is null.")

} }
} }


@@ -734,8 +790,28 @@ func CloudbrainGetLog(ctx *context.APIContext) {
content = result["Content"].(string) content = result["Content"].(string)
} }


if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
content = content + ctx.Data["existStr"].(string)
if (job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference)) && job.Type == models.TypeCloudBrainOne && job.Status == string(models.JobFailed) {
if ctx.Data["existStr"] != nil {
if baseLine == "" && order == "desc" && result["Lines"].(int) == 0 {
result["Lines"] = 1
result["EndLine"] = 1
content = content + ctx.Data["existStr"].(string)
}

if result["Lines"].(int) == 0 && result["StartLine"] == result["EndLine"] && result["StartLine"].(int) != 0 {
content = content + ctx.Data["existStr"].(string)
result["Lines"] = 1
result["StartLine"] = result["StartLine"].(int) - 1
}
if result["Lines"].(int) == 1 && result["StartLine"] == result["EndLine"] {
result["Lines"] = 0
result["StartLine"] = result["StartLine"].(int) + 1
}
}
} else {
if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
content = content + ctx.Data["existStr"].(string)
}
} }


logFileName := result["FileName"] logFileName := result["FileName"]


+ 38
- 62
routers/api/v1/repo/cloudbrain_dashboard.go View File

@@ -103,86 +103,62 @@ func GetAllCloudbrainsOverview(ctx *context.Context) {
}) })
} }
func GetOverviewDuration(ctx *context.Context) { func GetOverviewDuration(ctx *context.Context) {
recordCloudbrain, err := models.GetRecordBeginTime()
if err != nil {
log.Error("Can not get recordCloudbrain", err)
ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
return
}
recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix
now := time.Now()
endTime := now
var workServerNumber int64
var cardNum int64
durationSum := 0
cardDurationSum := 0


durationAllSum := int64(0)
cardDuSum := int64(0)
cloudBrainOneCardDuSum := 0
cloudBrainTwoCardDuSum := 0
c2NetCardDuSum := 0
cDNetCardDuSum := 0


cloudBrainOneCardDuSum := int64(0)
cloudBrainTwoCardDuSum := int64(0)
c2NetCardDuSum := int64(0)
cDNetCardDuSum := int64(0)
cloudBrainOneDuration := 0
cloudBrainTwoDuration := 0
c2NetDuration := 0
cDCenterDuration := 0


cloudBrainOneDuration := int64(0)
cloudBrainTwoDuration := int64(0)
c2NetDuration := int64(0)
cDCenterDuration := int64(0)

cloudbrains, _, err := models.CloudbrainAllKanBan(&models.CloudbrainsOptions{
Type: models.TypeCloudBrainAll,
BeginTimeUnix: int64(recordBeginTime),
EndTimeUnix: endTime.Unix(),
})
cloudbrainTypeDuration, err := models.GetCloudbrainTypeCardDuration()
if err != nil { if err != nil {
ctx.ServerError("Get cloudbrains failed:", err)
log.Error("GetCloudbrainTypeCardDuration err!", err)
return return
} }
models.LoadSpecs4CloudbrainInfo(cloudbrains)

for _, cloudbrain := range cloudbrains {
cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain)
if cloudbrain.Cloudbrain.Spec != nil {
cardNum = int64(cloudbrain.Cloudbrain.Spec.AccCardsNum)
} else {
cardNum = 1
for _, result := range cloudbrainTypeDuration {
if result.Type == models.TypeCloudBrainOne {
cloudBrainOneDuration = result.DurationSum
cloudBrainOneCardDuSum = result.CardDurationSum
} }
if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
workServerNumber = int64(cloudbrain.Cloudbrain.WorkServerNumber)
} else {
workServerNumber = 1
if result.Type == models.TypeCloudBrainTwo {
cloudBrainTwoDuration = result.DurationSum
cloudBrainTwoCardDuSum = result.CardDurationSum
} }
duration := models.ConvertStrToDuration(cloudbrain.TrainJobDuration)
CardDuration := workServerNumber * int64(cardNum) * duration

if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
cloudBrainOneDuration += duration
cloudBrainOneCardDuSum += CardDuration
} else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
cloudBrainTwoDuration += duration
cloudBrainTwoCardDuSum += CardDuration
} else if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
c2NetDuration += duration
c2NetCardDuSum += CardDuration
} else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter {
cDCenterDuration += duration
cDNetCardDuSum += CardDuration
if result.Type == models.TypeC2Net {
c2NetDuration = result.DurationSum
c2NetCardDuSum = result.CardDurationSum
} }

durationAllSum += duration
cardDuSum += CardDuration
if result.Type == models.TypeCDCenter {
cDCenterDuration = result.DurationSum
cDNetCardDuSum = result.CardDurationSum
}
}
cloudbrainAllDuration, err := models.GetCloudbrainAllCardDuration()
if err != nil {
log.Error("GetCloudbrainAllCardDuration err!", err)
return
} }
durationSum = cloudbrainAllDuration.DurationSum
cardDurationSum = cloudbrainAllDuration.CardDurationSum

ctx.JSON(http.StatusOK, map[string]interface{}{ ctx.JSON(http.StatusOK, map[string]interface{}{
"cloudBrainOneCardDuSum": cloudBrainOneCardDuSum, "cloudBrainOneCardDuSum": cloudBrainOneCardDuSum,
"cloudBrainTwoCardDuSum": cloudBrainTwoCardDuSum, "cloudBrainTwoCardDuSum": cloudBrainTwoCardDuSum,
"c2NetCardDuSum": c2NetCardDuSum, "c2NetCardDuSum": c2NetCardDuSum,
"cDNetCardDuSum": cDNetCardDuSum, "cDNetCardDuSum": cDNetCardDuSum,
"cardDuSum": cardDuSum,
"cardDuSum": cardDurationSum,


"cloudBrainOneDuration": cloudBrainOneDuration, "cloudBrainOneDuration": cloudBrainOneDuration,
"cloudBrainTwoDuration": cloudBrainTwoDuration, "cloudBrainTwoDuration": cloudBrainTwoDuration,
"c2NetDuration": c2NetDuration, "c2NetDuration": c2NetDuration,
"cDCenterDuration": cDCenterDuration, "cDCenterDuration": cDCenterDuration,
"durationSum": durationAllSum,
"durationSum": durationSum,
}) })
} }


@@ -647,7 +623,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) {
} }


jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark),
string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)}
string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)}
for _, v := range jobTypeList { for _, v := range jobTypeList {
if _, ok := cloudOneJobTypeRes[v]; !ok { if _, ok := cloudOneJobTypeRes[v]; !ok {
cloudOneJobTypeRes[v] = 0 cloudOneJobTypeRes[v] = 0
@@ -780,7 +756,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) {
var jobTypes []string var jobTypes []string
jobTypeNot := false jobTypeNot := false
if jobType == string(models.JobTypeBenchmark) { if jobType == string(models.JobTypeBenchmark) {
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset))
} else if jobType != "all" && jobType != "" { } else if jobType != "all" && jobType != "" {
jobTypes = append(jobTypes, jobType) jobTypes = append(jobTypes, jobType)
} }


+ 1
- 1
routers/api/v1/repo/images.go View File

@@ -88,7 +88,7 @@ func getModelArtsImages(ctx *context.APIContext) {
} }


func getC2netNpuImages(ctx *context.APIContext) { func getC2netNpuImages(ctx *context.APIContext) {
images, err := grampus.GetImages(grampus.ProcessorTypeNPU)
images, err := grampus.GetImages(grampus.ProcessorTypeNPU, string(models.JobTypeTrain))
var npuImageInfos []NPUImageINFO var npuImageInfos []NPUImageINFO
if err != nil { if err != nil {
log.Error("GetImages failed:", err.Error()) log.Error("GetImages failed:", err.Error())


+ 14
- 0
routers/api/v1/repo/mlops.go View File

@@ -69,3 +69,17 @@ func GetRight(ctx *context.APIContext) {
}) })


} }

func GetCloudBrainJobId(ctx *context.APIContext) {
cloudbrains, err := models.GetCloudbrainsByDisplayJobName(ctx.Repo.Repository.ID, ctx.Query("jobType"), ctx.Query("name"))
if err != nil {
log.Warn("get cloudbrain by display name failed", err)
ctx.JSON(http.StatusOK, map[string]string{"jobId": ""})
return
}
if len(cloudbrains) > 0 {
ctx.JSON(http.StatusOK, map[string]string{"jobId": cloudbrains[0].JobID})
return
}
ctx.JSON(http.StatusOK, map[string]string{"jobId": ""})
}

+ 1
- 1
routers/api/v1/repo/modelarts.go View File

@@ -39,7 +39,7 @@ func GetModelArtsNotebook2(ctx *context.APIContext) {


ID := ctx.Params(":id") ID := ctx.Params(":id")


job,err := cloudbrain.GetCloudBrainByIdOrJobId(ID)
job, err := cloudbrain.GetCloudBrainByIdOrJobId(ID, "id")


if err != nil { if err != nil {
ctx.NotFound(err) ctx.NotFound(err)


+ 51
- 0
routers/api/v1/repo/modelmanage.go View File

@@ -4,8 +4,10 @@ import (
"net/http" "net/http"


"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/convert"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/storage"
api "code.gitea.io/gitea/modules/structs"
routerRepo "code.gitea.io/gitea/routers/repo" routerRepo "code.gitea.io/gitea/routers/repo"
) )


@@ -43,12 +45,32 @@ func QueryModelById(ctx *context.APIContext) {
routerRepo.QueryModelById(ctx.Context) routerRepo.QueryModelById(ctx.Context)
} }


func QueryModelByName(ctx *context.APIContext) {
log.Info("QueryModelByName by api.")
routerRepo.ShowSingleModel(ctx.Context)
}

func QueryModelListForPredict(ctx *context.APIContext) { func QueryModelListForPredict(ctx *context.APIContext) {
log.Info("QueryModelListForPredict by api.") log.Info("QueryModelListForPredict by api.")
ctx.Context.SetParams("isOnlyThisRepo", "true") ctx.Context.SetParams("isOnlyThisRepo", "true")
routerRepo.QueryModelListForPredict(ctx.Context) routerRepo.QueryModelListForPredict(ctx.Context)
} }


func QueryTrainJobList(ctx *context.APIContext) {
result, err := routerRepo.QueryTrainJobListApi(ctx.Context)
if err != nil {
log.Info("query error." + err.Error())
ctx.JSON(http.StatusOK, nil)
} else {
re := make([]*api.Cloudbrain, 0)
for _, task := range result {
conRe := convert.ToCloudBrain(task)
re = append(re, conRe)
}
ctx.JSON(http.StatusOK, re)
}
}

func QueryTrainModelList(ctx *context.APIContext) { func QueryTrainModelList(ctx *context.APIContext) {
result, err := routerRepo.QueryTrainModelFileById(ctx.Context) result, err := routerRepo.QueryTrainModelFileById(ctx.Context)
if err != nil { if err != nil {
@@ -58,6 +80,21 @@ func QueryTrainModelList(ctx *context.APIContext) {
ctx.JSON(http.StatusOK, re) ctx.JSON(http.StatusOK, re)
} }


func QueryTrainJobVersionList(ctx *context.APIContext) {
result, err := routerRepo.QueryTrainJobVersionListApi(ctx.Context)
if err != nil {
log.Info("query error." + err.Error())
ctx.JSON(http.StatusOK, nil)
} else {
re := make([]*api.Cloudbrain, 0)
for _, task := range result {
conRe := convert.ToCloudBrain(task)
re = append(re, conRe)
}
ctx.JSON(http.StatusOK, re)
}
}

func convertFileFormat(result []storage.FileInfo) []FileInfo { func convertFileFormat(result []storage.FileInfo) []FileInfo {
re := make([]FileInfo, 0) re := make([]FileInfo, 0)
if result != nil { if result != nil {
@@ -89,6 +126,11 @@ func CreateModelConvert(ctx *context.APIContext) {
routerRepo.SaveModelConvert(ctx.Context) routerRepo.SaveModelConvert(ctx.Context)
} }


func StopModelConvert(ctx *context.APIContext) {
log.Info("StopModelConvert by api.")
routerRepo.StopModelConvertApi(ctx.Context)
}

func ShowModelConvertPage(ctx *context.APIContext) { func ShowModelConvertPage(ctx *context.APIContext) {
log.Info("ShowModelConvertPage by api.") log.Info("ShowModelConvertPage by api.")
modelResult, count, err := routerRepo.GetModelConvertPageData(ctx.Context) modelResult, count, err := routerRepo.GetModelConvertPageData(ctx.Context)
@@ -114,3 +156,12 @@ func QueryModelConvertById(ctx *context.APIContext) {
ctx.JSON(http.StatusOK, nil) ctx.JSON(http.StatusOK, nil)
} }
} }

func QueryModelConvertByName(ctx *context.APIContext) {
modelResult, err := routerRepo.GetModelConvertByName(ctx.Context)
if err == nil {
ctx.JSON(http.StatusOK, modelResult)
} else {
ctx.JSON(http.StatusOK, nil)
}
}

+ 8
- 6
routers/api/v1/repo/repo_dashbord.go View File

@@ -601,7 +601,7 @@ func getSummaryFileName(ctx *context.Context, beginTime time.Time, endTime time.
func allProjectsPeroidHeader(ctx *context.Context) map[string]string { func allProjectsPeroidHeader(ctx *context.Context) map[string]string {


return map[string]string{"A1": ctx.Tr("admin.repos.id"), "B1": ctx.Tr("admin.repos.projectName"), "C1": ctx.Tr("repo.owner"), "D1": ctx.Tr("admin.repos.isPrivate"), "E1": ctx.Tr("admin.repos.openi"), "F1": ctx.Tr("admin.repos.visit"), "G1": ctx.Tr("admin.repos.download"), "H1": ctx.Tr("admin.repos.pr"), "I1": ctx.Tr("admin.repos.commit"), return map[string]string{"A1": ctx.Tr("admin.repos.id"), "B1": ctx.Tr("admin.repos.projectName"), "C1": ctx.Tr("repo.owner"), "D1": ctx.Tr("admin.repos.isPrivate"), "E1": ctx.Tr("admin.repos.openi"), "F1": ctx.Tr("admin.repos.visit"), "G1": ctx.Tr("admin.repos.download"), "H1": ctx.Tr("admin.repos.pr"), "I1": ctx.Tr("admin.repos.commit"),
"J1": ctx.Tr("admin.repos.watches"), "K1": ctx.Tr("admin.repos.stars"), "L1": ctx.Tr("admin.repos.forks"), "M1": ctx.Tr("admin.repos.issues"), "N1": ctx.Tr("admin.repos.closedIssues"), "O1": ctx.Tr("admin.repos.contributor"), "P1": ctx.Tr("admin.repos.isFork"), "Q1": ctx.Tr("admin.repos.isMirror"), "R1": ctx.Tr("admin.repos.create")}
"J1": ctx.Tr("admin.repos.watches"), "K1": ctx.Tr("admin.repos.stars"), "L1": ctx.Tr("admin.repos.forks"), "M1": ctx.Tr("admin.repos.issues"), "N1": ctx.Tr("admin.repos.closedIssues"), "O1": ctx.Tr("admin.repos.contributor"), "P1": ctx.Tr("admin.repos.numDataset"), "Q1": ctx.Tr("admin.repos.numCloudbrain"), "R1": ctx.Tr("admin.repos.numModel"), "S1": ctx.Tr("admin.repos.numModelConvert"), "T1": ctx.Tr("admin.repos.isFork"), "U1": ctx.Tr("admin.repos.isMirror"), "V1": ctx.Tr("admin.repos.create")}


} }


@@ -619,11 +619,13 @@ func allProjectsPeriodSummaryValues(row int, rs *ProjectSummaryBaseData, ctx *co
} }


func allProjectsPeroidValues(row int, rs *models.RepoStatistic, ctx *context.Context) map[string]string { func allProjectsPeroidValues(row int, rs *models.RepoStatistic, ctx *context.Context) map[string]string {

return map[string]string{getCellName("A", row): strconv.FormatInt(rs.RepoID, 10), getCellName("B", row): rs.DisplayName(), getCellName("C", row): rs.OwnerName, getCellName("D", row): getBoolDisplay(rs.IsPrivate, ctx), getCellName("E", row): strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), return map[string]string{getCellName("A", row): strconv.FormatInt(rs.RepoID, 10), getCellName("B", row): rs.DisplayName(), getCellName("C", row): rs.OwnerName, getCellName("D", row): getBoolDisplay(rs.IsPrivate, ctx), getCellName("E", row): strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64),
getCellName("F", row): strconv.FormatInt(rs.NumVisits, 10), getCellName("G", row): strconv.FormatInt(rs.NumDownloads, 10), getCellName("H", row): strconv.FormatInt(rs.NumPulls, 10), getCellName("I", row): strconv.FormatInt(rs.NumCommits, 10), getCellName("F", row): strconv.FormatInt(rs.NumVisits, 10), getCellName("G", row): strconv.FormatInt(rs.NumDownloads, 10), getCellName("H", row): strconv.FormatInt(rs.NumPulls, 10), getCellName("I", row): strconv.FormatInt(rs.NumCommits, 10),
getCellName("J", row): strconv.FormatInt(rs.NumWatches, 10), getCellName("K", row): strconv.FormatInt(rs.NumStars, 10), getCellName("L", row): strconv.FormatInt(rs.NumForks, 10), getCellName("M", row): strconv.FormatInt(rs.NumIssues, 10), getCellName("J", row): strconv.FormatInt(rs.NumWatches, 10), getCellName("K", row): strconv.FormatInt(rs.NumStars, 10), getCellName("L", row): strconv.FormatInt(rs.NumForks, 10), getCellName("M", row): strconv.FormatInt(rs.NumIssues, 10),
getCellName("N", row): strconv.FormatInt(rs.NumClosedIssues, 10), getCellName("O", row): strconv.FormatInt(rs.NumContributor, 10), getCellName("P", row): getBoolDisplay(rs.IsFork, ctx), getCellName("Q", row): getBoolDisplay(rs.IsMirror, ctx), getCellName("R", row): time.Unix(int64(rs.RepoCreatedUnix), 0).Format(CREATE_TIME_FORMAT),
getCellName("N", row): strconv.FormatInt(rs.NumClosedIssues, 10), getCellName("O", row): strconv.FormatInt(rs.NumContributor, 10), getCellName("P", row): strconv.FormatInt(rs.NumDatasetFile, 10), getCellName("Q", row): strconv.FormatInt(rs.NumCloudbrain, 10), getCellName("R", row): strconv.FormatInt(rs.NumModels, 10), getCellName("S", row): strconv.FormatInt(rs.NumModelConvert, 10), getCellName("T", row): getBoolDisplay(rs.IsFork, ctx), getCellName("U", row): getBoolDisplay(rs.IsMirror, ctx), getCellName("V", row): time.Unix(int64(rs.RepoCreatedUnix), 0).Format(CREATE_TIME_FORMAT),
} }

} }


func allProjectsOpenIHeader() map[string]string { func allProjectsOpenIHeader() map[string]string {
@@ -804,11 +806,11 @@ func generateOpenICountSql(latestDate string) string {
} }


func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string {
sql := "SELECT A.repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " +
sql := "SELECT A.repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor,num_models,num_model_convert,num_cloudbrain,num_dataset_file FROM " +
"(SELECT repo_id,sum(num_visits) as num_visits " + "(SELECT repo_id,sum(num_visits) as num_visits " +
" FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," +
"(SELECT repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor from public.repo_statistic where date='" + latestDate + "') B" +
"(SELECT repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor,num_models,num_model_convert,num_cloudbrain,num_dataset_file from public.repo_statistic where date='" + latestDate + "') B" +
" where A.repo_id=B.repo_id" " where A.repo_id=B.repo_id"


if q != "" { if q != "" {
@@ -828,8 +830,8 @@ func generateTypeAllOpenISql(latestDate string, page int, pageSize int) string {


func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string {


sql := "SELECT A.repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " +
"(SELECT repo_id,sum(num_watches_added) as num_watches,sum(num_visits) as num_visits, sum(num_downloads_added) as num_downloads,sum(num_pulls_added) as num_pulls,sum(num_commits_added) as num_commits,sum(num_stars_added) as num_stars,sum(num_forks_added) num_forks,sum(num_issues_added) as num_issues,sum(num_closed_issues_added) as num_closed_issues,sum(num_contributor_added) as num_contributor " +
sql := "SELECT A.repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor,num_models,num_model_convert,num_cloudbrain,num_dataset_file FROM " +
"(SELECT repo_id,sum(num_watches_added) as num_watches,sum(num_visits) as num_visits, sum(num_downloads_added) as num_downloads,sum(num_pulls_added) as num_pulls,sum(num_commits_added) as num_commits,sum(num_stars_added) as num_stars,sum(num_forks_added) num_forks,sum(num_issues_added) as num_issues,sum(num_closed_issues_added) as num_closed_issues,sum(num_contributor_added) as num_contributor,sum(num_models_added) as num_models,sum(num_model_convert_added) as num_model_convert,sum(num_dataset_file_added) as num_dataset_file, sum(num_cloudbrain_added) as num_cloudbrain " +
" FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," +
"(SELECT repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total from public.repo_statistic where date='" + latestDate + "') B" + "(SELECT repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total from public.repo_statistic where date='" + latestDate + "') B" +


+ 14
- 2
routers/api/v1/repo/topic.go View File

@@ -177,13 +177,25 @@ func AddTopic(ctx *context.APIContext) {
return return
} }


_, err = models.AddTopic(ctx.Repo.Repository.ID, topicName)
topic, err := models.AddTopic(ctx.Repo.Repository.ID, topicName)
if err != nil { if err != nil {
log.Error("AddTopic failed: %v", err) log.Error("AddTopic failed: %v", err)
ctx.InternalServerError(err) ctx.InternalServerError(err)
return return
} }

found := false
topicNames := make([]string, len(topics))
for i, t := range topics {
topicNames[i] = t.Name
if strings.EqualFold(topic.Name, t.Name) {
found = true
break
}
}
if !found && topic.Name != "" {
topicNames = append(topicNames, topic.Name)
}
models.UpdateRepoTopics(ctx.Repo.Repository.ID, topicNames)
ctx.Status(http.StatusNoContent) ctx.Status(http.StatusNoContent)
} }




+ 20
- 0
routers/api/v1/user/repo.go View File

@@ -5,6 +5,7 @@
package user package user


import ( import (
"code.gitea.io/gitea/modules/modelarts"
"net/http" "net/http"


"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
@@ -146,3 +147,22 @@ func ListOrgRepos(ctx *context.APIContext) {


listUserRepos(ctx, ctx.Org.Organization, ctx.IsSigned) listUserRepos(ctx, ctx.Org.Organization, ctx.IsSigned)
} }

func GetComputeNodes(ctx *context.APIContext) {
taskeType := ctx.QueryInt("type")
if taskeType == 2 {
ctx.JSON(http.StatusOK, []int{1})
} else {
modelarts.InitMultiNode()
if modelarts.MultiNodeConfig != nil {
for _, info := range modelarts.MultiNodeConfig.Info {
if isInOrg, _ := models.IsOrganizationMemberByOrgName(info.Org, ctx.User.ID); isInOrg {
ctx.JSON(http.StatusOK, info.Node)
return
}
}
}
ctx.JSON(http.StatusOK, []int{1})
}

}

+ 106
- 0
routers/home.go View File

@@ -7,6 +7,7 @@ package routers


import ( import (
"bytes" "bytes"
"code.gitea.io/gitea/routers/response"
"encoding/json" "encoding/json"
"net/http" "net/http"
"strconv" "strconv"
@@ -43,6 +44,8 @@ const (
tplHomeTerm base.TplName = "terms" tplHomeTerm base.TplName = "terms"
tplHomePrivacy base.TplName = "privacy" tplHomePrivacy base.TplName = "privacy"
tplResoruceDesc base.TplName = "resource_desc" tplResoruceDesc base.TplName = "resource_desc"
tplRepoSquare base.TplName = "explore/repos/square"
tplRepoSearch base.TplName = "explore/repos/search"
) )


// Home render home page // Home render home page
@@ -296,6 +299,109 @@ func ExploreRepos(ctx *context.Context) {
}) })
} }


func GetRepoSquarePage(ctx *context.Context) {
ctx.Data["SquareBanners"] = repository.GetBanners()
ctx.Data["SquareTopics"] = repository.GetTopics()
ctx.Data["SquareRecommendRepos"] = repository.GetRecommendRepos()

repos, _ := repository.GetPreferredRepos()
ctx.Data["SquarePreferredRepos"] = repos
ctx.HTML(200, tplRepoSquare)
}
func GetRepoSearchPage(ctx *context.Context) {
ctx.Data["SquareTopics"] = repository.GetTopics()
ctx.HTML(200, tplRepoSearch)
}

func RepoSquare(ctx *context.Context) {
var result []*models.Repository4Card
var err error
switch ctx.Query("type") {
case "preferred":
result, err = repository.GetPreferredRepos()
case "incubation":
result, err = repository.GetIncubationRepos()
case "hot-paper":
result, err = repository.GetHotPaperRepos()
default:
result, err = repository.GetPreferredRepos()
}
if err != nil {
ctx.JSON(http.StatusOK, response.ResponseError(err))
return
}
resultMap := make(map[string]interface{}, 0)
resultMap["Repos"] = result
ctx.JSON(http.StatusOK, response.SuccessWithData(resultMap))
}

func ActiveUser(ctx *context.Context) {
var err error
var currentUserId int64
if ctx.User != nil {
currentUserId = ctx.User.ID
}
result, err := repository.GetActiveUser4Square(currentUserId)
if err != nil {
log.Error("ActiveUser err. %v", err)
ctx.JSON(http.StatusOK, response.Success())
return
}
resultMap := make(map[string]interface{}, 0)
resultMap["Users"] = result
ctx.JSON(http.StatusOK, response.SuccessWithData(resultMap))
}
func ActiveOrg(ctx *context.Context) {
result, err := repository.GetActiveOrgs()
if err != nil {
log.Error("ActiveOrg err. %v", err)
ctx.JSON(http.StatusOK, response.Success())
return
}
resultMap := make(map[string]interface{}, 0)
resultMap["Orgs"] = result
ctx.JSON(http.StatusOK, response.SuccessWithData(resultMap))
}

func RepoFind(ctx *context.Context) {
keyword := strings.Trim(ctx.Query("q"), " ")
topic := strings.Trim(ctx.Query("topic"), " ")
sort := strings.Trim(ctx.Query("sort"), " ")
page := ctx.QueryInt("page")
pageSize := ctx.QueryInt("pageSize")
if pageSize == 0 {
pageSize = 15
}
if pageSize > 100 {
ctx.JSON(http.StatusOK, response.ServerError("pageSize illegal"))
return
}
if page <= 0 {
page = 1
}

var ownerID int64
if ctx.User != nil && !ctx.User.IsAdmin {
ownerID = ctx.User.ID
}

result, err := repository.FindRepos(repository.FindReposOptions{
ListOptions: models.ListOptions{Page: page, PageSize: pageSize},
Actor: ctx.User,
Sort: sort,
Keyword: keyword,
Topic: topic,
Private: ctx.User != nil,
OwnerID: ownerID,
})
if err != nil {
log.Error("RepoFind error. %v", err)
ctx.JSON(http.StatusOK, response.ResponseError(err))
return
}
ctx.JSON(http.StatusOK, response.SuccessWithData(result))
}

func ExploreDatasets(ctx *context.Context) { func ExploreDatasets(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("explore") ctx.Data["Title"] = ctx.Tr("explore")
ctx.Data["PageIsExplore"] = true ctx.Data["PageIsExplore"] = true


+ 3
- 0
routers/private/internal.go View File

@@ -6,6 +6,7 @@
package private package private


import ( import (
"code.gitea.io/gitea/services/repository"
"strings" "strings"


"code.gitea.io/gitea/routers/admin" "code.gitea.io/gitea/routers/admin"
@@ -55,7 +56,9 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/task/history_handle/duration", repo.HandleTaskWithNoDuration) m.Post("/task/history_handle/duration", repo.HandleTaskWithNoDuration)
m.Post("/task/history_handle/aicenter", repo.HandleTaskWithAiCenter) m.Post("/task/history_handle/aicenter", repo.HandleTaskWithAiCenter)
m.Post("/resources/specification/handle_historical_task", admin.RefreshHistorySpec) m.Post("/resources/specification/handle_historical_task", admin.RefreshHistorySpec)
m.Post("/repos/cnt_stat/handle_historical_task", admin.RefreshHistorySpec)
m.Post("/duration_statisctic/history_handle", repo.CloudbrainUpdateHistoryData) m.Post("/duration_statisctic/history_handle", repo.CloudbrainUpdateHistoryData)
m.Post("/square/repo/stat/refresh", repository.RefreshRepoStatData)


}, CheckInternalToken) }, CheckInternalToken)
} }

+ 36
- 5
routers/repo/ai_model_convert.go View File

@@ -573,13 +573,10 @@ func deleteCloudBrainTask(task *models.AiModelConvert) {
} }
} }


func StopModelConvert(ctx *context.Context) {
id := ctx.Params(":id")
log.Info("stop model convert start.id=" + id)
func stopModelConvert(id string) error {
job, err := models.QueryModelConvertById(id) job, err := models.QueryModelConvertById(id)
if err != nil { if err != nil {
ctx.ServerError("Not found task.", err)
return
return err
} }
if job.IsGpuTrainTask() { if job.IsGpuTrainTask() {
err = cloudbrain.StopJob(job.CloudBrainTaskId) err = cloudbrain.StopJob(job.CloudBrainTaskId)
@@ -600,6 +597,35 @@ func StopModelConvert(ctx *context.Context) {
err = models.UpdateModelConvert(job) err = models.UpdateModelConvert(job)
if err != nil { if err != nil {
log.Error("UpdateModelConvert failed:", err) log.Error("UpdateModelConvert failed:", err)
return err
}
return nil
}

func StopModelConvertApi(ctx *context.Context) {
id := ctx.Query("id")
log.Info("stop model convert start.id=" + id)
err := stopModelConvert(id)
if err == nil {
ctx.JSON(200, map[string]string{
"code": "0",
"msg": "succeed",
})
} else {
ctx.JSON(200, map[string]string{
"code": "1",
"msg": err.Error(),
})
}
}

func StopModelConvert(ctx *context.Context) {
id := ctx.Params(":id")
log.Info("stop model convert start.id=" + id)
err := stopModelConvert(id)
if err != nil {
ctx.ServerError("Not found task.", err)
return
} }
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelmanage/convert_model") ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelmanage/convert_model")
} }
@@ -732,6 +758,11 @@ func GetModelConvertById(ctx *context.Context) (*models.AiModelConvert, error) {
return models.QueryModelConvertById(id) return models.QueryModelConvertById(id)
} }


func GetModelConvertByName(ctx *context.Context) ([]*models.AiModelConvert, error) {
name := ctx.Query("name")
return models.QueryModelConvertByName(name, ctx.Repo.Repository.ID)
}

func GetModelConvertPageData(ctx *context.Context) ([]*models.AiModelConvert, int64, error) { func GetModelConvertPageData(ctx *context.Context) ([]*models.AiModelConvert, int64, error) {
page := ctx.QueryInt("page") page := ctx.QueryInt("page")
if page <= 0 { if page <= 0 {


+ 59
- 27
routers/repo/ai_model_manage.go View File

@@ -11,6 +11,8 @@ import (
"regexp" "regexp"
"strings" "strings"


"code.gitea.io/gitea/services/repository"

"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
@@ -170,10 +172,17 @@ func updateStatus(id string, modelSize int64, status int, modelPath string, stat
if len(statusDesc) > 400 { if len(statusDesc) > 400 {
statusDesc = statusDesc[0:400] statusDesc = statusDesc[0:400]
} }
m, _ := models.QueryModelById(id)
err := models.ModifyModelStatus(id, modelSize, status, modelPath, statusDesc) err := models.ModifyModelStatus(id, modelSize, status, modelPath, statusDesc)
if err != nil { if err != nil {
log.Info("update status error." + err.Error()) log.Info("update status error." + err.Error())
} }
if m != nil {
if modelSize > 0 && m.Size == 0 {
go repository.ResetRepoModelNum(m.RepoId)
}
}

} }


func SaveNewNameModel(ctx *context.Context) { func SaveNewNameModel(ctx *context.Context) {
@@ -308,13 +317,14 @@ func getSize(files []storage.FileInfo) int64 {
func UpdateModelSize(modeluuid string) { func UpdateModelSize(modeluuid string) {
model, err := models.QueryModelById(modeluuid) model, err := models.QueryModelById(modeluuid)
if err == nil { if err == nil {
var size int64
if model.Type == models.TypeCloudBrainOne { if model.Type == models.TypeCloudBrainOne {
if strings.HasPrefix(model.Path, setting.Attachment.Minio.Bucket+"/"+Model_prefix) { if strings.HasPrefix(model.Path, setting.Attachment.Minio.Bucket+"/"+Model_prefix) {
files, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, model.Path[len(setting.Attachment.Minio.Bucket)+1:]) files, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, model.Path[len(setting.Attachment.Minio.Bucket)+1:])
if err != nil { if err != nil {
log.Info("Failed to query model size from minio. id=" + modeluuid) log.Info("Failed to query model size from minio. id=" + modeluuid)
} }
size := getSize(files)
size = getSize(files)
models.ModifyModelSize(modeluuid, size) models.ModifyModelSize(modeluuid, size)
} }
} else if model.Type == models.TypeCloudBrainTwo { } else if model.Type == models.TypeCloudBrainTwo {
@@ -323,10 +333,13 @@ func UpdateModelSize(modeluuid string) {
if err != nil { if err != nil {
log.Info("Failed to query model size from obs. id=" + modeluuid) log.Info("Failed to query model size from obs. id=" + modeluuid)
} }
size := getSize(files)
size = getSize(files)
models.ModifyModelSize(modeluuid, size) models.ModifyModelSize(modeluuid, size)
} }
} }
if model.Size == 0 && size > 0 {
go repository.ResetRepoModelNum(model.RepoId)
}
} else { } else {
log.Info("not found model,uuid=" + modeluuid) log.Info("not found model,uuid=" + modeluuid)
} }
@@ -441,13 +454,14 @@ func DeleteModelFile(ctx *context.Context) {
fileName := ctx.Query("fileName") fileName := ctx.Query("fileName")
model, err := models.QueryModelById(id) model, err := models.QueryModelById(id)
if err == nil { if err == nil {
var totalSize int64
if model.ModelType == MODEL_LOCAL_TYPE { if model.ModelType == MODEL_LOCAL_TYPE {
if model.Type == models.TypeCloudBrainOne { if model.Type == models.TypeCloudBrainOne {
bucketName := setting.Attachment.Minio.Bucket bucketName := setting.Attachment.Minio.Bucket
objectName := model.Path[len(bucketName)+1:] + fileName objectName := model.Path[len(bucketName)+1:] + fileName
log.Info("delete bucket=" + bucketName + " path=" + objectName) log.Info("delete bucket=" + bucketName + " path=" + objectName)
if strings.HasPrefix(model.Path, bucketName+"/"+Model_prefix) { if strings.HasPrefix(model.Path, bucketName+"/"+Model_prefix) {
totalSize := storage.MinioGetFilesSize(bucketName, []string{objectName})
totalSize = storage.MinioGetFilesSize(bucketName, []string{objectName})
err := storage.Attachments.DeleteDir(objectName) err := storage.Attachments.DeleteDir(objectName)
if err != nil { if err != nil {
log.Info("Failed to delete model. id=" + id) log.Info("Failed to delete model. id=" + id)
@@ -467,7 +481,7 @@ func DeleteModelFile(ctx *context.Context) {
objectName := model.Path[len(setting.Bucket)+1:] + fileName objectName := model.Path[len(setting.Bucket)+1:] + fileName
log.Info("delete bucket=" + setting.Bucket + " path=" + objectName) log.Info("delete bucket=" + setting.Bucket + " path=" + objectName)
if strings.HasPrefix(model.Path, bucketName+"/"+Model_prefix) { if strings.HasPrefix(model.Path, bucketName+"/"+Model_prefix) {
totalSize := storage.ObsGetFilesSize(bucketName, []string{objectName})
totalSize = storage.ObsGetFilesSize(bucketName, []string{objectName})
err := storage.ObsRemoveObject(bucketName, objectName) err := storage.ObsRemoveObject(bucketName, objectName)
if err != nil { if err != nil {
log.Info("Failed to delete model. id=" + id) log.Info("Failed to delete model. id=" + id)
@@ -484,6 +498,9 @@ func DeleteModelFile(ctx *context.Context) {
} }
} }
} }
if (model.Size - totalSize) <= 0 {
go repository.ResetRepoModelNum(model.RepoId)
}
} }
ctx.JSON(200, map[string]string{ ctx.JSON(200, map[string]string{
"code": "0", "code": "0",
@@ -552,6 +569,9 @@ func deleteModelByID(ctx *context.Context, id string) error {
} }
} }
} }
if model.Size > 0 {
go repository.ResetRepoModelNum(model.RepoId)
}
} }
} }
return err return err
@@ -691,36 +711,42 @@ func downloadFromCloudBrainTwo(path string, task *models.AiModelManage, ctx *con
} }


func QueryTrainJobVersionList(ctx *context.Context) { func QueryTrainJobVersionList(ctx *context.Context) {

VersionListTasks, err := QueryTrainJobVersionListApi(ctx)
if err != nil {
ctx.JSON(200, nil)
} else {
ctx.JSON(200, VersionListTasks)
}
}

func QueryTrainJobVersionListApi(ctx *context.Context) ([]*models.Cloudbrain, error) {
log.Info("query train job version list. start.") log.Info("query train job version list. start.")
JobID := ctx.Query("jobId") JobID := ctx.Query("jobId")
if JobID == "" { if JobID == "" {
JobID = ctx.Query("JobId") JobID = ctx.Query("JobId")
} }

VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID)

log.Info("query return count=" + fmt.Sprint(count)) log.Info("query return count=" + fmt.Sprint(count))


return VersionListTasks, err
}

func QueryTrainJobList(ctx *context.Context) {
VersionListTasks, err := QueryTrainJobListApi(ctx)
if err != nil { if err != nil {
ctx.ServerError("QueryTrainJobList:", err)
ctx.JSON(200, nil)
} else { } else {
ctx.JSON(200, VersionListTasks) ctx.JSON(200, VersionListTasks)
} }
} }


func QueryTrainJobList(ctx *context.Context) {
log.Info("query train job list. start.")
func QueryTrainJobListApi(ctx *context.Context) ([]*models.Cloudbrain, error) {
repoId := ctx.QueryInt64("repoId") repoId := ctx.QueryInt64("repoId")

VersionListTasks, count, err := models.QueryModelTrainJobList(repoId) VersionListTasks, count, err := models.QueryModelTrainJobList(repoId)
log.Info("query return count=" + fmt.Sprint(count)) log.Info("query return count=" + fmt.Sprint(count))


if err != nil {
ctx.ServerError("QueryTrainJobList:", err)
} else {
ctx.JSON(200, VersionListTasks)
}

return VersionListTasks, err
} }


func QueryTrainModelFileById(ctx *context.Context) ([]storage.FileInfo, error) { func QueryTrainModelFileById(ctx *context.Context) ([]storage.FileInfo, error) {
@@ -1300,19 +1326,25 @@ func QueryModelFileForPredict(ctx *context.Context) {


func QueryModelFileByID(id string) []storage.FileInfo { func QueryModelFileByID(id string) []storage.FileInfo {
model, err := models.QueryModelById(id) model, err := models.QueryModelById(id)
if err == nil {
if model.Type == models.TypeCloudBrainTwo {
prefix := model.Path[len(setting.Bucket)+1:]
fileinfos, _ := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix)
return fileinfos
} else if model.Type == models.TypeCloudBrainOne {
prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:]
fileinfos, _ := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, prefix)
return fileinfos
}
} else {
if err != nil {
log.Error("no such model!", err.Error()) log.Error("no such model!", err.Error())
return nil
}
return QueryModelFileByModel(model)
}

func QueryModelFileByModel(model *models.AiModelManage) []storage.FileInfo {

if model.Type == models.TypeCloudBrainTwo {
prefix := model.Path[len(setting.Bucket)+1:]
fileinfos, _ := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix)
return fileinfos
} else if model.Type == models.TypeCloudBrainOne {
prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:]
fileinfos, _ := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, prefix)
return fileinfos
} }

return nil return nil
} }




+ 4
- 1
routers/repo/aisafety.go View File

@@ -847,6 +847,9 @@ func createForGPU(ctx *context.Context, jobName string) error {
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
os.RemoveAll(codePath) os.RemoveAll(codePath)


gitRepo, _ := git.OpenRepository(repo.RepoPath())
commitID, _ := gitRepo.GetBranchCommitID(cloudbrain.DefaultBranchName)

if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil {
log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"])
return errors.New("system error") return errors.New("system error")
@@ -891,7 +894,7 @@ func createForGPU(ctx *context.Context, jobName string) error {
BranchName: cloudbrain.DefaultBranchName, BranchName: cloudbrain.DefaultBranchName,
BootFile: BootFile, BootFile: BootFile,
Params: Params, Params: Params,
CommitID: "",
CommitID: commitID,
ModelName: modelName, ModelName: modelName,
ModelVersion: modelVersion, ModelVersion: modelVersion,
CkptName: CkptName, CkptName: CkptName,


+ 3
- 0
routers/repo/attachment.go View File

@@ -29,6 +29,7 @@ import (
"code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/upload" "code.gitea.io/gitea/modules/upload"
"code.gitea.io/gitea/modules/worker" "code.gitea.io/gitea/modules/worker"
repo_service "code.gitea.io/gitea/services/repository"
gouuid "github.com/satori/go.uuid" gouuid "github.com/satori/go.uuid"
) )


@@ -180,6 +181,7 @@ func DeleteAttachment(ctx *context.Context) {
ctx.Error(500, fmt.Sprintf("DeleteAttachment: %v", err)) ctx.Error(500, fmt.Sprintf("DeleteAttachment: %v", err))
return return
} }
go repo_service.DecreaseRepoDatasetNum(attach.DatasetID)


attachjson, _ := json.Marshal(attach) attachjson, _ := json.Marshal(attach)
labelmsg.SendDeleteAttachToLabelSys(string(attachjson)) labelmsg.SendDeleteAttachToLabelSys(string(attachjson))
@@ -894,6 +896,7 @@ func CompleteMultipart(ctx *context.Context) {
return return
} }
attachment.UpdateDatasetUpdateUnix() attachment.UpdateDatasetUpdateUnix()
go repo_service.IncreaseRepoDatasetNum(dataset.ID)
repository, _ := models.GetRepositoryByID(dataset.RepoID) repository, _ := models.GetRepositoryByID(dataset.RepoID)
notification.NotifyOtherTask(ctx.User, repository, fmt.Sprint(repository.IsPrivate, attachment.IsPrivate), attachment.Name, models.ActionUploadAttachment) notification.NotifyOtherTask(ctx.User, repository, fmt.Sprint(repository.IsPrivate, attachment.IsPrivate), attachment.Name, models.ActionUploadAttachment)
if attachment.DatasetID != 0 { if attachment.DatasetID != 0 {


+ 225
- 91
routers/repo/cloudbrain.go View File

@@ -81,6 +81,7 @@ var (


const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types" const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types"
const CLONE_FILE_PREFIX = "file:///" const CLONE_FILE_PREFIX = "file:///"
const README = "README"


var benchmarkTypesMap = make(map[string]*models.BenchmarkTypes, 0) var benchmarkTypesMap = make(map[string]*models.BenchmarkTypes, 0)


@@ -284,7 +285,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
} }
var datasetInfos map[string]models.DatasetInfo var datasetInfos map[string]models.DatasetInfo
var datasetNames string var datasetNames string
//var
var attachSize int
if uuids != "" { if uuids != "" {
datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids) datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids)
if err != nil { if err != nil {
@@ -293,6 +294,18 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
return return
} }

if jobType == string(models.JobTypeDebug) {
for _, infos := range datasetInfos {
attachSize += infos.Size
}
if attachSize > int(setting.DebugAttachSize*1000*1000*1000) {
log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugAttachSize)) // GB
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize", int(setting.DebugAttachSize*1000*1000*1000)), tpl, &form)
return
}
}
} }


command := cloudbrain.GetCloudbrainDebugCommand() command := cloudbrain.GetCloudbrainDebugCommand()
@@ -373,13 +386,19 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
} }


if form.ModelName != "" { //使用预训练模型训练 if form.ModelName != "" { //使用预训练模型训练
_, err := models.QueryModelByPath(form.PreTrainModelUrl)
if err != nil {
log.Error("Can not find model", err)
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("repo.modelconvert.manage.model_not_exist"), tpl, &form)
return
}
req.ModelName = form.ModelName req.ModelName = form.ModelName
req.LabelName = form.LabelName req.LabelName = form.LabelName
req.CkptName = form.CkptName req.CkptName = form.CkptName
req.ModelVersion = form.ModelVersion req.ModelVersion = form.ModelVersion
req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl
req.PreTrainModelUrl = form.PreTrainModelUrl req.PreTrainModelUrl = form.PreTrainModelUrl

} }


_, err = cloudbrain.GenerateTask(req) _, err = cloudbrain.GenerateTask(req)
@@ -411,8 +430,13 @@ func loadCodeAndMakeModelPath(repo *models.Repository, codePath string, branchNa
return "cloudbrain.load_code_failed" return "cloudbrain.load_code_failed"
} }


return initModelPath(jobName, resultPath)

}

func initModelPath(jobName string, resultPath string) string {
modelPath := setting.JobPath + jobName + resultPath + "/" modelPath := setting.JobPath + jobName + resultPath + "/"
err = mkModelPath(modelPath)
err := mkModelPath(modelPath)
if err != nil { if err != nil {
return "cloudbrain.load_code_failed" return "cloudbrain.load_code_failed"
} }
@@ -691,6 +715,17 @@ func CloudBrainRestart(ctx *context.Context) {
break break
} }
} }
if !HasModelFile(task) {
resultCode = "-1"
errorMsg = ctx.Tr("repo.debug.manage.model_not_exist")
break
}

if hasDatasetDeleted(task) {
resultCode = "-1"
errorMsg = ctx.Tr("repo.debug.manage.dataset_not_exist")
break
}


err = cloudbrain.RestartTask(ctx, task, &ID) err = cloudbrain.RestartTask(ctx, task, &ID)
if err != nil { if err != nil {
@@ -712,6 +747,40 @@ func CloudBrainRestart(ctx *context.Context) {
} }




func hasDatasetDeleted(task *models.Cloudbrain) bool {
if task.Uuid == "" {
return false
}
uuids := strings.Split(task.Uuid, ";")
attachs, _ := models.GetAttachmentsByUUIDs(uuids)
return len(attachs) < len(uuids)
}

func HasModelFile(task *models.Cloudbrain) bool {
if task.PreTrainModelUrl == "" {
return true
}

model, err := models.QueryModelByPath(task.PreTrainModelUrl)
if err != nil {
log.Error("Can not find model", err)
return false
}

fileInfos := QueryModelFileByModel(model)
isFind := false
if fileInfos != nil {
for _, fileInfo := range fileInfos {
if fileInfo.FileName == task.CkptName {
isFind = true
break
}
}

}
return isFind
}

func getOldJobPath(task *models.Cloudbrain) string { func getOldJobPath(task *models.Cloudbrain) string {
return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName
} }
@@ -786,7 +855,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo


if task.JobType == string(models.JobTypeBenchmark) { if task.JobType == string(models.JobTypeBenchmark) {
task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
} else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
} else if models.IsModelBenchMarkJobType(task.JobType) {
task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
task.BenchmarkTypeName = task.JobType task.BenchmarkTypeName = task.JobType
ctx.Data["BenchmarkTypeName"] = task.JobType ctx.Data["BenchmarkTypeName"] = task.JobType
@@ -854,10 +923,13 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo
func CloudBrainDebug(ctx *context.Context) { func CloudBrainDebug(ctx *context.Context) {
task := ctx.Cloudbrain task := ctx.Cloudbrain
debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
if task.BootFile!=""{
ctx.Redirect(getFileUrl(debugUrl,task.BootFile))


}else{
if ctx.QueryTrim("file") != "" {
ctx.Redirect(getFileUrl(debugUrl, ctx.QueryTrim("file")))
} else {
if task.BootFile != "" {
go cloudbrainTask.UploadNotebookFiles(task)
}
ctx.Redirect(debugUrl) ctx.Redirect(debugUrl)
} }


@@ -1581,6 +1653,21 @@ func CloudBrainDownloadModel(ctx *context.Context) {
ctx.Resp.Header().Set("Cache-Control", "max-age=0") ctx.Resp.Header().Set("Cache-Control", "max-age=0")
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
} }

func CloudBrainDownloadMultiModel(ctx *context.Context) {
parentDir := ctx.Query("parentDir")
jobName := ctx.Query("jobName")
filePath := "jobs/" + jobName + "/model/" + parentDir
allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
if err == nil {
returnFileName := jobName + ".zip"
MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
}
}

func CloudBrainDownloadInferenceResult(ctx *context.Context) { func CloudBrainDownloadInferenceResult(ctx *context.Context) {
parentDir := ctx.Query("parentDir") parentDir := ctx.Query("parentDir")
fileName := ctx.Query("fileName") fileName := ctx.Query("fileName")
@@ -1617,6 +1704,8 @@ func GetRate(ctx *context.Context) {
ctx.Redirect(setting.Snn4imagenetServerHost) ctx.Redirect(setting.Snn4imagenetServerHost)
} else if job.JobType == string(models.JobTypeBrainScore) { } else if job.JobType == string(models.JobTypeBrainScore) {
ctx.Redirect(setting.BrainScoreServerHost) ctx.Redirect(setting.BrainScoreServerHost)
} else if job.JobType == string(models.JobTypeSnn4Ecoset) {
ctx.Redirect(setting.Snn4EcosetServerHost)
} else { } else {
log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
} }
@@ -1758,7 +1847,7 @@ func mkPathAndReadMeFile(path string, text string) error {
return err return err
} }


fileName := path + "README"
fileName := path + README
f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
if err != nil { if err != nil {
log.Error("OpenFile failed", err.Error()) log.Error("OpenFile failed", err.Error())
@@ -1816,6 +1905,7 @@ func SyncCloudbrainStatus() {
if task.JobType == string(models.JobTypeModelSafety) { if task.JobType == string(models.JobTypeModelSafety) {
continue continue
} }

if task.Type == models.TypeCloudBrainOne { if task.Type == models.TypeCloudBrainOne {


task, err = cloudbrainTask.SyncCloudBrainOneStatus(task) task, err = cloudbrainTask.SyncCloudBrainOneStatus(task)
@@ -1824,32 +1914,7 @@ func SyncCloudbrainStatus() {
continue continue
} }


if task.Status != string(models.JobWaiting) {
if task.Duration >= setting.MaxDuration && task.JobType == string(models.JobTypeDebug) {
log.Info("begin to stop job(%s), because of the duration", task.DisplayJobName)
err = cloudbrain.StopJob(task.JobID)
if err != nil {
log.Error("StopJob(%s) failed:%v", task.DisplayJobName, err)
continue
}
oldStatus := task.Status
task.Status = string(models.JobStopped)
if task.EndTime == 0 {
task.EndTime = timeutil.TimeStampNow()
}
task.ComputeAndSetDuration()
if oldStatus != task.Status {
notification.NotifyChangeCloudbrainStatus(task, oldStatus)
}
err = models.UpdateJob(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
continue
}
}

}
} else if task.Type == models.TypeCloudBrainTwo {
} else if task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter {
if task.JobType == string(models.JobTypeDebug) { if task.JobType == string(models.JobTypeDebug) {
err := modelarts.HandleNotebookInfo(task) err := modelarts.HandleNotebookInfo(task)
if err != nil { if err != nil {
@@ -1866,48 +1931,77 @@ func SyncCloudbrainStatus() {
log.Error("task.JobType(%s) is error:%s", task.DisplayJobName, task.JobType) log.Error("task.JobType(%s) is error:%s", task.DisplayJobName, task.JobType)
} }
} else if task.Type == models.TypeC2Net { } else if task.Type == models.TypeC2Net {
result, err := grampus.GetJob(task.JobID)
if err != nil {
log.Error("GetTrainJob(%s) failed:%v", task.DisplayJobName, err)
continue
}

if result != nil {
if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 {
task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
if task.JobType == string(models.JobTypeDebug) {
cloudbrainTask.SyncGrampusNotebookStatus(task)
} else {
result, err := grampus.GetJob(task.JobID)
if err != nil {
log.Error("GetTrainJob(%s) failed:%v", task.DisplayJobName, err)
continue
} }
oldStatus := task.Status
task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
task.Duration = result.JobInfo.RunSec


if task.Duration < 0 {
task.Duration = 0
}
task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
if result != nil {
if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 {
task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
}
oldStatus := task.Status
task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
task.Duration = result.JobInfo.RunSec

if task.Duration < 0 {
task.Duration = 0
}
task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)


if task.StartTime == 0 && result.JobInfo.StartedAt > 0 {
task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
if task.StartTime == 0 && result.JobInfo.StartedAt > 0 {
task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
}
if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
task.EndTime = task.StartTime.Add(task.Duration)
}
task.CorrectCreateUnix()
if oldStatus != task.Status {
notification.NotifyChangeCloudbrainStatus(task, oldStatus)
if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource {
if len(result.JobInfo.Tasks[0].CenterID) == 1 {
urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID))
}
}
}
err = models.UpdateJob(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
continue
}
} }
if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
task.EndTime = task.StartTime.Add(task.Duration)
}
} else {
log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
}
if task.Status != string(models.JobWaiting) {
if task.Duration >= setting.MaxDuration && task.JobType == string(models.JobTypeDebug) {
log.Info("begin to stop job(%s), because of the duration", task.DisplayJobName)
err = cloudbrainTask.StopDebugJob(task)
if err != nil {
log.Error("StopJob(%s) failed:%v", task.DisplayJobName, err)
continue
} }
task.CorrectCreateUnix()
oldStatus := task.Status
task.Status = string(models.JobStopped)
if task.EndTime == 0 {
task.EndTime = timeutil.TimeStampNow()
}
task.ComputeAndSetDuration()
if oldStatus != task.Status { if oldStatus != task.Status {
notification.NotifyChangeCloudbrainStatus(task, oldStatus) notification.NotifyChangeCloudbrainStatus(task, oldStatus)
if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource {
if len(result.JobInfo.Tasks[0].CenterID) == 1 {
urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID))
}
}
} }
err = models.UpdateJob(task) err = models.UpdateJob(task)
if err != nil { if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
continue continue
} }
} }
} else {
log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)

} }
} }


@@ -2084,7 +2178,7 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) {
} }


var jobTypes []string var jobTypes []string
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeModelSafety))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset), string(models.JobTypeModelSafety))
ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
ListOptions: models.ListOptions{ ListOptions: models.ListOptions{
Page: page, Page: page,
@@ -2117,14 +2211,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) {
ciTasks[i].BenchmarkTypeName = "" ciTasks[i].BenchmarkTypeName = ""
if ciTasks[i].JobType == string(models.JobTypeBenchmark) { if ciTasks[i].JobType == string(models.JobTypeBenchmark) {
ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
} else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) {
} else if models.IsModelBenchMarkJobType(ciTasks[i].JobType) {
ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType


if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) { if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) {
ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost
} else {
} else if ciTasks[i].JobType == string(models.JobTypeBrainScore) {
ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost
} else {
ciTasks[i].BenchmarkTypeRankLink = setting.Snn4EcosetServerHost
} }


} }
@@ -2474,7 +2570,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
displayJobName := form.DisplayJobName displayJobName := form.DisplayJobName
jobName := util.ConvertDisplayJobNameToJobName(displayJobName) jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
image := form.Image image := form.Image
uuid := form.Attachment
jobType := form.JobType jobType := form.JobType
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
branchName := cloudbrain.DefaultBranchName branchName := cloudbrain.DefaultBranchName
@@ -2516,7 +2611,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
return return
} }


if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
if !models.IsModelBenchMarkJobType(jobType) {
log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx, jobType) cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr("jobtype error", tpl, &form) ctx.RenderWithErr("jobtype error", tpl, &form)
@@ -2545,29 +2640,41 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
mkModelPath(modelPath) mkModelPath(modelPath)
uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/") uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/")


snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath
if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "", ctx.User.Name)
uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/")
command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, trimSpaceNewlineInString(form.Description))
downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, benchmarkPath, "", "", ctx.User.Name)
uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/")
command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description))


} }
benchmarkChildTypeID := 0 benchmarkChildTypeID := 0
brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "", ctx.User.Name)
uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/")
downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, benchmarkPath, "", "", ctx.User.Name)
uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/")
benchmarkChildTypeID = form.BenchmarkChildTypeID benchmarkChildTypeID = form.BenchmarkChildTypeID
command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description))
command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description))
} }
var uuid string
var datasetInfos map[string]models.DatasetInfo
var datasetNames string
if setting.IsSnn4EcosetEnabled && jobType == string(models.JobTypeSnn4Ecoset) {
downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, benchmarkPath, "", "", ctx.User.Name)
uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/")
command = fmt.Sprintf(cloudbrain.Snn4EcosetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description))

attachment, err := getEcosetAttachment()
if err != nil {
log.Error("load benchmark code failed", err)
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
return
}
uuid = attachment.UUID
datasetInfos, datasetNames, _ = models.GetDatasetInfo(uuid)


datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
if err != nil {
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
return
} }

spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{
JobType: models.JobTypeBenchmark, JobType: models.JobTypeBenchmark,
ComputeResource: models.GPU, ComputeResource: models.GPU,
@@ -2599,8 +2706,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
JobType: jobType, JobType: jobType,
Description: form.Description, Description: form.Description,
BranchName: branchName, BranchName: branchName,
@@ -2612,6 +2717,14 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
Spec: spec, Spec: spec,
} }
if form.ModelName != "" {
req.ModelName = form.ModelName
req.LabelName = form.LabelName
req.CkptName = form.CkptName
req.ModelVersion = form.ModelVersion
req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl
req.PreTrainModelUrl = form.PreTrainModelUrl
}


_, err = cloudbrain.GenerateTask(req) _, err = cloudbrain.GenerateTask(req)
if err != nil { if err != nil {
@@ -2623,6 +2736,21 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
} }


func getEcosetAttachment() (*models.Attachment, error) {
ecosetRepo, err := models.GetRepositoryByOwnerAndName(setting.Snn4EcosetOwner, setting.Snn4EcosetName)
if err != nil {
return nil, err
}

datasetInfo, err := models.GetDatasetByRepo(ecosetRepo)
if err != nil {
return nil, err
}

return models.GetAttachmentByDatasetIdFileName(setting.Snn4AttachmentName, datasetInfo.ID)

}

func getBrainRegion(benchmarkChildTypeID int) string { func getBrainRegion(benchmarkChildTypeID int) string {
values := []string{"V1", "V2", "V4", "IT"} values := []string{"V1", "V2", "V4", "IT"}
return values[benchmarkChildTypeID] return values[benchmarkChildTypeID]
@@ -2683,18 +2811,24 @@ func InferenceCloudBrainJobShow(ctx *context.Context) {
cloudBrainShow(ctx, tplCloudBrainInferenceJobShow, models.JobTypeInference) cloudBrainShow(ctx, tplCloudBrainInferenceJobShow, models.JobTypeInference)
} }


func DownloadInferenceResultFile(ctx *context.Context) {
func DownloadGPUInferenceResultFile(ctx *context.Context) {
var jobID = ctx.Params(":jobid") var jobID = ctx.Params(":jobid")
var versionName = ctx.Query("version_name")
task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
task, err := models.GetCloudbrainByJobID(jobID)
if err != nil { if err != nil {
log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
return return
} }

allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, task.ResultUrl)
returnFileName := task.DisplayJobName + ".zip"
MinioDownloadManyFile(task.ResultUrl, ctx, returnFileName, allFile)
parentDir := ctx.Query("parentDir")
filePath := "jobs/" + task.JobName + "/result/" + parentDir
log.Info("prefix=" + filePath)
allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
if err == nil {
returnFileName := task.DisplayJobName + ".zip"
MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
}
} }


func getInferenceJobCommand(form auth.CreateCloudBrainInferencForm) (string, error) { func getInferenceJobCommand(form auth.CreateCloudBrainInferencForm) (string, error) {


+ 73
- 113
routers/repo/cloudbrain_statistic.go View File

@@ -29,17 +29,16 @@ func CloudbrainDurationStatisticHour() {
statisticTime = currentTime statisticTime = currentTime
} }


err = models.DeleteCloudbrainDurationStatistic(timeutil.TimeStamp(statisticTime.Add(-1*time.Hour).Unix()), timeutil.TimeStamp(currentTime.Unix()))
err = models.DeleteCloudbrainDurationStatistic(timeutil.TimeStamp(statisticTime.Unix()), timeutil.TimeStamp(currentTime.Unix()))
if err != nil { if err != nil {
log.Error("DeleteCloudbrainDurationStatistic failed", err) log.Error("DeleteCloudbrainDurationStatistic failed", err)
} }
statisticTime = statisticTime.Add(+1 * time.Hour)
for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) { for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) {
countEach := summaryDurationStat(statisticTime) countEach := summaryDurationStat(statisticTime)
count += countEach count += countEach
statisticTime = statisticTime.Add(+1 * time.Hour) statisticTime = statisticTime.Add(+1 * time.Hour)
} }
log.Info("summaryDurationStat count: %v", count)
} }
} }
func UpdateDurationStatisticHistoryData(beginTime time.Time, endTime time.Time) int64 { func UpdateDurationStatisticHistoryData(beginTime time.Time, endTime time.Time) int64 {
@@ -65,15 +64,18 @@ func summaryDurationStat(statisticTime time.Time) int64 {


ciTasks, err := models.GetCloudbrainByTime(beginTime, endTime) ciTasks, err := models.GetCloudbrainByTime(beginTime, endTime)
if err != nil { if err != nil {
log.Info("GetCloudbrainByTime err: %v", err)
log.Error("GetCloudbrainByTime err: %v", err)
return 0 return 0
} }
models.LoadSpecs4CloudbrainInfo(ciTasks)
cloudBrainCenterCodeAndCardTypeInfo, cloudbrainMap := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime)
err = models.LoadSpecs4CloudbrainInfo(ciTasks)
if err != nil {
log.Error("LoadSpecs4CloudbrainInfo err: %v", err)
}
cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, int(beginTime), int(endTime))


resourceQueues, err := models.GetCanUseCardInfo() resourceQueues, err := models.GetCanUseCardInfo()
if err != nil { if err != nil {
log.Info("GetCanUseCardInfo err: %v", err)
log.Error("GetCanUseCardInfo err: %v", err)
return 0 return 0
} }


@@ -85,56 +87,45 @@ func summaryDurationStat(statisticTime time.Time) int64 {
cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType] += resourceQueue.CardsTotalNum * 1 * 60 * 60 cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType] += resourceQueue.CardsTotalNum * 1 * 60 * 60
} }
} }

for centerCode, CardTypes := range cloudBrainCenterCodeAndCardTypeInfo {
for cardType, cardDuration := range CardTypes {
cloudbrainTable := cloudbrainMap[centerCode+"/"+cardType]
if cloudbrainTable != nil {
if _, ok := cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType]; !ok {
cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType] = 0
}
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTimeUnix: dateTimeUnix,
DayTime: dayTime,
HourTime: hourTime,
Cluster: cloudbrainTable.Cluster,
AiCenterName: GetAiCenterNameByCode(centerCode, "zh-CN"),
AiCenterCode: centerCode,
AccCardType: cardType,
CardsUseDuration: cardDuration,
CardsTotalDuration: cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType],
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
}
count++
delete(cardsTotalDurationMap, cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType)
}
}
}

for key, cardsTotalDuration := range cardsTotalDurationMap { for key, cardsTotalDuration := range cardsTotalDurationMap {
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTimeUnix: dateTimeUnix,
DayTime: dayTime,
HourTime: hourTime,
Cluster: strings.Split(key, "/")[0],
AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"),
AiCenterCode: strings.Split(key, "/")[1],
AccCardType: strings.Split(key, "/")[2],
CardsUseDuration: 0,
CardsTotalDuration: cardsTotalDuration,
CardsTotalNum: cardsTotalDuration / 1 / 60 / 60,
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
if _, ok := cloudBrainCenterCodeAndCardTypeInfo[strings.Split(key, "/")[0]+"/"+strings.Split(key, "/")[1]][strings.Split(key, "/")[2]]; ok {
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTimeUnix: dateTimeUnix,
DayTime: dayTime,
HourTime: hourTime,
Cluster: strings.Split(key, "/")[0],
AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"),
AiCenterCode: strings.Split(key, "/")[1],
AccCardType: strings.Split(key, "/")[2],
CardsUseDuration: cloudBrainCenterCodeAndCardTypeInfo[strings.Split(key, "/")[0]+"/"+strings.Split(key, "/")[1]][strings.Split(key, "/")[2]],
CardsTotalDuration: cardsTotalDuration,
CardsTotalNum: cardsTotalDuration / 1 / 60 / 60,
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
}
count++
} else {
cloudbrainDurationStat := models.CloudbrainDurationStatistic{
DateTimeUnix: dateTimeUnix,
DayTime: dayTime,
HourTime: hourTime,
Cluster: strings.Split(key, "/")[0],
AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"),
AiCenterCode: strings.Split(key, "/")[1],
AccCardType: strings.Split(key, "/")[2],
CardsUseDuration: 0,
CardsTotalDuration: cardsTotalDuration,
CardsTotalNum: cardsTotalDuration / 1 / 60 / 60,
CreatedUnix: timeutil.TimeStampNow(),
}
if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
}
count++
} }
count++
} }

log.Info("finish summary cloudbrainDurationStat")
return count return count
} }


@@ -153,33 +144,21 @@ func GetAiCenterNameByCode(centerCode string, language string) string {
return aiCenterName return aiCenterName
} }


func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) (map[string]map[string]int, map[string]*models.Cloudbrain) {
func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, hourBeginTime int, hourEndTime int) map[string]map[string]int {
var WorkServerNumber int var WorkServerNumber int
var AccCardsNum int var AccCardsNum int
cloudbrainMap := make(map[string]*models.Cloudbrain)
cloudBrainCenterCodeAndCardType := make(map[string]map[string]int) cloudBrainCenterCodeAndCardType := make(map[string]map[string]int)
for _, cloudbrain := range ciTasks { for _, cloudbrain := range ciTasks {
if cloudbrain.Cloudbrain.StartTime == 0 {
cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix
}
if cloudbrain.Cloudbrain.EndTime == 0 {
cloudbrain.Cloudbrain.EndTime = timeutil.TimeStamp(time.Now().Unix())
}
cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain)
if cloudbrain.Cloudbrain.Spec != nil {
if _, ok := cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if cloudbrain.Cloudbrain.Spec != nil {
cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = &cloudbrain.Cloudbrain
}
}
}

cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain) cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain)
if cloudbrain.Cloudbrain.StartTime == 0 { if cloudbrain.Cloudbrain.StartTime == 0 {
cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix
} }
if cloudbrain.Cloudbrain.EndTime == 0 { if cloudbrain.Cloudbrain.EndTime == 0 {
cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix
if cloudbrain.Cloudbrain.Status == string(models.JobRunning) {
cloudbrain.Cloudbrain.EndTime = timeutil.TimeStamp(time.Now().Unix())
} else {
cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.StartTime + timeutil.TimeStamp(cloudbrain.Cloudbrain.Duration)
}
} }
if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber
@@ -191,55 +170,36 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be
} else { } else {
AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum
} }
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter]; !ok {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter] = make(map[string]int)
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter]; !ok {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter] = make(map[string]int)
} }
taskStartTime := int(cloudbrain.Cloudbrain.StartTime)
taskEndTime := int(cloudbrain.Cloudbrain.EndTime)
if cloudbrain.Cloudbrain.Spec != nil { if cloudbrain.Cloudbrain.Spec != nil {
if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) && cloudbrain.Cloudbrain.DeletedAt.IsZero() {
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
} else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = 0
}
} else {
if int64(cloudbrain.Cloudbrain.StartTime) < beginTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
} else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += 0
}
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if taskStartTime < hourBeginTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (taskEndTime - hourBeginTime)
} else if taskStartTime < hourBeginTime && taskEndTime > hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (hourEndTime - hourBeginTime)
} else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (taskEndTime - taskStartTime)
} else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime > hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (hourEndTime - taskStartTime)
} }
} else { } else {
if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime))
} else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
}
} else {
if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime))
} else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime))
} else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime))
}
if taskStartTime < hourBeginTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (taskEndTime - hourBeginTime)
} else if taskStartTime < hourBeginTime && taskEndTime > hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (hourEndTime - hourBeginTime)
} else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime >= hourBeginTime && taskEndTime <= hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (taskEndTime - taskStartTime)
} else if taskStartTime >= hourBeginTime && taskStartTime <= hourEndTime && taskEndTime > hourEndTime {
cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Cluster+"/"+cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (hourEndTime - taskStartTime)
} }
} }
} }
} }

return cloudBrainCenterCodeAndCardType, cloudbrainMap
return cloudBrainCenterCodeAndCardType
} }


func CloudbrainUpdateHistoryData(ctx *context.Context) { func CloudbrainUpdateHistoryData(ctx *context.Context) {


+ 1
- 0
routers/repo/dataset.go View File

@@ -373,6 +373,7 @@ func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) {
} }


data, err := json.Marshal(datasets) data, err := json.Marshal(datasets)
log.Info("datakey", string(data))
if err != nil { if err != nil {
log.Error("json.Marshal failed:", err.Error()) log.Error("json.Marshal failed:", err.Error())
ctx.JSON(200, map[string]string{ ctx.JSON(200, map[string]string{


+ 668
- 81
routers/repo/grampus.go View File

@@ -44,14 +44,37 @@ import (


const ( const (
tplGrampusTrainJobShow base.TplName = "repo/grampus/trainjob/show" tplGrampusTrainJobShow base.TplName = "repo/grampus/trainjob/show"
tplGrampusNotebookShow base.TplName = "repo/grampus/notebook/show"


//GPU //GPU
tplGrampusNotebookGPUNew base.TplName = "repo/grampus/notebook/gpu/new"
tplGrampusTrainJobGPUNew base.TplName = "repo/grampus/trainjob/gpu/new" tplGrampusTrainJobGPUNew base.TplName = "repo/grampus/trainjob/gpu/new"


//NPU //NPU
tplGrampusNotebookNPUNew base.TplName = "repo/grampus/notebook/npu/new"
tplGrampusTrainJobNPUNew base.TplName = "repo/grampus/trainjob/npu/new" tplGrampusTrainJobNPUNew base.TplName = "repo/grampus/trainjob/npu/new"
) )


func GrampusNotebookNew(ctx *context.Context) {
ctx.Data["IsCreate"] = true
notebookType := ctx.QueryInt("type")
processType := grampus.ProcessorTypeGPU
if notebookType == 1 {
processType = grampus.ProcessorTypeNPU
}
err := grampusNotebookNewDataPrepare(ctx, processType)
if err != nil {
ctx.ServerError("get new notebook-job info failed", err)
return
}
if processType == grampus.ProcessorTypeGPU {
ctx.HTML(http.StatusOK, tplGrampusNotebookGPUNew)
} else {
ctx.HTML(http.StatusOK, tplGrampusNotebookNPUNew)
}

}

func GrampusTrainJobGPUNew(ctx *context.Context) { func GrampusTrainJobGPUNew(ctx *context.Context) {
ctx.Data["IsCreate"] = true ctx.Data["IsCreate"] = true
err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
@@ -72,57 +95,262 @@ func GrampusTrainJobNPUNew(ctx *context.Context) {
} }
ctx.HTML(200, tplGrampusTrainJobNPUNew) ctx.HTML(200, tplGrampusTrainJobNPUNew)
} }
func GrampusNotebookCreate(ctx *context.Context, form auth.CreateGrampusNotebookForm) {
ctx.Data["IsCreate"] = true
displayJobName := form.DisplayJobName
jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
uuid := form.Attachment
description := form.Description
repo := ctx.Repo.Repository
branchName := form.BranchName
image := strings.TrimSpace(form.Image)


func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error {
codeStoragePath := setting.CBCodePathPrefix + jobName + cloudbrain.CodeMountPath + "/"

tpl := tplGrampusNotebookGPUNew
processType := grampus.ProcessorTypeGPU
computeSource := models.GPUResource
computeSourceSimple := models.GPU
if form.Type == 1 {
tpl = tplGrampusNotebookNPUNew
processType = grampus.ProcessorTypeNPU
computeSource = models.NPUResource
computeSourceSimple = models.NPU
codeStoragePath = grampus.JobPath + jobName + modelarts.CodePath
}

lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeDebug), displayJobName))
defer lock.UnLock()
isOk, err := lock.Lock(models.CloudbrainKeyDuration)
if !isOk {
log.Error("lock processed failed:%v", err, ctx.Data["MsgID"])
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tpl, &form)
return
}

if !jobNamePattern.MatchString(displayJobName) {
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
return
}

//check count limit
count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeC2Net, string(models.JobTypeDebug), computeSource)
if err != nil {
log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"])
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr("system error", tpl, &form)
return
} else {
if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr("you have already a running or waiting task, can not create more", tpl, &form)
return
}
}

//check whether the task name in the project is duplicated
tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeDebug), displayJobName)
if err == nil {
if len(tasks) != 0 {
log.Error("the job name did already exist", ctx.Data["MsgID"])
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr("the job name did already exist", tpl, &form)
return
}
} else {
if !models.IsErrJobNotExist(err) {
log.Error("system error, %v", err, ctx.Data["MsgID"])
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr("system error", tpl, &form)
return
}
}

//check specification
spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{
JobType: models.JobTypeDebug,
ComputeResource: computeSourceSimple,
Cluster: models.C2NetCluster,
})
if err != nil || spec == nil {
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr("Resource specification not available", tpl, &form)
return
}

if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) {
log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID)
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tpl, &form)
return
}

var datasetInfos map[string]models.DatasetInfo
var datasetNames string
//var
if uuid != "" {
datasetInfos, datasetNames, err = models.GetDatasetInfo(uuid, computeSourceSimple)
if err != nil {
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
return
}
}

//prepare code and out path
codeLocalPath := setting.JobPath + jobName + cloudbrain.CodeMountPath + "/"
_, err = ioutil.ReadDir(codeLocalPath)
if err == nil {
os.RemoveAll(codeLocalPath)
}

if err := downloadZipCode(ctx, codeLocalPath, branchName); err != nil {
log.Error("downloadZipCode failed, server timed out: %s (%v)", repo.FullName(), err)
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form)
return
}

if processType == grampus.ProcessorTypeGPU {
if err := uploadCodeToMinio(codeLocalPath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil {
log.Error("Failed to uploadCodeToMinio: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"])
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form)
return
}

} else {

if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil {
log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err)
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form)
return
}
}

commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)

req := &grampus.GenerateNotebookJobReq{
JobName: jobName,
DisplayJobName: displayJobName,
ComputeResource: computeSource,
ProcessType: processType,
ImageUrl: image,
ImageId: form.ImageID,
Description: description,
Uuid: uuid,
CommitID: commitID,
BranchName: branchName,
DatasetNames: datasetNames,
DatasetInfos: datasetInfos,
Spec: spec,
CodeStoragePath: codeStoragePath,
CodeName: strings.ToLower(repo.Name),
}

if form.ModelName != "" { //使用预训练模型训练

_, err := models.QueryModelByPath(form.PreTrainModelUrl)
if err != nil {
log.Error("Can not find model", err)
grampusNotebookNewDataPrepare(ctx, processType)
ctx.RenderWithErr(ctx.Tr("repo.modelconvert.manage.model_not_exist"), tpl, &form)
return
}
req.ModelName = form.ModelName
req.LabelName = form.LabelName
req.CkptName = form.CkptName
req.ModelVersion = form.ModelVersion
req.PreTrainModelUrl = form.PreTrainModelUrl
req.PreTrainModelPath = getPreTrainModelPath(form.PreTrainModelUrl, form.CkptName)

}

_, err = grampus.GenerateNotebookJob(ctx, req)
if err != nil {
log.Error("GenerateNotebookJob failed:%v", err.Error(), ctx.Data["MsgID"])
grampusTrainJobNewDataPrepare(ctx, processType)
ctx.RenderWithErr(err.Error(), tpl, &form)
return
}
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all")
}
func grampusNotebookNewDataPrepare(ctx *context.Context, processType string) error {
ctx.Data["PageIsCloudBrain"] = true ctx.Data["PageIsCloudBrain"] = true


var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name)
ctx.Data["display_job_name"] = displayJobName ctx.Data["display_job_name"] = displayJobName


//get valid images //get valid images
images, err := grampus.GetImages(processType)
if processType == grampus.ProcessorTypeNPU {
images, err := grampus.GetImages(processType, string(models.JobTypeDebug))
if err != nil {
log.Error("GetImages failed:", err.Error())
} else {
ctx.Data["images"] = images.Infos
}
}
//prepare available specs
computeResourceSimple := models.GPU
datasetType := models.TypeCloudBrainOne
computeResource := models.GPUResource
if processType == grampus.ProcessorTypeNPU {
computeResourceSimple = models.NPU
datasetType = models.TypeCloudBrainTwo
computeResource = models.NPUResource
}

prepareGrampusSpecs(ctx, computeResourceSimple, models.JobTypeDebug)

//get branches
branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
if err != nil { if err != nil {
log.Error("GetImages failed:", err.Error())
log.Error("GetBranches error:", err.Error())
} else { } else {
ctx.Data["images"] = images.Infos
ctx.Data["branches"] = branches
} }


grampus.InitSpecialPool()
ctx.Data["branchName"] = ctx.Repo.BranchName


ctx.Data["GPUEnabled"] = true
ctx.Data["NPUEnabled"] = true
includeCenters := make(map[string]struct{})
excludeCenters := make(map[string]struct{})
if grampus.SpecialPools != nil {
for _, pool := range grampus.SpecialPools.Pools {
if pool.IsExclusive {
if !IsUserInOrgPool(ctx.User.ID, pool) {
ctx.Data[pool.Type+"Enabled"] = false
}
} else {
if strings.Contains(strings.ToLower(processType), strings.ToLower(pool.Type)) {
if IsUserInOrgPool(ctx.User.ID, pool) {
for _, center := range pool.Pool {
includeCenters[center.Queue] = struct{}{}
}
} else {
for _, center := range pool.Pool {
excludeCenters[center.Queue] = struct{}{}
}
ctx.Data["datasetType"] = datasetType
waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeC2Net, computeResource, models.JobTypeDebug)
ctx.Data["WaitCount"] = waitCount
NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeC2Net, string(models.JobTypeDebug), computeResource)
ctx.Data["NotStopTaskCount"] = NotStopTaskCount


}
ctx.Data["code_path"] = cloudbrain.CodeMountPath
ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
ctx.Data["model_path"] = cloudbrain.ModelMountPath


}
return nil
}


}
func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error {
ctx.Data["PageIsCloudBrain"] = true

var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name)
ctx.Data["display_job_name"] = displayJobName

//get valid images
if processType == grampus.ProcessorTypeNPU {
images, err := grampus.GetImages(processType, string(models.JobTypeTrain))
if err != nil {
log.Error("GetImages failed:", err.Error())
} else {
ctx.Data["images"] = images.Infos
} }
} }


//prepare available specs //prepare available specs
if processType == grampus.ProcessorTypeNPU { if processType == grampus.ProcessorTypeNPU {
prepareGrampusTrainSpecs(ctx, models.NPU)
prepareGrampusSpecs(ctx, models.NPU)
} else if processType == grampus.ProcessorTypeGPU { } else if processType == grampus.ProcessorTypeGPU {
prepareGrampusTrainSpecs(ctx, models.GPU)
prepareGrampusSpecs(ctx, models.GPU)
} }


//get branches //get branches
@@ -201,55 +429,19 @@ func GrampusTrainJobVersionNew(ctx *context.Context) {
} }
} }


func prepareGrampusTrainSpecs(ctx *context.Context, computeResource string) {
func prepareGrampusSpecs(ctx *context.Context, computeResource string, jobType ...models.JobType) {
tempJobType := models.JobTypeTrain
if len(jobType) > 0 {
tempJobType = jobType[0]
}
noteBookSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{ noteBookSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{
JobType: models.JobTypeTrain,
JobType: tempJobType,
ComputeResource: computeResource, ComputeResource: computeResource,
Cluster: models.C2NetCluster, Cluster: models.C2NetCluster,
}) })
ctx.Data["Specs"] = noteBookSpecs ctx.Data["Specs"] = noteBookSpecs
} }


func getFilterSpecBySpecialPool(specs *models.GetGrampusResourceSpecsResult, includeCenters map[string]struct{}, excludeCenters map[string]struct{}) []models.GrampusSpec {
if len(includeCenters) == 0 && len(excludeCenters) == 0 {
return specs.Infos
}
var grampusSpecs []models.GrampusSpec
for _, info := range specs.Infos {
if isInIncludeCenters(info, includeCenters) || (len(excludeCenters) != 0 && isNotAllInExcludeCenters(info, excludeCenters)) {
grampusSpecs = append(grampusSpecs, info)
}

}
return grampusSpecs
}

func isInIncludeCenters(grampusSpec models.GrampusSpec, centers map[string]struct{}) bool {
for _, center := range grampusSpec.Centers {
if _, ok := centers[center.ID]; ok {
return true
}
}
return false
}
func isNotAllInExcludeCenters(grampusSpec models.GrampusSpec, centers map[string]struct{}) bool {
for _, center := range grampusSpec.Centers {
if _, ok := centers[center.ID]; !ok {
return true
}
}
return false
}

func IsUserInOrgPool(userId int64, pool *models.SpecialPool) bool {
org, _ := models.GetOrgByName(pool.Org)
if org != nil {
isOrgMember, _ := models.IsOrganizationMember(org.ID, userId)
return isOrgMember
}
return false
}

func grampusParamCheckCreateTrainJob(form auth.CreateGrampusTrainJobForm) error { func grampusParamCheckCreateTrainJob(form auth.CreateGrampusTrainJobForm) error {
if !strings.HasSuffix(strings.TrimSpace(form.BootFile), ".py") { if !strings.HasSuffix(strings.TrimSpace(form.BootFile), ".py") {
log.Error("the boot file(%s) must be a python file", form.BootFile) log.Error("the boot file(%s) must be a python file", form.BootFile)
@@ -721,30 +913,64 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
} }


func GetGrampusNotebook(ctx *context.APIContext) {
var (
err error
)

ID := ctx.Params(":id")
job, err := models.GetCloudbrainByID(ID)
if err != nil {
ctx.NotFound("", err)
log.Error("GetCloudbrainByID failed:", err)
return
}

jobAfter, err := cloudbrainTask.SyncGrampusNotebookStatus(job)

aiCenterName := cloudbrainService.GetAiCenterShow(jobAfter.AiCenter, ctx.Context)

if err != nil {
ctx.NotFound(err)
log.Error("Sync cloud brain one status failed:", err)
return
}

ctx.JSON(http.StatusOK, map[string]interface{}{
"ID": ID,
"JobName": jobAfter.JobName,
"JobStatus": jobAfter.Status,
"AiCenter": aiCenterName,
"CreatedTime": jobAfter.CreatedUnix.Format("2006-01-02 15:04:05"),
"CompletedTime": jobAfter.UpdatedUnix.Format("2006-01-02 15:04:05"),
"JobDuration": jobAfter.TrainJobDuration,
})
}

func GrampusStopJob(ctx *context.Context) { func GrampusStopJob(ctx *context.Context) {
var ID = ctx.Params(":jobid")
var ID = ctx.Params(":id")
var resultCode = "0" var resultCode = "0"
var errorMsg = "" var errorMsg = ""
var status = "" var status = ""


task := ctx.Cloudbrain task := ctx.Cloudbrain
for { for {
if task.Status == string(models.GrampusStatusStopped) || task.Status == string(models.GrampusStatusFailed) || task.Status == string(models.GrampusStatusSucceeded) {
if task.Status == models.GrampusStatusStopped || task.Status == models.GrampusStatusFailed || task.Status == models.GrampusStatusSucceeded {
log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
resultCode = "-1" resultCode = "-1"
errorMsg = "system error"
errorMsg = ctx.Tr("cloudbrain.Already_stopped")
break break
} }


res, err := grampus.StopJob(task.JobID)
res, err := grampus.StopJob(task.JobID, task.JobType)
if err != nil { if err != nil {
log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"]) log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
resultCode = strconv.Itoa(res.ErrorCode) resultCode = strconv.Itoa(res.ErrorCode)
errorMsg = res.ErrorMsg
errorMsg = ctx.Tr("cloudbrain.Stopped_failed")
break break
} }
oldStatus := task.Status oldStatus := task.Status
task.Status = string(models.GrampusStatusStopped)
task.Status = getStopJobResponseStatus(res)
if task.EndTime == 0 { if task.EndTime == 0 {
task.EndTime = timeutil.TimeStampNow() task.EndTime = timeutil.TimeStampNow()
} }
@@ -773,6 +999,33 @@ func GrampusStopJob(ctx *context.Context) {
}) })
} }


func getStopJobResponseStatus(res *models.GrampusStopJobResponse) string {
newStatus := models.GrampusStatusStopping
if res.Status != "" {
newStatus = grampus.TransTrainJobStatus(res.Status)
}
return newStatus
}

func GrampusNotebookDel(ctx *context.Context) {
var listType = ctx.Query("listType")
if err := deleteGrampusJob(ctx); err != nil {
log.Error("deleteGrampusJob failed: %v", err, ctx.Data["msgID"])
ctx.ServerError(err.Error(), err)
return
}

var isAdminPage = ctx.Query("isadminpage")
var isHomePage = ctx.Query("ishomepage")
if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
} else if isHomePage == "true" {
ctx.Redirect(setting.AppSubURL + "/cloudbrains")
} else {
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=" + listType)
}
}

func GrampusTrainJobDel(ctx *context.Context) { func GrampusTrainJobDel(ctx *context.Context) {
var listType = ctx.Query("listType") var listType = ctx.Query("listType")
if err := deleteGrampusJob(ctx); err != nil { if err := deleteGrampusJob(ctx); err != nil {
@@ -795,9 +1048,9 @@ func GrampusTrainJobDel(ctx *context.Context) {
func deleteGrampusJob(ctx *context.Context) error { func deleteGrampusJob(ctx *context.Context) error {
task := ctx.Cloudbrain task := ctx.Cloudbrain


if task.Status != string(models.GrampusStatusStopped) && task.Status != string(models.GrampusStatusSucceeded) && task.Status != string(models.GrampusStatusFailed) {
if task.Status != models.GrampusStatusStopped && task.Status != models.GrampusStatusSucceeded && task.Status != models.GrampusStatusFailed {
log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"]) log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
return errors.New("the job has not been stopped")
return errors.New(ctx.Tr("cloudbrain.Not_Stopped"))
} }


err := models.DeleteJob(task) err := models.DeleteJob(task)
@@ -815,6 +1068,166 @@ func deleteGrampusJob(ctx *context.Context) error {
return nil return nil
} }


type NotebookDataset struct {
DatasetUrl string `json:"dataset_url"`
}

func GrampusNotebookShow(ctx *context.Context) {
ctx.Data["PageIsCloudBrain"] = true

var task *models.Cloudbrain
task, err := models.GetCloudbrainByIDWithDeleted(ctx.Params(":id"))
if err != nil {
log.Error("GetCloudbrainByID failed:" + err.Error())
ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
return
}
task.ContainerIp = ""

if task.DeletedAt.IsZero() && cloudbrainTask.IsTaskNotStop(task) { //normal record
result, err := grampus.GetNotebookJob(task.JobID)
if err != nil {
log.Error("GetJob failed:" + err.Error())
ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
return
}

if result != nil {
if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 {
task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
}
oldStatus := task.Status
task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
if task.Status != oldStatus || task.Status == models.GrampusStatusRunning {
task.Duration = result.JobInfo.RunSec
if task.Duration < 0 {
task.Duration = 0
}
task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)

if task.StartTime == 0 && result.JobInfo.StartedAt > 0 {
task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
}
if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
task.EndTime = task.StartTime.Add(task.Duration)
}
task.CorrectCreateUnix()
if oldStatus != task.Status {
notification.NotifyChangeCloudbrainStatus(task, oldStatus)
if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource {
if len(result.JobInfo.Tasks[0].CenterID) == 1 {
urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID))
}
}
}
}
err = models.UpdateJob(task)
if err != nil {
log.Error("UpdateJob failed:" + err.Error())
}
}
}

if len(task.Parameters) > 0 {
var parameters models.Parameters
err := json.Unmarshal([]byte(task.Parameters), &parameters)
if err != nil {
log.Error("Failed to Unmarshal Parameters: %s (%v)", task.Parameters, err)
ctx.ServerError("system error", err)
return
}

if len(parameters.Parameter) > 0 {
paramTemp := ""
for _, Parameter := range parameters.Parameter {
param := Parameter.Label + " = " + Parameter.Value + "; "
paramTemp = paramTemp + param
}
task.Parameters = paramTemp[:len(paramTemp)-2]
} else {
task.Parameters = ""
}
}
user, err := models.GetUserByID(task.UserID)
if err == nil {
task.User = user
}

prepareSpec4Show(ctx, task)

ctx.Data["task"] = task
ctx.Data["datasetDownload"] = getDatasetDownloadInfo(ctx, task)
ctx.Data["modelDownload"] = getModelDownloadInfo(ctx, task)
ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task)
ctx.Data["ai_center"] = cloudbrainService.GetAiCenterShow(task.AiCenter, ctx)
ctx.Data["code_path"] = cloudbrain.CodeMountPath
ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
ctx.Data["model_path"] = cloudbrain.ModelMountPath
ctx.HTML(http.StatusOK, tplGrampusNotebookShow)
}

func getDatasetDownloadInfo(ctx *context.Context, task *models.Cloudbrain) []*models.DatasetDownload {
datasetDownload := make([]*models.DatasetDownload, 0)
if ctx.IsSigned {
if task.Uuid != "" && task.UserID == ctx.User.ID {
if task.IsGPUTask() {
return GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false)
} else {
datasetDownload = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false)
datasetObsUrlList := make([]NotebookDataset, 0)
_ = json.Unmarshal([]byte(task.DataUrl), &datasetObsUrlList)

for _, datasetInfo := range datasetDownload {

for _, datasetObs := range datasetObsUrlList {
log.Info("datasetObsUrl:" + datasetObs.DatasetUrl + "datasetName:" + datasetInfo.DatasetName)
if strings.Contains(datasetObs.DatasetUrl, datasetInfo.DatasetName) {
datasetInfo.DatasetDownloadLink = datasetObs.DatasetUrl
break
}
}

}

}

}
}

return datasetDownload
}

func getModelDownloadInfo(ctx *context.Context, task *models.Cloudbrain) *models.ModelDownload {
var modelDownload models.ModelDownload
if ctx.IsSigned {
if task.ModelName != "" && task.UserID == ctx.User.ID {
if task.IsNPUTask() {
modelDownload = models.ModelDownload{
Name: task.CkptName,
DownloadLink: "",
IsDelete: false,
}
if !HasModelFile(task) {
modelDownload.IsDelete = true
}
datasetObsUrlList := make([]NotebookDataset, 0)
_ = json.Unmarshal([]byte(task.DataUrl), &datasetObsUrlList)
for _, datasetObs := range datasetObsUrlList {
if strings.Contains(datasetObs.DatasetUrl, task.CkptName) {
modelDownload.DownloadLink = datasetObs.DatasetUrl
break
}
}

}

}

}

return &modelDownload
}

func GrampusTrainJobShow(ctx *context.Context) { func GrampusTrainJobShow(ctx *context.Context) {
ctx.Data["PageIsCloudBrain"] = true ctx.Data["PageIsCloudBrain"] = true


@@ -826,7 +1239,7 @@ func GrampusTrainJobShow(ctx *context.Context) {
return return
} }
task.ContainerIp = "" task.ContainerIp = ""
task.User, _ = models.GetUserByID(task.UserID)
if task.DeletedAt.IsZero() { //normal record if task.DeletedAt.IsZero() { //normal record
result, err := grampus.GetJob(task.JobID) result, err := grampus.GetJob(task.JobID)
if err != nil { if err != nil {
@@ -895,6 +1308,7 @@ func GrampusTrainJobShow(ctx *context.Context) {
taskList := make([]*models.Cloudbrain, 0) taskList := make([]*models.Cloudbrain, 0)
taskList = append(taskList, task) taskList = append(taskList, task)
prepareSpec4Show(ctx, task) prepareSpec4Show(ctx, task)
ctx.Data["version_list_task"] = taskList ctx.Data["version_list_task"] = taskList
ctx.Data["datasetDownload"] = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false) ctx.Data["datasetDownload"] = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false)
ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task) ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task)
@@ -1158,3 +1572,176 @@ func HandleTaskWithAiCenter(ctx *context.Context) {
r["updateCounts"] = updateCounts r["updateCounts"] = updateCounts
ctx.JSON(http.StatusOK, response.SuccessWithData(r)) ctx.JSON(http.StatusOK, response.SuccessWithData(r))
} }

func GrampusNotebookDebug(ctx *context.Context) {

result, err := grampus.GetNotebookJob(ctx.Cloudbrain.JobID)

if err != nil {
ctx.RenderWithErr(err.Error(), tplDebugJobIndex, nil)
return
}
if len(result.JobInfo.Tasks) > 0 {

ctx.Redirect(result.JobInfo.Tasks[0].Url + "?token=" + result.JobInfo.Tasks[0].Token)
return
}
ctx.NotFound("Can not find the job.", nil)

}

func GrampusNotebookRestart(ctx *context.Context) {
var id = ctx.Params(":id")
var resultCode = "-1"
var errorMsg = ""
var status = ""
var spec *models.Specification

task := ctx.Cloudbrain
if ctx.Written() {
return
}

for {

if task.Status != models.GrampusStatusStopped && task.Status != models.GrampusStatusSucceeded && task.Status != models.GrampusStatusFailed {
log.Error("the job(%s) is not stopped", task.JobName, ctx.Data["MsgID"])
errorMsg = "the job is not stopped"
break
}

count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeC2Net, string(models.JobTypeDebug), task.ComputeResource)

if err != nil {
log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"])
errorMsg = "system error"
break
} else {
if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
resultCode = "2"
errorMsg = ctx.Tr("repo.cloudbrain.morethanonejob")
break
}
}

oldSpec, err := resource.GetCloudbrainSpec(task.ID)
if err != nil || oldSpec == nil {
log.Error("NotebookManage GetCloudbrainSpec error.%v", err)
errorMsg = "Resource specification not available"
break
}

computeSourceSimple := models.GPU
action := models.ActionCreateGrampusGPUDebugTask
if task.ComputeResource == models.NPUResource {
computeSourceSimple = models.NPU
action = models.ActionCreateGrampusNPUDebugTask
}
spec, err = resource.GetAndCheckSpec(ctx.User.ID, oldSpec.ID, models.FindSpecsOptions{
JobType: models.JobType(task.JobType),
ComputeResource: computeSourceSimple,
Cluster: models.C2NetCluster,
})
if err != nil || spec == nil {
log.Error("NotebookManage GetAndCheckSpec error.task.id = %d", task.ID)
errorMsg = "Resource specification not support any more"
break
}
if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) {
log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID)
errorMsg = ctx.Tr("points.insufficient_points_balance")
break
}
if task.IsGPUTask() {
if _, err := os.Stat(getOldJobPath(task)); err != nil {
log.Error("Can not find job minio path", err)
resultCode = "-1"
errorMsg = ctx.Tr("cloudbrain.result_cleared")
break
}
}

if !HasModelFile(task) { //使用预训练模型训练
errorMsg = ctx.Tr("repo.debug.manage.model_not_exist")
break
}
if hasDatasetDeleted(task) {
errorMsg = ctx.Tr("repo.debug.manage.dataset_not_exist")
break
}

createTime := timeutil.TimeStampNow()

res, err := grampus.RestartNotebookJob(task.JobID)
if err != nil {
log.Error("ManageNotebook2(%s) failed:%v", task.DisplayJobName, err.Error(), ctx.Data["MsgID"])
errorMsg = ctx.Tr("repo.debug_again_fail")
break
}

if res.GrampusResult.ErrorCode != 0 || res.NewId == "" {
log.Error("ManageNotebook2 failed:" + res.GrampusResult.ErrorMsg)
errorMsg = ctx.Tr("repo.debug_again_fail")
if res.GrampusResult.ErrorCode == 5005 {
errorMsg = ctx.Tr("repo.debug_again_fail_forever")
}

break
}

newTask := &models.Cloudbrain{
Status: res.Status,
UserID: task.UserID,
RepoID: task.RepoID,
JobID: res.NewId,
JobName: task.JobName,
DisplayJobName: task.DisplayJobName,
JobType: task.JobType,
Type: task.Type,
Uuid: task.Uuid,
Image: task.Image,
ImageID: task.ImageID,
EngineID: task.EngineID,
CommitID: task.CommitID,
EngineName: task.EngineName,
IsLatestVersion: "1",
BranchName: task.BranchName,
DatasetName: task.DatasetName,
ComputeResource: task.ComputeResource,
Description: task.Description,
CreatedUnix: createTime,
UpdatedUnix: createTime,
Spec: spec,
ModelName: task.ModelName,
ModelVersion: task.ModelVersion,
LabelName: task.LabelName,
PreTrainModelUrl: task.PreTrainModelUrl,
CkptName: task.CkptName,
WorkServerNumber: 1,
}

err = models.RestartCloudbrain(task, newTask)
if err != nil {
log.Error("RestartCloudbrain(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"])
errorMsg = "system error"
break
}

id = strconv.FormatInt(newTask.ID, 10)

status = res.Status
resultCode = "0"

notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, id, newTask.DisplayJobName, action)

break
}

ctx.JSON(200, map[string]string{
"result_code": resultCode,
"error_msg": errorMsg,
"status": status,
"id": id,
})
}

+ 170
- 38
routers/repo/modelarts.go View File

@@ -218,6 +218,22 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm
return return
} }
} }

var datasetInfos map[string]models.DatasetInfo
var attachSize int
if uuid != "" {
datasetInfos, _, err = models.GetDatasetInfo(uuid)
for _, infos := range datasetInfos {
attachSize += infos.Size
}
if attachSize > int(setting.DebugAttachSize*1000*1000*1000) {
log.Error("The DatasetSize exceeds the limit (%d)", int(setting.DebugAttachSize)) //GB
notebookNewDataPrepare(ctx)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize", int(setting.DebugAttachSize*1000*1000*1000)), tplModelArtsNotebookNew, &form)
return
}
}

var aiCenterCode = models.AICenterOfCloudBrainTwo var aiCenterCode = models.AICenterOfCloudBrainTwo
if setting.ModelartsCD.Enabled { if setting.ModelartsCD.Enabled {
aiCenterCode = models.AICenterOfChengdu aiCenterCode = models.AICenterOfChengdu
@@ -239,10 +255,37 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm
return return
} }


req := cloudbrain.GenerateModelArtsNotebookReq{
DisplayJobName: displayJobName,
JobName: jobName,
Description: description,
Uuid: uuid,
ImageId: imageId,
Spec: spec,
BootFile: "",
AutoStopDurationMs: modelarts.AutoStopDurationMs,
}

if form.ModelName != "" { //使用预训练模型训练
_, err := models.QueryModelByPath(form.PreTrainModelUrl)
if err != nil {
log.Error("Can not find model", err)
notebookNewDataPrepare(ctx)
ctx.RenderWithErr(ctx.Tr("repo.modelconvert.manage.model_not_exist"), tplModelArtsNotebookNew, &form)
return
}
req.ModelName = form.ModelName
req.LabelName = form.LabelName
req.CkptName = form.CkptName
req.ModelVersion = form.ModelVersion
req.PreTrainModelUrl = form.PreTrainModelUrl

}

if setting.ModelartsCD.Enabled { if setting.ModelartsCD.Enabled {
_, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, uuid, description, imageId, spec, "", modelarts.AutoStopDurationMs)
_, err = modelarts_cd.GenerateNotebook(ctx, req)
} else { } else {
_, err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, uuid, description, imageId, spec, "", modelarts.AutoStopDurationMs)
_, err = modelarts.GenerateNotebook2(ctx, req)
} }


if err != nil { if err != nil {
@@ -279,11 +322,17 @@ func NotebookShow(ctx *context.Context) {


} }


datasetDownload := make([]models.DatasetDownload, 0)
datasetDownload := make([]*models.DatasetDownload, 0)
var modelDownload models.ModelDownload
if ctx.IsSigned { if ctx.IsSigned {
if task.Uuid != "" && task.UserID == ctx.User.ID { if task.Uuid != "" && task.UserID == ctx.User.ID {
datasetDownload = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, true) datasetDownload = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, true)
} }
if task.ModelName != "" && task.UserID == ctx.User.ID {
modelDownload = GetModelDownload(task)

}

} }
user, err := models.GetUserByID(task.UserID) user, err := models.GetUserByID(task.UserID)
if err == nil { if err == nil {
@@ -304,6 +353,7 @@ func NotebookShow(ctx *context.Context) {
} }
ctx.Data["duration"] = task.TrainJobDuration ctx.Data["duration"] = task.TrainJobDuration
ctx.Data["datasetDownload"] = datasetDownload ctx.Data["datasetDownload"] = datasetDownload
ctx.Data["modelDownload"] = modelDownload
ctx.Data["task"] = task ctx.Data["task"] = task
ctx.Data["ID"] = ID ctx.Data["ID"] = ID
ctx.Data["jobName"] = task.JobName ctx.Data["jobName"] = task.JobName
@@ -311,8 +361,25 @@ func NotebookShow(ctx *context.Context) {
ctx.HTML(200, tplModelArtsNotebookShow) ctx.HTML(200, tplModelArtsNotebookShow)
} }


func GetCloudBrainDataSetInfo(uuid string, datasetname string, isNeedDown bool) []models.DatasetDownload {
datasetDownload := make([]models.DatasetDownload, 0)
func GetModelDownload(task *models.Cloudbrain) models.ModelDownload {
index := strings.Index(task.PreTrainModelUrl, "/")
key := task.PreTrainModelUrl[index+1:] + task.CkptName
url, _ := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key)
modelDownload := models.ModelDownload{
Name: task.CkptName,
DownloadLink: url,
IsDelete: false,
}

if !HasModelFile(task) {
log.Warn("Can not get model by path:" + task.PreTrainModelUrl)
modelDownload.IsDelete = true
}
return modelDownload
}

func GetCloudBrainDataSetInfo(uuid string, datasetname string, isNeedDown bool) []*models.DatasetDownload {
datasetDownload := make([]*models.DatasetDownload, 0)
if len(uuid) == 0 { if len(uuid) == 0 {
return datasetDownload return datasetDownload
} }
@@ -349,7 +416,7 @@ func GetCloudBrainDataSetInfo(uuid string, datasetname string, isNeedDown bool)
} }
} }


datasetDownload = append(datasetDownload, models.DatasetDownload{
datasetDownload = append(datasetDownload, &models.DatasetDownload{
DatasetName: name, DatasetName: name,
DatasetDownloadLink: url, DatasetDownloadLink: url,
RepositoryLink: link, RepositoryLink: link,
@@ -388,9 +455,13 @@ func NotebookDebug2(ctx *context.Context) {
ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil) ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil)
return return
} }
if task.BootFile != "" {
ctx.Redirect(getFileUrl(result.Url, task.BootFile) + "?token=" + result.Token)

if ctx.QueryTrim("file") != "" {
ctx.Redirect(getFileUrl(result.Url, ctx.QueryTrim("file")) + "?token=" + result.Token)
} else { } else {
if task.BootFile != "" {
go cloudbrainTask.UploadNotebookFiles(task)
}
ctx.Redirect(result.Url + "?token=" + result.Token) ctx.Redirect(result.Url + "?token=" + result.Token)
} }


@@ -412,7 +483,7 @@ func getFileUrl(url string, filename string) string {
} }
} }


return url + middle + path.Base(filename)
return url + middle + filename
} }


func NotebookRestart(ctx *context.Context) { func NotebookRestart(ctx *context.Context) {
@@ -476,6 +547,16 @@ func NotebookRestart(ctx *context.Context) {
errorMsg = ctx.Tr("points.insufficient_points_balance") errorMsg = ctx.Tr("points.insufficient_points_balance")
break break
} }
if !HasModelFile(task) { //使用预训练模型训练
errorMsg = ctx.Tr("repo.debug.manage.model_not_exist")
break
}

if hasDatasetDeleted(task) {
errorMsg = ctx.Tr("repo.debug.manage.dataset_not_exist")
break
}

createTime := timeutil.TimeStampNow() createTime := timeutil.TimeStampNow()
param := models.NotebookAction{ param := models.NotebookAction{
Action: models.ActionStart, Action: models.ActionStart,
@@ -511,21 +592,26 @@ func NotebookRestart(ctx *context.Context) {
} }


newTask := &models.Cloudbrain{ newTask := &models.Cloudbrain{
Status: res.Status,
UserID: task.UserID,
RepoID: task.RepoID,
JobID: task.JobID,
JobName: task.JobName,
DisplayJobName: task.DisplayJobName,
JobType: task.JobType,
Type: task.Type,
Uuid: task.Uuid,
Image: task.Image,
ComputeResource: task.ComputeResource,
Description: task.Description,
CreatedUnix: createTime,
UpdatedUnix: createTime,
Spec: spec,
Status: res.Status,
UserID: task.UserID,
RepoID: task.RepoID,
JobID: task.JobID,
JobName: task.JobName,
DisplayJobName: task.DisplayJobName,
JobType: task.JobType,
Type: task.Type,
Uuid: task.Uuid,
Image: task.Image,
ComputeResource: task.ComputeResource,
Description: task.Description,
CreatedUnix: createTime,
UpdatedUnix: createTime,
Spec: spec,
ModelName: task.ModelName,
ModelVersion: task.ModelVersion,
LabelName: task.LabelName,
PreTrainModelUrl: task.PreTrainModelUrl,
CkptName: task.CkptName,
} }


err = models.RestartCloudbrain(task, newTask) err = models.RestartCloudbrain(task, newTask)
@@ -564,21 +650,11 @@ func NotebookStop(ctx *context.Context) {
if task.Status != string(models.ModelArtsRunning) { if task.Status != string(models.ModelArtsRunning) {
log.Error("the job(%s) is not running", task.JobName, ctx.Data["MsgID"]) log.Error("the job(%s) is not running", task.JobName, ctx.Data["MsgID"])
resultCode = "-1" resultCode = "-1"
errorMsg = "the job is not running"
errorMsg = ctx.Tr("cloudbrain.Already_stopped")
break break
} }


param := models.NotebookAction{
Action: models.ActionStop,
}

var err error
var res *models.NotebookActionResult
if task.Type == models.TypeCloudBrainTwo {
res, err = modelarts.ManageNotebook2(task.JobID, param)
} else if task.Type == models.TypeCDCenter {
res, err = modelarts_cd.ManageNotebook(task.JobID, param)
}
err, res := StopModelArtsNotebook(task)


if err != nil { if err != nil {
log.Error("ManageNotebook2(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"]) log.Error("ManageNotebook2(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"])
@@ -619,6 +695,21 @@ func NotebookStop(ctx *context.Context) {
}) })
} }


func StopModelArtsNotebook(task *models.Cloudbrain) (error, *models.NotebookActionResult) {
param := models.NotebookAction{
Action: models.ActionStop,
}

var err error
var res *models.NotebookActionResult
if task.Type == models.TypeCloudBrainTwo {
res, err = modelarts.ManageNotebook2(task.JobID, param)
} else if task.Type == models.TypeCDCenter {
res, err = modelarts_cd.ManageNotebook(task.JobID, param)
}
return err, res
}

func NotebookDel(ctx *context.Context) { func NotebookDel(ctx *context.Context) {
var listType = ctx.Query("debugListType") var listType = ctx.Query("debugListType")
task := ctx.Cloudbrain task := ctx.Cloudbrain
@@ -1791,7 +1882,7 @@ func TrainJobShow(ctx *context.Context) {
return return
} }
ctx.Data["canNewJob"] = canNewJob ctx.Data["canNewJob"] = canNewJob
datasetList := make([][]models.DatasetDownload, 0)
datasetList := make([][]*models.DatasetDownload, 0)
//将运行参数转化为epoch_size = 3, device_target = Ascend的格式 //将运行参数转化为epoch_size = 3, device_target = Ascend的格式
for i, task := range VersionListTasks { for i, task := range VersionListTasks {


@@ -2518,7 +2609,8 @@ func inferenceJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModel
ctx.Data["datasetType"] = models.TypeCloudBrainTwo ctx.Data["datasetType"] = models.TypeCloudBrainTwo
waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "")
ctx.Data["WaitCount"] = waitCount ctx.Data["WaitCount"] = waitCount

NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeInference))
ctx.Data["NotStopTaskCount"] = NotStopTaskCount
return nil return nil
} }
func InferenceJobShow(ctx *context.Context) { func InferenceJobShow(ctx *context.Context) {
@@ -2582,6 +2674,46 @@ func InferenceJobShow(ctx *context.Context) {
ctx.HTML(http.StatusOK, tplModelArtsInferenceJobShow) ctx.HTML(http.StatusOK, tplModelArtsInferenceJobShow)
} }


func MultiModelDownload(ctx *context.Context) {
var (
err error
)
jobID := ctx.Params(":jobid")
versionName := ctx.Query("version_name")
parentDir := ctx.Query("parent_dir")

task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
if err != nil {
log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", task.JobName, err.Error())
return
}

if task.ComputeResource == models.NPUResource {
path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir), "/")
path = strings.TrimSuffix(path, "/")
path += "/"
allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path)
if err == nil {
returnFileName := task.DisplayJobName + ".zip"
ObsDownloadManyFile(path, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
}
} else if task.ComputeResource == models.GPUResource {
filePath := setting.CBCodePathPrefix + task.JobName + cloudbrain.ModelMountPath + "/" + parentDir
allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath)
if err == nil {
returnFileName := task.DisplayJobName + ".zip"
MinioDownloadManyFile(filePath, ctx, returnFileName, allFile)
} else {
log.Info("error,msg=" + err.Error())
ctx.ServerError("no file to download.", err)
}
}

}

func ModelDownload(ctx *context.Context) { func ModelDownload(ctx *context.Context) {
var ( var (
err error err error


+ 11
- 2
routers/repo/repo_statistic.go View File

@@ -75,7 +75,7 @@ func RepoStatisticDaily(date string) {
if repo.NumIssues != 0 { if repo.NumIssues != 0 {
issueFixedRate = float32(repo.NumClosedIssues) / float32(repo.NumIssues) issueFixedRate = float32(repo.NumClosedIssues) / float32(repo.NumIssues)
} else { } else {
issueFixedRate = 1.0
issueFixedRate = float32(setting.RadarMap.ProjectHealth0IssueCloseRatio)
} }


var numVersions int64 var numVersions int64
@@ -124,7 +124,7 @@ func RepoStatisticDaily(date string) {
NumDevMonths: numDevMonths, NumDevMonths: numDevMonths,
RepoSize: repo.Size, RepoSize: repo.Size,
DatasetSize: datasetSize, DatasetSize: datasetSize,
NumModels: 0,
NumModels: repo.ModelCnt,
NumWikiViews: numWikiViews, NumWikiViews: numWikiViews,
NumCommits: numCommits, NumCommits: numCommits,
NumIssues: int64(repo.NumIssues), NumIssues: int64(repo.NumIssues),
@@ -135,6 +135,9 @@ func RepoStatisticDaily(date string) {
NumCommitsGrowth: numCommitsGrowth, NumCommitsGrowth: numCommitsGrowth,
NumCommitLinesGrowth: numCommitLinesGrowth, NumCommitLinesGrowth: numCommitLinesGrowth,
NumContributorsGrowth: numContributorsGrowth, NumContributorsGrowth: numContributorsGrowth,
NumCloudbrain: repo.AiTaskCnt,
NumDatasetFile: repo.DatasetCnt,
NumModelConvert: models.QueryModelConvertCountByRepoID(repo.ID),
} }


dayBeforeDate := t.AddDate(0, 0, -1).Format("2006-01-02") dayBeforeDate := t.AddDate(0, 0, -1).Format("2006-01-02")
@@ -155,6 +158,10 @@ func RepoStatisticDaily(date string) {
repoStat.NumIssuesAdded = repoStat.NumIssues - repoStatisticBefore.NumIssues repoStat.NumIssuesAdded = repoStat.NumIssues - repoStatisticBefore.NumIssues
repoStat.NumPullsAdded = repoStat.NumPulls - repoStatisticBefore.NumPulls repoStat.NumPullsAdded = repoStat.NumPulls - repoStatisticBefore.NumPulls
repoStat.NumContributorAdded = repoStat.NumContributor - repoStatisticBefore.NumContributor repoStat.NumContributorAdded = repoStat.NumContributor - repoStatisticBefore.NumContributor
repoStat.NumModelsAdded = repoStat.NumModels - repoStatisticBefore.NumModels
repoStat.NumCloudbrainAdded = repoStat.NumCloudbrain - repoStatisticBefore.NumCloudbrain
repoStat.NumModelConvertAdded = repoStat.NumModelConvert - repoStatisticBefore.NumModelConvert
repoStat.NumDatasetFileAdded = repoStat.NumDatasetFile - repoStatisticBefore.NumDatasetFile
} }
} }
day4MonthsAgo := t.AddDate(0, -4, 0) day4MonthsAgo := t.AddDate(0, -4, 0)
@@ -166,6 +173,8 @@ func RepoStatisticDaily(date string) {
repoStat.NumIssuesGrowth = repoStat.NumIssues - repoStatisticFourMonthsAgo.NumIssues repoStat.NumIssuesGrowth = repoStat.NumIssues - repoStatisticFourMonthsAgo.NumIssues
} }


models.SyncStatDataToRepo(repo)

if _, err = models.InsertRepoStat(&repoStat); err != nil { if _, err = models.InsertRepoStat(&repoStat); err != nil {
log.Error("InsertRepoStat failed(%s): %v", projectName, err) log.Error("InsertRepoStat failed(%s): %v", projectName, err)
log.Error("failed statistic: %s", projectName) log.Error("failed statistic: %s", projectName)


+ 12
- 0
routers/repo/user_data_analysis.go View File

@@ -21,6 +21,7 @@ import (
const ( const (
PAGE_SIZE = 2000 PAGE_SIZE = 2000
Excel_File_Path = "/useranalysis/" Excel_File_Path = "/useranalysis/"
USER_YEAR = 2022
) )


func getUserMetricsExcelHeader(ctx *context.Context) map[string]string { func getUserMetricsExcelHeader(ctx *context.Context) map[string]string {
@@ -104,6 +105,7 @@ func getExcelHeader(ctx *context.Context) map[string]string {
excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime")) excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime"))
excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum")) excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum"))
excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount")) excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount"))
excelHeader = append(excelHeader, ctx.Tr("user.static.ModelConvertCount"))


excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser")) excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser"))
excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset")) excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset"))
@@ -178,6 +180,8 @@ func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *mode
tmp = tmp + 1 tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount)
tmp = tmp + 1 tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser) xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser)
tmp = tmp + 1 tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset) xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset)
@@ -256,6 +260,8 @@ func writeExcelPage(row int, xlsx *excelize.File, sheetName string, userRecord *
tmp = tmp + 1 tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount) xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CommitModelCount)
tmp = tmp + 1 tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.ModelConvertCount)
tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser) xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.FocusOtherUser)
tmp = tmp + 1 tmp = tmp + 1
xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset) xlsx.SetCellValue(sheetName, getColumn(tmp)+rows, userRecord.CollectDataset)
@@ -714,6 +720,12 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) {
log.Info("startTime time:" + startTime.Format("2006-01-02 15:04:05")) log.Info("startTime time:" + startTime.Format("2006-01-02 15:04:05"))
log.Info("endTime time:" + endTime.Format("2006-01-02 15:04:05")) log.Info("endTime time:" + endTime.Format("2006-01-02 15:04:05"))
warnEmailMessage := "用户统计信息入库失败,请尽快定位。" warnEmailMessage := "用户统计信息入库失败,请尽快定位。"

startYear := time.Date(USER_YEAR, 1, 1, 0, 0, 0, 1, t.Location())
endYear := startYear.AddDate(1, 0, 0)

models.RefreshUserYearTable(startYear, endYear)

//query wiki data //query wiki data
log.Info("start to time count data") log.Info("start to time count data")
wikiMap, err := queryWikiCountMap(startTime, endTime) wikiMap, err := queryWikiCountMap(startTime, endTime)


+ 33
- 15
routers/routes/routes.go View File

@@ -371,7 +371,18 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/images/custom", repo.GetCustomImages) m.Get("/images/custom", repo.GetCustomImages)
m.Get("/images/star", repo.GetStarImages) m.Get("/images/star", repo.GetStarImages)


m.Get("/repos", routers.ExploreRepos)
m.Group("/repos", func() {
//m.Get("", routers.ExploreRepos)
m.Get("", routers.GetRepoSearchPage)
m.Group("/square", func() {
m.Get("", routers.GetRepoSquarePage)
m.Get("/tab", routers.RepoSquare)
m.Get("/active-user", routers.ActiveUser)
m.Get("/active-org", routers.ActiveOrg)
})

m.Get("/search", routers.RepoFind)
})
m.Get("/datasets", routers.ExploreDatasets) m.Get("/datasets", routers.ExploreDatasets)
m.Get("/users", routers.ExploreUsers) m.Get("/users", routers.ExploreUsers)
m.Get("/organizations", routers.ExploreOrganizations) m.Get("/organizations", routers.ExploreOrganizations)
@@ -1175,6 +1186,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate)
m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
m.Get("/download_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadModel) m.Get("/download_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadModel)
m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel)
}) })
m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainNew) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainNew)
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate)
@@ -1198,6 +1210,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel)
//m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) //m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels)
m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel) m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel)
m.Get("/download_multi_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadMultiModel)
//m.Get("/get_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo.GetLogFromModelDir) //m.Get("/get_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo.GetLogFromModelDir)
//m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) //m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainTrainJobVersionNew) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainTrainJobVersionNew)
@@ -1210,20 +1223,34 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/:jobid", func() { m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.InferenceCloudBrainJobShow) m.Get("", reqRepoCloudBrainReader, repo.InferenceCloudBrainJobShow)
m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadInferenceResult) m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadInferenceResult)
m.Get("/downloadall", repo.DownloadInferenceResultFile)
m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadMultiModel)
m.Get("/downloadall", cloudbrain.AdminOrJobCreaterRightForTrain, repo.DownloadGPUInferenceResultFile)
}) })
m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceCloudBrainJobNew) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceCloudBrainJobNew)
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainInferencForm{}), repo.CloudBrainInferenceJobCreate) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainInferencForm{}), repo.CloudBrainInferenceJobCreate)
}) })
}, context.RepoRef()) }, context.RepoRef())
m.Group("/grampus", func() { m.Group("/grampus", func() {
m.Group("/notebook", func() {
m.Group("/:id", func() {
m.Get("", reqRepoCloudBrainReader, repo.GrampusNotebookShow)
m.Get("/debug", reqWechatBind, cloudbrain.AdminOrJobCreaterRight, repo.GrampusNotebookDebug)
m.Post("/restart", reqWechatBind, cloudbrain.AdminOrJobCreaterRight, repo.GrampusNotebookRestart)
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.GrampusStopJob)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.GrampusNotebookDel)
})

m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.GrampusNotebookNew)
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateGrampusNotebookForm{}), repo.GrampusNotebookCreate)
})

m.Group("/train-job", func() { m.Group("/train-job", func() {
m.Group("/:jobid", func() { m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.GrampusTrainJobShow) m.Get("", reqRepoCloudBrainReader, repo.GrampusTrainJobShow)
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.GrampusStopJob)
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusStopJob)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusTrainJobDel) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusTrainJobDel)
m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload) m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload)
m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.MultiModelDownload)
m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.GrampusTrainJobVersionNew) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.GrampusTrainJobVersionNew)
m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateGrampusTrainJobForm{}), repo.GrampusTrainJobVersionCreate) m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateGrampusTrainJobForm{}), repo.GrampusTrainJobVersionCreate)
}) })
@@ -1291,16 +1318,6 @@ func RegisterRoutes(m *macaron.Macaron) {


m.Group("/modelarts", func() { m.Group("/modelarts", func() {
m.Group("/notebook", func() { m.Group("/notebook", func() {
/* v1.0
m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.NotebookShow)
m.Get("/debug", cloudbrain.AdminOrJobCreaterRight, repo.NotebookDebug)
m.Post("/:action", reqRepoCloudBrainWriter, repo.NotebookManage)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookDel)
})
m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew)
m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate)
*/
m.Group("/:id", func() { m.Group("/:id", func() {
m.Get("", reqRepoCloudBrainReader, repo.NotebookShow) m.Get("", reqRepoCloudBrainReader, repo.NotebookShow)
m.Get("/debug", cloudbrain.AdminOrJobCreaterRight, repo.NotebookDebug2) m.Get("/debug", cloudbrain.AdminOrJobCreaterRight, repo.NotebookDebug2)
@@ -1319,6 +1336,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobStop) m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobStop)
m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobDel) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobDel)
m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload) m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload)
m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.MultiModelDownload)
m.Get("/download_log_file", cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobDownloadLogFile) m.Get("/download_log_file", cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobDownloadLogFile)
m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, context.PointAccount(), repo.TrainJobNewVersion) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, context.PointAccount(), repo.TrainJobNewVersion)
m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
@@ -1334,7 +1352,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/:jobid", func() { m.Group("/:jobid", func() {
m.Get("", reqRepoCloudBrainReader, repo.InferenceJobShow) m.Get("", reqRepoCloudBrainReader, repo.InferenceJobShow)
m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ResultDownload) m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ResultDownload)
m.Get("/downloadall", repo.DownloadMultiResultFile)
m.Get("/downloadall", cloudbrain.AdminOrJobCreaterRightForTrain, repo.DownloadMultiResultFile)
}) })
m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceJobNew) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceJobNew)
m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsInferenceJobForm{}), repo.InferenceJobCreate) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsInferenceJobForm{}), repo.InferenceJobCreate)


+ 1
- 1
routers/user/home.go View File

@@ -779,7 +779,7 @@ func Cloudbrains(ctx *context.Context) {
var jobTypes []string var jobTypes []string
jobTypeNot := false jobTypeNot := false
if jobType == string(models.JobTypeBenchmark) { if jobType == string(models.JobTypeBenchmark) {
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset))
} else if jobType != "all" && jobType != "" { } else if jobType != "all" && jobType != "" {
jobTypes = append(jobTypes, jobType) jobTypes = append(jobTypes, jobType)
} }


+ 14
- 14
services/cloudbrain/clear.go View File

@@ -14,21 +14,21 @@ import (


func ClearCloudbrainResultSpace() { func ClearCloudbrainResultSpace() {
log.Info("clear cloudbrain one result space begin.") log.Info("clear cloudbrain one result space begin.")
if !setting.ClearStrategy.Enabled{
if !setting.ClearStrategy.Enabled {
return return
} }


tasks, err := models.GetCloudBrainOneStoppedNotDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize)
tasks, err := models.GetGPUStoppedNotDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize)
if err != nil { if err != nil {
log.Warn("Failed to get cloudbrain, clear result failed.", err) log.Warn("Failed to get cloudbrain, clear result failed.", err)
return return
} }
debugTasks, err := models.GetCloudBrainOneStoppedDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.DebugJobSize)
debugTasks, err := models.GetGPUStoppedDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.DebugJobSize)
if err != nil { if err != nil {
log.Warn("Failed to get debug cloudbrain.", err) log.Warn("Failed to get debug cloudbrain.", err)


} }
tasks=append(tasks,debugTasks...)
tasks = append(tasks, debugTasks...)


if err != nil { if err != nil {
log.Warn("Failed to get cloudbrain, clear result failed.", err) log.Warn("Failed to get cloudbrain, clear result failed.", err)
@@ -38,7 +38,7 @@ func ClearCloudbrainResultSpace() {
for _, task := range tasks { for _, task := range tasks {
err := DeleteCloudbrainOneJobStorage(task.JobName) err := DeleteCloudbrainOneJobStorage(task.JobName)
if err == nil { if err == nil {
log.Info("clear job in cloudbrain table:"+task.JobName)
log.Info("clear job in cloudbrain table:" + task.JobName)
ids = append(ids, task.ID) ids = append(ids, task.ID)
} }
} }
@@ -69,10 +69,10 @@ func clearMinioHistoryTrashFile() {
SortModTimeAscend(miniofiles) SortModTimeAscend(miniofiles)
for _, file := range miniofiles { for _, file := range miniofiles {


if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
if file.Name() != "" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {


has,err:=models.IsCloudbrainExistByJobName(file.Name())
if err==nil && !has {
has, err := models.IsCloudbrainExistByJobName(file.Name())
if err == nil && !has {
dirPath := setting.CBCodePathPrefix + file.Name() + "/" dirPath := setting.CBCodePathPrefix + file.Name() + "/"
log.Info("clear job in minio trash:" + file.Name()) log.Info("clear job in minio trash:" + file.Name())
storage.Attachments.DeleteDir(dirPath) storage.Attachments.DeleteDir(dirPath)
@@ -90,7 +90,7 @@ func clearMinioHistoryTrashFile() {
} }
} }


func clearLocalHistoryTrashFile() {
func clearLocalHistoryTrashFile() {
files, err := ioutil.ReadDir(setting.JobPath) files, err := ioutil.ReadDir(setting.JobPath)
processCount := 0 processCount := 0
if err != nil { if err != nil {
@@ -99,11 +99,11 @@ func clearLocalHistoryTrashFile() {
SortModTimeAscend(files) SortModTimeAscend(files)
for _, file := range files { for _, file := range files {
//清理n天前的历史垃圾数据,清理job目录 //清理n天前的历史垃圾数据,清理job目录
if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
has,err:=models.IsCloudbrainExistByJobName(file.Name())
if err==nil && !has{
if file.Name() != "" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
has, err := models.IsCloudbrainExistByJobName(file.Name())
if err == nil && !has {
os.RemoveAll(setting.JobPath + file.Name()) os.RemoveAll(setting.JobPath + file.Name())
log.Info("clear job in local trash:"+file.Name())
log.Info("clear job in local trash:" + file.Name())
processCount++ processCount++
} }
if processCount == setting.ClearStrategy.BatchSize { if processCount == setting.ClearStrategy.BatchSize {
@@ -127,7 +127,7 @@ func SortModTimeAscend(files []os.FileInfo) {


func DeleteCloudbrainOneJobStorage(jobName string) error { func DeleteCloudbrainOneJobStorage(jobName string) error {


if jobName==""{
if jobName == "" {
return nil return nil
} }
//delete local //delete local


+ 12
- 2
services/cloudbrain/cloudbrainTask/count.go View File

@@ -34,7 +34,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s
ComputeResource: models.GPUResource, ComputeResource: models.GPUResource,
}, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): {
CloudBrainTypes: []int{models.TypeCloudBrainOne}, CloudBrainTypes: []int{models.TypeCloudBrainOne},
JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet},
JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet, models.JobTypeSnn4Ecoset},
NotFinalStatuses: CloudbrainOneNotFinalStatuses, NotFinalStatuses: CloudbrainOneNotFinalStatuses,
ComputeResource: models.GPUResource, ComputeResource: models.GPUResource,
}, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): {
@@ -62,11 +62,21 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s
JobType: []models.JobType{models.JobTypeTrain}, JobType: []models.JobType{models.JobTypeTrain},
NotFinalStatuses: GrampusNotFinalStatuses, NotFinalStatuses: GrampusNotFinalStatuses,
ComputeResource: models.NPUResource, ComputeResource: models.NPUResource,
}, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.GPUResource: {
CloudBrainTypes: []int{models.TypeC2Net},
JobType: []models.JobType{models.JobTypeDebug},
NotFinalStatuses: GrampusNotFinalStatuses,
ComputeResource: models.GPUResource,
}, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.NPUResource: {
CloudBrainTypes: []int{models.TypeC2Net},
JobType: []models.JobType{models.JobTypeDebug},
NotFinalStatuses: GrampusNotFinalStatuses,
ComputeResource: models.NPUResource,
}} }}


func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) {
jobNewType := jobType jobNewType := jobType
if jobType == string(models.JobTypeSnn4imagenet) || jobType == string(models.JobTypeBrainScore) {
if models.IsModelBenchMarkJobType(jobType) {
jobNewType = string(models.JobTypeBenchmark) jobNewType = string(models.JobTypeBenchmark)
} }




+ 252
- 29
services/cloudbrain/cloudbrainTask/notebook.go View File

@@ -4,6 +4,9 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"path" "path"
"strings"

"code.gitea.io/gitea/modules/notebook"


"code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/modelarts"
"code.gitea.io/gitea/modules/modelarts_cd" "code.gitea.io/gitea/modules/modelarts_cd"
@@ -29,6 +32,9 @@ import (
) )


const NoteBookExtension = ".ipynb" const NoteBookExtension = ".ipynb"
const CPUType = 0
const GPUType = 1
const NPUType = 2


func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) {


@@ -66,7 +72,7 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
} }


//create repo if not exist //create repo if not exist
repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName)
repo, _ := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName)
if repo == nil { if repo == nil {
repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{ repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{
Name: setting.FileNoteBook.ProjectName, Name: setting.FileNoteBook.ProjectName,
@@ -80,17 +86,220 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp
AutoInit: true, AutoInit: true,
DefaultBranch: "master", DefaultBranch: "master",
}) })
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName)))
return
}
} else {

noteBook, _ := models.GetWaitOrRunFileNotebookByRepo(repo.ID, getCloudbrainType(option.Type))
if noteBook != nil {

if isRepoConfilcts(option, noteBook) {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_repo_conflict")))
return
}

if isNotebookSpecMath(option, noteBook) {
if !isRepoMatch(option, noteBook) {
err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName)
if err != nil {
log.Error("download code failed", err)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
return
}
}
if !isRepoFileMatch(option, noteBook) {
noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName)
noteBook.BranchName += ";" + option.BranchName
noteBook.Description += ";" + getDescription(option)

err := models.UpdateJob(noteBook)
if err != nil {
log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"])
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error()))
return
}
}

ctx.JSON(http.StatusOK, models.BaseMessageApi{
Code: 0,
Message: noteBook.JobID,
})
return
}

}
}

if option.Type <= GPUType {
cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo)
} else {
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo)
}

}
func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOption) {
if ctx.Written() {
return
}

if path.Ext(option.File) != NoteBookExtension {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong")))
return
}

isNotebookFileExist, _ := isNoteBookFileExist(ctx, option)
if !isNotebookFileExist {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist")))
return
} }

task, err := models.GetCloudbrainByJobID(option.JobId)
if err != nil { if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo",setting.FileNoteBook.ProjectName)))
log.Error("job not found:"+option.JobId, err)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Job id may not be right. can not find job."))
return return
} }
if option.Type <= 1 {
cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo)
if task.BootFile == "" || task.Status != string(models.ModelArtsRunning) {
log.Warn("Boot file is empty or status is running. ")
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Boot file is empty or status is running."))
return
}
if !isRepoFileMatch(option, task) {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("can not math repo file."))
return
}
debugBaseUrl, token, err := getBaseUrlAndToken(task)
if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error()))
return
}

if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName), task, token) {
ctx.JSON(http.StatusOK, models.BaseOKMessageApi)
} else { } else {
modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("upload failed."))

}

}

func getBaseUrlAndToken(task *models.Cloudbrain) (string, string, error) {
var debugBaseUrl string
var token string
if task.Type == models.TypeCloudBrainOne {
debugBaseUrl = setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName + "/lab"

} else {
var result *models.GetNotebook2Result
var err error
if task.Type == models.TypeCloudBrainTwo {
result, err = modelarts.GetNotebook2(task.JobID)
} else if task.Type == models.TypeCDCenter {
result, err = modelarts_cd.GetNotebook(task.JobID)
}
if err != nil || result == nil || result.Status != string(models.ModelArtsRunning) || result.Url == "" {
log.Error("notebook job not found:"+task.JobID, err)
return "", "", fmt.Errorf("can not get job or job is invalid.")
}

debugBaseUrl = result.Url
token = result.Token

}
return debugBaseUrl, token, nil
}

func uploadNotebookFileIfCannotBroswer(debugBaseUrl string, bootFile string, task *models.Cloudbrain, token string) bool {
c := &notebook.NotebookContent{
Url: debugBaseUrl,
Path: bootFile,
PathType: "file",
Token: token,
}
if c.IsNotebookFileCanBrowser() {
return true
} else {
c.SetCookiesAndCsrf()
c.UploadNoteBookFile(task)
return c.IsNotebookFileCanBrowser()
}

}

func isNotebookSpecMath(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
if option.Type == NPUType || option.Type == CPUType {
return true
}
spec, err := models.GetCloudbrainSpecByID(book.ID)
if err != nil {
log.Warn("can not get spec ", err)
return false
}
return spec.AccCardsNum > 0
}

func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")
branches := strings.Split(book.BranchName, ";")

for i, bootFile := range bootFiles {
splits := strings.Split(bootFile, "/")
if len(splits) >= 3 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName && branches[i] != option.BranchName {
return true
}
}
}

return false

}

func isRepoMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")

for _, bootFile := range bootFiles {
splits := strings.Split(bootFile, "/")
if len(splits) >= 3 {
if splits[0] == option.OwnerName && splits[1] == option.ProjectName {
return true
}
}
}
return false

}

func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool {
bootFiles := strings.Split(book.BootFile, ";")
branches := strings.Split(book.BranchName, ";")

for i, bootFile := range bootFiles {
if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName) == bootFile {
return true
}
} }


return false

}
func UploadNotebookFiles(task *models.Cloudbrain) {
if task.Status == string(models.JobRunning) && task.BootFile != "" {

debugBaseUrl, token, err := getBaseUrlAndToken(task)
if err != nil {
log.Error("can not get base url:", err)
return
}
bootFiles := strings.Split(task.BootFile, ";")

for _, bootFile := range bootFiles {
uploadNotebookFileIfCannotBroswer(debugBaseUrl, bootFile, task, token)
}

}
} }


func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) {
@@ -131,17 +340,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot
} else { } else {
if count >= 1 { if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
ctx.JSON(http.StatusOK,models.BaseMessageApi{
Code: 2,
ctx.JSON(http.StatusOK, models.BaseMessageApi{
Code: 2,
Message: ctx.Tr("repo.cloudbrain.morethanonejob"), Message: ctx.Tr("repo.cloudbrain.morethanonejob"),
}) })
return return
} }
} }


errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName)
if errStr != "" {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist")))
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName)
if err != nil {
log.Error("download code failed", err)
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
return return
} }
command := cloudbrain.GetCloudbrainDebugCommand() command := cloudbrain.GetCloudbrainDebugCommand()
@@ -185,7 +395,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot
JobType: jobType, JobType: jobType,
Description: getDescription(option), Description: getDescription(option),
BranchName: option.BranchName, BranchName: option.BranchName,
BootFile: option.File,
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName),
Params: "{\"parameter\":[]}", Params: "{\"parameter\":[]}",
CommitID: "", CommitID: "",
BenchmarkTypeID: 0, BenchmarkTypeID: 0,
@@ -206,8 +416,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot


} }


func getCodePath(jobName string) string {
return setting.JobPath + jobName + cloudbrain.CodeMountPath
func getCloudbrainType(optionType int) int {
if optionType < 1 {
return models.TypeCloudBrainOne
}
if setting.ModelartsCD.Enabled {
return models.TypeCDCenter
}
return models.TypeCloudBrainTwo
}

func getCodePath(jobName string, repo *models.Repository) string {
return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name
} }


func getDescription(option api.CreateFileNotebookJobOption) string { func getDescription(option api.CreateFileNotebookJobOption) string {
@@ -237,8 +457,8 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
} else { } else {
if count >= 1 { if count >= 1 {
log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
ctx.JSON(http.StatusOK,models.BaseMessageApi{
Code: 2,
ctx.JSON(http.StatusOK, models.BaseMessageApi{
Code: 2,
Message: ctx.Tr("repo.cloudbrain.morethanonejob"), Message: ctx.Tr("repo.cloudbrain.morethanonejob"),
}) })
return return
@@ -260,7 +480,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
} }
} }


err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName)
err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName)
if err != nil { if err != nil {
ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed")))
return return
@@ -291,10 +511,22 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote
} }


var jobId string var jobId string
req := cloudbrain.GenerateModelArtsNotebookReq{
DisplayJobName: displayJobName,
JobName: jobName,
Description: getDescription(option),
ImageId: setting.FileNoteBook.ImageIdNPU,
Spec: spec,
BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName),
AutoStopDurationMs: modelarts.AutoStopDurationMs / 4,
BranchName: option.BranchName,
}

if setting.ModelartsCD.Enabled { if setting.ModelartsCD.Enabled {
jobId, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPUCD, spec, option.File,modelarts.AutoStopDurationMs/4)
req.ImageId = setting.FileNoteBook.ImageIdNPUCD
jobId, err = modelarts_cd.GenerateNotebook(ctx, req)
} else { } else {
jobId, err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPU, spec, option.File,modelarts.AutoStopDurationMs/4)
jobId, err = modelarts.GenerateNotebook2(ctx, req)
} }


if err != nil { if err != nil {
@@ -336,17 +568,8 @@ func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobO
return true, nil return true, nil
} }


func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string {
err := downloadCode(repo, codePath, branchName)
if err != nil {
return "cloudbrain.load_code_failed"
}

err = uploadOneFileToMinio(codePath, filePath, jobName, cloudbrain.CodeMountPath+"/")
if err != nil {
return "cloudbrain.load_code_failed"
}
return ""
func getBootFile(filePath string, ownerName string, projectName string) string {
return ownerName + "/" + projectName + "/" + filePath
} }


func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) { func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) {


+ 78
- 0
services/cloudbrain/cloudbrainTask/sync_status.go View File

@@ -3,9 +3,13 @@ package cloudbrainTask
import ( import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/cloudbrain"
"code.gitea.io/gitea/modules/grampus"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/modelarts"
"code.gitea.io/gitea/modules/modelarts_cd"
"code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"net/http" "net/http"
"strconv" "strconv"
) )
@@ -58,6 +62,55 @@ func SyncCloudBrainOneStatus(task *models.Cloudbrain) (*models.Cloudbrain, error


} }


func SyncGrampusNotebookStatus(job *models.Cloudbrain) (*models.Cloudbrain, error) {
result, err := grampus.GetNotebookJob(job.JobID)
if err != nil {

log.Error("GetJob(%s) failed:%v", job.JobName, err)

return job, err
}

if job.StartTime == 0 && result.JobInfo.StartedAt > 0 {
job.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
}
oldStatus := job.Status
job.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
job.Duration = result.JobInfo.RunSec
job.TrainJobDuration = models.ConvertDurationToStr(job.Duration)

if job.EndTime == 0 && models.IsTrainJobTerminal(job.Status) && job.StartTime > 0 {
job.EndTime = job.StartTime.Add(job.Duration)
}
job.CorrectCreateUnix()

if len(job.AiCenter) == 0 {
if len(result.JobInfo.Tasks) > 0 {
if len(result.JobInfo.Tasks[0].CenterID) > 0 && len(result.JobInfo.Tasks[0].CenterName) > 0 {
job.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
}
}
}

if job.Status != models.GrampusStatusWaiting {
if oldStatus != job.Status {
notification.NotifyChangeCloudbrainStatus(job, oldStatus)
}
if job.ComputeResource == models.NPUResource {
job.TrainUrl = result.JobInfo.Tasks[0].CodeUrl
job.DataUrl = result.JobInfo.Tasks[0].DataUrl
}
err = models.UpdateJob(job)
if err != nil {
log.Error("UpdateJob failed:", err)
return nil, err
}
}

return job, nil

}

func isNoteBookReady(task *models.Cloudbrain) bool { func isNoteBookReady(task *models.Cloudbrain) bool {
if task.JobType != string(models.JobTypeDebug) { if task.JobType != string(models.JobTypeDebug) {
return true return true
@@ -90,3 +143,28 @@ func isNoteBookReady(task *models.Cloudbrain) bool {
return false return false


} }

func StopDebugJob(task *models.Cloudbrain) error {
param := models.NotebookAction{
Action: models.ActionStop,
}
var err error = nil

if task.JobType == string(models.JobTypeDebug) {
if task.Type == models.TypeCloudBrainOne {
return cloudbrain.StopJob(task.JobID)
} else if task.Type == models.TypeCloudBrainTwo {
_, err = modelarts.ManageNotebook2(task.JobID, param)

} else if task.Type == models.TypeCDCenter {
_, err = modelarts_cd.ManageNotebook(task.JobID, param)

} else if task.Type == models.TypeC2Net {
_, err = grampus.StopJob(task.JobID, task.JobType)

}

}
return err

}

+ 88
- 0
services/repository/contributor.go View File

@@ -0,0 +1,88 @@
package repository

import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/redis/redis_client"
"code.gitea.io/gitea/modules/redis/redis_key"
"encoding/json"
"github.com/patrickmn/go-cache"
"time"
)

var repoContributorCache = cache.New(5*time.Minute, 1*time.Minute)

type ContributorCacheVal struct {
Contributors []*models.ContributorInfo
Total int
}

func GetRepoTopNContributors(repo *models.Repository, N int) ([]*models.ContributorInfo, int) {
val, _ := redis_client.Get(redis_key.RepoTopNContributors(repo.ID, N))
if val != "" {
log.Debug("Get RepoTopNContributors from redis,repo.ID = %d value = %v", repo.ID, val)
temp := &ContributorCacheVal{}
json.Unmarshal([]byte(val), temp)
return temp.Contributors, temp.Total
}

contributorInfos, total := getRepoTopNContributorsFromDisk(repo, N)
log.Debug("Get RepoTopNContributors from disk,repo.ID = %d ", repo.ID)
jsonVal, err := json.Marshal(&ContributorCacheVal{Contributors: contributorInfos, Total: total})
if err == nil {
redis_client.Setex(redis_key.RepoTopNContributors(repo.ID, N), string(jsonVal), 2*time.Minute)
}
return contributorInfos, total
}

func getRepoTopNContributorsFromDisk(repo *models.Repository, N int) ([]*models.ContributorInfo, int) {
contributorInfos := make([]*models.ContributorInfo, 0)

branchName := GetDefaultBranchName(repo)
if branchName == "" {
return contributorInfos, 0
}

contributors, err := git.GetContributors(repo.RepoPath(), branchName)
if err == nil && contributors != nil {
contributorInfoHash := make(map[string]*models.ContributorInfo)
for _, c := range contributors {
if len(contributorInfos) >= N {
break
}
if c.Email == "" {
continue
}
// get user info from committer email
user, err := models.GetUserByActivateEmail(c.Email)
if err == nil {
// committer is system user, get info through user's primary email
if existedContributorInfo, ok := contributorInfoHash[user.Email]; ok {
// existed: same primary email, different committer name
existedContributorInfo.CommitCnt += c.CommitCnt
} else {
// new committer info
var newContributor = &models.ContributorInfo{
user.RelAvatarLink(), user.Name, user.Email, c.CommitCnt,
}
contributorInfos = append(contributorInfos, newContributor)
contributorInfoHash[user.Email] = newContributor
}
} else {
// committer is not system user
if existedContributorInfo, ok := contributorInfoHash[c.Email]; ok {
// existed: same primary email, different committer name
existedContributorInfo.CommitCnt += c.CommitCnt
} else {
var newContributor = &models.ContributorInfo{
"", "", c.Email, c.CommitCnt,
}
contributorInfos = append(contributorInfos, newContributor)
contributorInfoHash[c.Email] = newContributor
}
}
}
}
return contributorInfos, len(contributors)
}

+ 51
- 6
services/repository/repository.go View File

@@ -5,18 +5,19 @@
package repository package repository


import ( import (
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"

"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/notification"
repo_module "code.gitea.io/gitea/modules/repository" repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
pull_service "code.gitea.io/gitea/services/pull" pull_service "code.gitea.io/gitea/services/pull"
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"
"xorm.io/xorm"
) )


const SHELL_FLAG_ON = 1 const SHELL_FLAG_ON = 1
@@ -328,3 +329,47 @@ func IsUploadFileInvalidErr(err error) bool {
_, ok := err.(UploadFileInvalidErr) _, ok := err.(UploadFileInvalidErr)
return ok return ok
} }

func IncreaseRepoDatasetNum(datasetID int64, engines ...*xorm.Engine) error {
dataset, err := models.GetDatasetByID(datasetID)
if err != nil {
return err
}
return models.OperateRepoDatasetNum(dataset.RepoID, 1, engines...)
}

func IncreaseRepoModelNum(repoId int64, engines ...*xorm.Engine) error {
return models.OperateRepoModelNum(repoId, 1, engines...)
}

func ResetRepoModelNum(repoId int64) error {
return models.ResetRepoModelNum(repoId)
}

func DecreaseRepoDatasetNum(datasetID int64, engines ...*xorm.Engine) error {
dataset, err := models.GetDatasetByID(datasetID)
if err != nil {
return err
}
return models.OperateRepoDatasetNum(dataset.RepoID, -1, engines...)
}

func DecreaseRepoModelNum(repoId int64, engines ...*xorm.Engine) error {
return models.OperateRepoModelNum(repoId, -1, engines...)
}

func GetDefaultBranchName(repo *models.Repository) string {
gitRepo, err := git.OpenRepository(repo.RepoPath())
if err != nil {
return ""
}
defer gitRepo.Close()
if len(repo.DefaultBranch) > 0 && gitRepo.IsBranchExist(repo.DefaultBranch) {
return repo.DefaultBranch
}
brs, _, err := gitRepo.GetBranches(0, 0)
if len(brs) > 0 {
return brs[0]
}
return ""
}

+ 315
- 0
services/repository/square.go View File

@@ -0,0 +1,315 @@
package repository

import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"encoding/json"
"github.com/patrickmn/go-cache"
"time"
)

var repoSquareCache = cache.New(2*time.Minute, 1*time.Minute)

const (
RREFERED_CACHE = "PreferredRepos"
REPO_BANNER_CACHE = "RepoBanner"
TOPICS_CACHE = "RepoTopics"
RECOMMEND_CACHE = "RecommendRepos"
)

func GetBanners() []map[string]string {
v, success := repoSquareCache.Get(REPO_BANNER_CACHE)
if success {
log.Debug("GetBanners from cache,value = %v", v)
if v == nil {
return nil
}
r := v.([]map[string]string)
return r
}
repoMap := getMapContent("repos/square_banner")
repoSquareCache.Set(REPO_BANNER_CACHE, repoMap, 1*time.Minute)
return repoMap
}

func GetTopics() []string {
v, success := repoSquareCache.Get(TOPICS_CACHE)
if success {
log.Debug("GetTopics from cache,value = %v", v)
if v == nil {
return nil
}
r := v.([]string)
return r
}
topics := getArrayContent("repos/recommend_topics")
repoSquareCache.Set(TOPICS_CACHE, topics, 1*time.Minute)
return topics
}

func getMapContent(fileName string) []map[string]string {
url := setting.RecommentRepoAddr + fileName
result, err := RecommendContentFromPromote(url)
remap := make([]map[string]string, 0)
if err == nil {
json.Unmarshal([]byte(result), &remap)
}
return remap
}

func getArrayContent(fileName string) []string {
url := setting.RecommentRepoAddr + fileName
result, err := RecommendContentFromPromote(url)
r := make([]string, 0)
if err == nil {
json.Unmarshal([]byte(result), &r)
}
return r
}

func GetRecommendRepos() []map[string]interface{} {
v, success := repoSquareCache.Get(RECOMMEND_CACHE)
if success {
log.Debug("GetRecommendRepos from cache,value = %v", v)
if v == nil {
return nil
}
r := v.([]map[string]interface{})
return r
}
repoMap := getMapContent("home/projects")
r, _ := GetRecommendRepoFromPromote(repoMap)
repoSquareCache.Set(RECOMMEND_CACHE, r, 1*time.Minute)
return r
}

func GetPreferredRepos() ([]*models.Repository4Card, error) {
v, success := repoSquareCache.Get(RREFERED_CACHE)
if success {
log.Debug("GetPreferredRepos from cache,value = %v", v)
if v == nil {
return nil, nil
}
r := v.([]*models.Repository4Card)
return r, nil
}

repos, err := models.GetSelectedRepos(models.FindSelectedReposOpts{
ListOptions: models.ListOptions{
PageSize: 10,
Page: 1,
},
OnlyPublic: true,
})
if err != nil {
return nil, err
}
result := make([]*models.Repository4Card, len(repos))
for i, r := range repos {
result[i] = r.ToCardFormat()
}

repoSquareCache.Set(RREFERED_CACHE, result, 1*time.Minute)
return result, nil
}

func GetIncubationRepos() ([]*models.Repository4Card, error) {
org, err := models.GetOrgByName(setting.IncubationSourceOrgName)
if models.IsErrOrgNotExist(err) {
return make([]*models.Repository4Card, 0), nil
}
if err != nil {
return nil, err
}
repos, err := models.GetSelectedRepos(models.FindSelectedReposOpts{
ListOptions: models.ListOptions{
PageSize: 10,
Page: 1,
},
OrgId: org.ID,
OnlyPublic: true,
})
if err != nil {
return nil, err
}
result := make([]*models.Repository4Card, len(repos))
for i, r := range repos {
result[i] = r.ToCardFormat()
}
return result, nil
}

func GetHotPaperRepos() ([]*models.Repository4Card, error) {
rlist, _, err := models.SearchRepository(&models.SearchRepoOptions{
ListOptions: models.ListOptions{
Page: 1,
PageSize: 10,
},
OrderBy: models.SearchOrderByLastMonthVisitsReverse + "," + models.SearchOrderByRecentUpdated,
TopicOnly: true,
TopicName: setting.PaperRepoTopicName,
AllPublic: true,
})
if err != nil {
return nil, err
}
result := make([]*models.Repository4Card, len(rlist))
for i, r := range rlist {
result[i] = r.ToCardFormat()
}
return result, nil
}

type FindReposOptions struct {
models.ListOptions
Actor *models.User
Sort string
Keyword string
Topic string
Private bool
OwnerID int64
}

func FindRepos(opts FindReposOptions) (*models.FindReposResponse, error) {

var (
repos []*models.Repository
count int64
err error
orderBy models.SearchOrderBy
)

switch opts.Sort {
//1.近期热门:按最近1个月浏览量倒序排序,最近1个月浏览量>最近更新>项目名称升序
case "mostpopular":
orderBy = models.SearchOrderByLastMonthVisitsReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically
//2.近期活跃:按提交增长量(最近4个月commit数)倒序排序,提交增长量>最近更新>项目名称升序。
case "mostactive":
orderBy = models.SearchOrderByLastFourMonthCommitsReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically
//3.最近更新:按最近更新>项目名称升序排序。
case "recentupdate":
orderBy = models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically
//4.最近创建:按项目创建时间排序,最近的排前面。最近创建>项目名称升序。
case "newest":
orderBy = models.SearchOrderByNewest + "," + models.SearchOrderByAlphabetically
//5.点赞最多:按点赞数倒序排序。点赞数>最近更新>项目名称升序。
case "moststars":
orderBy = models.SearchOrderByStarsReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically
//6.派生最多:按派生数倒序排序。派生数>最近更新>项目名称升序。
case "mostforks":
orderBy = models.SearchOrderByForksReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically
//7.数据集最多:按项目包含的数据集文件数量倒序排序,数据集文件数>最近更新>项目名称升序。
case "mostdatasets":
orderBy = models.SearchOrderByDatasetCntReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically
//8.AI任务最多:按项目包含的AI任务数量倒序排序,AI任务数>最近更新>项目名称升序。
case "mostaitasks":
orderBy = models.SearchOrderByAiTaskCntReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically
//9.模型最多:按项目包含的模型数量倒序排序,模型大小为0则不统计。模型数>最近更新>项目名称升序。
case "mostmodels":
orderBy = models.SearchOrderByModelCntReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically

default:
orderBy = models.SearchOrderByLastMonthVisitsReverse + "," + models.SearchOrderByRecentUpdated + "," + models.SearchOrderByAlphabetically
}

repos, count, err = models.SearchRepository(&models.SearchRepoOptions{
ListOptions: opts.ListOptions,
Actor: opts.Actor,
OrderBy: orderBy,
Private: opts.Private,
Keyword: opts.Keyword,
OwnerID: opts.OwnerID,
AllPublic: true,
AllLimited: true,
TopicName: opts.Topic,
IncludeDescription: setting.UI.SearchRepoDescription,
})
if err != nil {
log.Error("FindRepos error when SearchRepository.%v", err)
return nil, err
}
result := make([]*models.Repository4Card, len(repos))
for i, r := range repos {
t := r.ToCardFormat()
contributors, _ := GetRepoTopNContributors(r, 6)
t.Contributors = contributors
result[i] = t
}

return &models.FindReposResponse{
Repos: result,
Total: count,
Page: opts.Page,
PageSize: opts.PageSize,
}, nil
}

type ActiveUser struct {
User *models.User4Front
Followed bool
ShowButton bool
}

func GetActiveUser4Square(currentUserId int64) ([]*ActiveUser, error) {
result := make([]*ActiveUser, 0)
userIds, err := models.QueryLast30DaysHighestIndexUsers(5)
if err != nil {
log.Error("ActiveUser err. %v", err)
return result, err
}
if len(userIds) == 0 {
return result, nil
}

users, err := models.GetUsersByIDs(userIds)
if err != nil {
return result, nil
}
usersMap := make(map[int64]*models.User)
for _, v := range users {
usersMap[v.ID] = v
}

for i := 0; i < len(userIds); i++ {
userId := userIds[i]
user := usersMap[userId]
if user == nil {
continue
}
isFollowed := false
if currentUserId != 0 {
isFollowed = models.IsFollowing(currentUserId, userId)
}
a := &ActiveUser{
Followed: isFollowed,
User: user.ToFrontFormat(),
ShowButton: currentUserId != userId,
}
result = append(result, a)
}
return result, nil
}

func GetActiveOrgs() ([]*models.User4Front, error) {
orgScores, err := models.FindTopNOpenIOrgs(5)
if err != nil {
return nil, err
}
orgs := make([]*models.User4Front, len(orgScores))
for i, v := range orgScores {
orgs[i] = v.ToFrontFormat()
}
return orgs, nil
}

func RefreshRepoStatData() {
repos, err := models.GetAllRepositories()
if err != nil {
log.Error("RefreshRepoStatData GetAllRepositories failed: %v", err.Error())
return
}
for _, repo := range repos {
models.SyncStatDataToRepo(repo)
}
}

+ 1
- 1
services/socketwrap/clientManager.go View File

@@ -10,7 +10,7 @@ import (
"github.com/elliotchance/orderedmap" "github.com/elliotchance/orderedmap"
) )


var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35}
var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 39, 40}


type ClientsManager struct { type ClientsManager struct {
Clients *orderedmap.OrderedMap Clients *orderedmap.OrderedMap


+ 19
- 19
templates/admin/cloudbrain/list.tmpl View File

@@ -98,7 +98,7 @@
<div class="two wide column nowrap" style="width:10% !important;"> <div class="two wide column nowrap" style="width:10% !important;">
{{if eq .JobType "DEBUG"}} {{if eq .JobType "DEBUG"}}
<a class="title" <a class="title"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain/{{$JobID}}{{else}}/modelarts/notebook/{{$JobID}}{{end}}"
href="{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{$JobID}}"
title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px"> title="{{.DisplayJobName}}" style="font-size: 14px;padding-right:0px">
<span class="fitted" <span class="fitted"
style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span> style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
@@ -135,13 +135,13 @@
</div> </div>
<!-- 集群 --> <!-- 集群 -->
<div class="one wide column text center nowrap" style="width:6% !important;"> <div class="one wide column text center nowrap" style="width:6% !important;">
<span style="font-size: 12px;" class="cluster_{{.DisplayJobName}}_{{$JobID}}">{{if .Cluster}}{{.Cluster}}{{else}}--{{end}}</span>
<span style="font-size: 12px;" class="cluster_{{.DisplayJobName}}_{{$JobID}}">{{if .Cluster}}{{.Cluster}}{{else}}--{{end}}</span>
</div> </div>
<!-- 任务状态 --> <!-- 任务状态 -->
<div class="two wide column text center nowrap" <div class="two wide column text center nowrap"
style="width: 6% !important;"> style="width: 6% !important;">
<span class="job-status" id="{{$JobID}}" <span class="job-status" id="{{$JobID}}"
data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "DEBUG"}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{else if eq .JobType "INFERENCE"}}/modelarts/inference-job{{else if eq .JobType "TRAIN"}}/modelarts/train-job{{else if eq .JobType "BENCHMARK" "MODELSAFETY"}}/cloudbrain{{end}}'
data-repopath='{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "DEBUG"}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}{{else if eq .JobType "INFERENCE"}}/modelarts/inference-job{{else if eq .JobType "TRAIN"}}/modelarts/train-job{{else if eq .JobType "BENCHMARK" "MODELSAFETY"}}/cloudbrain{{end}}'
data-jobid="{{$JobID}}" data-version="{{.VersionName}}"> data-jobid="{{$JobID}}" data-version="{{.VersionName}}">
<span><i id="{{$JobID}}-icon" style="vertical-align: middle;" <span><i id="{{$JobID}}-icon" style="vertical-align: middle;"
class="{{.Status}}"></i><span id="{{$JobID}}-text" class="{{.Status}}"></i><span id="{{$JobID}}-text"
@@ -170,7 +170,7 @@
</div> </div>
<!-- 智算中心 --> <!-- 智算中心 -->
<div class="one wide column text center nowrap" style="width:8% !important;"> <div class="one wide column text center nowrap" style="width:8% !important;">
<span style="font-size: 12px;" class="aicenter_{{.DisplayJobName}}_{{$JobID}}" title="{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}">{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}</span>
<span style="font-size: 12px;" id="cluster-{{$JobID}}" class="aicenter_{{.DisplayJobName}}_{{$JobID}}" title="{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}">{{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}</span>
</div> </div>
<!-- XPU类型 --> <!-- XPU类型 -->
<div class="one wide column text center nowrap" style="width:8% !important;"> <div class="one wide column text center nowrap" style="width:8% !important;">
@@ -234,7 +234,7 @@
<a style="margin: 0 1rem;" id="ai-debug-{{$JobID}}" <a style="margin: 0 1rem;" id="ai-debug-{{$JobID}}"
class='ui basic ai_debug {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING"}}disabled {{else}}blue {{end}}button' class='ui basic ai_debug {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING"}}disabled {{else}}blue {{end}}button'
data-jobid="{{$JobID}}" data-jobid="{{$JobID}}"
data-repopath='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}/{{$JobID}}/'>
data-repopath='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{$JobID}}/'>
{{$.i18n.Tr "repo.debug"}} {{$.i18n.Tr "repo.debug"}}
</a> </a>
{{else}} {{else}}
@@ -242,7 +242,7 @@
<a id="ai-debug-{{$JobID}}" <a id="ai-debug-{{$JobID}}"
class='ui basic ai_debug {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING"}} disabled {{else}}blue {{end}}button' class='ui basic ai_debug {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING"}} disabled {{else}}blue {{end}}button'
data-jobid="{{$JobID}}" data-jobid="{{$JobID}}"
data-repopath='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}/{{$JobID}}/'>
data-repopath='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{$JobID}}/'>
{{$.i18n.Tr "repo.debug_again"}} {{$.i18n.Tr "repo.debug_again"}}
</a> </a>
{{end}} {{end}}
@@ -268,8 +268,8 @@
{{$.CsrfTokenHtml}} {{$.CsrfTokenHtml}}
<a style="padding: 0.5rem 1rem;" id="ai-stop-{{$JobID}}" <a style="padding: 0.5rem 1rem;" id="ai-stop-{{$JobID}}"
class='ui basic ai_stop {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "SUCCEEDED" "STOPPED" "STOPPING"}}disabled {{else}} blue {{end}}button' class='ui basic ai_stop {{if eq .Status "KILLED" "FAILED" "START_FAILED" "KILLING" "COMPLETED" "SUCCEEDED" "STOPPED" "STOPPING"}}disabled {{else}} blue {{end}}button'
data-repopath='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else if eq .JobType "BENCHMARK" }}/cloudbrain/benchmark{{else if eq .ComputeResource "NPU" }}/modelarts/notebook{{end}}/{{$JobID}}/stop'
data-jobid="{{$JobID}}">
data-repopath='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else if eq .JobType "BENCHMARK" }}/cloudbrain/benchmark{{else if eq .ComputeResource "NPU" }}/modelarts/notebook{{end}}{{end}}/{{$JobID}}/stop'
data-jobid="{{$JobID}}" data-bootfile="{{.BootFile}}">
{{$.i18n.Tr "repo.stop"}} {{$.i18n.Tr "repo.stop"}}
</a> </a>
</form> </form>
@@ -304,17 +304,17 @@
</a> </a>
</form> </form>
{{else}} {{else}}
<form class="ui compact buttons" id="delForm-{{$JobID}}"
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .ComputeResource "NPU"}}/modelarts/notebook{{else}}/cloudbrain{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}{{else if eq .JobType "INFERENCE"}}{{if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{end}}{{end}}/{{$JobID}}/del?isadminpage=true'
method="post">
{{$.CsrfTokenHtml}}
<a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}"
data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}/modelarts/inference-job/{{$JobID}}/del_version?isadminpage=true"
data-version="" class="ui basic ai_delete blue button"
style="border-radius: .28571429rem;">
{{$.i18n.Tr "repo.delete"}}
</a>
</form>
<form class="ui compact buttons" id="delForm-{{$JobID}}"
action='{{AppSubUrl}}/{{.Repo.OwnerName}}/{{.Repo.Name}}{{if eq .JobType "BENCHMARK"}}/cloudbrain/benchmark{{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}}/cloudbrain{{else if eq .JobType "DEBUG"}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}{{else if eq .JobType "TRAIN"}}{{if eq .Cloudbrain.Type 1}}/modelarts/train-job{{else if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{else if eq .Cloudbrain.Type 2}}/grampus/train-job{{end}}{{else if eq .JobType "INFERENCE"}}{{if eq .Cloudbrain.Type 0}}/cloudbrain/train-job{{end}}{{end}}/{{$JobID}}/del?ishomepage=true'
method="post">
{{$.CsrfTokenHtml}}
<a style="padding: 0.5rem 1rem;margin-left:0.2rem" id="ai-delete-{{$JobID}}"
data-repopath="{{.Repo.OwnerName}}/{{.Repo.Name}}/modelarts/inference-job/{{$JobID}}/del_version?ishomepage=true"
data-version="{{.VersionName}}" class="ui basic ai_delete blue button"
style="border-radius: .28571429rem;">
{{$.i18n.Tr "repo.delete"}}
</a>
</form>
{{end}} {{end}}
</div> </div>
</div> </div>


+ 5
- 5
templates/base/head_navbar.tmpl View File

@@ -35,7 +35,7 @@
</div> </div>
</div> </div>
</div> </div>
<a class="item" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a> <a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a>
<div class="ui simple dropdown item" > <div class="ui simple dropdown item" >
{{.i18n.Tr "repo.model_manager"}} {{.i18n.Tr "repo.model_manager"}}
@@ -48,7 +48,7 @@
{{.i18n.Tr "explore"}} {{.i18n.Tr "explore"}}
<i class="dropdown icon"></i> <i class="dropdown icon"></i>
<div class="menu"> <div class="menu">
<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>
<!--<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>-->
<a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a> <a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a> <a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a>
{{if .IsOperator}} {{if .IsOperator}}
@@ -75,7 +75,7 @@
</div> </div>
</div> </div>


<a class="item" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a> <a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a>
<div class="ui simple dropdown item" > <div class="ui simple dropdown item" >
{{.i18n.Tr "repo.model_manager"}} {{.i18n.Tr "repo.model_manager"}}
@@ -89,7 +89,7 @@
{{.i18n.Tr "explore"}} {{.i18n.Tr "explore"}}
<i class="dropdown icon"></i> <i class="dropdown icon"></i>
<div class="menu" > <div class="menu" >
<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>
<!--<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>-->
<a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a> <a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a> <a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a>
{{if .IsOperator}} {{if .IsOperator}}
@@ -100,7 +100,7 @@
</div> </div>
</div> </div>
{{else if .IsLandingPageExplore}} {{else if .IsLandingPageExplore}}
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "home"}}</a>
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "home"}}</a>
{{else if .IsLandingPageOrganizations}} {{else if .IsLandingPageOrganizations}}
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "home"}}</a> <a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "home"}}</a>
{{end}} {{end}}


+ 5
- 5
templates/base/head_navbar_fluid.tmpl View File

@@ -32,7 +32,7 @@
</div> </div>
</div> </div>
</div> </div>
<a class="item" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a> <a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a>
<div class="ui simple dropdown item" > <div class="ui simple dropdown item" >
{{.i18n.Tr "repo.model_manager"}} {{.i18n.Tr "repo.model_manager"}}
@@ -45,7 +45,7 @@
{{.i18n.Tr "explore"}} {{.i18n.Tr "explore"}}
<i class="dropdown icon"></i> <i class="dropdown icon"></i>
<div class="menu"> <div class="menu">
<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>
<!--<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>-->
<a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a> <a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a> <a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a>
{{if .IsOperator}} {{if .IsOperator}}
@@ -71,7 +71,7 @@
</div> </div>
</div> </div>
</div> </div>
<a class="item" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a> <a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a>
<div class="ui simple dropdown item" > <div class="ui simple dropdown item" >
{{.i18n.Tr "repo.model_manager"}} {{.i18n.Tr "repo.model_manager"}}
@@ -84,7 +84,7 @@
{{.i18n.Tr "explore"}} {{.i18n.Tr "explore"}}
<i class="dropdown icon"></i> <i class="dropdown icon"></i>
<div class="menu"> <div class="menu">
<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>
<!--<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>-->
<a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a> <a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a> <a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a>
{{if .IsOperator}} {{if .IsOperator}}
@@ -95,7 +95,7 @@
</div> </div>
</div> </div>
{{else if .IsLandingPageExplore}} {{else if .IsLandingPageExplore}}
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "home"}}</a>
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "home"}}</a>
{{else if .IsLandingPageOrganizations}} {{else if .IsLandingPageOrganizations}}
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "home"}}</a> <a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "home"}}</a>
{{end}} {{end}}


+ 5
- 5
templates/base/head_navbar_home.tmpl View File

@@ -24,7 +24,7 @@
</div> </div>
</div> </div>
</div> </div>
<a class="item" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a> <a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a>
<div class="ui simple dropdown item" > <div class="ui simple dropdown item" >
{{.i18n.Tr "repo.model_manager"}} {{.i18n.Tr "repo.model_manager"}}
@@ -37,7 +37,7 @@
{{.i18n.Tr "explore"}} {{.i18n.Tr "explore"}}
<i class="dropdown icon"></i> <i class="dropdown icon"></i>
<div class="menu"> <div class="menu">
<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>
<!--<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>-->
<a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a> <a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a> <a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a>
{{if .IsOperator}} {{if .IsOperator}}
@@ -64,7 +64,7 @@
</div> </div>
</div> </div>


<a class="item" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a> <a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a>
<div class="ui simple dropdown item" > <div class="ui simple dropdown item" >
{{.i18n.Tr "repo.model_manager"}} {{.i18n.Tr "repo.model_manager"}}
@@ -77,7 +77,7 @@
{{.i18n.Tr "explore"}} {{.i18n.Tr "explore"}}
<i class="dropdown icon"></i> <i class="dropdown icon"></i>
<div class="menu"> <div class="menu">
<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>
<!--<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>-->
<a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a> <a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a> <a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a>
{{if .IsOperator}} {{if .IsOperator}}
@@ -88,7 +88,7 @@
</div> </div>
</div> </div>
{{else if .IsLandingPageExplore}} {{else if .IsLandingPageExplore}}
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "home"}}</a>
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "home"}}</a>
{{else if .IsLandingPageOrganizations}} {{else if .IsLandingPageOrganizations}}
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "home"}}</a> <a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "home"}}</a>
{{end}} {{end}}


+ 4
- 4
templates/base/head_navbar_pro.tmpl View File

@@ -34,7 +34,7 @@
</div> </div>
</div> </div>


<a class="item" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "custom.head.project"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a> <a class="item" href="{{AppSubUrl}}/explore/datasets">{{.i18n.Tr "custom.head.dataset"}}</a>
<div class="ui simple dropdown item" > <div class="ui simple dropdown item" >
{{.i18n.Tr "repo.model_manager"}} {{.i18n.Tr "repo.model_manager"}}
@@ -47,7 +47,7 @@
{{.i18n.Tr "explore"}} {{.i18n.Tr "explore"}}
<i class="dropdown icon"></i> <i class="dropdown icon"></i>
<div class="menu"> <div class="menu">
<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>
<!--<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>-->
<a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a> <a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a> <a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a>
{{if .IsOperator}} {{if .IsOperator}}
@@ -87,7 +87,7 @@
{{.i18n.Tr "explore"}} {{.i18n.Tr "explore"}}
<i class="dropdown icon"></i> <i class="dropdown icon"></i>
<div class="menu" > <div class="menu" >
<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>
<!--<a class="item" href="{{AppSubUrl}}/explore/users">{{.i18n.Tr "explore.users"}}</a>-->
<a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a> <a class="item" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "explore.organizations"}}</a>
<a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a> <a class="item" href="{{AppSubUrl}}/explore/images">{{.i18n.Tr "explore.images"}}</a>
{{if .IsOperator}} {{if .IsOperator}}
@@ -98,7 +98,7 @@
</div> </div>
</div> </div>
{{else if .IsLandingPageExplore}} {{else if .IsLandingPageExplore}}
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/repos">{{.i18n.Tr "home"}}</a>
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/repos/square">{{.i18n.Tr "home"}}</a>
{{else if .IsLandingPageOrganizations}} {{else if .IsLandingPageOrganizations}}
<a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "home"}}</a> <a class="item {{if .PageIsExplore}}active{{end}}" href="{{AppSubUrl}}/explore/organizations">{{.i18n.Tr "home"}}</a>
{{end}} {{end}}


+ 2
- 2
templates/custom/select_model.tmpl View File

@@ -1,6 +1,6 @@
<input type="hidden" id="ai_model_version" name="model_version" value="{{$.model_version}}"> <input type="hidden" id="ai_model_version" name="model_version" value="{{$.model_version}}">
<div class="inline min_title fields" style="{{if not .job_name}}width: 96.8%{{else}}width: 94.8%{{end}};">
<label class="{{if not .job_name}}label-fix-width{{end}}" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.infer_job.select_model"}}</label>&nbsp;
<div class="inline min_title required fields" style="width: 94%;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.infer_job.select_model"}}</label>
<div class="six wide field"> <div class="six wide field">
<div class="ui fluid search selection dropdown" id="select_model"> <div class="ui fluid search selection dropdown" id="select_model">
<input type="hidden" name="model_name" required value="{{$.model_name}}"> <input type="hidden" name="model_name" required value="{{$.model_name}}">


+ 2
- 2
templates/custom/task_wait_count.tmpl View File

@@ -1,7 +1,7 @@
<div style="display:inline-block;"> <div style="display:inline-block;">
<div style="display:flex;align-items:center;color:#f2711c;"> <div style="display:flex;align-items:center;color:#f2711c;">
<i class="ri-error-warning-line" style="margin-right: 0.5rem; font-size: 14px"></i> <i class="ri-error-warning-line" style="margin-right: 0.5rem; font-size: 14px"></i>
<span style="font-size: 12px">{{.i18n.Tr "repo.wait_count_start"}} <span class="__task_wait_count__">{{.WaitCount}}</span> {{.i18n.Tr "repo.wait_count_end"}}</span>
<span style="font-size: 12px">{{.i18n.Tr "repo.wait_count_start"}} <span class="__task_wait_count__">{{if not .WaitCount}}1{{else}}{{addOne .WaitCount}}{{end}}</span> {{.i18n.Tr "repo.wait_count_end"}}</span>
</div> </div>
</div> </div>
<script> <script>
@@ -12,7 +12,7 @@
var specsSelEl = $('select#__specs__'); var specsSelEl = $('select#__specs__');
var seldOption = specsSelEl.find('option:selected'); var seldOption = specsSelEl.find('option:selected');
var queueCode = seldOption.attr('queueCode'); var queueCode = seldOption.attr('queueCode');
$('span.__task_wait_count__').text(queuesDetail[queueCode] || 0);
$('span.__task_wait_count__').text((queuesDetail[queueCode] || 0) +1);
}; };
$('body').on('change', 'select#__specs__', function(e) { $('body').on('change', 'select#__specs__', function(e) {
changeSpecs(); changeSpecs();


+ 2
- 2
templates/explore/navbar.tmpl View File

@@ -1,6 +1,6 @@
<div class="tablet only mobile only sixteen wide mobile sixteen wide tablet column row"> <div class="tablet only mobile only sixteen wide mobile sixteen wide tablet column row">
<div class="ui secondary pointing tabular top attached borderless menu navbar"> <div class="ui secondary pointing tabular top attached borderless menu navbar">
<a class="{{if .PageIsExploreRepositories}}active{{end}} item" href="{{AppSubUrl}}/explore/repos">
<a class="{{if .PageIsExploreRepositories}}active{{end}} item" href="{{AppSubUrl}}/explore/repos/square">
{{svg "octicon-repo" 16}} {{.i18n.Tr "explore.repos"}} {{svg "octicon-repo" 16}} {{.i18n.Tr "explore.repos"}}
</a> </a>
<a class="{{if .PageIsDatasets}}active{{end}} item" href="{{AppSubUrl}}/explore/datasets"> <a class="{{if .PageIsDatasets}}active{{end}} item" href="{{AppSubUrl}}/explore/datasets">
@@ -24,7 +24,7 @@
<div class="computer only three wide computer column"> <div class="computer only three wide computer column">
<div class="ui grid"> <div class="ui grid">
<div class="sixteen wide column ui secondary sticky pointing tabular vertical menu"> <div class="sixteen wide column ui secondary sticky pointing tabular vertical menu">
<a class="{{if .PageIsExploreRepositories}}active{{end}} item" href="{{AppSubUrl}}/explore/repos">
<a class="{{if .PageIsExploreRepositories}}active{{end}} item" href="{{AppSubUrl}}/explore/repos/square">
{{svg "octicon-repo" 16}} {{.i18n.Tr "explore.repos"}} {{svg "octicon-repo" 16}} {{.i18n.Tr "explore.repos"}}
</a> </a>
<a class="{{if .PageIsDatasets}}active{{end}} item" href="{{AppSubUrl}}/explore/datasets"> <a class="{{if .PageIsDatasets}}active{{end}} item" href="{{AppSubUrl}}/explore/datasets">


+ 8
- 0
templates/explore/repos/search.tmpl View File

@@ -0,0 +1,8 @@
{{template "base/head_home" .}}
<link rel="stylesheet" href="{{StaticUrlPrefix}}/css/vp-repos-search.css?v={{MD5 AppVer}}" />
<script>
var staticSquareTopics = {{ .SquareTopics }};
</script>
<div id="__vue-root"></div>
<script src="{{StaticUrlPrefix}}/js/vp-repos-search.js?v={{MD5 AppVer}}"></script>
{{template "base/footer" .}}

+ 16
- 0
templates/explore/repos/square.tmpl View File

@@ -0,0 +1,16 @@
{{template "base/head_home" .}}
{{ if .SquareBanners }}
{{ range .SquareBanners }}
<img preload style="height:0;width:0;position:absolute;left:-2000px;" src="{{.src}}" />
{{ end }}
{{ end }}
<link rel="stylesheet" href="{{StaticUrlPrefix}}/css/vp-repos-square.css?v={{MD5 AppVer}}" />
<script>
var staticSquareBanners = {{ .SquareBanners }};
var staticSquarePreferredRepos = {{ .SquarePreferredRepos }};
var staticSquareTopics = {{ .SquareTopics }};
var staticSquareRecommendRepos = {{ .SquareRecommendRepos }};
</script>
<div id="__vue-root"></div>
<script src="{{StaticUrlPrefix}}/js/vp-repos-square.js?v={{MD5 AppVer}}"></script>
{{template "base/footer" .}}

+ 123
- 257
templates/repo/cloudbrain/benchmark/new.tmpl View File

@@ -1,37 +1,9 @@
{{template "base/head" .}} {{template "base/head" .}}
<style>
.unite {
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
}

.title {
font-size: 16px !important;
padding-left: 3rem !important;
}

.min_title{
font-size: 14px !important;
margin-bottom: 2rem !important;
}

.width81 {
margin-left: 1.5rem;
width: 81% !important;
}

.width48 {
width: 48.5% !important;
}

.nowrapx {
white-space: nowrap !important;
}
</style>
{{template "custom/global_mask" .}} {{template "custom/global_mask" .}}
<div class="repository"> <div class="repository">
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="ui container"> <div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-repo-link="{{.RepoLink}}" data-flag-model="true"></div>
{{if eq .NotStopTaskCount 0}} {{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
@@ -78,21 +50,12 @@
onkeydown="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)"
onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea> onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea>
</div> </div>

<!--<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.gpu_type"}}</label>
<select id="cloudbrain_gpu_type" class="ui search dropdown width48" placeholder="选择GPU类型"
name="gpu_type">
{{range .benchmark_gpu_types}}
<option value="{{.Queue}}">{{.Value}}</option>
{{end}}
</select>
</div>-->
<div class="required unite min_title two inline fields" style="margin-left: 80px;">
<div class="required min_title two inline fields" style="margin-left: 80px;">
<div class="required ten wide field" style="width: 26.5% !important;"> <div class="required ten wide field" style="width: 26.5% !important;">
<label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.task_type"}}</label>&nbsp; <label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.task_type"}}</label>&nbsp;
<select id="cloudbrain_job_type" class="ui search dropdown job_type"
<select id="cloudbrain_job_type" class="ui search dropdown job_type"
placeholder="select {{.i18n.Tr "cloudbrain.task_type"}}" name="job_type"> placeholder="select {{.i18n.Tr "cloudbrain.task_type"}}" name="job_type">
<option value="SNN4ECOSET">SNN4ECOSET</option>
<option value="SNN4IMAGENET">SNN4IMAGENET</option> <option value="SNN4IMAGENET">SNN4IMAGENET</option>
<option value="BRAINSCORE">BRAINSCORE</option> <option value="BRAINSCORE">BRAINSCORE</option>
</select> </select>
@@ -116,22 +79,10 @@
<a id="benchmark_model_example" href="https://openi.pcl.ac.cn/BDIP/snn4imagenet" <a id="benchmark_model_example" href="https://openi.pcl.ac.cn/BDIP/snn4imagenet"
target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a> target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
</div> </div>
{{template "custom/select_model" .}}
<div id="images-new-cb"> <div id="images-new-cb">
</div> </div>
{{template "custom/select_dataset_train" .}}
<!--<div class="required min_title inline field" style="margin-top:2rem;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="cloudbrain_resource_spec" class="ui search dropdown"
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px'
name="resource_spec_id">
{{range .benchmark_resource_specs}}
<option name="resource_spec_id" value="{{.Id}}">
{{$.i18n.Tr "cloudbrain.gpu_num"}}:{{.GpuNum}},{{$.i18n.Tr "cloudbrain.cpu_num"}}:{{.CpuNum}},{{$.i18n.Tr "cloudbrain.memory"}}(MB):{{.MemMiB}},{{$.i18n.Tr "cloudbrain.shared_memory"}}(MB):{{.ShareMemMiB}}
</option>
{{end}}
</select>
</div>-->
<div class="required min_title inline field" style="margin-top:2rem;"> <div class="required min_title inline field" style="margin-top:2rem;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="__specs__" class="ui search dropdown width48" <select id="__specs__" class="ui search dropdown width48"
@@ -156,7 +107,7 @@
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}"> <button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}} {{.i18n.Tr "repo.cloudbrain.new"}}
</button> </button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
<a class="ui button" href="{{.RepoLink}}/cloudbrain/benchmark">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div> </div>
</div> </div>
</form> </form>
@@ -199,20 +150,11 @@
onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea> onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea>
</div> </div>


<!--<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.gpu_type"}}</label>
<select id="cloudbrain_gpu_type" class="ui search dropdown" placeholder="选择GPU类型"
style='width:385px' name="gpu_type">
{{range .benchmark_gpu_types}}
<option value="{{.Queue}}">{{.Value}}</option>
{{end}}
</select>
</div>-->
<div class="required unite inline min_title fields" style="width: 90%;margin-left: 5.7rem;">&nbsp;
<div class="required inline min_title fields" style="width: 90%;margin-left: 5.7rem;">&nbsp;
<div class="required eight wide field"> <div class="required eight wide field">
<label style="font-weight: normal;white-space: nowrap;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_type"}}</label> <label style="font-weight: normal;white-space: nowrap;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_type"}}</label>


<select class="ui fluid selection search dropdown" id="benchmark_types_id"
<select class="ui fluid selection search dropdown benchmark_types_id"
name="benchmark_types_id"> name="benchmark_types_id">
{{range .benchmark_types}} {{range .benchmark_types}}
{{if eq .Id $.benchmarkTypeID}} {{if eq .Id $.benchmarkTypeID}}
@@ -226,26 +168,13 @@
<div class="eight wide field" id="engine_name"> <div class="eight wide field" id="engine_name">
<input type="hidden" id="benchmark_child_types_id_hidden" name="benchmark_child_types_id_hidden" value="{{.benchmark_child_types_id_hidden}}"> <input type="hidden" id="benchmark_child_types_id_hidden" name="benchmark_child_types_id_hidden" value="{{.benchmark_child_types_id_hidden}}">
<label style="font-weight: normal;white-space: nowrap;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_child_type"}}</label> <label style="font-weight: normal;white-space: nowrap;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_child_type"}}</label>
<select class="ui fluid selection dropdown nowrapx" id="benchmark_child_types_id" style='width: 100%;' name="benchmark_child_types_id">
<select class="ui fluid selection dropdown " id="benchmark_child_types_id" style='width: 100%;' name="benchmark_child_types_id">
</select> </select>
</div> </div>
</div> </div>
<div id="images-new-cb"> <div id="images-new-cb">
</div> </div>


<!--<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="cloudbrain_resource_spec" class="ui search dropdown"
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px'
name="resource_spec_id">
{{range .benchmark_resource_specs}}
<option name="resource_spec_id" value="{{.Id}}">
{{$.i18n.Tr "cloudbrain.gpu_num"}}:{{.GpuNum}},{{$.i18n.Tr "cloudbrain.cpu_num"}}:{{.CpuNum}},{{$.i18n.Tr "cloudbrain.memory"}}(MB):{{.MemMiB}},{{$.i18n.Tr "cloudbrain.shared_memory"}}(MB):{{.ShareMemMiB}}
</option>
{{end}}
</select>
</div>-->

<div class="required min_title inline field"> <div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="__specs__" class="ui search dropdown width48" <select id="__specs__" class="ui search dropdown width48"
@@ -282,197 +211,134 @@
target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a> target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
</div> </div>



<div class="inline unite min_title field">
<label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
</div>
</form>
{{else if eq .benchmarkMode "aisafety"}}
<form id="form_id" class="ui form alogrithm_form" action="{{.Link}}?benchmarkMode=alogrithm" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="update">
<input type="hidden" name="job_type" value="BENCHMARK">
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.cloudbrain.benchmark.evaluate_scenes"}}</label>
<div class="ui blue small menu compact selectcloudbrain">
<a class="item alogrithm_benchmark"
href="{{.Link}}?benchmarkMode=alogrithm">{{.i18n.Tr "repo.cloudbrain.benchmark.algorithm"}}</a>
<a class="item model_benchmark"
href="{{.Link}}?benchmarkMode=model">{{.i18n.Tr "repo.cloudbrain.benchmark.model"}}</a>
<a class="active item aisafety_benchmark"
href="{{.RepoLink}}/modelsafety/create_gpu">模型安全评测</a>
</div>
</div>

<div>
<div class="min_title inline field" style="margin-top:-10px;">
<label class="label-fix-width" style="font-weight: normal;"></label>
{{template "custom/task_wait_count" .}}
</div>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input style="width: 80%;" name="display_job_name" id="trainjob_job_name"
placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.display_job_name}}"
tabindex="3" autofocus required maxlength="36">
<span class="tooltips" style="display: block;margin-left: 11.5rem;">{{.i18n.Tr "repo.cloudbrain_jobname_err"}}</span>
</div>
<div class="min_title inline field">
<label class="label-fix-width" style="font-weight: normal;"
for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="254"
placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}
onchange="this.value=this.value.substring(0, 255)"
onkeydown="this.value=this.value.substring(0, 255)"
onkeyup="this.value=this.value.substring(0, 255)">{{.description}}</textarea>
</div>


<div id="images-new-cb">
</div>

<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="__specs__" class="ui search dropdown width48"
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px' ovalue="{{.spec_id}}"
name="spec_id">
</select>
<span><i class="question circle icon link"></i></span>
<a href="{{AppSubUrl}}/resource_desc" target="_blank">{{.i18n.Tr "custom.resource_description"}}</a>
</div>

<div class="inline min_title field required">
<label class="label-fix-width" style="font-weight: normal;">推理程序</label>
<input disabled="disabled" style="width: 33.5%;" name="test_file" id="test_file" value="test.py"
tabindex="3" autofocus required maxlength="254">
<a id="test_href_id" href="https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark"
target="_blank">{{.i18n.Tr "cloudbrain.view_sample"}}</a>
</div>


<div class="inline unite min_title field">
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;"></label> <label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}"> <button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}} {{.i18n.Tr "repo.cloudbrain.new"}}
</button> </button>
<a class="ui button" href="/">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
<a class="ui button" href="{{.RepoLink}}/cloudbrain/benchmark">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div> </div>
</div> </div>
</form> </form>

{{end}} {{end}}
</div> </div>
</div> </div>
</div> </div>
{{template "base/footer" .}} {{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script> <script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script> <script>
let form = document.getElementById('form_id');
let createFlag = false
form.onsubmit = function (e) {
if(createFlag) return false
createFlag = true
}
let repolink = {{.RepoLink }}
let url_href = window.location.pathname.split('create')[0]
$(".ui.button").attr('href', url_href)
$('.menu .item')
.tab();
$('#benchmark_types_id').change(function () {
setChildType();
})
$(document).ready(() => {
$('.ui.search.dropdown.job_type').dropdown({
onChange: function (value, text, $selectedItem) {
if (value === "BRAINSCORE") {
$('#brainscore_child_type').css('display', 'block')
$('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/similarity2brain_ann')
} else {
$('#brainscore_child_type').css('display', 'none')
$('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4imagenet')
}
}
})
})
function setChildType() {
let type_id = $('#benchmark_types_id').val();
if (type_id == 3) {
$('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
$('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
} else {
$('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
$('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
}
let child_selected_id = $('#benchmark_child_types_id_hidden').val();
$.get(`${repolink}/cloudbrain/benchmark/get_child_types?benchmark_type_id=${type_id}`, (data) => {
const n_length = data['child_types'].length
let html = ''
for (let i = 0; i < n_length; i++) {
if (child_selected_id == data['child_types'][i].id) {
html += `<option value="${data['child_types'][i].id}" selected="true">${data['child_types'][i].value}</option>`;
} else {
html += `<option value="${data['child_types'][i].id}">${data['child_types'][i].value}</option>`;
}
}
document.getElementById("benchmark_child_types_id").innerHTML = html;
})
}
document.onreadystatechange = function () {
if (document.readyState === "complete") {
if ($('input[name=benchmarkMode]').val() === 'alogrithm' || $('input[name=benchmarkMode]').val() === '') {
setChildType();
}
}
}
var isValidate = false;
function validate() {
$('.ui.form')
.form({
on: 'blur',
fields: {
image: {
identifier: 'image',
rules: [
{
type: 'empty',
promt: ''
}
]
},
display_job_name: {
identifier: 'display_job_name',
rules: [
{
type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
promt: ''
}
]
},
spec_id: {
identifier: 'spec_id',
rules: [{ type: 'empty' }]
}
},
onSuccess: function () {
// $('.ui.page.dimmer').dimmer('show')
document.getElementById("mask").style.display = "block"
isValidate = true;
},
onFailure: function (e) {
isValidate = false;
return false;
}
})
}
// let repolink = {{.RepoLink }}
// $('#benchmark_types_id').change(function () {
// console.log("----")
// // setChildType();
// })
// $("#benchmark_types_id").dropdown({
// onChange:function (value, text, $selectedItem){
// console.log(value)
// }
// })
// $(document).ready(() => {
// $('.ui.search.dropdown.job_type').dropdown({
// onChange: function (value, text, $selectedItem) {
// console.log(value)
// if (value === "BRAINSCORE") {
// $('#brainscore_child_type').css('display', 'block')
// $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/similarity2brain_ann')
// } else {
// $('#brainscore_child_type').css('display', 'none')
// $('#benchmark_model_example').attr('href', 'https://openi.pcl.ac.cn/BDIP/snn4imagenet')
// }
// }
// })
// })
// function setChildType(type_id=1) {
// if (type_id == 3) {
// $('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
// $('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_MOT_benchmark');
// } else {
// $('#train_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
// $('#test_href_id').attr('href', 'https://openi.pcl.ac.cn/CV_benchmark/CV_reID_benchmark');
// }
// let child_selected_id = $('#benchmark_child_types_id_hidden').val();
// $.get(`${repolink}/cloudbrain/benchmark/get_child_types?benchmark_type_id=${type_id}`, (data) => {
// const n_length = data['child_types'].length
// let html = ''
// for (let i = 0; i < n_length; i++) {
// if (child_selected_id == data['child_types'][i].id) {
// html += `<option value="${data['child_types'][i].id}" selected="true">${data['child_types'][i].value}</option>`;
// } else {
// html += `<option value="${data['child_types'][i].id}">${data['child_types'][i].value}</option>`;
// }
// }
// document.getElementById("benchmark_child_types_id").innerHTML = html;
// })
// }
// $(document).ready(function (){
// console.log("00")
// $(".ui.selection.dropdown.benchmark_types_id").dropdown({
// onChange:function (value, text, $selectedItem){
// console.log(value)
// setChildType(value)
// }
// })
// })
// document.onreadystatechange = function () {
// if (document.readyState === "complete") {
// if ($('input[name=benchmarkMode]').val() === 'alogrithm' || $('input[name=benchmarkMode]').val() === '') {
// setChildType();
// }
// }
// }
// var isValidate = false;
// function validate() {
// $('.ui.form')
// .form({
// on: 'blur',
// fields: {
// image: {
// identifier: 'image',
// rules: [
// {
// type: 'empty',
// promt: ''
// }
// ]
// },
// display_job_name: {
// identifier: 'display_job_name',
// rules: [
// {
// type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
// promt: ''
// }
// ]
// },
// spec_id: {
// identifier: 'spec_id',
// rules: [{ type: 'empty' }]
// }
// },
// onSuccess: function () {
// // $('.ui.page.dimmer').dimmer('show')
// document.getElementById("mask").style.display = "block"
// isValidate = true;
// },
// onFailure: function (e) {
// isValidate = false;
// return false;
// }
// })
// }


validate()
$('.ui.create_train_job.green.button').click(function (e) {
validate()
})
// validate()
// $('.ui.create_train_job.green.button').click(function (e) {
// validate()
// })


;(function() { ;(function() {
var SPECS = {{ .benchmark_specs }}; var SPECS = {{ .benchmark_specs }};


+ 31
- 218
templates/repo/cloudbrain/benchmark/show.tmpl View File

@@ -1,200 +1,4 @@
{{template "base/head" .}} {{template "base/head" .}}
<style>
.according-panel-heading {
box-sizing: border-box;
padding: 8px 16px;
color: #252b3a;
background-color: #f2f5fc;
line-height: 1.5;
cursor: pointer;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
}

.accordion-panel-title {
margin-top: 0;
margin-bottom: 0;
color: #252b3a;
}

.accordion-panel-title-content {
vertical-align: middle;
display: inline-block;
width: calc(100% - 32px);
cursor: default;
}

.acc-margin-bottom {
margin-bottom: 5px;
}

.title_text {
font-size: 12px;
}

.ac-display-inblock {
display: inline-block;
}

.cti-mgRight-sm {
margin-right: 8px;
}

.ac-text-normal {
font-size: 14px;
color: #575d6c;
}

.uc-accordionTitle-black {
color: #333;
}

.accordion-border {
border: 1px solid #cce2ff;
}

.padding0 {
padding: 0 !important;
}

.content-pad {
padding: 15px 35px;
}

.content-margin {
margin: 10px 5px;
}

.tab_2_content {
min-height: 425px;
margin-left: 10px;
}

.ac-grid {
display: block;
*zoom: 1;
}

.ac-grid-col {
float: left;
width: 100%;
}

.ac-grid-col2 .ac-grid-col {
width: 50%;
}

.ti-form {
text-align: left;
max-width: 100%;
vertical-align: middle;
}

.ti-form>tbody {
font-size: 12px;
}

.ti-form>tbody,
.ti-form>tbody>tr {
vertical-align: inherit;
}

.ti-text-form-label {

padding-bottom: 20px;
padding-right: 20px;
color: #8a8e99;
font-size: 12px;
white-space: nowrap !important;
width: 80px;
line-height: 30px;
}

.ti-text-form-content {
line-height: 30px;
padding-bottom: 20px;
}

.ti-form>tbody>tr>td {
vertical-align: top;
white-space: normal;
}

td,
th {
padding: 0;
}

.ac-grid-col .text-span {
width: 450px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}

.redo-color {
color: #3291F8;
}

.ti-action-menu-item:not(:last-child) {
margin-right: 10px;
padding-right: 11px;
text-decoration: none !important;
color: #526ecc;
cursor: pointer;
display: inline-block;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
position: relative;
}

.ti-action-menu-item:not(:last-child):after {
content: "";
display: inline-block;
position: absolute;
height: 12px;
right: 0;
top: 50%;
-webkit-transform: translateY(-6px);
-ms-transform: translateY(-6px);
-o-transform: translateY(-6px);
transform: translateY(-6px);
border-right: 1px solid #dfe1e6;
}

.text-width80 {
width: 100px;
line-height: 30px;
}

.border-according {
border: 1px solid #dfe1e6;
}

.disabled {
cursor: default;
pointer-events: none;
color: rgba(0, 0, 0, .6) !important;
opacity: .45 !important;
}

.pad20 {

border: 0px !important;
}

.model_file_bread {
margin-bottom: -0.5rem !important;
padding-left: 1rem;
padding-top: 0.5rem;
}
</style>
<div id="mask"> <div id="mask">
<div id="loadingPage"> <div id="loadingPage">
<div class="rect1"></div> <div class="rect1"></div>
@@ -402,7 +206,7 @@
<table class="ti-form"> <table class="ti-form">
<tbody class="ti-text-form"> <tbody class="ti-text-form">


{{if eq .JobType "BENCHMARK"}}
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_train"}} {{$.i18n.Tr "repo.cloudbrain.benchmark.evaluate_train"}}
@@ -410,11 +214,7 @@


<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w"> <div class="text-span text-span-w">
{{if eq .JobType "BENCHMARK"}}
train.py train.py
{{else}}
--
{{end}}
</div> </div>
</td> </td>
</tr> </tr>
@@ -424,16 +224,39 @@
</td> </td>


<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w">
{{if eq .JobType "BENCHMARK"}}
test.py test.py
{{else}}
--
{{end}}
</div> </div>
</td> </td>
</tr> </tr>
{{else}}
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.model_name"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .ModelName}}{{.ModelName}}{{else}}--{{end}}</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelconvert.modelversion"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .ModelVersion}}{{.ModelVersion}}{{else}}--{{end}}</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.infer_job_model_file"}}
</td>


<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .CkptName}}{{.CkptName}}{{else}}--{{end}}</div>
</td>
</tr>
{{end}}
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.description"}} {{$.i18n.Tr "repo.modelarts.train_job.description"}}
@@ -467,17 +290,7 @@
<div class="text-span text-span-w"></div> <div class="text-span text-span-w"></div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.model_manager"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-mirror">
{{.DatasetName}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_creator"}} {{$.i18n.Tr "repo.cloudbrain_creator"}}
@@ -520,7 +333,7 @@
</div> </div>
</div> </div>
</div> </div>
<div style="clear: both;"></div>
</div> </div>
</div> </div>
<div class="ui tab" data-tab="second{{$k}}"> <div class="ui tab" data-tab="second{{$k}}">


+ 7
- 11
templates/repo/cloudbrain/inference/new.tmpl View File

@@ -1,9 +1,5 @@
{{template "base/head" .}} {{template "base/head" .}}
<style> <style>
.unite{
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
}


.title{ .title{
font-size: 16px !important; font-size: 16px !important;
@@ -44,7 +40,7 @@
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
{{template "custom/alert_cb" .}} {{template "custom/alert_cb" .}}
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-queue="{{.QueuesDetail}}" data-queue-start="{{.i18n.Tr "repo.wait_count_start"}}" data-queue-end="{{.i18n.Tr "repo.wait_count_end"}}"></div>
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}"></div>
<h4 class="ui top attached header"> <h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new_infer"}} {{.i18n.Tr "repo.modelarts.train_job.new_infer"}}
</h4> </h4>
@@ -63,7 +59,7 @@
<input type="hidden" id="failed_train_url" value="{{$.train_url}}"> <input type="hidden" id="failed_train_url" value="{{$.train_url}}">
<input type="hidden" id="fail_dataset_name" value="{{$.dataset_name}}"> <input type="hidden" id="fail_dataset_name" value="{{$.dataset_name}}">
<input type="hidden" id="fail_dataset_uuid" value="{{$.attachment}}"> <input type="hidden" id="fail_dataset_uuid" value="{{$.attachment}}">
<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<h4 class="title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field"> <div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain"> <div class="ui blue mini menu compact selectcloudbrain">
@@ -100,7 +96,7 @@
<span class="tooltips" style="margin-left:11.5rem;display: block;">{{.i18n.Tr "repo.cloudbrain_jobname_err"}}</span> <span class="tooltips" style="margin-left:11.5rem;display: block;">{{.i18n.Tr "repo.cloudbrain_jobname_err"}}</span>
</div> </div>


<div class="unite min_title inline field">
<div class="min_title inline field">
<label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label> <label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
{{if .description}} {{if .description}}
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}>{{.description}}</textarea> <textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}>{{.description}}</textarea>
@@ -111,10 +107,10 @@
<div class="ui divider"></div> <div class="ui divider"></div>


<!-- 模型相关配置 --> <!-- 模型相关配置 -->
<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>
<div class="required unite inline min_title fields" style="width: 96.8%;">
<div class="required eight wide field">
<label style="font-weight: normal;white-space: nowrap;width: 210px;text-align: right;">{{.i18n.Tr "repo.modelarts.infer_job.select_model"}}</label>
<h4 class="title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>
<div class="required inline min_title fields" style="width: 94%;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.infer_job.select_model"}}</label>
<div class="six wide field">
<div class="ui fluid search selection dropdown loading " id="select_model"> <div class="ui fluid search selection dropdown loading " id="select_model">
<input type="hidden" name="model_name" required> <input type="hidden" name="model_name" required>
<div class="text"></div> <div class="text"></div>


+ 12
- 199
templates/repo/cloudbrain/inference/show.tmpl View File

@@ -1,200 +1,5 @@
{{template "base/head" .}} {{template "base/head" .}}
<style> <style>
.according-panel-heading {
box-sizing: border-box;
padding: 8px 16px;
color: #252b3a;
background-color: #f2f5fc;
line-height: 1.5;
cursor: pointer;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
}

.accordion-panel-title {
margin-top: 0;
margin-bottom: 0;
color: #252b3a;
}

.accordion-panel-title-content {
vertical-align: middle;
display: inline-block;
width: calc(100% - 32px);
cursor: default;
}

.acc-margin-bottom {
margin-bottom: 5px;
}

.title_text {
font-size: 12px;
}

.ac-display-inblock {
display: inline-block;
}

.cti-mgRight-sm {
margin-right: 8px;
}

.ac-text-normal {
font-size: 14px;
color: #575d6c;
}

.uc-accordionTitle-black {
color: #333;
}

.accordion-border {
border: 1px solid #cce2ff;
}

.padding0 {
padding: 0 !important;
}

.content-pad {
padding: 15px 35px;
}

.content-margin {
margin: 10px 5px;
}

.tab_2_content {
min-height: 360px;
margin-left: 10px;
}

.ac-grid {
display: block;
*zoom: 1;
}

.ac-grid-col {
float: left;
width: 100%;
}

.ac-grid-col2 .ac-grid-col {
width: 50%;
}

.ti-form {
text-align: left;
max-width: 100%;
vertical-align: middle;
}

.ti-form>tbody {
font-size: 12px;
}

.ti-form>tbody,
.ti-form>tbody>tr {
vertical-align: inherit;
}

.ti-text-form-label {

padding-bottom: 20px;
padding-right: 20px;
color: #8a8e99;
font-size: 12px;
white-space: nowrap !important;
width: 80px;
line-height: 30px;
}

.ti-text-form-content {
line-height: 30px;
padding-bottom: 20px;
}

.ti-form>tbody>tr>td {
vertical-align: top;
white-space: normal;
}

td,
th {
padding: 0;
}

.info_text {
padding-bottom: 20px;
padding-right: 20px;
font-size: 12px;
}

.ac-grid-col .text-span {
width: 450px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}

.redo-color {
color: #3291F8;
}

.ti-action-menu-item:not(:last-child) {
margin-right: 10px;
padding-right: 11px;
text-decoration: none !important;
color: #526ecc;
cursor: pointer;
display: inline-block;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
position: relative;
}

.ti-action-menu-item:not(:last-child):after {
content: "";
display: inline-block;
position: absolute;
height: 12px;
right: 0;
top: 50%;
-webkit-transform: translateY(-6px);
-ms-transform: translateY(-6px);
-o-transform: translateY(-6px);
transform: translateY(-6px);
border-right: 1px solid #dfe1e6;
}

.text-width80 {
width: 100px;
line-height: 30px;
}

.border-according {
border: 1px solid #dfe1e6;
}

.disabled {
cursor: default;
pointer-events: none;
color: rgba(0, 0, 0, .6) !important;
opacity: .45 !important;
}

.pad20 {

border: 0px !important;
}

.model_file_bread { .model_file_bread {
margin-bottom: -0.5rem !important; margin-bottom: -0.5rem !important;
padding-left: 1rem; padding-left: 1rem;
@@ -566,10 +371,18 @@
<div class="ui tab" data-tab="four"> <div class="ui tab" data-tab="four">
<input type="hidden" name="model{{.VersionName}}" value="-1"> <input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1"> <input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>

<div style="display: flex;justify-content: space-between;">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>
</div>
<a id="{{.VersionName}}-result-down" style="padding-right: 1%;display: none;"
class='{{if $.canDownload}}ti-download-file{{else}}disabled{{end}} file-info'
href="{{$.RepoLink}}/cloudbrain/inference-job/{{.JobID}}/downloadall?version_name={{.VersionName}}">
<i class="ri-download-cloud-2-line"></i>
<span style="margin-left: 0.3rem;">{{$.i18n.Tr "repo.all_result_download"}}</span>
</a>
</div> </div>
<div id="dir_list{{.VersionName}}"> <div id="dir_list{{.VersionName}}">




+ 142
- 191
templates/repo/cloudbrain/new.tmpl View File

@@ -13,72 +13,102 @@
.inline.required.field.cloudbrain_brainscore { .inline.required.field.cloudbrain_brainscore {
display: none; display: none;
} }

.width80{
width: 80.7% !important;
}


</style> </style>


{{template "custom/global_mask" .}} {{template "custom/global_mask" .}}

<div class="repository"> <div class="repository">
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-queue="{{.QueuesDetail}}" data-queue-start="{{.i18n.Tr "repo.wait_count_start"}}" data-queue-end="{{.i18n.Tr "repo.wait_count_end"}}"></div>
{{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}}
{{end}}
<div class="ui negative message" id="messageInfo" style="display:none;">
<p></p>
</div>
{{template "custom/alert_cb" .}}
<div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-exceed-size="{{DebugAttachSize}}"></div>
{{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}}
{{end}}
<div class="ui negative message" id="messageInfo" style="display:none;">
<p></p>
</div>
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new_debug"}}
</h4>
{{template "custom/alert_cb" .}}
<div class="ui attached segment">
<form id="form_id" class="ui form" action="{{.Link}}" method="post"> <form id="form_id" class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}} {{.CsrfTokenHtml}}
<input type="hidden" name='isBranches' value="{{.Branches}}"> <input type="hidden" name='isBranches' value="{{.Branches}}">
<h3 class="ui top attached header">
{{.i18n.Tr "repo.cloudbrain.new"}}
</h3>
<div class="ui attached segment">
<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue small menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/cloudbrain/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16"
height="16">
<path fill="none" d="M0 0h24v24H0z" />
<path
d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z" />
</svg>
CPU/GPU
</a>
<a class="item" href="{{.RepoLink}}/modelarts/notebook/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16"
height="16">
<path fill="none" d="M0 0h24v24H0z" />
<path
d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z" />
</svg>
Ascend NPU</a>
</div>
</div>
<div class="inline field">
<label></label>
{{template "custom/task_wait_count" .}}
<input type="hidden" id="ai_image_name" value="{{.image}}">
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/cloudbrain/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_openi"}}
</a>
<a class="item" href="{{.RepoLink}}/grampus/notebook/create?type=0">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_c2net"}}(Beta)
</a>
</div> </div>
<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.task_name"}}</label>
<input name="display_job_name" id="cloudbrain_job_name"
placeholder="{{.i18n.Tr "cloudbrain.task_name"}}" value="{{.display_job_name}}" tabindex="3"
autofocus required maxlength="255" onkeyup="this.value=this.value.replace(/[, ]/g,'')">
</div>
<div class="inline required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue small menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/cloudbrain/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16"
height="16">
<path fill="none" d="M0 0h24v24H0z" />
<path
d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z" />
</svg>
CPU/GPU
</a>
<a class="item" href="{{.RepoLink}}/modelarts/notebook/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16"
height="16">
<path fill="none" d="M0 0h24v24H0z" />
<path
d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z" />
</svg>
Ascend NPU</a>
</div> </div>

<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.task_type"}}</label>
<select id="cloudbrain_job_type" class="ui search dropdown"
placeholder="{{.i18n.Tr "cloudbrain.task_type"}}" style='width:385px' name="job_type">
<option name="job_type" value="DEBUG">DEBUG</option>
</select>
</div>
<div class="min_title inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
{{template "custom/task_wait_count" .}}
<div style="display: flex;align-items: center;margin-left: 156px;margin-top: 0.5rem;">
<i class="ri-error-warning-line" style="color: #f2711c;margin-right: 0.5rem;"></i>
<span style="color: #888;font-size: 12px;">{{.i18n.Tr "cloudbrain.new_debug_gpu_tooltips" "/code" "/dataset" "/pretrainmodel" "/model" | Safe}}</span>
</div> </div>
</div>
<div class="inline min_title required field" style="margin-bottom: 0rem !important;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.task_name"}}</label>
<input name="display_job_name" id="cloudbrain_job_name" style="width: 60%;"
placeholder="{{.i18n.Tr "cloudbrain.task_name"}}" value="{{.display_job_name}}" tabindex="3"
autofocus required maxlength="255" onkeyup="this.value=this.value.replace(/[, ]/g,'')">
</div>
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 2rem;">{{.i18n.Tr "repo.cloudbrain_jobname_err"}}</span>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
{{if .description}}
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}>{{.description}}</textarea>
{{else}}
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)"></textarea>
{{end}}
</div>
<div class="ui divider"></div>


<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.task_type"}}</label>
<select id="cloudbrain_job_type" class="ui search dropdown width80"
placeholder="{{.i18n.Tr "cloudbrain.task_type"}}" name="job_type">
<option name="job_type" value="DEBUG">DEBUG</option>
</select>
</div>
<div class="inline required field cloudbrain_benchmark"> <div class="inline required field cloudbrain_benchmark">
<label style="vertical-align: top; margin-top:9px">数据集类别</label> <label style="vertical-align: top; margin-top:9px">数据集类别</label>
<select class="ui search dropdown" multiple="multiple" id="cloudbrain_benchmark_category" <select class="ui search dropdown" multiple="multiple" id="cloudbrain_benchmark_category"
@@ -94,27 +124,27 @@
</div> </div>
</div> </div>
<input id="store_category" type="hidden" name="get_benchmark_category"> <input id="store_category" type="hidden" name="get_benchmark_category">
<div class="inline required field">
<label>{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2 {{if not .Branches}}error{{end}}" id="code_version"
name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branch_name }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{else}}
<option name="branch_name" value="{{.branchName}}">{{.branchName}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branchName }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{end}}
</select>
</div>
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2" id="code_version"
name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branch_name }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{else}}
<option name="branch_name" value="{{.branchName}}">{{.branchName}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branchName }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{end}}
</select>
</div>
<!--<div class="inline required field"> <!--<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.gpu_type"}}</label> <label>{{.i18n.Tr "cloudbrain.gpu_type"}}</label>
<select id="cloudbrain_gpu_type" class="ui search dropdown gpu-type" placeholder="选择GPU类型" <select id="cloudbrain_gpu_type" class="ui search dropdown gpu-type" placeholder="选择GPU类型"
@@ -124,14 +154,14 @@
{{end}} {{end}}
</select> </select>
</div>--> </div>-->
{{template "custom/select_model" .}}
<div id="images-new-cb">


<div id="images-new-cb">

</div>
<div id="select-multi-dataset">

</div>
</div>
<div id="select-multi-dataset">


</div>
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 1rem;"></span>
<!--<div class="inline required field"> <!--<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.resource_specification"}}</label> <label>{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="cloudbrain_resource_spec" class="ui search dropdown" <select id="cloudbrain_resource_spec" class="ui search dropdown"
@@ -144,42 +174,27 @@
{{end}} {{end}}
</select> </select>
</div>--> </div>-->

<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="__specs__" class="ui search dropdown"
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px' ovalue="{{.spec_id}}"
{{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}}
name="spec_id">
</select>
<span><i class="question circle icon link"></i></span>
<a href="{{AppSubUrl}}/resource_desc" target="_blank">{{.i18n.Tr "custom.resource_description"}}</a>
{{if .CloudBrainPaySwitch}}
<div class="cloudbrain_resource_spec_blance_tip" style="padding:0 5px;margin:6px 0;margin-left:265px;font-size:12px;width: 50%!important;">
<span>{{$.i18n.Tr "points.balance_of_points"}}<span style="color:red;margin: 0 3px">{{.PointAccount.Balance}}</span>{{$.i18n.Tr "points.points"}}</span><span>{{$.i18n.Tr "points.expected_time"}}<span style="color:red;margin: 0 3px" class="can-use-time"></span>{{$.i18n.Tr "points.hours"}}</span>
<span style="float:right;">
<i class="question circle icon link" data-position="right center" data-variation="mini"></i>
<a href="{{AppSubUrl}}/reward/point/rule" target="_blank">{{$.i18n.Tr "points.points_acquisition_instructions"}}</a>
</span>
</div>
{{end}}
</div>

<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.dataset_storage_path"}}</label>
<input name="dataset_path" id="cloudbrain_dataset_path" value="{{.dataset_path}}" tabindex="3"
disabled autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.model_storage_path"}}</label>
<input name="model_path" id="cloudbrain_model_path" value="{{.model_path}}" tabindex="3"
disabled autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.code_storage_path"}}</label>
<input name="code_path" id="cloudbrain_code_path" value="{{.code_path}}" tabindex="3" disabled
autofocus required maxlength="255" readonly="readonly">
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="__specs__" class="ui search dropdown width48"
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px' ovalue="{{.spec_id}}"
{{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}}
name="spec_id">
</select>
<span><i class="question circle icon link"></i></span>
<a href="{{AppSubUrl}}/resource_desc" target="_blank">{{.i18n.Tr "custom.resource_description"}}</a>
{{if .CloudBrainPaySwitch}}
<div class="cloudbrain_resource_spec_blance_tip" style="padding:0 5px;margin:6px 0;margin-left:265px;font-size:12px;width: 50%!important;">
<span>{{$.i18n.Tr "points.balance_of_points"}}<span style="color:red;margin: 0 3px">{{.PointAccount.Balance}}</span>{{$.i18n.Tr "points.points"}}</span><span>{{$.i18n.Tr "points.expected_time"}}<span style="color:red;margin: 0 3px" class="can-use-time"></span>{{$.i18n.Tr "points.hours"}}</span>
<span style="float:right;">
<i class="question circle icon link" data-position="right center" data-variation="mini"></i>
<a href="{{AppSubUrl}}/reward/point/rule" target="_blank">{{$.i18n.Tr "points.points_acquisition_instructions"}}</a>
</span>
</div> </div>
{{end}}
</div>
<div class="inline required field cloudbrain_benchmark"> <div class="inline required field cloudbrain_benchmark">
<label>{{.i18n.Tr "cloudbrain.benchmark_path"}}</label> <label>{{.i18n.Tr "cloudbrain.benchmark_path"}}</label>
<input name="benchmark_path" id="cloudbrain_benchmark_path" value="{{.benchmark_path}}" <input name="benchmark_path" id="cloudbrain_benchmark_path" value="{{.benchmark_path}}"
@@ -199,22 +214,25 @@
<label>{{.i18n.Tr "cloudbrain.start_command"}}</label> <label>{{.i18n.Tr "cloudbrain.start_command"}}</label>
<textarea name="command" rows="10" readonly="readonly">{{.command}}</textarea> <textarea name="command" rows="10" readonly="readonly">{{.command}}</textarea>
</div> </div>

<div class="inline field">
<label></label>
<button class="ui green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel"
href="{{.RepoLink}}/debugjob?debugListType=all">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
<div class="inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel"
href="{{.RepoLink}}/debugjob?debugListType=all">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div> </div>
</form> </form>
</div> </div>
</div> </div>
</div> </div>
{{template "base/footer" .}} {{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script> <script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script> <script>
let form = document.getElementById('form_id'); let form = document.getElementById('form_id');
$('#messageInfo').css('display', 'none') $('#messageInfo').css('display', 'none')
@@ -223,63 +241,6 @@
context.value = '' context.value = ''
$(".icon.icons").css("visibility", "hidden") $(".icon.icons").css("visibility", "hidden")
} }
var isValidate = false;
function validate(){
$('.ui.form').form({
on: 'blur',
fields: {
display_job_name:{
identifier : 'display_job_name',
rules: [
{
type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
}
]
},
spec_id: {
identifier: 'spec_id',
rules: [{ type: 'empty' }]
}
},
onSuccess: function(){
isValidate = true;
},
onFailure: function(e){
isValidate = false;
return false;
}
})
}
validate();
let createFlag = false
form.onsubmit = function (e) {
if (!isValidate) return false;
if(createFlag) return false
let value_task = $("input[name='display_job_name']").val()
let value_image = $("input[name='image']").val()
let value_data = $("input[name='attachment']").val()
let re = /^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/
let flag = re.test(value_task)
if (!flag) {
$('#messageInfo').css('display', 'block')
let str = '只能以小写字母或数字开头且只包含小写字母、数字、_和-,不能以_结尾,最长36个字符。'
$('#messageInfo p').text(str)
return false
}
let min_value_task = value_task.toLowerCase()
$("input[name='display_job_name']").attr("value", min_value_task)
createFlag = true
document.getElementById("mask").style.display = "block"
}

// 页面加载完毕后遮罩层隐藏
document.onreadystatechange = function () {
if (document.readyState === "complete") {
document.getElementById("mask").style.display = "none"
}

}

$('#cloudbrain_benchmark_category') $('#cloudbrain_benchmark_category')
.dropdown({ .dropdown({
placeholder: "选择数据集类别", placeholder: "选择数据集类别",
@@ -314,16 +275,6 @@
} }
}) })
}) })

$('.ui.green.button').click(function () {
if (!$('input[name="isBranches"]').val()) {
return false
}
selected_value = $("#cloudbrain_benchmark_category").val()
$('#store_category').attr("value", selected_value)
validate();
})

;(function() { ;(function() {
var SPECS = {{ .debug_specs }}; var SPECS = {{ .debug_specs }};
var showPoint = {{ .CloudBrainPaySwitch }}; var showPoint = {{ .CloudBrainPaySwitch }};


+ 71
- 255
templates/repo/cloudbrain/show.tmpl View File

@@ -1,208 +1,4 @@
{{template "base/head" .}} {{template "base/head" .}}
<style>
.according-panel-heading {
box-sizing: border-box;
padding: 8px 16px;
color: #252b3a;
background-color: #f2f5fc;
line-height: 1.5;
cursor: pointer;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
}

.accordion-panel-title {
margin-top: 0;
margin-bottom: 0;
color: #252b3a;
}

.accordion-panel-title-content {
vertical-align: middle;
display: inline-block;
width: calc(100% - 32px);
cursor: default;
}

.acc-margin-bottom {
margin-bottom: 5px;
}

.title_text {
font-size: 12px;
}

.ac-display-inblock {
display: inline-block;
}

.cti-mgRight-sm {
margin-right: 8px;
}

.ac-text-normal {
font-size: 14px;
color: #575d6c;
}

.uc-accordionTitle-black {
color: #333;
}

.accordion-border {
border: 1px solid #cce2ff;
}

.padding0 {
padding: 0 !important;
}

.content-pad {
padding: 15px 35px;
}

.content-margin {
margin: 10px 5px;
}

.tab_2_content {
min-height: 420px;
margin-left: 10px;
}

.ac-grid {
display: block;
*zoom: 1;
}

.ac-grid-col {
float: left;
width: 100%;
}

.ac-grid-col2 .ac-grid-col {
width: 50%;
}

.ti-form {
text-align: left;
max-width: 100%;
vertical-align: middle;
}

.ti-form>tbody {
font-size: 12px;
}

.ti-form>tbody,
.ti-form>tbody>tr {
vertical-align: inherit;
}

.info_text {
padding-bottom: 20px;
padding-right: 20px;
font-size: 12px;
}

.ti-text-form-label {

padding-bottom: 20px;
padding-right: 20px;
color: #8a8e99;
font-size: 12px;
white-space: nowrap !important;
width: 80px;
line-height: 30px;
}

.ti-text-form-content {
line-height: 30px;
padding-bottom: 20px;
}

.ti-form>tbody>tr>td {
vertical-align: top;
white-space: normal;
}

td,
th {
padding: 0;
}

.ac-grid-col .text-span {
width: 450px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}

.redo-color {
color: #3291F8;
}

.ti-action-menu-item:not(:last-child) {
margin-right: 10px;
padding-right: 11px;
text-decoration: none !important;
color: #526ecc;
cursor: pointer;
display: inline-block;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
position: relative;
}

.ti-action-menu-item:not(:last-child):after {
content: "";
display: inline-block;
position: absolute;
height: 12px;
right: 0;
top: 50%;
-webkit-transform: translateY(-6px);
-ms-transform: translateY(-6px);
-o-transform: translateY(-6px);
transform: translateY(-6px);
border-right: 1px solid #dfe1e6;
}

.text-width80 {
width: 100px;
line-height: 30px;
}

.border-according {
border: 1px solid #dfe1e6;
}

.disabled {
cursor: default;
pointer-events: none;
color: rgba(0, 0, 0, .6) !important;
opacity: .45 !important;
}

.pad20 {

border: 0px !important;
}

.model_file_bread {
margin-bottom: -0.5rem !important;
padding-left: 1rem;
padding-top: 0.5rem;
}
</style>


<div id="mask"> <div id="mask">
<div id="loadingPage"> <div id="loadingPage">
<div class="rect1"></div> <div class="rect1"></div>
@@ -308,6 +104,17 @@
</div> </div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.code_version"}}
</td>
<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-code">
{{.BranchName}}
<span style="margin-left:1rem" class="ui label">{{SubStr .CommitID 0 10}}</span>
</div>
</td>
</tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.computing_resources"}} {{$.i18n.Tr "repo.modelarts.computing_resources"}}
@@ -321,52 +128,71 @@
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.task_type"}}
{{$.i18n.Tr "repo.modelarts.createtime"}}
</td> </td>

<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w" <div class="text-span text-span-w"
id="{{.VersionName}}-computeresource">
{{.JobType}}
id="{{.VersionName}}-createtime">
{{TimeSinceUnix1 .CreatedUnix}}
</div> </div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.code_version"}}
{{$.i18n.Tr "repo.cloudbrain.time.starttime"}}
</td> </td>

<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-code">
{{.BranchName}}
<span style="margin-left:1rem" class="ui label">{{SubStr .CommitID 0 10}}</span>
<div class="text-span text-span-w"
id="{{.VersionName}}-startTime">
{{if not (eq .StartTime 0)}}
{{TimeSinceUnix1 .StartTime}}
{{else}}
--
{{end}}
</div> </div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.gpu_type"}}
{{$.i18n.Tr "repo.cloudbrain.time.endtime"}}
</td> </td>


<td class="ti-text-form-content resorce_type">
<div class="text-span text-span-w"></div>
<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-EndTime">
{{if not (eq .EndTime 0)}}
{{TimeSinceUnix1 .EndTime}}
{{else}}
--
{{end}}
</div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.createtime"}}
{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}
</td> </td>


<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w" <div class="text-span text-span-w"
id="{{.VersionName}}-createtime">
{{TimeSinceUnix1 .CreatedUnix}}
id="{{.VersionName}}-duration">
{{$.duration}}
</div> </div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.description"}}
</td>





<td class="ti-text-form-content">
<div class="text-span-new" id="model_description">
{{.Description}}
</div>
</td>
</tr>
</tbody> </tbody>
</table> </table>
</div> </div>
@@ -392,8 +218,6 @@
</div> </div>
</td> </td>
</tr> </tr>

<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.standard"}} {{$.i18n.Tr "repo.modelarts.train_job.standard"}}
@@ -405,24 +229,29 @@
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.dataset_storage_path"}}
{{$.i18n.Tr "repo.modelarts.model_name"}}
</td> </td>


<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w" id="dataset_storage_path">
{{$.dataset_path}}
</div>
<div class="text-span text-span-w">{{if .ModelName}}{{.ModelName}}{{else}}--{{end}}</div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.model_storage_path"}}
{{$.i18n.Tr "repo.modelconvert.modelversion"}}
</td> </td>


<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w" id="model_storage_path">
{{$.model_path}}
</div>
<div class="text-span text-span-w">{{if .ModelVersion}}{{.ModelVersion}}{{else}}--{{end}}</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.infer_job_model_file"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .CkptName}}{{.CkptName}}{{else}}--{{end}}</div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
@@ -438,56 +267,43 @@
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain.time.starttime"}}
{{$.i18n.Tr "cloudbrain.dataset_storage_path"}}
</td> </td>


<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-startTime">
{{if not (eq .StartTime 0)}}
{{TimeSinceUnix1 .StartTime}}
{{else}}
--
{{end}}
<div class="text-span text-span-w" id="dataset_storage_path">
{{$.dataset_path}}
</div> </div>
</td> </td>
</tr> </tr>

<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain.time.endtime"}}
{{$.i18n.Tr "cloudbrain.model_storage_path"}}
</td> </td>

<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-EndTime">
{{if not (eq .EndTime 0)}}
{{TimeSinceUnix1 .EndTime}}
{{else}}
--
{{end}}
<div class="text-span text-span-w" id="code_storage_path">
/pretrainmodel
</div> </div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}
{{$.i18n.Tr "cloudbrain.output_storage_path"}}
</td> </td>


<td class="ti-text-form-content"> <td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-duration">
{{$.duration}}
<div class="text-span text-span-w" id="model_storage_path">
{{$.model_path}}
</div> </div>
</td> </td>
</tr> </tr>

</tbody> </tbody>
</table> </table>
</div> </div>
</div> </div>
</div> </div>
<div style="clear:both"> <div style="clear:both">
{{if $.datasetDownload}}
<table style="border:none" class="ui fixed small stackable table"> <table style="border:none" class="ui fixed small stackable table">
<thead> <thead>
<tr><th style="color: #8a8e99;font-size:12px" class="three wide left aligned">{{$.i18n.Tr "dataset.file"}}</th> <tr><th style="color: #8a8e99;font-size:12px" class="three wide left aligned">{{$.i18n.Tr "dataset.file"}}</th>
@@ -508,6 +324,7 @@


</tbody> </tbody>
</table> </table>
{{end}}
</div> </div>
</div> </div>
</div> </div>
@@ -609,6 +426,5 @@
shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}},
}); });
$('td.ti-text-form-content.spec div').text(specStr); $('td.ti-text-form-content.spec div').text(specStr);
SPEC && $('td.ti-text-form-content.resorce_type div').text(getListValueWithKey(ACC_CARD_TYPE, SPEC.AccCardType));
})(); })();
</script> </script>

+ 2
- 16
templates/repo/cloudbrain/trainjob/new.tmpl View File

@@ -1,11 +1,5 @@
{{template "base/head" .}} {{template "base/head" .}}
<style> <style>
.train-job-title {
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 16px !important;
padding-left: 3rem !important;
}
.min_title{ .min_title{
font-size: 14px !important; font-size: 14px !important;
margin-bottom: 2rem !important; margin-bottom: 2rem !important;
@@ -57,20 +51,12 @@
text-align: center; text-align: center;
color: #C2C7CC; color: #C2C7CC;
} }
.label-fix-width{
width: 140px !important;
text-align: right;
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 14px !important;
}

</style> </style>
{{template "custom/global_mask" .}} {{template "custom/global_mask" .}}
<div class="repository"> <div class="repository">
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="ui container"> <div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-queue="{{.QueuesDetail}}" data-queue-start="{{.i18n.Tr "repo.wait_count_start"}}" data-queue-end="{{.i18n.Tr "repo.wait_count_end"}}"></div>
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}"></div>
{{if eq .NotStopTaskCount 0}} {{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}} {{template "base/alert" .}}
{{end}} {{end}}
@@ -85,7 +71,7 @@
<input type="hidden" name="action" value="update"> <input type="hidden" name="action" value="update">
<input type="hidden" id="ai_image_name" value="{{.image}}"> <input type="hidden" id="ai_image_name" value="{{.image}}">
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4> <h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required unite min_title inline field">
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain"> <div class="ui blue mini menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/cloudbrain/train-job/create"> <a class="active item" href="{{.RepoLink}}/cloudbrain/train-job/create">


+ 24
- 198
templates/repo/cloudbrain/trainjob/show.tmpl View File

@@ -1,201 +1,6 @@
{{template "base/head" .}} {{template "base/head" .}}
<link rel="stylesheet" href="/self/ztree/css/zTreeStyle/zTreeStyle.css" type="text/css"> <link rel="stylesheet" href="/self/ztree/css/zTreeStyle/zTreeStyle.css" type="text/css">
<style> <style>
.according-panel-heading {
box-sizing: border-box;
padding: 8px 16px;
color: #252b3a;
background-color: #f2f5fc;
line-height: 1.5;
cursor: pointer;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
}

.accordion-panel-title {
margin-top: 0;
margin-bottom: 0;
color: #252b3a;
}

.accordion-panel-title-content {
vertical-align: middle;
display: inline-block;
width: calc(100% - 32px);
cursor: default;
}

.acc-margin-bottom {
margin-bottom: 5px;
}

.title_text {
font-size: 12px;
}

.ac-display-inblock {
display: inline-block;
}

.cti-mgRight-sm {
margin-right: 8px;
}

.ac-text-normal {
font-size: 14px;
color: #575d6c;
}

.uc-accordionTitle-black {
color: #333;
}

.accordion-border {
border: 1px solid #cce2ff;
}

.padding0 {
padding: 0 !important;
}

.content-pad {
padding: 15px 35px;
}

.content-margin {
margin: 10px 5px;
}

.tab_2_content {
min-height: 360px;
margin-left: 10px;
}

.ac-grid {
display: block;
*zoom: 1;
}

.ac-grid-col {
float: left;
width: 100%;
}

.ac-grid-col2 .ac-grid-col {
width: 50%;
}

.ti-form {
text-align: left;
max-width: 100%;
vertical-align: middle;
}

.ti-form>tbody {
font-size: 12px;
}

.ti-form>tbody,
.ti-form>tbody>tr {
vertical-align: inherit;
}

.ti-text-form-label {

padding-bottom: 20px;
padding-right: 20px;
color: #8a8e99;
font-size: 12px;
white-space: nowrap !important;
width: 80px;
line-height: 30px;
}

.ti-text-form-content {
line-height: 30px;
padding-bottom: 20px;
}

.ti-form>tbody>tr>td {
vertical-align: top;
white-space: normal;
}

td,
th {
padding: 0;
}

.info_text {
padding-bottom: 20px;
padding-right: 20px;
font-size: 12px;
}

.ac-grid-col .text-span {
width: 450px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}

.redo-color {
color: #3291F8;
}

.ti-action-menu-item:not(:last-child) {
margin-right: 10px;
padding-right: 11px;
text-decoration: none !important;
color: #526ecc;
cursor: pointer;
display: inline-block;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
position: relative;
}

.ti-action-menu-item:not(:last-child):after {
content: "";
display: inline-block;
position: absolute;
height: 12px;
right: 0;
top: 50%;
-webkit-transform: translateY(-6px);
-ms-transform: translateY(-6px);
-o-transform: translateY(-6px);
transform: translateY(-6px);
border-right: 1px solid #dfe1e6;
}

.text-width80 {
width: 100px;
line-height: 30px;
}

.border-according {
border: 1px solid #dfe1e6;
}

.disabled {
cursor: default;
pointer-events: none;
color: rgba(0, 0, 0, .6) !important;
opacity: .45 !important;
}

.pad20 {

border: 0px !important;
}

.model_file_bread { .model_file_bread {
margin-bottom: -0.5rem !important; margin-bottom: -0.5rem !important;
padding-left: 1rem; padding-left: 1rem;
@@ -318,7 +123,17 @@
</div> </div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_creator"}}
</td>


<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-mirror">
{{.User.Name}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.start_time"}} {{$.i18n.Tr "repo.modelarts.train_job.start_time"}}
@@ -472,6 +287,7 @@
</div> </div>
</div> </div>
<div style="clear:both"> <div style="clear:both">
{{if $.datasetDownload}}
<table style="border:none" class="ui fixed small stackable table"> <table style="border:none" class="ui fixed small stackable table">
<thead> <thead>
<tr><th style="color: #8a8e99;font-size:12px" class="three wide left aligned">{{$.i18n.Tr "dataset.file"}}</th> <tr><th style="color: #8a8e99;font-size:12px" class="three wide left aligned">{{$.i18n.Tr "dataset.file"}}</th>
@@ -492,6 +308,7 @@


</tbody> </tbody>
</table> </table>
{{end}}
</div> </div>
</div> </div>


@@ -544,9 +361,18 @@
<div class="ui tab" data-tab="four{{$k}}"> <div class="ui tab" data-tab="four{{$k}}">
<input type="hidden" name="model{{.VersionName}}" value="-1"> <input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1"> <input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">result</div>
<div class="divider"> / </div>
<div style="display: flex;justify-content: space-between;">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">result</div>
<div class="divider"> / </div>
</div>
<a id="{{.VersionName}}-result-down" style="padding-right: 1%;display: none;"
class='{{if $.canDownload}}ti-download-file{{else}}disabled{{end}} file-info'
href="{{$.RepoLink}}/cloudbrain/train-job/{{.JobID}}/download_multi_model?version_name={{.VersionName}}&jobName={{.JobName}}">
<i class="ri-download-cloud-2-line"></i>
<span style="margin-left: 0.3rem;">{{$.i18n.Tr "repo.all_result_download"}}</span>
</a>
</div> </div>
<div id="dir_list{{.VersionName}}"> <div id="dir_list{{.VersionName}}">
</div> </div>


+ 20
- 16
templates/repo/debugjob/index.tmpl View File

@@ -195,7 +195,7 @@
<span>{{$.i18n.Tr "repo.modelarts.createtime"}}</span> <span>{{$.i18n.Tr "repo.modelarts.createtime"}}</span>
</div> </div>
<div class="two wide column text center"> <div class="two wide column text center">
<span>{{$.i18n.Tr "repo.modelarts.computing_resources"}}</span>
<span>{{$.i18n.Tr "repo.modelarts.cluster.computing_resources"}}</span>
</div> </div>
<div class="one wide column text center"> <div class="one wide column text center">
<span>{{$.i18n.Tr "repo.cloudbrain_creator"}}</span> <span>{{$.i18n.Tr "repo.cloudbrain_creator"}}</span>
@@ -211,7 +211,7 @@
<!-- 任务名 --> <!-- 任务名 -->
<div class="four wide column"> <div class="four wide column">
<a class="title" <a class="title"
href='{{if eq .ComputeResource "CPU/GPU"}}{{$.RepoLink}}/cloudbrain/{{.Cloudbrain.ID}}{{else}}{{$.RepoLink}}/modelarts/notebook/{{.Cloudbrain.ID}}{{end}}'
href='{{$.RepoLink}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{.Cloudbrain.ID}}/'
title="{{.DisplayJobName}}" style="font-size: 14px;"> title="{{.DisplayJobName}}" style="font-size: 14px;">
<span class="fitted text_over" <span class="fitted text_over"
style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span> style="width: 90%;vertical-align: middle;">{{.DisplayJobName}}</span>
@@ -220,8 +220,8 @@
<div class="two wide column text center"> <div class="two wide column text center">
<!--任务状态 --> <!--任务状态 -->
<span class="job-status" id="{{.Cloudbrain.ID}}" <span class="job-status" id="{{.Cloudbrain.ID}}"
data-repopath="{{$.RepoRelPath}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}"
data-jobid="{{.Cloudbrain.ID}}" data-resource="{{.ComputeResource}}" data-bootfile="{{.BootFile}}">
data-repopath="{{$.RepoRelPath}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}"
data-jobid="{{.Cloudbrain.ID}}" data-resource="{{.ComputeResource}}">
<span><i id="{{.Cloudbrain.ID}}-icon" style="vertical-align: middle;" <span><i id="{{.Cloudbrain.ID}}-icon" style="vertical-align: middle;"
class="{{.Status}}"></i><span id="{{.Cloudbrain.ID}}-text" class="{{.Status}}"></i><span id="{{.Cloudbrain.ID}}-text"
style="margin-left: 0.4em;font-size: 12px;">{{.Status}}</span></span> style="margin-left: 0.4em;font-size: 12px;">{{.Status}}</span></span>
@@ -234,9 +234,14 @@
</div> </div>
<div class="two wide column text center"> <div class="two wide column text center">
<!-- 任务计算资源 --> <!-- 任务计算资源 -->
<span style="font-size: 12px;margin-left: 0.4rem;"
class="">
{{.ComputeResource}}</span>
<span style="font-size: 12px;margin-left: 0.4rem;">
{{if eq .Cloudbrain.Type 2}}
{{$.i18n.Tr "cloudbrain.resource_cluster_c2net_simple"}}
{{else}}
{{$.i18n.Tr "cloudbrain.resource_cluster_openi_simple"}}
{{end}}
{{.ComputeResource}}
</span>
</div> </div>
<div class="one wide column text center"> <div class="one wide column text center">
{{if .User.Name}} {{if .User.Name}}
@@ -262,7 +267,7 @@
<a style="margin: 0 1rem;" id="ai-debug-{{.Cloudbrain.ID}}" <a style="margin: 0 1rem;" id="ai-debug-{{.Cloudbrain.ID}}"
class='ui basic ai_debug {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING"}}disabled {{else}}blue {{end}}button' class='ui basic ai_debug {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING"}}disabled {{else}}blue {{end}}button'
data-jobid="{{.Cloudbrain.ID}}" data-jobid="{{.Cloudbrain.ID}}"
data-repopath='{{$.RepoLink}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}/{{.Cloudbrain.ID}}/'>
data-repopath='{{$.RepoLink}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{.Cloudbrain.ID}}/'>
{{$.i18n.Tr "repo.debug"}} {{$.i18n.Tr "repo.debug"}}
</a> </a>
{{else}} {{else}}
@@ -270,7 +275,7 @@
<a id="ai-debug-{{.Cloudbrain.ID}}" <a id="ai-debug-{{.Cloudbrain.ID}}"
class='ui basic ai_debug {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING"}} disabled {{else}}blue {{end}}button' class='ui basic ai_debug {{if eq .Status "CREATING" "STOPPING" "WAITING" "STARTING"}} disabled {{else}}blue {{end}}button'
data-jobid="{{.Cloudbrain.ID}}" data-jobid="{{.Cloudbrain.ID}}"
data-repopath='{{$.RepoLink}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}/{{.Cloudbrain.ID}}/'
data-repopath='{{$.RepoLink}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{.Cloudbrain.ID}}/'
data-linkpath='{{$.Link}}'> data-linkpath='{{$.Link}}'>
{{$.i18n.Tr "repo.debug_again"}} {{$.i18n.Tr "repo.debug_again"}}
</a> </a>
@@ -295,9 +300,8 @@
{{if .CanDel}} {{if .CanDel}}
<a id="ai-stop-{{.Cloudbrain.ID}}" <a id="ai-stop-{{.Cloudbrain.ID}}"
class='ui basic ai_stop {{if eq .Status "STOPPED" "FAILED" "START_FAILED" "STOPPING" "CREATING" "STARTING" "SUCCEEDED" "CREATE_FAILED" "DELETED"}}disabled {{else}}blue {{end}}button' class='ui basic ai_stop {{if eq .Status "STOPPED" "FAILED" "START_FAILED" "STOPPING" "CREATING" "STARTING" "SUCCEEDED" "CREATE_FAILED" "DELETED"}}disabled {{else}}blue {{end}}button'
data-repopath="{{$.RepoLink}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}/{{.Cloudbrain.ID}}/stop"
data-jobid="{{.Cloudbrain.ID}}"
{{if .BootFile}}data-bootfile="{{.BootFile}}"{{end}}>
data-repopath='{{$.RepoLink}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{.Cloudbrain.ID}}/stop'
data-jobid="{{.Cloudbrain.ID}}">
{{$.i18n.Tr "repo.stop"}} {{$.i18n.Tr "repo.stop"}}
</a> </a>
@@ -309,7 +313,7 @@
</form> </form>
<!-- 删除 --> <!-- 删除 -->
<form id="delForm-{{.Cloudbrain.ID}}" <form id="delForm-{{.Cloudbrain.ID}}"
action="{{if eq .ComputeResource "CPU/GPU"}}{{$.RepoLink}}/cloudbrain{{else}}{{$.RepoLink}}/modelarts/notebook{{end}}/{{.Cloudbrain.ID}}/del"
action="{{$.RepoLink}}{{if eq .Cloudbrain.Type 2}}/grampus/notebook{{else}}{{if eq .ComputeResource "CPU/GPU"}}/cloudbrain{{else}}/modelarts/notebook{{end}}{{end}}/{{.Cloudbrain.ID}}/del"
method="post"> method="post">
<input type="hidden" name="debugListType" value="{{$.ListType}}"> <input type="hidden" name="debugListType" value="{{$.ListType}}">
{{$.CsrfTokenHtml}} {{$.CsrfTokenHtml}}
@@ -328,7 +332,7 @@
</div> </div>
{{if not .BootFile}} {{if not .BootFile}}
<div class="ui compact buttons" <div class="ui compact buttons"
style="{{if eq .ComputeResource "CPU/GPU"}} visibility: visible {{else}} visibility: hidden{{end}}">
style="{{if and (ne .Cloudbrain.Type 2) (eq .ComputeResource "CPU/GPU")}} visibility: visible {{else}} visibility: hidden{{end}}">
<div class="ui dropdown" id="model_more" <div class="ui dropdown" id="model_more"
style="padding: .58928571em 1.125em .58928571em;"> style="padding: .58928571em 1.125em .58928571em;">
<div class="text">{{$.i18n.Tr "repo.more"}}</div> <div class="text">{{$.i18n.Tr "repo.more"}}</div>
@@ -427,7 +431,7 @@
</div> </div>


<div class="ui modal debug-again-alert"> <div class="ui modal debug-again-alert">
<div class="ui message" style="background-color: rgba(242, 113, 28, 0.05);border: 1px solid rgba(242, 113, 28, 1);border-radius: 5px;">
<div class="ui message" style="background-color: rgba(242, 113, 28, 0.05);border: 1px solid rgba(242, 113, 28, 1);border-radius: 5px;">
<div style="display: flex;align-items: center;"> <div style="display: flex;align-items: center;">
<i class="ri-information-line" style="font-size: 35px;color: rgba(242, 113, 28, 1);;"></i> <i class="ri-information-line" style="font-size: 35px;color: rgba(242, 113, 28, 1);;"></i>
<div style="text-align: left;margin-left: 1rem;"> <div style="text-align: left;margin-left: 1rem;">
@@ -477,4 +481,4 @@
}) })
}) })


</script>
</script>

+ 162
- 0
templates/repo/grampus/notebook/gpu/new.tmpl View File

@@ -0,0 +1,162 @@
{{template "base/head" .}}
{{template "custom/global_mask" .}}

<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-exceed-size="{{DebugAttachSize}}"></div>
{{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}}
{{end}}
<div class="ui negative message" id="messageInfo" style="display:none;">
<p></p>
</div>
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new_debug"}}
</h4>
{{template "custom/alert_cb" .}}
<div class="ui attached segment">
<form id="form_id" class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="type" value="0">
<input type="hidden" id="ai_image_name" value="{{.image}}">
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="item" href="{{.RepoLink}}/cloudbrain/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_openi"}}
</a>
<a class="active item" href="{{.RepoLink}}/grampus/notebook/create?type=0">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_c2net"}}(Beta)
</a>
</div>
</div>
<div class="inline required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue small menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/grampus/notebook/create?type=0">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16"
height="16">
<path fill="none" d="M0 0h24v24H0z" />
<path
d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z" />
</svg>
CPU/GPU
</a>
<a class="item" href="{{.RepoLink}}/grampus/notebook/create?type=1">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16"
height="16">
<path fill="none" d="M0 0h24v24H0z" />
<path
d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z" />
</svg>
Ascend NPU</a>
</div>
</div>
<div class="min_title inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
{{template "custom/task_wait_count" .}}
<div style="display: flex;align-items: center;margin-left: 156px;margin-top: 0.5rem;">
<i class="ri-error-warning-line" style="color: #f2711c;margin-right: 0.5rem;"></i>
<span style="color: #888;font-size: 12px;">{{.i18n.Tr "cloudbrain.new_debug_gpu_tooltips1" "/code" "/dataset" "/pretrainmodel" | Safe}}</span>
</div>
</div>
<div class="inline min_title required field" style="margin-bottom: 0rem !important;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.task_name"}}</label>
<input name="display_job_name" id="cloudbrain_job_name" style="width: 60%;"
placeholder="{{.i18n.Tr "cloudbrain.task_name"}}" value="{{.display_job_name}}" tabindex="3"
autofocus required maxlength="255" onkeyup="this.value=this.value.replace(/[, ]/g,'')">
</div>
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 2rem;">{{.i18n.Tr "repo.cloudbrain_jobname_err"}}</span>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
{{if .description}}
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}>{{.description}}</textarea>
{{else}}
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)"></textarea>
{{end}}
</div>
<div class="ui divider"></div>
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2" id="code_version"
name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branch_name }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{else}}
<option name="branch_name" value="{{.branchName}}">{{.branchName}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branchName }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{end}}
</select>
</div>
{{template "custom/select_model" .}}
<div id="images-new-cb">

</div>
<div id="select-multi-dataset">

</div>
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 1rem;"></span>
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_specification"}}</label>
<select id="__specs__" class="ui search dropdown width48"
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px' ovalue="{{.spec_id}}"
{{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}}
name="spec_id">
</select>
<span><i class="question circle icon link"></i></span>
<a href="{{AppSubUrl}}/resource_desc" target="_blank">{{.i18n.Tr "custom.resource_description"}}</a>
{{if .CloudBrainPaySwitch}}
<div class="cloudbrain_resource_spec_blance_tip" style="padding:0 5px;margin:6px 0;margin-left:265px;font-size:12px;width: 50%!important;">
<span>{{$.i18n.Tr "points.balance_of_points"}}<span style="color:red;margin: 0 3px">{{.PointAccount.Balance}}</span>{{$.i18n.Tr "points.points"}}</span><span>{{$.i18n.Tr "points.expected_time"}}<span style="color:red;margin: 0 3px" class="can-use-time"></span>{{$.i18n.Tr "points.hours"}}</span>
<span style="float:right;">
<i class="question circle icon link" data-position="right center" data-variation="mini"></i>
<a href="{{AppSubUrl}}/reward/point/rule" target="_blank">{{$.i18n.Tr "points.points_acquisition_instructions"}}</a>
</span>
</div>
{{end}}
</div>
<div class="inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel"
href="{{.RepoLink}}/debugjob?debugListType=all">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
</form>
</div>
</div>
</div>
{{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script>
;(function() {
var SPECS = {{ .Specs }};
var showPoint = {{ .CloudBrainPaySwitch }};
window.renderSpecsSelect($('#__specs__'), SPECS, showPoint, {
gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}},
free: {{$.i18n.Tr "cloudbrain.free"}},
point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}},
memory: {{$.i18n.Tr "cloudbrain.memory"}},
shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}},
});
})();
</script>

+ 164
- 0
templates/repo/grampus/notebook/npu/new.tmpl View File

@@ -0,0 +1,164 @@
{{template "base/head" .}}
{{template "custom/global_mask" .}}
<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-exceed-size="{{DebugAttachSize}}"></div>
{{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}}
{{end}}
<div class="ui negative message" id="messageInfo" style="display: none;">
<p></p>
</div>
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new_debug"}}
</h4>
{{template "custom/alert_cb" .}}
<div class="ui attached segment">
<form class="ui form" id="form_id" action="{{.Link}}" method="post">
<input type="hidden" name="type" value="1">
<input type="hidden" name="image" value="">
{{.CsrfTokenHtml}}
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="item" href="{{.RepoLink}}/cloudbrain/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_openi"}}
</a>
<a class="active item" href="{{.RepoLink}}/grampus/notebook/create?type=1">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_c2net"}}(Beta)
</a>
</div>
</div>
<div class="inline required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue small menu compact selectcloudbrain">
<a class="item" href="{{.RepoLink}}/grampus/notebook/create?type=0">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
CPU/GPU</a>
<a class="active item" href="{{.RepoLink}}/grampus/notebook/create?type=1">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
Ascend NPU</a>
</div>
</div>
<div class="inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
{{template "custom/task_wait_count" .}}
</div>
<div class="inline min_title required field" style="margin-bottom: 0rem !important;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.task_name"}}</label>
<input name="display_job_name" id="cloudbrain_job_name" placeholder="任务名称" style="width: 60%;" value="{{.display_job_name}}" tabindex="3" autofocus required maxlength="36" onkeyup="this.value=this.value.replace(/[, ]/g,'')">
</div>
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 2rem;">{{.i18n.Tr "repo.cloudbrain_jobname_err"}}</span>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
{{if .description}}
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}>{{.description}}</textarea>
{{else}}
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)"></textarea>
{{end}}
</div>
<div class="ui divider"></div>
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.code_version"}}</label>
<select class="ui dropdown width80 left2" id="code_version"
name="branch_name">
{{if .branch_name}}
<option name="branch_name" value="{{.branch_name}}">{{.branch_name}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branch_name }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{else}}
<option name="branch_name" value="{{.branchName}}">{{.branchName}}</option>
{{range $k, $v :=.Branches}}
{{ if ne $v $.branchName }}
<option name="branch_name" value="{{$v}}">{{$v}}</option>
{{end}}
{{end}}
{{end}}
</select>
</div>
{{template "custom/select_model" .}}
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.mirror"}}</label>
<select class="ui search dropdown cloudbrain_image width48" placeholder="{{.i18n.Tr "cloudbrain.choose_mirror"}}" style='width:385px' name="image_id">
{{if .image_id}}
{{range .images}}
{{if eq $.image_id .ID}}
<option value="{{.ID}}">{{.Name}}</option>
{{end}}
{{end}}
{{range .images}}
{{if ne $.image_id .ID}}
<option value="{{.ID}}">{{.Name}}</option>
{{end}}
{{end}}
{{else}}
{{range .images}}
<option name="image_id" value="{{.ID}}">{{.Name}}</option>
{{end}}
{{end}}
</select>
</div>
<div id="select-multi-dataset">

</div>
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 1rem;"></span>
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.specification"}}</label>
<select id="__specs__" class="ui search dropdown width48" ovalue="{{.spec_id}}"
{{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}}
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px' name="spec_id"></select>
<span><i class="question circle icon link"></i></span>
<a href="{{AppSubUrl}}/resource_desc" target="_blank">{{.i18n.Tr "custom.resource_description"}}</a>
{{if .CloudBrainPaySwitch}}
<div class="cloudbrain_resource_spec_blance_tip" style="padding:0 5px;margin:6px 0;margin-left:265px;font-size:12px;width: 50%!important;">
<span>{{$.i18n.Tr "points.balance_of_points"}}<span style="color:red;margin: 0 3px">{{.PointAccount.Balance}}</span>{{$.i18n.Tr "points.points"}}</span><span>{{$.i18n.Tr "points.expected_time"}}<span style="color:red;margin: 0 3px" class="can-use-time"></span>{{$.i18n.Tr "points.hours"}}</span>
<span style="float:right;">
<i class="question circle icon link" data-position="right center" data-variation="mini"></i>
<a href="{{AppSubUrl}}/reward/point/rule" target="_blank">{{$.i18n.Tr "points.points_acquisition_instructions"}}</a>
</span>
</div>
{{end}}
</div>
<div class="inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel" href="{{.RepoLink}}/debugjob?debugListType=all">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
</form>
</div>
</div>
</div>
{{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script>
;(function() {
var SPECS = {{ .Specs }};
var showPoint = {{ .CloudBrainPaySwitch }};
window.renderSpecsSelect($('#__specs__'), SPECS, showPoint, {
gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}},
free: {{$.i18n.Tr "cloudbrain.free"}},
point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}},
memory: {{$.i18n.Tr "cloudbrain.memory"}},
shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}},
});
})();
</script>

+ 439
- 0
templates/repo/grampus/notebook/show.tmpl View File

@@ -0,0 +1,439 @@
{{template "base/head" .}}
<div class="repository">
{{template "repo/header" .}}
<div class="ui container">
<h4 class="ui header" id="vertical-segment">
<div class="ui breadcrumb">
<a class="section" href="{{.RepoLink}}/debugjob?debugListType=all">
{{.i18n.Tr "repo.cloudbrain"}}
</a>
<div class="divider"> / </div>
<a class="section backTodeBug" href="{{.RepoLink}}/debugjob?debugListType=all">
{{$.i18n.Tr "repo.modelarts.notebook"}}
</a>
<div class="divider"> / </div>
{{with .task}}
<div class="active section">{{.DisplayJobName}}</div>
{{end}}
</div>
</h4>
{{with .task}}
<div class="ui accordion border-according" id="accordion" data-repopath="{{$.RepoRelPath}}/grampus/notebook"
data-jobid="{{.ID}}" data-version="">
<div class="active title padding0">
<div class="according-panel-heading">
<div class="accordion-panel-title">
<!-- <i class="dropdown icon"></i> -->
<span class="accordion-panel-title-content">
<span>
<div class="ac-display-inblock title_text acc-margin-bottom">
<span class="cti-mgRight-sm">
{{TimeSinceUnix1 .CreatedUnix}}
</span>
<span class="cti-mgRight-sm">{{$.i18n.Tr "repo.modelarts.status"}}:
<span id="{{.VersionName}}-status-span"><i id="icon"
style="vertical-align: middle;" class="{{.Status}}"></i><span id="text"
style="margin-left: 0.4em;font-size: 12px;">{{.Status}}</span></span>
</span>
<span
class="cti-mgRight-sm">{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}:</span>
<span class="cti-mgRight-sm uc-accordionTitle-black"
id="{{.VersionName}}-duration-span">{{ConvertDurationToStr .Duration}}</span>
<span class="refresh-status" data-tooltip="刷新" style="cursor: pointer;" data-inverted="" data-version="{{.VersionName}}">
<i class="redo icon redo-color"></i>
</span>
</div>
</span>
</span>
</div>
</div>
</div>
<div class="active content">
<div class="content-pad">
<div class="ui pointing secondary menu" style="border-bottom: 1px solid rgba(34,36,38,.15);">
<a class="active item" data-tab="first">{{$.i18n.Tr "repo.modelarts.train_job.config"}}</a>
</div>
<div class="ui tab active" data-tab="first">
<div style="padding-top: 10px;">
<div class="tab_2_content">
<div class="ac-grid ac-grid-col2">
<div class="ac-grid-col">
<table class="ti-form">
<tbody class="ti-text-form">
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_task"}}
</td>
<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.DisplayJobName}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.status"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-status">
{{.Status}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_creator"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{.User.Name}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.code_version"}}
</td>
<td class="ti-text-form-content">
<div class="text-span text-span-w">
{{if .BranchName}}
{{.BranchName}}
{{else}}
--
{{end}}
<span style="margin-left:1rem" class="ui label">{{SubStr .CommitID 0 10}}</span>
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.computing_resources"}}
</td>
<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-computeresource">
{{.ComputeResource}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.createtime"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-createtime">
{{TimeSinceUnix1 .CreatedUnix}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain.time.starttime"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">
<div class="text-span text-span-w"
id="{{.VersionName}}-startTime">
{{if not (eq .StartTime 0)}}
{{TimeSinceUnix1 .StartTime}}
{{else}}
--
{{end}}
</div>
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain.time.endtime"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-EndTime">
{{if not (eq .EndTime 0)}}
{{TimeSinceUnix1 .EndTime}}
{{else}}
--
{{end}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w"
id="{{.VersionName}}-duration">
{{if not (eq .Duration 0)}}
{{ConvertDurationToStr .Duration}}
{{else}}
--
{{end}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.description"}}
</td>

<td class="ti-text-form-content">
<div class="text-span-new" id="model_description">
{{.Description}}
</div>
</td>
</tr>

</tbody>
</table>
</div>
<div class="ac-grid-col">
<table class="ti-form">
<tbody class="ti-text-form">

<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.mirror"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-mirror">
<span class="ui poping up clipboard" data-position="top center" id="clipboard-btn-image" style="cursor:pointer"
data-clipboard-text="{{.EngineName}}"
data-success="{{$.i18n.Tr "repo.copied"}}"
data-error="{{$.i18n.Tr "repo.copied_error"}}"
data-content="{{$.i18n.Tr "repo.copy"}}"
data-variation="inverted tiny"
>
<span title="{{.EngineName}}">{{.EngineName}}</span>
</span>
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.train_job.standard"}}
</td>

<td class="ti-text-form-content spec">
<div class="text-span text-span-w"></div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.grampus.train_job.ai_center"}}
</td>
<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-ai_center">
{{if $.ai_center}}{{$.ai_center}}{{else}}--{{end}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.model_name"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .ModelName}}{{.ModelName}}{{else}}--{{end}}</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelconvert.modelversion"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .ModelVersion}}{{.ModelVersion}}{{else}}--{{end}}</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.infer_job_model_file"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w">{{if .CkptName}}{{.CkptName}}{{else}}--{{end}}</div>
</td>
</tr>
{{if eq .ComputeResource "CPU/GPU"}}
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.code_storage_path"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="code_storage_path">
{{$.code_path}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.dataset_storage_path"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="dataset_storage_path">
{{$.dataset_path}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.model_storage_path"}}
</td>
<td class="ti-text-form-content">
<div class="text-span text-span-w" id="code_storage_path">
/pretrainmodel
</div>
</td>
</tr>
{{end}}
{{if eq .ComputeResource "NPU"}}
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "cloudbrain.code_obs_address"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-trainUrl">
<span class="ui poping up clipboard" data-position="top center" id="clipboard-btn-trainUrl" style="cursor:pointer"
data-clipboard-text="{{.TrainUrl}}"
data-success="{{$.i18n.Tr "repo.copied"}}"
data-error="{{$.i18n.Tr "repo.copied_error"}}"
data-content="{{$.i18n.Tr "repo.copy"}}"
data-variation="inverted tiny"
>
<span title="{{.TrainUrl}}">
{{if .TrainUrl}}
{{.TrainUrl}}
{{else}}
--
{{end}}
</span>
</span>
</div>
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
</div>
</div>
<div style="clear:both">
{{if $.datasetDownload}}
<table style="border:none" class="ui fixed small stackable table">
<thead>
<tr><th style="color: #8a8e99;font-size:12px" class="three wide left aligned">{{$.i18n.Tr "dataset.file"}}</th>
{{if eq .ComputeResource "NPU"}}
<th style="color: #8a8e99;font-size:12px"class="eleven wide">{{$.i18n.Tr "dataset.download_url"}}</th>
<th style="color: #8a8e99;font-size:12px" class="two wide center aligned">{{$.i18n.Tr "dataset.download_oper"}}</th>
{{end}}
{{$Resource := .ComputeResource}}
</tr></thead>
<tbody>
{{range $k,$v := $.datasetDownload}}
<tr>
<td class="dataset_nowrap_two_line">
{{if eq .IsDelete true}}
{{.DatasetName}}({{$.i18n.Tr "dataset.file_deleted"}})
{{else}}
<a href="{{.RepositoryLink}}" target="_blank">{{.DatasetName}}</a>
{{end}}
</td>
{{if eq $Resource "NPU"}}
<td><div class="dataset_nowrap_two_line">{{.DatasetDownloadLink}}</div></td>
<td class="center aligned"><a class="ui poping up clipboard" id="clipboard-btn-dataset-{{$k}}" data-original="{{$.i18n.Tr "repo.copy_link"}}" data-success="{{$.i18n.Tr "repo.copy_link_success"}}" data-error="{{$.i18n.Tr "repo.copy_link_error"}}" data-content="{{$.i18n.Tr "repo.copy_link"}}" data-variation="inverted tiny" data-clipboard-text="{{.DatasetDownloadLink}}">{{if .DatasetDownloadLink}}{{$.i18n.Tr "dataset.download_copy"}}{{end}}</a></td>
{{end}}
</tr>
{{end}}
</tbody>
{{end}}
</table>
{{if and (eq .ComputeResource "NPU") ($.modelDownload.Name)}}
<table style="border:none" class="ui fixed small stackable table">
<thead>
<tr>
<th style="color: #8a8e99;font-size:12px" class="three wide left aligned">{{$.i18n.Tr "repo.modelarts.infer_job_model_file"}}</th>
<th style="color: #8a8e99;font-size:12px"class="eleven wide">{{$.i18n.Tr "dataset.download_model_url"}}</th>
<th style="color: #8a8e99;font-size:12px" class="two wide center aligned">{{$.i18n.Tr "dataset.download_oper"}}</th>
</tr>
</thead>
<tbody>
<tr>
<td class="dataset_nowrap_two_line">
{{if eq $.modelDownload.IsDelete true}}
{{$.modelDownload.Name}}({{$.i18n.Tr "dataset.file_deleted"}})
{{else}}
<a href="{{$.RepoLink}}/modelmanage/show_model_info?name={{.ModelName}}" target="_blank">{{$.modelDownload.Name}}</a>
{{end}}
</td>
<td><div class="dataset_nowrap_two_line">{{$.modelDownload.DownloadLink}}</div></td>
<td class="center aligned"><a class="ui poping up clipboard" id="clipboard-btn-model" data-original="{{$.i18n.Tr "repo.copy_link"}}" data-success="{{$.i18n.Tr "repo.copy_link_success"}}" data-error="{{$.i18n.Tr "repo.copy_link_error"}}" data-content="{{$.i18n.Tr "repo.copy_link"}}" data-variation="inverted tiny" data-clipboard-text="{{$.modelDownload.DownloadLink}}">{{if $.modelDownload.DownloadLink}}{{$.i18n.Tr "dataset.download_copy"}}{{end}}</a></td>
</tr>
</tbody>
</table>
{{end}}
</div>
</div>
</div>


</div>
</div>
</div>
{{end}}
{{template "base/paginate" .}}
</div>
<!-- 确认模态框 -->
<div id="deletemodel">
<div class="ui basic modal">
<div class="ui icon header">
<i class="trash icon"></i> {{.i18n.Tr "cloudbrain.delete_task"}}
</div>

<div class="content">
<p>{{.i18n.Tr "cloudbrain.task_delete_confirm"}}</p>
</div>
<div class="actions">
<div class="ui red basic inverted cancel button">
<i class="remove icon"></i> {{.i18n.Tr "cloudbrain.operate_cancel"}}
</div>
<div class="ui green basic inverted ok button">
<i class="checkmark icon"></i> {{.i18n.Tr "cloudbrain.operate_confirm"}}
</div>
</div>
</div>
</div>


</div>
{{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script>
;(function() {
var SPEC = {{ .Spec }};
var showPoint = false;
var specStr = window.renderSpecStr(SPEC, showPoint, {
gpu_memory: {{$.i18n.Tr "cloudbrain.gpu_memory"}},
free: {{$.i18n.Tr "cloudbrain.free"}},
point_hr: {{$.i18n.Tr "cloudbrain.point_hr"}},
memory: {{$.i18n.Tr "cloudbrain.memory"}},
shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}},
});
$('td.ti-text-form-content.spec div').text(specStr);
})();
</script>

+ 2
- 8
templates/repo/grampus/trainjob/gpu/new.tmpl View File

@@ -1,12 +1,6 @@
{{template "base/head" .}} {{template "base/head" .}}
<style> <style>


.train-job-title {
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 16px !important;
padding-left: 3rem !important;
}
.min_title{ .min_title{
font-size: 14px !important; font-size: 14px !important;
margin-bottom: 2rem !important; margin-bottom: 2rem !important;
@@ -96,14 +90,14 @@
<div class="required min_title inline field"> <div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain"> <div class="ui blue mini menu compact selectcloudbrain">
<a {{if.GPUEnabled}}class="active item" href="{{.RepoLink}}/grampus/train-job/gpu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}}>
<a class="active item" href="{{.RepoLink}}/grampus/train-job/gpu/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"> <svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/> <path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/> <path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg> </svg>
CPU/GPU CPU/GPU
</a> </a>
<a {{if.NPUEnabled}}class="item" href="{{.RepoLink}}/grampus/train-job/npu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}} >
<a class="item" href="{{.RepoLink}}/grampus/train-job/npu/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"> <svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/> <path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/> <path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>


+ 2
- 15
templates/repo/grampus/trainjob/npu/new.tmpl View File

@@ -1,11 +1,5 @@
{{template "base/head" .}} {{template "base/head" .}}
<style> <style>
.train-job-title {
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 16px !important;
padding-left: 3rem !important;
}
.min_title{ .min_title{
font-size: 14px !important; font-size: 14px !important;
margin-bottom: 2rem !important; margin-bottom: 2rem !important;
@@ -45,13 +39,6 @@
text-align: center; text-align: center;
color: #C2C7CC; color: #C2C7CC;
} }
.label-fix-width{
width: 140px !important;
text-align: right;
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
font-size: 14px !important;
}
</style> </style>
{{template "custom/global_mask" .}} {{template "custom/global_mask" .}}
<div class="repository"> <div class="repository">
@@ -89,14 +76,14 @@
<div class="required min_title inline field"> <div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain"> <div class="ui blue mini menu compact selectcloudbrain">
<a {{if.GPUEnabled}}class="item" href="{{.RepoLink}}/grampus/train-job/gpu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}}>
<a class="item" href="{{.RepoLink}}/grampus/train-job/gpu/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"> <svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/> <path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/> <path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg> </svg>
CPU/GPU CPU/GPU
</a> </a>
<a {{if.NPUEnabled}}class="active item" href="{{.RepoLink}}/grampus/train-job/npu/create"{{else}}href="javascript:return false;" class="item disabled" {{end}} >
<a class="active item" href="{{.RepoLink}}/grampus/train-job/npu/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"> <svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/> <path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/> <path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>


+ 23
- 193
templates/repo/grampus/trainjob/show.tmpl View File

@@ -1,195 +1,6 @@
{{template "base/head" .}} {{template "base/head" .}}
<link rel="stylesheet" href="/self/ztree/css/zTreeStyle/zTreeStyle.css" type="text/css"> <link rel="stylesheet" href="/self/ztree/css/zTreeStyle/zTreeStyle.css" type="text/css">
<style> <style>
.according-panel-heading {
box-sizing: border-box;
padding: 8px 16px;
color: #252b3a;
background-color: #f2f5fc;
line-height: 1.5;
cursor: pointer;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
}

.accordion-panel-title {
margin-top: 0;
margin-bottom: 0;
color: #252b3a;
}

.accordion-panel-title-content {
vertical-align: middle;
display: inline-block;
width: calc(100% - 32px);
cursor: default;
}

.acc-margin-bottom {
margin-bottom: 5px;
}

.title_text {
font-size: 12px;
}

.ac-display-inblock {
display: inline-block;
}

.cti-mgRight-sm {
margin-right: 8px;
}

.ac-text-normal {
font-size: 14px;
color: #575d6c;
}

.uc-accordionTitle-black {
color: #333;
}

.accordion-border {
border: 1px solid #cce2ff;
}

.padding0 {
padding: 0 !important;
}

.content-pad {
padding: 15px 35px;
}

.content-margin {
margin: 10px 5px;
}

.tab_2_content {
min-height: 360px;
margin-left: 10px;
}

.ac-grid {
display: block;
*zoom: 1;
}

.ac-grid-col {
float: left;
width: 100%;
}

.ac-grid-col2 .ac-grid-col {
width: 50%;
}

.ti-form {
text-align: left;
max-width: 100%;
vertical-align: middle;
}

.ti-form>tbody {
font-size: 12px;
}

.ti-form>tbody,
.ti-form>tbody>tr {
vertical-align: inherit;
}

.ti-text-form-label {

padding-bottom: 20px;
padding-right: 20px;
color: #8a8e99;
font-size: 12px;
white-space: nowrap !important;
width: 80px;
line-height: 30px;
}

.ti-text-form-content {
line-height: 30px;
padding-bottom: 20px;
}

.ti-form>tbody>tr>td {
vertical-align: top;
white-space: normal;
}

td,
th {
padding: 0;
}

.ac-grid-col .text-span {
width: 450px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}

.redo-color {
color: #3291F8;
}

.ti-action-menu-item:not(:last-child) {
margin-right: 10px;
padding-right: 11px;
text-decoration: none !important;
color: #526ecc;
cursor: pointer;
display: inline-block;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
position: relative;
}

.ti-action-menu-item:not(:last-child):after {
content: "";
display: inline-block;
position: absolute;
height: 12px;
right: 0;
top: 50%;
-webkit-transform: translateY(-6px);
-ms-transform: translateY(-6px);
-o-transform: translateY(-6px);
transform: translateY(-6px);
border-right: 1px solid #dfe1e6;
}

.text-width80 {
width: 100px;
line-height: 30px;
}

.border-according {
border: 1px solid #dfe1e6;
}

.disabled {
cursor: default;
pointer-events: none;
color: rgba(0, 0, 0, .6) !important;
opacity: .45 !important;
}

.pad20 {

border: 0px !important;
}

.model_file_bread { .model_file_bread {
margin-bottom: -0.5rem !important; margin-bottom: -0.5rem !important;
padding-left: 1rem; padding-left: 1rem;
@@ -310,6 +121,17 @@
</div> </div>
</td> </td>
</tr> </tr>
<tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.cloudbrain_creator"}}
</td>

<td class="ti-text-form-content">
<div class="text-span text-span-w" id="{{.VersionName}}-mirror">
{{.User.Name}}
</div>
</td>
</tr>
<tr class="ti-no-ng-animate"> <tr class="ti-no-ng-animate">
<td class="ti-no-ng-animate ti-text-form-label text-width80"> <td class="ti-no-ng-animate ti-text-form-label text-width80">
{{$.i18n.Tr "repo.modelarts.run_version"}} {{$.i18n.Tr "repo.modelarts.run_version"}}
@@ -575,10 +397,18 @@
<div class="ui tab" data-tab="third{{$k}}"> <div class="ui tab" data-tab="third{{$k}}">
<input type="hidden" name="model{{.VersionName}}" value="-1"> <input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1"> <input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>

<div style="display: flex;justify-content: space-between;">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">{{.VersionName}}</div>
<div class="divider"> / </div>
</div>
<a id="{{.VersionName}}-result-down" style="padding-right: 1%;display: none;"
class='{{if $.canDownload}}ti-download-file{{else}}disabled{{end}} file-info'
href="{{$.RepoLink}}/grampus/train-job/{{.JobID}}/download_multi_model?version_name={{.VersionName}}">
<i class="ri-download-cloud-2-line"></i>
<span style="margin-left: 0.3rem;">{{$.i18n.Tr "repo.all_result_download"}}</span>
</a>
</div> </div>
<div id="dir_list{{.VersionName}}"> <div id="dir_list{{.VersionName}}">
</div> </div>


+ 6
- 11
templates/repo/modelarts/inferencejob/new.tmpl View File

@@ -1,10 +1,5 @@
{{template "base/head" .}} {{template "base/head" .}}
<style> <style>
.unite{
font-family: SourceHanSansSC-medium !important;
color: rgba(16, 16, 16, 100) !important;
}

.title{ .title{
font-size: 16px !important; font-size: 16px !important;
padding-left: 3rem !important; padding-left: 3rem !important;
@@ -64,7 +59,7 @@
<input type="hidden" id="fail_dataset_name" value="{{$.dataset_name}}"> <input type="hidden" id="fail_dataset_name" value="{{$.dataset_name}}">
<input type="hidden" id="fail_dataset_uuid" value="{{$.attachment}}"> <input type="hidden" id="fail_dataset_uuid" value="{{$.attachment}}">


<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<h4 class="title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field"> <div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue mini menu compact selectcloudbrain"> <div class="ui blue mini menu compact selectcloudbrain">
@@ -111,10 +106,10 @@
</div> </div>
<div class="ui divider"></div> <div class="ui divider"></div>
<!-- 模型相关配置 --> <!-- 模型相关配置 -->
<h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>
<div class="required unite inline min_title fields" style="width: 96.8%;">
<div class="required eight wide field">
<label style="font-weight: normal;white-space: nowrap;width: 210px;text-align: right;">{{.i18n.Tr "repo.modelarts.infer_job.select_model"}}</label>
<h4 class="title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>
<div class="required inline min_title fields" style="width: 94%;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.infer_job.select_model"}}</label>
<div class="six wide field">
<div class="ui fluid search selection dropdown loading " id="select_model"> <div class="ui fluid search selection dropdown loading " id="select_model">
<input type="hidden" name="model_name" required> <input type="hidden" name="model_name" required>
<div class="text"></div> <div class="text"></div>
@@ -148,7 +143,7 @@
</span> </span>
</div> </div>
<!-- AI引擎 --> <!-- AI引擎 -->
<div class="required inline min_title fields" style="width: 95%;">
<div class="required inline min_title fields" style="width: 92.5%;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</label> <label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</label>
<div class="field" style="flex: 1.5;"> <div class="field" style="flex: 1.5;">
<select class="ui dropdown width" id="trainjob_engines"> <select class="ui dropdown width" id="trainjob_engines">


+ 12
- 159
templates/repo/modelarts/inferencejob/show.tmpl View File

@@ -1,160 +1,5 @@
{{template "base/head" .}} {{template "base/head" .}}
<style> <style>
.according-panel-heading{
box-sizing: border-box;
padding: 8px 16px;
color: #252b3a;
background-color: #f2f5fc;
line-height: 1.5;
cursor: pointer;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
}
.accordion-panel-title {
margin-top: 0;
margin-bottom: 0;
color: #252b3a;
}
.accordion-panel-title-content{
vertical-align: middle;
display: inline-block;
width: calc(100% - 32px);
cursor: default;
}
.acc-margin-bottom {
margin-bottom: 5px;
}
.title_text {
font-size: 12px;
}
.ac-display-inblock {
display: inline-block;
}
.cti-mgRight-sm {
margin-right: 8px;
}
.ac-text-normal {
font-size: 14px;
color: #575d6c;
}
.uc-accordionTitle-black {
color: #333;
}
.accordion-border{
border:1px solid #cce2ff;
}
.padding0{
padding: 0 !important;
}
.content-pad{
padding: 15px 35px;
}
.content-margin{
margin:10px 5px ;
}
.tab_2_content {
min-height: 360px;
margin-left: 10px;
}
.ac-grid {
display: block;
*zoom: 1;
}
.ac-grid-col {
float: left;
width: 100%;
}
.ac-grid-col2 .ac-grid-col {
width: 50%;
}
.ti-form {
text-align: left;
max-width: 100%;
vertical-align: middle;
}
.ti-form>tbody {
font-size: 12px;
}
.ti-form>tbody, .ti-form>tbody>tr {
vertical-align: inherit;
}
.ti-text-form-label {

padding-bottom: 20px;
padding-right: 20px;
color: #8a8e99;
font-size: 12px;
white-space: nowrap !important;
width: 80px;
line-height: 30px;
}
.ti-text-form-content{
line-height: 30px;
padding-bottom: 20px;
}
.ti-form>tbody>tr>td {
vertical-align: top;
white-space: normal;
}
td, th {
padding: 0;
}
.ac-grid-col .text-span {
width: 450px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.redo-color{
color: #3291F8;
}
.ti-action-menu-item:not(:last-child){
margin-right: 10px;
padding-right: 11px;
text-decoration: none!important;
color: #526ecc;
cursor: pointer;
display: inline-block;
-moz-user-select: none;
-webkit-user-select: none;
-ms-user-select: none;
-khtml-user-select: none;
user-select: none;
position: relative;
}
.ti-action-menu-item:not(:last-child):after {
content: "";
display: inline-block;
position: absolute;
height: 12px;
right: 0;
top: 50%;
-webkit-transform: translateY(-6px);
-ms-transform: translateY(-6px);
-o-transform: translateY(-6px);
transform: translateY(-6px);
border-right: 1px solid #dfe1e6;
}
.text-width80{
width: 100px;
line-height: 30px;
}
.border-according{
border: 1px solid #dfe1e6;
}
.disabled {
cursor: default;
pointer-events: none;
color: rgba(0,0,0,.6) !important;
opacity: .45 !important;
}
.pad20{

border:0px !important;
}
.model_file_bread{ .model_file_bread{
margin-bottom: -0.5rem !important; margin-bottom: -0.5rem !important;
padding-left: 1rem; padding-left: 1rem;
@@ -508,10 +353,18 @@ td, th {
<div class="ui tab" data-tab="third"> <div class="ui tab" data-tab="third">
<input type="hidden" name="model{{.VersionName}}" value="-1"> <input type="hidden" name="model{{.VersionName}}" value="-1">
<input type="hidden" name="modelback{{.VersionName}}" value="-1"> <input type="hidden" name="modelback{{.VersionName}}" value="-1">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">result</div>
<div class="divider"> / </div>

<div style="display: flex;justify-content: space-between;">
<div class='ui breadcrumb model_file_bread' id='file_breadcrumb{{.VersionName}}'>
<div class="active section">result</div>
<div class="divider"> / </div>
</div>
<a id="{{.VersionName}}-result-down" style="padding-right: 1%;display: none;"
class='{{if $.canDownload}}ti-download-file{{else}}disabled{{end}} file-info'
href="{{$.RepoLink}}/modelarts/inference-job/{{.JobID}}/downloadall?version_name={{.VersionName}}">
<i class="ri-download-cloud-2-line"></i>
<span style="margin-left: 0.3rem;">{{$.i18n.Tr "repo.all_result_download"}}</span>
</a>
</div> </div>
<div id="dir_list{{.VersionName}}"> <div id="dir_list{{.VersionName}}">




+ 114
- 185
templates/repo/modelarts/notebook/new.tmpl View File

@@ -1,208 +1,138 @@
{{template "base/head" .}} {{template "base/head" .}}
<style>
.inline.required.field.cloudbrain_benchmark {
display: none;
}
</style>

{{template "custom/global_mask" .}} {{template "custom/global_mask" .}}
<div class="repository"> <div class="repository">
{{template "repo/header" .}} {{template "repo/header" .}}
<div class="repository new repo ui middle very relaxed page grid">
<div class="column">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}"></div>
{{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}}
{{end}}
<div class="ui negative message" id="messageInfo" style="display: none;">
<p></p>
</div>
{{template "custom/alert_cb" .}}
<div class="ui container">
<div class="cloudbrain-type" style="display: none;" data-cloudbrain-type="{{.datasetType}}" data-repo-link="{{.RepoLink}}" data-flag-model="true" data-dataset-uuid="{{.attachment}}" data-dataset-name="{{.dataset_name}}" data-exceed-size="{{DebugAttachSize}}"></div>
{{if eq .NotStopTaskCount 0}}
{{template "base/alert" .}}
{{end}}
<div class="ui negative message" id="messageInfo" style="display: none;">
<p></p>
</div>
{{template "custom/alert_cb" .}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.modelarts.train_job.new_debug"}}
</h4>
<div class="ui attached segment">
<form class="ui form" id="form_id" action="{{.Link}}" method="post"> <form class="ui form" id="form_id" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}} {{.CsrfTokenHtml}}
<h3 class="ui top attached header">
{{.i18n.Tr "repo.cloudbrain.new"}}
</h3>
<div class="ui attached segment">
<!-- <br> -->
<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue small menu compact selectcloudbrain">
<a class="item" href="{{.RepoLink}}/cloudbrain/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
CPU/GPU</a>
<a class="active item" href="{{.RepoLink}}/modelarts/notebook/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
Ascend NPU</a>
</div>
<!-- <br> -->
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4>
<div class="required min_title inline field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label>
<div class="ui blue mini menu compact selectcloudbrain">
<a class="active item" href="{{.RepoLink}}/cloudbrain/create">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_openi"}}
</a>
<a class="item" href="{{.RepoLink}}/grampus/notebook/create?type=0">
<svg class="svg" sxmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16"><path fill="none" d="M0 0h24v24H0z"></path><path d="M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10zm-2.29-2.333A17.9 17.9 0 0 1 8.027 13H4.062a8.008 8.008 0 0 0 5.648 6.667zM10.03 13c.151 2.439.848 4.73 1.97 6.752A15.905 15.905 0 0 0 13.97 13h-3.94zm9.908 0h-3.965a17.9 17.9 0 0 1-1.683 6.667A8.008 8.008 0 0 0 19.938 13zM4.062 11h3.965A17.9 17.9 0 0 1 9.71 4.333 8.008 8.008 0 0 0 4.062 11zm5.969 0h3.938A15.905 15.905 0 0 0 12 4.248 15.905 15.905 0 0 0 10.03 11zm4.259-6.667A17.9 17.9 0 0 1 15.973 11h3.965a8.008 8.008 0 0 0-5.648-6.667z"></path></svg>
{{.i18n.Tr "cloudbrain.resource_cluster_c2net"}}(Beta)
</a>
</div> </div>
<div class="inline field">
<label></label>
{{template "custom/task_wait_count" .}}
</div>
<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.task_name"}}</label>
<input name="display_job_name" id="cloudbrain_job_name" placeholder="任务名称" value="{{.display_job_name}}" tabindex="3" autofocus required maxlength="36" onkeyup="this.value=this.value.replace(/[, ]/g,'')">
</div>
<div class="inline required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.compute_resource"}}</label>
<div class="ui blue small menu compact selectcloudbrain">
<a class="item" href="{{.RepoLink}}/cloudbrain/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
CPU/GPU</a>
<a class="active item" href="{{.RepoLink}}/modelarts/notebook/create">
<svg class="svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="16" height="16">
<path fill="none" d="M0 0h24v24H0z"/>
<path d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z"/>
</svg>
Ascend NPU</a>
</div> </div>

<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.mirror"}}</label>
<select id="cloudbrain_image" class="ui search dropdown" placeholder="{{.i18n.Tr "cloudbrain.choose_mirror"}}" style='width:385px' name="image_id">
</div>
<div class="inline field">
<label class="label-fix-width" style="font-weight: normal;"></label>
{{template "custom/task_wait_count" .}}
</div>
<div class="required min_title inline field" style="margin-bottom: 0rem !important;">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.job_name"}}</label>
<input style="width: 60%;" name="display_job_name" id="display_job_name"
placeholder={{.i18n.Tr "repo.modelarts.train_job.job_name"}} value="{{.display_job_name}}"
tabindex="3" onkeyup="this.value=this.value.replace(/[, ]/g,'')" autofocus required
maxlength="36">
</div>
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 2rem;">{{.i18n.Tr "repo.cloudbrain_jobname_err"}}</span>
<div class="inline min_title field">
<label class="label-fix-width" style="font-weight: normal;" for="description">{{.i18n.Tr "repo.modelarts.train_job.description"}}</label>
{{if .description}}
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}}>{{.description}}</textarea>
{{else}}
<textarea style="width: 80%;" id="description" name="description" rows="3" maxlength="255" placeholder={{.i18n.Tr "repo.modelarts.train_job.new_place"}} onchange="this.value=this.value.substring(0, 255)" onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)"></textarea>
{{end}}
</div>
<div class="ui divider"></div>
<h4 class="train-job-title ui header ">{{.i18n.Tr "repo.modelarts.train_job.parameter_setting"}}:</h4>
{{template "custom/select_model" .}}
<div class="inline min_title required field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.mirror"}}</label>
<select id="cloudbrain_image" class="ui search dropdown width48" placeholder="{{.i18n.Tr "cloudbrain.choose_mirror"}}" name="image_id">
{{if .image_id}}
{{range .images}} {{range .images}}
<option name="image_id" value="{{.Id}}">{{.Value}}</option>
{{if eq $.image_id .Id}}
<option value="{{.Id}}">{{.Value}}</option>
{{end}}
{{end}} {{end}}
</select>
</div>
<div id="select-multi-dataset">

</div>

<!--<div class="inline required field">
<label>工作环境</label>
<input name="de" id="cloudbrain_de" value="{{.env}}" tabindex="3" disabled autofocus required maxlength="255" readonly="readonly">
</div>
<div class="inline required field">
<label>类型</label>
<input name="job_type" id="cloudbrain_job_type" value="{{.notebook_type}}" tabindex="3" disabled autofocus required maxlength="255" readonly="readonly">
</div> -->
<!--<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.specification"}}</label>
<select id="cloudbrain_flavor" class="ui search dropdown" placeholder="选择规格" style='width:385px' name="flavor">
{{range .flavors}}
<option name="flavor" value="{{.Value}}">{{.Desc}}</option>

{{range .images}}
{{if ne $.image_id .Id}}
<option value="{{.Id}}">{{.Value}}</option>
{{end}}
{{end}}
{{else}}
{{range .images}}
<option name="image_id" value="{{.Id}}">{{.Value}}</option>
{{end}} {{end}}
</select>
</div>-->
<div class="inline required field">
<label>{{.i18n.Tr "cloudbrain.specification"}}</label>
<select id="__specs__" class="ui search dropdown" ovalue="{{.spec_id}}"
{{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}}
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px' name="spec_id"></select>
<span><i class="question circle icon link"></i></span>
<a href="{{AppSubUrl}}/resource_desc" target="_blank">{{.i18n.Tr "custom.resource_description"}}</a>
{{if .CloudBrainPaySwitch}}
<div class="cloudbrain_resource_spec_blance_tip" style="padding:0 5px;margin:6px 0;margin-left:265px;font-size:12px;width: 50%!important;">
<span>{{$.i18n.Tr "points.balance_of_points"}}<span style="color:red;margin: 0 3px">{{.PointAccount.Balance}}</span>{{$.i18n.Tr "points.points"}}</span><span>{{$.i18n.Tr "points.expected_time"}}<span style="color:red;margin: 0 3px" class="can-use-time"></span>{{$.i18n.Tr "points.hours"}}</span>
<span style="float:right;">
<i class="question circle icon link" data-position="right center" data-variation="mini"></i>
<a href="{{AppSubUrl}}/reward/point/rule" target="_blank">{{$.i18n.Tr "points.points_acquisition_instructions"}}</a>
</span>
</div>
{{end}} {{end}}
</select>
</div>
<div id="select-multi-dataset">

</div>
<span class="tooltips" style="margin-left: 11.5rem;margin-bottom: 1rem;"></span>
<div class="inline required min_title field">
<label class="label-fix-width" style="font-weight: normal;">{{.i18n.Tr "cloudbrain.specification"}}</label>
<select id="__specs__" class="ui search dropdown width48" ovalue="{{.spec_id}}"
{{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}}
placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px' name="spec_id"></select>
<span><i class="question circle icon link"></i></span>
<a href="{{AppSubUrl}}/resource_desc" target="_blank">{{.i18n.Tr "custom.resource_description"}}</a>
{{if .CloudBrainPaySwitch}}
<div class="cloudbrain_resource_spec_blance_tip" style="padding:0 5px;margin:6px 0;margin-left:265px;font-size:12px;width: 50%!important;">
<span>{{$.i18n.Tr "points.balance_of_points"}}<span style="color:red;margin: 0 3px">{{.PointAccount.Balance}}</span>{{$.i18n.Tr "points.points"}}</span><span>{{$.i18n.Tr "points.expected_time"}}<span style="color:red;margin: 0 3px" class="can-use-time"></span>{{$.i18n.Tr "points.hours"}}</span>
<span style="float:right;">
<i class="question circle icon link" data-position="right center" data-variation="mini"></i>
<a href="{{AppSubUrl}}/reward/point/rule" target="_blank">{{$.i18n.Tr "points.points_acquisition_instructions"}}</a>
</span>
</div> </div>
<!--<div class="inline required field">
<label>数据集存放路径</label>
<input name="dataset_path" id="cloudbrain_dataset_path" value="{{.dataset_path}}" tabindex="3" disabled autofocus required maxlength="255" readonly="readonly">
</div> -->
<div class="inline field">
<label>{{.i18n.Tr "cloudbrain.description"}}</label>
<input name="description" id="cloudbrain_description" tabindex="3" autofocus maxlength="255">
</div>
<div class="inline field">
<label></label>
<button class="ui green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel" href="{{.RepoLink}}/debugjob?debugListType=all">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div>
{{end}}
</div>
<div class="inline field">
<label class="label-fix-width"></label>
<button class="ui create_train_job green button {{if eq .NotStopTaskCount 1}}disabled{{end}}">
{{.i18n.Tr "repo.cloudbrain.new"}}
</button>
<a class="ui button cancel" href="{{.RepoLink}}/debugjob?debugListType=all">{{.i18n.Tr "repo.cloudbrain.cancel"}}</a>
</div> </div>
</form> </form>
</div> </div>
</div> </div>
</div> </div>
{{template "base/footer" .}} {{template "base/footer" .}}
<script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script> <script src="{{StaticUrlPrefix}}/js/specsuse.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script src="{{StaticUrlPrefix}}/js/cloudbrainNew.js?v={{MD5 AppVer}}" type="text/javascript"></script>
<script> <script>
// 判断必填选项是否填写正确
let form = document.getElementById('form_id');

$('#messageInfo').css('display','none')

var isValidate = false;
function validate(){
$('.ui.form').form({
on: 'blur',
fields: {
display_job_name:{
identifier : 'display_job_name',
rules: [
{
type: 'regExp[/^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/]',
}
]
},
spec_id: {
identifier: 'spec_id',
rules: [{ type: 'empty' }]
}
},
onSuccess: function(){
isValidate = true;
},
onFailure: function(e){
isValidate = false;
return false;
}
})
}
validate();
let createFlag = false
form.onsubmit = function(e){
if(!isValidate) return false;
if(createFlag) return false;
let value_task = $("input[name='display_job_name']").val()
let re = /^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$/
let flag = re.test(value_task)
if(!flag){
$('#messageInfo').css('display','block')
let str = '只能以小写字母或数字开头且只包含小写字母、数字、_和-,不能以_结尾,最长36个字符。'
$('#messageInfo p').text(str)
return false
}
let min_value_task = value_task.toLowerCase()
$("input[name='display_job_name']").attr("value",min_value_task)
document.getElementById("mask").style.display = "block"
createFlag = true

}
// 点击按钮后遮罩层显示
// function showmask() {
// document.getElementById("mask").style.display = "block"
// }

// 页面加载完毕后遮罩层隐藏
document.onreadystatechange = function() {
if (document.readyState === "complete") {
document.getElementById("mask").style.display = "none"
}
}
$('select.dropdown')
.dropdown();

$(function() {
$("#cloudbrain_job_type").change(function() {
if ($(this).val() == 'BENCHMARK') {
$(".cloudbrain_benchmark").show();
} else {
$(".cloudbrain_benchmark").hide();
}
})
})
$(document).ready(function(){
$(document).keydown(function(event){
if(event.keyCode==13){
event.preventDefault();
}
});
});

;(function() { ;(function() {
var SPECS = {{ .Specs }}; var SPECS = {{ .Specs }};
var showPoint = {{ .CloudBrainPaySwitch }}; var showPoint = {{ .CloudBrainPaySwitch }};
@@ -214,5 +144,4 @@
shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}},
}); });
})(); })();
console.log("-------------:",{{.NotStopTaskCount}})
</script> </script>

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save