From 30d16031764ea5bfce1a469893c444f937cafb16 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Mon, 26 Sep 2022 14:09:37 +0800 Subject: [PATCH 001/187] #2943 add specification api --- models/resource_specification.go | 26 ++++++++++++++ routers/admin/resources.go | 2 +- routers/api/v1/api.go | 4 +++ routers/api/v1/repo/spec.go | 36 +++++++++++++++++++ routers/response/response.go | 6 +++- routers/response/response_list.go | 2 ++ .../resource/resource_specification.go | 12 +++++++ 7 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 routers/api/v1/repo/spec.go diff --git a/models/resource_specification.go b/models/resource_specification.go index 2da8d015d..f7a7dccbd 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -194,6 +194,32 @@ func (Specification) TableName() string { return "resource_specification" } +func (s *Specification) ToShow() *SpecificationShow { + return &SpecificationShow{ + ID: s.ID, + AccCardsNum: s.AccCardsNum, + AccCardType: s.AccCardType, + CpuCores: s.CpuCores, + MemGiB: s.MemGiB, + GPUMemGiB: s.GPUMemGiB, + ShareMemGiB: s.ShareMemGiB, + ComputeResource: s.ComputeResource, + UnitPrice: s.UnitPrice, + } +} + +type SpecificationShow struct { + ID int64 + AccCardsNum int + AccCardType string + CpuCores int + MemGiB float32 + GPUMemGiB float32 + ShareMemGiB float32 + ComputeResource string + UnitPrice int +} + func InsertResourceSpecification(r ResourceSpecification) (int64, error) { return x.Insert(&r) } diff --git a/routers/admin/resources.go b/routers/admin/resources.go index 8a8c55f86..bbdae2609 100644 --- a/routers/admin/resources.go +++ b/routers/admin/resources.go @@ -182,7 +182,7 @@ func UpdateResourceSpecification(ctx *context.Context, req models.ResourceSpecif if err != nil { log.Error("UpdateResourceSpecification error. %v", err) - ctx.JSON(http.StatusOK, response.ResponseError(err)) + ctx.JSON(http.StatusOK, response.ResponseBizError(err)) return } ctx.JSON(http.StatusOK, response.Success()) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 0b941b400..30ca7751b 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -689,6 +689,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/migrate", reqToken(), bind(auth.MigrateRepoForm{}), repo.Migrate) + m.Group("/specification", func() { + m.Get("", repo.GetResourceSpec) + }, reqToken()) + m.Group("/:username/:reponame", func() { m.Combo("").Get(reqAnyRepoReader(), repo.Get). Delete(reqToken(), reqOwner(), repo.Delete). diff --git a/routers/api/v1/repo/spec.go b/routers/api/v1/repo/spec.go new file mode 100644 index 000000000..1fac207c9 --- /dev/null +++ b/routers/api/v1/repo/spec.go @@ -0,0 +1,36 @@ +package repo + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/routers/response" + "code.gitea.io/gitea/services/cloudbrain/resource" +) + +func GetResourceSpec(ctx *context.APIContext) { + jobType := ctx.Query("jobType") + computeResource := ctx.Query("compute") + cluster := ctx.Query("cluster") + aiCenterCode := ctx.Query("center") + if jobType == "" || computeResource == "" || cluster == "" { + log.Info("GetResourceSpec api.param error") + ctx.JSON(200, response.ResponseBizError(response.PARAM_ERROR)) + return + } + specs, err := resource.FindAvailableSpecs4Show(ctx.User.ID, models.FindSpecsOptions{ + JobType: models.JobType(jobType), + ComputeResource: computeResource, + Cluster: cluster, + AiCenterCode: aiCenterCode, + }) + if err != nil { + log.Error("GetResourceSpec api error. %v", err) + ctx.JSON(200, response.ResponseError(err)) + return + } + + specMap := make(map[string]interface{}, 0) + specMap["Specs"] = specs + ctx.JSON(200, response.SuccessWithData(specMap)) +} diff --git a/routers/response/response.go b/routers/response/response.go index ccd6be445..ff654e5dc 100644 --- a/routers/response/response.go +++ b/routers/response/response.go @@ -24,10 +24,14 @@ func ServerError(msg string) *AiforgeResponse { return &AiforgeResponse{Code: RESPONSE_CODE_ERROR_DEFAULT, Msg: msg} } -func ResponseError(err *BizError) *AiforgeResponse { +func ResponseBizError(err *BizError) *AiforgeResponse { return &AiforgeResponse{Code: err.Code, Msg: err.Err} } +func ResponseError(err error) *AiforgeResponse { + return &AiforgeResponse{Code: RESPONSE_CODE_ERROR_DEFAULT, Msg: err.Error()} +} + func SuccessWithData(data interface{}) *AiforgeResponse { return &AiforgeResponse{Code: RESPONSE_CODE_SUCCESS, Msg: RESPONSE_MSG_SUCCESS, Data: data} } diff --git a/routers/response/response_list.go b/routers/response/response_list.go index 6514f3edd..7807813c6 100644 --- a/routers/response/response_list.go +++ b/routers/response/response_list.go @@ -1,5 +1,7 @@ package response +var PARAM_ERROR = &BizError{Code: 9001, Err: "param error"} + var RESOURCE_QUEUE_NOT_AVAILABLE = &BizError{Code: 1001, Err: "resource queue not available"} var SPECIFICATION_NOT_EXIST = &BizError{Code: 1002, Err: "specification not exist"} var SPECIFICATION_NOT_AVAILABLE = &BizError{Code: 1003, Err: "specification not available"} diff --git a/services/cloudbrain/resource/resource_specification.go b/services/cloudbrain/resource/resource_specification.go index b68abbb88..c0b20b21a 100644 --- a/services/cloudbrain/resource/resource_specification.go +++ b/services/cloudbrain/resource/resource_specification.go @@ -210,6 +210,18 @@ func FindAvailableSpecs(userId int64, opts models.FindSpecsOptions) ([]*models.S return specs, err } +func FindAvailableSpecs4Show(userId int64, opts models.FindSpecsOptions) ([]*models.SpecificationShow, error) { + specs, err := FindAvailableSpecs(userId, opts) + if err != nil { + return nil, err + } + result := make([]*models.SpecificationShow, len(specs)) + for i, v := range specs { + result[i] = v.ToShow() + } + return result, nil +} + func filterExclusiveSpecs(r []*models.Specification, userId int64) []*models.Specification { specs := make([]*models.Specification, 0, len(r)) specMap := make(map[int64]string, 0) From 5455a2b277119d25c487d49cf825a7c84d860189 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 24 Oct 2022 17:00:14 +0800 Subject: [PATCH 002/187] =?UTF-8?q?=E9=80=82=E9=85=8Drestful=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E5=B0=86=E4=BA=A4=E4=BA=92=E7=9A=84=E5=B1=9E?= =?UTF-8?q?=E6=80=A7=E5=85=A8=E9=83=A8=E6=9B=B4=E6=94=B9=E4=B8=BA=E5=B0=8F?= =?UTF-8?q?=E5=86=99=E5=BC=80=E5=A4=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 116 +++++++------- routers/api/v1/api.go | 3 + routers/api/v1/repo/modelmanage.go | 30 ++++ routers/repo/ai_model_convert.go | 32 ++-- routers/repo/ai_model_manage.go | 54 ++++--- templates/repo/modelmanage/convertIndex.tmpl | 22 +-- templates/repo/modelmanage/index.tmpl | 54 +++---- templates/repo/modelmanage/showinfo.tmpl | 106 ++++++------- web_src/js/components/Model.vue | 156 +++++++++---------- 9 files changed, 311 insertions(+), 262 deletions(-) create mode 100644 routers/api/v1/repo/modelmanage.go diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index a88da8fe5..7005b7923 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -12,67 +12,67 @@ import ( ) type AiModelManage struct { - ID string `xorm:"pk"` - Name string `xorm:"INDEX NOT NULL"` - Version string `xorm:"NOT NULL"` - VersionCount int `xorm:"NOT NULL DEFAULT 0"` - New int `xorm:"NOT NULL"` - Type int `xorm:"NOT NULL"` - Size int64 `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Label string `xorm:"varchar(1000)"` - Path string `xorm:"varchar(400) NOT NULL"` - DownloadCount int `xorm:"NOT NULL DEFAULT 0"` - Engine int64 `xorm:"NOT NULL DEFAULT 0"` - Status int `xorm:"NOT NULL DEFAULT 0"` - StatusDesc string `xorm:"varchar(500)"` - Accuracy string `xorm:"varchar(1000)"` - AttachmentId string `xorm:"NULL"` - RepoId int64 `xorm:"INDEX NULL"` - CodeBranch string `xorm:"varchar(400) NULL"` - CodeCommitID string `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - UserName string - UserRelAvatarLink string - TrainTaskInfo string `xorm:"text NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` - IsCanOper bool - IsCanDelete bool + ID string `xorm:"pk" json:"id"` + Name string `xorm:"INDEX NOT NULL" json:"name"` + Version string `xorm:"NOT NULL" json:"version"` + VersionCount int `xorm:"NOT NULL DEFAULT 0" json:"versionCount"` + New int `xorm:"NOT NULL" json:"new"` + Type int `xorm:"NOT NULL" json:"type"` + Size int64 `xorm:"NOT NULL" json:"size"` + Description string `xorm:"varchar(2000)" json:"description"` + Label string `xorm:"varchar(1000)" json:"label"` + Path string `xorm:"varchar(400) NOT NULL" json:"path"` + DownloadCount int `xorm:"NOT NULL DEFAULT 0" json:"downloadCount"` + Engine int64 `xorm:"NOT NULL DEFAULT 0" json:"engine"` + Status int `xorm:"NOT NULL DEFAULT 0" json:"status"` + StatusDesc string `xorm:"varchar(500)" json:"statusDesc"` + Accuracy string `xorm:"varchar(1000)" json:"accuracy"` + AttachmentId string `xorm:"NULL" json:"attachmentId"` + RepoId int64 `xorm:"INDEX NULL" json:"repoId"` + CodeBranch string `xorm:"varchar(400) NULL" json:"codeBranch"` + CodeCommitID string `xorm:"NULL" json:"codeCommitID"` + UserId int64 `xorm:"NOT NULL" json:"userId"` + UserName string `json:"userName"` + UserRelAvatarLink string `json:"userRelAvatarLink"` + TrainTaskInfo string `xorm:"text NULL" json:"trainTaskInfo"` + CreatedUnix timeutil.TimeStamp `xorm:"created" json:"createdUnix"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated" json:"updatedUnix"` + IsCanOper bool `json:"isCanOper"` + IsCanDelete bool `json:"isCanDelete"` } type AiModelConvert struct { - ID string `xorm:"pk"` - Name string `xorm:"INDEX NOT NULL"` - Status string `xorm:"NULL"` - StatusResult string `xorm:"NULL"` - SrcEngine int `xorm:"NOT NULL DEFAULT 0"` - RepoId int64 `xorm:"INDEX NULL"` - ModelId string `xorm:"NOT NULL"` - ModelName string `xorm:"NULL"` - ModelVersion string `xorm:"NOT NULL"` - ModelPath string `xorm:"NULL"` - DestFormat int `xorm:"NOT NULL DEFAULT 0"` - NetOutputFormat int `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - CloudBrainTaskId string `xorm:"NULL"` - ModelArtsVersionId string `xorm:"NULL"` - ContainerID string - ContainerIp string - RunTime int64 `xorm:"NULL"` - TrainJobDuration string - InputShape string `xorm:"varchar(2000)"` - InputDataFormat string `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Path string `xorm:"varchar(400) NOT NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp - UserName string - UserRelAvatarLink string - IsCanOper bool - IsCanDelete bool + ID string `xorm:"pk" json:"id"` + Name string `xorm:"INDEX NOT NULL" json:"name"` + Status string `xorm:"NULL" json:"status"` + StatusResult string `xorm:"NULL" json:"statusResult"` + SrcEngine int `xorm:"NOT NULL DEFAULT 0" json:"srcEngine"` + RepoId int64 `xorm:"INDEX NULL" json:"repoId"` + ModelId string `xorm:"NOT NULL" json:"modelId"` + ModelName string `xorm:"NULL" json:"modelName"` + ModelVersion string `xorm:"NOT NULL" json:"modelVersion"` + ModelPath string `xorm:"NULL" json:"modelPath"` + DestFormat int `xorm:"NOT NULL DEFAULT 0" json:"destFormat"` + NetOutputFormat int `xorm:"NULL" json:"netOutputFormat"` + UserId int64 `xorm:"NOT NULL" json:"userId"` + CloudBrainTaskId string `xorm:"NULL" json:"cloudBrainTaskId"` + ModelArtsVersionId string `xorm:"NULL" json:"modelArtsVersionId"` + ContainerID string `json:"containerID"` + ContainerIp string `json:"containerIp"` + RunTime int64 `xorm:"NULL" json:"runTime"` + TrainJobDuration string `json:"trainJobDuration"` + InputShape string `xorm:"varchar(2000)" json:"inputShape"` + InputDataFormat string `xorm:"NOT NULL" json:"inputDataFormat"` + Description string `xorm:"varchar(2000)" json:"description"` + Path string `xorm:"varchar(400) NOT NULL" json:"path"` + CreatedUnix timeutil.TimeStamp `xorm:"created" json:"createdUnix"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated" json:"updatedUnix"` + StartTime timeutil.TimeStamp `json:"startTime"` + EndTime timeutil.TimeStamp `json:"endTime"` + UserName string `json:"userName"` + UserRelAvatarLink string `json:"userRelAvatarLink"` + IsCanOper bool `json:"isCanOper"` + IsCanDelete bool `json:"isCanDelete"` } type AiModelQueryOptions struct { diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8e1d725ed..7667cfcfa 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -947,6 +947,9 @@ func RegisterRoutes(m *macaron.Macaron) { }) }, reqRepoReader(models.UnitTypeCloudBrain)) m.Group("/modelmanage", func() { + m.Post("/create_new_model", repo.CreateNewModel) + m.Get("/show_model_api", repo.ShowModelManageApi) + m.Delete("/delete_model", repo.DeleteModel) m.Get("/:id", repo.GetCloudbrainModelConvertTask) m.Get("/:id/log", repo.CloudbrainForModelConvertGetLog) m.Get("/:id/modelartlog", repo.TrainJobForModelConvertGetLog) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go new file mode 100644 index 000000000..fce7cd886 --- /dev/null +++ b/routers/api/v1/repo/modelmanage.go @@ -0,0 +1,30 @@ +package repo + +import ( + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + routerRepo "code.gitea.io/gitea/routers/repo" +) + +func CreateNewModel(ctx *context.APIContext) { + log.Info("create new model by api.") + ctx.Context.SetParams("JobId", ctx.Query("jobId")) + ctx.Context.SetParams("VersionName", ctx.Query("versionName")) + ctx.Context.SetParams("Name", ctx.Query("name")) + ctx.Context.SetParams("Version", ctx.Query("version")) + ctx.Context.SetParams("Engine", ctx.Query("engine")) + ctx.Context.SetParams("modelSelectedFile", ctx.Query("modelSelectedFile")) + ctx.Context.SetParams("Label", ctx.Query("label")) + ctx.Context.SetParams("Description", ctx.Query("description")) + + routerRepo.SaveModel(ctx.Context) + +} + +func ShowModelManageApi(ctx *context.APIContext) { + routerRepo.ShowModelPageInfo(ctx.Context) +} + +func DeleteModel(ctx *context.APIContext) { + routerRepo.DeleteModel(ctx.Context) +} diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 9a5874956..27ff0091f 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -72,27 +72,27 @@ func SaveModelConvert(ctx *context.Context) { log.Info("save model convert start.") if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.no_operate_right"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.no_operate_right"), }) return } name := ctx.Query("name") desc := ctx.Query("desc") modelId := ctx.Query("modelId") - modelPath := ctx.Query("ModelFile") - SrcEngine := ctx.QueryInt("SrcEngine") + modelPath := ctx.Query("modelFile") + SrcEngine := ctx.QueryInt("srcEngine") InputShape := ctx.Query("inputshape") InputDataFormat := ctx.Query("inputdataformat") - DestFormat := ctx.QueryInt("DestFormat") - NetOutputFormat := ctx.QueryInt("NetOutputFormat") + DestFormat := ctx.QueryInt("destFormat") + NetOutputFormat := ctx.QueryInt("netOutputFormat") task, err := models.QueryModelById(modelId) if err != nil { log.Error("no such model!", err.Error()) ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.model_not_exist"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.model_not_exist"), }) return } @@ -103,8 +103,8 @@ func SaveModelConvert(ctx *context.Context) { if convert.Name == name { log.Info("convert.Name=" + name + " convert.id=" + convert.ID) ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.create_error1"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.create_error1"), }) return } @@ -117,8 +117,8 @@ func SaveModelConvert(ctx *context.Context) { if isRunningTask(convert.Status) { log.Info("convert.Status=" + convert.Status + " convert.id=" + convert.ID) ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.create_error2"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.create_error2"), }) return } @@ -148,7 +148,7 @@ func SaveModelConvert(ctx *context.Context) { go goCreateTask(modelConvert, ctx, task) ctx.JSON(200, map[string]string{ - "result_code": "0", + "code": "0", }) } @@ -588,11 +588,11 @@ func StopModelConvert(ctx *context.Context) { } func ShowModelConvertInfo(ctx *context.Context) { - ctx.Data["ID"] = ctx.Query("ID") + ctx.Data["ID"] = ctx.Query("id") ctx.Data["isModelManage"] = true ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage) - job, err := models.QueryModelConvertById(ctx.Query("ID")) + job, err := models.QueryModelConvertById(ctx.Query("id")) if err == nil { if job.TrainJobDuration == "" { job.TrainJobDuration = "00:00:00" @@ -741,7 +741,7 @@ func ModelConvertDownloadModel(ctx *context.Context) { ctx.ServerError("Not found task.", err) return } - AllDownload := ctx.QueryBool("AllDownload") + AllDownload := ctx.QueryBool("allDownload") if AllDownload { if job.IsGpuTrainTask() { path := setting.CBCodePathPrefix + job.ID + "/model/" diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 5b358b83b..6385cc674 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -175,7 +175,7 @@ func SaveNewNameModel(ctx *context.Context) { ctx.Error(403, ctx.Tr("repo.model_noright")) return } - name := ctx.Query("Name") + name := ctx.Query("name") if name == "" { ctx.Error(500, fmt.Sprintf("name or version is null.")) return @@ -197,13 +197,13 @@ func SaveModel(ctx *context.Context) { return } log.Info("save model start.") - JobId := ctx.Query("JobId") - VersionName := ctx.Query("VersionName") - name := ctx.Query("Name") - version := ctx.Query("Version") - label := ctx.Query("Label") - description := ctx.Query("Description") - engine := ctx.QueryInt("Engine") + JobId := ctx.Query("jobId") + VersionName := ctx.Query("versionName") + name := ctx.Query("name") + version := ctx.Query("version") + label := ctx.Query("label") + description := ctx.Query("description") + engine := ctx.QueryInt("engine") modelSelectedFile := ctx.Query("modelSelectedFile") log.Info("engine=" + fmt.Sprint(engine) + " modelSelectedFile=" + modelSelectedFile) @@ -293,13 +293,17 @@ func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir func DeleteModel(ctx *context.Context) { log.Info("delete model start.") - id := ctx.Query("ID") + id := ctx.Query("id") err := deleteModelByID(ctx, id) if err != nil { - ctx.JSON(500, err.Error()) + re := map[string]string{ + "code": "-1", + } + re["msg"] = err.Error() + ctx.JSON(200, re) } else { ctx.JSON(200, map[string]string{ - "result_code": "0", + "code": "0", }) } } @@ -356,7 +360,7 @@ func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, in func DownloadMultiModelFile(ctx *context.Context) { log.Info("DownloadMultiModelFile start.") - id := ctx.Query("ID") + id := ctx.Query("id") log.Info("id=" + id) task, err := models.QueryModelById(id) if err != nil { @@ -489,7 +493,10 @@ func downloadFromCloudBrainTwo(path string, task *models.AiModelManage, ctx *con func QueryTrainJobVersionList(ctx *context.Context) { log.Info("query train job version list. start.") - JobID := ctx.Query("JobID") + JobID := ctx.Query("jobId") + if JobID == "" { + JobID = ctx.Query("JobId") + } VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) @@ -521,7 +528,10 @@ func QueryTrainModelList(ctx *context.Context) { log.Info("query train job list. start.") jobName := ctx.Query("jobName") taskType := ctx.QueryInt("type") - VersionName := ctx.Query("VersionName") + VersionName := ctx.Query("versionName") + if VersionName == "" { + VersionName = ctx.Query("VersionName") + } if taskType == models.TypeCloudBrainTwo { objectkey := path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionName) + "/" modelDbResult, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, objectkey) @@ -614,7 +624,7 @@ func DownloadSingleModelFile(ctx *context.Context) { } func ShowModelInfo(ctx *context.Context) { - ctx.Data["ID"] = ctx.Query("ID") + ctx.Data["ID"] = ctx.Query("id") ctx.Data["name"] = ctx.Query("name") ctx.Data["isModelManage"] = true ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage) @@ -830,8 +840,8 @@ func ModifyModel(id string, description string) error { func ModifyModelInfo(ctx *context.Context) { log.Info("modify model start.") - id := ctx.Query("ID") - description := ctx.Query("Description") + id := ctx.Query("id") + description := ctx.Query("description") task, err := models.QueryModelById(id) if err != nil { @@ -896,7 +906,10 @@ func QueryModelListForPredict(ctx *context.Context) { } func QueryModelFileForPredict(ctx *context.Context) { - id := ctx.Query("ID") + id := ctx.Query("id") + if id == "" { + id = ctx.Query("ID") + } model, err := models.QueryModelById(id) if err == nil { if model.Type == models.TypeCloudBrainTwo { @@ -917,7 +930,10 @@ func QueryModelFileForPredict(ctx *context.Context) { } func QueryOneLevelModelFile(ctx *context.Context) { - id := ctx.Query("ID") + id := ctx.Query("id") + if id == "" { + id = ctx.Query("ID") + } parentDir := ctx.Query("parentDir") model, err := models.QueryModelById(id) if err != nil { diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 92eefca2e..36b763174 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -366,7 +366,7 @@ data['desc']= $('#Description').val() data['modelId'] = $('#ModelVersion').val() - data['SrcEngine'] = $('#SrcEngine').val(); + data['srcEngine'] = $('#SrcEngine').val(); data['inputshape']= $('#inputshape').val(); if(inputshapeNotValid(data['inputshape'])){ @@ -379,10 +379,10 @@ } data['inputdataformat']= $('#inputdataformat').val(); - data['DestFormat'] = $('#DestFormat').val(); - data['NetOutputFormat']= $('#NetOutputFormat').val(); - data['ModelFile'] = $('#ModelFile').val(); - if(data['ModelFile']==""){ + data['destFormat'] = $('#DestFormat').val(); + data['netOutputFormat']= $('#NetOutputFormat').val(); + data['modelFile'] = $('#ModelFile').val(); + if(data['modelFile']==""){ $('.ui.error.message').text("{{.i18n.Tr "repo.modelconvert.modelfileempty"}}") $('.ui.error.message').css('display','block') $("#ModelFile_Div").addClass("error") @@ -392,11 +392,11 @@ } $.post(`${repolink}/modelmanage/create_model_convert`,data,(result) => { console.log("result=" + result); - if(result.result_code ==0){ + if(result.code ==0){ $('.ui.modal.second').modal('hide'); window.location.reload(); }else{ - $('.ui.error.message').text(result.message) + $('.ui.error.message').text(result.msg) $('.ui.error.message').css('display','block') } }) @@ -509,7 +509,7 @@ if(modelId ==null || modelId ==""){ console.log("modelId is null"); }else{ - $.get(`${repolink}/modelmanage/query_modelfile_for_predict?ID=${modelId}`, (data) => { + $.get(`${repolink}/modelmanage/query_modelfile_for_predict?id=${modelId}`, (data) => { const n_length = data.length let file_html='' let firstFileName ='' @@ -578,13 +578,13 @@ let nameMap = modelData.nameMap let versionList = nameMap[modelName] if(versionList != null && versionList.length >0){ - if(versionList[0].Engine == engineOption){ + if(versionList[0].engine == engineOption){ return "selected=\"selected\""; }else{ - if((versionList[0].Engine==122 || versionList[0].Engine==37) && engineOption==2){ + if((versionList[0].engine==122 || versionList[0].engine==37) && engineOption==2){ return "selected=\"selected\""; } - if((versionList[0].Engine==121 || versionList[0].Engine==38) && engineOption==1){ + if((versionList[0].engine==121 || versionList[0].engine==38) && engineOption==1){ return "selected=\"selected\""; } } diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index b2994f0c2..9716637ed 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -138,20 +138,20 @@
- +
- +  
@@ -364,9 +364,9 @@ $("#task_name").removeClass("error") } - data['desc']= $('#Description').val() - data['modelId'] = $('#ModelVersion').val() - data['srcEngine'] = $('#SrcEngine').val(); + data['desc']= $('#description').val() + data['modelId'] = $('#modelVersion').val() + data['srcEngine'] = $('#srcEngine').val(); data['inputshape']= $('#inputshape').val(); if(inputshapeNotValid(data['inputshape'])){ @@ -379,9 +379,9 @@ } data['inputdataformat']= $('#inputdataformat').val(); - data['destFormat'] = $('#DestFormat').val(); - data['netOutputFormat']= $('#NetOutputFormat').val(); - data['modelFile'] = $('#ModelFile').val(); + data['destFormat'] = $('#destFormat').val(); + data['netOutputFormat']= $('#netOutputFormat').val(); + data['modelFile'] = $('#modelFile').val(); if(data['modelFile']==""){ $('.ui.error.message').text("{{.i18n.Tr "repo.modelconvert.modelfileempty"}}") $('.ui.error.message').css('display','block') @@ -456,7 +456,7 @@ $('#choice_version').dropdown({ onChange:function(value){ console.log("model version:" + value); - $('#choice_version input[name="ModelVersion"]').val(value) + $('#choice_version input[name="modelVersion"]').val(value) loadModelFile(value); } }) @@ -464,26 +464,26 @@ $('#choice_file').dropdown({ onChange:function(value){ console.log("model file:" + value); - $('#choice_file input[name="ModelFile"]').val(value) + $('#choice_file input[name="modelFile"]').val(value) } }) }) function srcEngineChanged(){ - var ele = window.document.getElementById("SrcEngine"); + var ele = window.document.getElementById("srcEngine"); var index=ele.selectedIndex; var options=ele.options; var option = options[index]; - console.log("SrcEngine value=" + option); + console.log("srcEngine value=" + option); let destFormatHtml = ""; let netOutputFormatHtml = ""; if(option==null || option =="undefined" || option.value == 0){ destFormatHtml += "" netOutputFormatHtml += ""; } - $('#DestFormat').html(destFormatHtml); - $('#NetOutputFormat').html(netOutputFormatHtml); + $('#destFormat').html(destFormatHtml); + $('#netOutputFormat').html(netOutputFormatHtml); } function loadModelList(){ @@ -550,25 +550,25 @@ n_length = versionList.length let train_html='' for (let i=0;i${versionList[i].Version}
` + train_html += `
${versionList[i].version}
` train_html += '
' } $("#model-version").append(train_html) - $('#choice_version .default.text').text(versionList[0].Version) - $('#choice_version input[name="ModelVersion"]').val(versionList[0].ID) - loadModelFile(versionList[0].ID); + $('#choice_version .default.text').text(versionList[0].version) + $('#choice_version input[name="modelVersion"]').val(versionList[0].id) + loadModelFile(versionList[0].id); } setEngineValue(value); } function setEngineValue(value){ - $('#SrcEngine').dropdown('clear'); + $('#srcEngine').dropdown('clear'); console.log("setEngineValue value=" + value); let html = "" html +=""; html +=""; html +=""; - $('#SrcEngine').html(html); + $('#srcEngine').html(html); srcEngineChanged(); } function getSelected(engineOption, modelName){ From ae1cf98f7713aa8f33a2a2271994ef0f5c3b87d5 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 25 Oct 2022 15:08:28 +0800 Subject: [PATCH 028/187] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E5=86=99=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/convertIndex.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index cc334707e..25891dbda 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -93,7 +93,7 @@
@@ -141,7 +141,7 @@ {{if .IsCanOper}} - + {{$.i18n.Tr "repo.modelconvert.download"}} {{else}} From 26d8cae2622e041242f58efc3c1519b08446a546 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 25 Oct 2022 17:17:38 +0800 Subject: [PATCH 029/187] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E5=86=99=E9=97=AE=E9=A2=98=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- templates/repo/modelmanage/convertIndex.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 25891dbda..d21cd0abf 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -526,7 +526,7 @@ } $("#model-file").append(file_html) $('#choice_file .default.text').text(firstFileName) - $('#choice_file input[name="ModelFile"]').val(firstFileName) + $('#choice_file input[name="modelFile"]').val(firstFileName) }) } From f453610611c17722b13d731105d055d9134cc9a4 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 25 Oct 2022 17:53:51 +0800 Subject: [PATCH 030/187] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/base_message.go | 6 ++ options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/api/v1/repo/cloudbrain.go | 46 +++++++- services/cloudbrain/cloudbrainTask/count.go | 22 ++-- services/cloudbrain/cloudbrainTask/train.go | 112 ++++++++++++++++++++ 6 files changed, 175 insertions(+), 13 deletions(-) diff --git a/models/base_message.go b/models/base_message.go index f0fa19bf2..f76e18175 100644 --- a/models/base_message.go +++ b/models/base_message.go @@ -29,3 +29,9 @@ func BaseErrorMessageApi(message string) BaseMessageApi { 1, message, } } + +type BaseMessageWithDataApi struct { + Code int `json:"code"` + Message string `json:"message"` + Data interface{} `json:"data"` +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 324930544..ce80a31fe 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -616,6 +616,7 @@ organization = Organizations uid = Uid u2f = Security Keys bind_wechat = Bind WeChat +no_wechat_bind = Can not do the operation, please bind WeChat first. wechat_bind = WeChat Binding bind_account_information = Bind account information bind_time = Bind Time diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index fd5596a2e..fdbafd814 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -621,6 +621,7 @@ organization=组织 uid=用户 ID u2f=安全密钥 wechat_bind = 微信绑定 +no_wechat_bind = 不能创建任务,请先绑定微信。 bind_wechat = 绑定微信 bind_account_information = 绑定账号信息 bind_time = 绑定时间 diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 58dd5b801..380de0ddb 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -33,6 +33,48 @@ import ( routerRepo "code.gitea.io/gitea/routers/repo" ) +func CloudBrainShow(ctx *context.APIContext) { + + task, err := models.GetCloudbrainByJobID(ctx.Params(":jobid")) + + if err != nil { + log.Info("error:" + err.Error()) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("repo.cloudbrain_query_fail")) + return + } + cloudbrainTask.PrepareSpec4Show(task) + task.ContainerIp = "" + if cloudbrainTask.IsTaskNotStop(task) { + cloudbrainTask.SyncTaskStatus(task) + } + + if task.TrainJobDuration == "" { + if task.Duration == 0 { + var duration int64 + if task.Status == string(models.JobWaiting) { + duration = 0 + } else if task.Status == string(models.JobRunning) { + duration = time.Now().Unix() - int64(task.CreatedUnix) + } else { + duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix) + } + task.Duration = duration + } + task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) + } + //to unify image output + if task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter { + task.ImageID = strconv.FormatInt(task.EngineID, 10) + task.Image = task.EngineName + + } else if task.Type == models.TypeC2Net { + task.Image = task.EngineName + } + + ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: task}) + +} + func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) { if option.Type == 2 { @@ -47,10 +89,10 @@ func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) func CreateCloudBrainInferenceTask(ctx *context.APIContext, option api.CreateTrainJobOption) { if option.Type == 0 { - cloudbrainTask.GrampusTrainJobGpuCreate(ctx.Context, option) + cloudbrainTask.CloudBrainInferenceJobCreate(ctx.Context, option) } if option.Type == 1 { - cloudbrainTask.GrampusTrainJobNpuCreate(ctx.Context, option) + cloudbrainTask.ModelArtsInferenceJobCreate(ctx.Context, option) } } diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index a9b254618..34b334445 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -14,28 +14,28 @@ type StatusInfo struct { ComputeResource string } -var cloudbrainOneNotFinalStatuses = []string{string(models.JobWaiting), string(models.JobRunning)} -var cloudbrainTwoNotFinalStatuses = []string{string(models.ModelArtsTrainJobInit), string(models.ModelArtsTrainJobImageCreating), string(models.ModelArtsTrainJobSubmitTrying), string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobRunning), string(models.ModelArtsTrainJobScaling), string(models.ModelArtsTrainJobCheckInit), string(models.ModelArtsTrainJobCheckRunning), string(models.ModelArtsTrainJobCheckRunningCompleted)} -var grampusTwoNotFinalStatuses = []string{models.GrampusStatusWaiting, models.GrampusStatusRunning} +var CloudbrainOneNotFinalStatuses = []string{string(models.JobWaiting), string(models.JobRunning)} +var CloudbrainTwoNotFinalStatuses = []string{string(models.ModelArtsTrainJobInit), string(models.ModelArtsTrainJobImageCreating), string(models.ModelArtsTrainJobSubmitTrying), string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobRunning), string(models.ModelArtsTrainJobScaling), string(models.ModelArtsTrainJobCheckInit), string(models.ModelArtsTrainJobCheckRunning), string(models.ModelArtsTrainJobCheckRunningCompleted)} +var GrampusNotFinalStatuses = []string{models.GrampusStatusWaiting, models.GrampusStatusRunning} var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, JobType: []models.JobType{models.JobTypeDebug}, - NotFinalStatuses: cloudbrainOneNotFinalStatuses, + NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, JobType: []models.JobType{models.JobTypeTrain}, - NotFinalStatuses: cloudbrainOneNotFinalStatuses, + NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeInference) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, JobType: []models.JobType{models.JobTypeInference}, - NotFinalStatuses: cloudbrainOneNotFinalStatuses, + NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet}, - NotFinalStatuses: cloudbrainOneNotFinalStatuses, + NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { CloudBrainTypes: []int{models.TypeCloudBrainTwo, models.TypeCDCenter}, @@ -45,22 +45,22 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s }, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { CloudBrainTypes: []int{models.TypeCloudBrainTwo}, JobType: []models.JobType{models.JobTypeTrain}, - NotFinalStatuses: cloudbrainTwoNotFinalStatuses, + NotFinalStatuses: CloudbrainTwoNotFinalStatuses, ComputeResource: models.NPUResource, }, string(models.JobTypeInference) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { CloudBrainTypes: []int{models.TypeCloudBrainTwo}, JobType: []models.JobType{models.JobTypeInference}, - NotFinalStatuses: cloudbrainTwoNotFinalStatuses, + NotFinalStatuses: CloudbrainTwoNotFinalStatuses, ComputeResource: models.NPUResource, }, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.GPUResource: { CloudBrainTypes: []int{models.TypeC2Net}, JobType: []models.JobType{models.JobTypeTrain}, - NotFinalStatuses: grampusTwoNotFinalStatuses, + NotFinalStatuses: GrampusNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.NPUResource: { CloudBrainTypes: []int{models.TypeC2Net}, JobType: []models.JobType{models.JobTypeTrain}, - NotFinalStatuses: grampusTwoNotFinalStatuses, + NotFinalStatuses: GrampusNotFinalStatuses, ComputeResource: models.NPUResource, }} diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index d31038536..6f836557c 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -12,6 +12,10 @@ import ( "regexp" "strings" + "code.gitea.io/gitea/modules/timeutil" + + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/git" @@ -680,3 +684,111 @@ func getPoolId() string { return resourcePools.Info[0].ID } + +func PrepareSpec4Show(task *models.Cloudbrain) { + s, err := resource.GetCloudbrainSpec(task.ID) + if err != nil { + log.Info("error:" + err.Error()) + return + } + task.Spec = s +} + +func IsTaskNotStop(task *models.Cloudbrain) bool { + statuses := CloudbrainOneNotFinalStatuses + if task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter { + statuses = CloudbrainTwoNotFinalStatuses + } else { + statuses = GrampusNotFinalStatuses + } + + for _, status := range statuses { + if task.Status == status { + return true + } + } + return false + +} + +func SyncTaskStatus(task *models.Cloudbrain) error { + if task.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(task.JobID) + if err != nil { + log.Info("error:" + err.Error()) + return fmt.Errorf("repo.cloudbrain_query_fail") + } + + if result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + + oldStatus := task.Status + task.Status = taskRes.TaskStatuses[0].State + + task.ContainerID = taskRes.TaskStatuses[0].ContainerID + models.ParseAndSetDurationFromCloudBrainOne(jobRes, task) + + if task.DeletedAt.IsZero() { //normal record + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } + err = models.UpdateJob(task) + if err != nil { + return fmt.Errorf("repo.cloudbrain_query_fail") + + } + } + + } else { + log.Info("error:" + err.Error()) + return fmt.Errorf("repo.cloudbrain_query_fail") + } + } else if task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter { + err := modelarts.HandleTrainJobInfo(task) + if err != nil { + return fmt.Errorf("repo.cloudbrain_query_fail") + } + + } else if task.Type == models.TypeC2Net { + result, err := grampus.GetJob(task.JobID) + if err != nil { + log.Error("GetJob failed:" + err.Error()) + return fmt.Errorf("repo.cloudbrain_query_fail") + } + + if result != nil { + if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 { + task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] + } + oldStatus := task.Status + task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) + if task.Status != result.JobInfo.Status || result.JobInfo.Status == models.GrampusStatusRunning { + task.Duration = result.JobInfo.RunSec + if task.Duration < 0 { + task.Duration = 0 + } + task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) + + if task.StartTime == 0 && result.JobInfo.StartedAt > 0 { + task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt) + } + if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 { + task.EndTime = task.StartTime.Add(task.Duration) + } + task.CorrectCreateUnix() + if oldStatus != task.Status { + notification.NotifyChangeCloudbrainStatus(task, oldStatus) + } + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob failed:" + err.Error()) + return fmt.Errorf("repo.cloudbrain_query_fail") + } + } + } + } + return nil + +} From 9e513d2ad79dd33ddb030ee5d44e9242df976453 Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 26 Oct 2022 10:59:25 +0800 Subject: [PATCH 031/187] =?UTF-8?q?=E6=8F=90=E4=BA=A4API=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/api.go | 5 +++ routers/api/v1/repo/modelmanage.go | 53 +++++++++++++++++++++++------- routers/repo/ai_model_convert.go | 29 ++++++++++++---- 3 files changed, 69 insertions(+), 18 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index b3694ced9..b03db6327 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -966,6 +966,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/create_new_model", repo.CreateNewModel) m.Get("/show_model_api", repo.ShowModelManageApi) m.Delete("/delete_model", repo.DeleteModel) + m.Get("/downloadall", repo.DownloadModel) + m.Get("/query_model_for_predict", repo.QueryModelListForPredict) + m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) + m.Post("/create_model_convert", repo.CreateModelConvert) + m.Get("/show_model_convert_page") m.Get("/:id", repo.GetCloudbrainModelConvertTask) m.Get("/:id/log", repo.CloudbrainForModelConvertGetLog) m.Get("/:id/modelartlog", repo.TrainJobForModelConvertGetLog) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index fce7cd886..b32726997 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -1,30 +1,61 @@ package repo import ( + "net/http" + "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" routerRepo "code.gitea.io/gitea/routers/repo" ) func CreateNewModel(ctx *context.APIContext) { - log.Info("create new model by api.") - ctx.Context.SetParams("JobId", ctx.Query("jobId")) - ctx.Context.SetParams("VersionName", ctx.Query("versionName")) - ctx.Context.SetParams("Name", ctx.Query("name")) - ctx.Context.SetParams("Version", ctx.Query("version")) - ctx.Context.SetParams("Engine", ctx.Query("engine")) - ctx.Context.SetParams("modelSelectedFile", ctx.Query("modelSelectedFile")) - ctx.Context.SetParams("Label", ctx.Query("label")) - ctx.Context.SetParams("Description", ctx.Query("description")) - + log.Info("CreateNewModel by api.") routerRepo.SaveModel(ctx.Context) - } func ShowModelManageApi(ctx *context.APIContext) { + log.Info("ShowModelManageApi by api.") routerRepo.ShowModelPageInfo(ctx.Context) } func DeleteModel(ctx *context.APIContext) { + log.Info("DeleteModel by api.") routerRepo.DeleteModel(ctx.Context) } + +func DownloadModel(ctx *context.APIContext) { + log.Info("DownloadModel by api.") + routerRepo.DownloadMultiModelFile(ctx.Context) +} + +func QueryModelListForPredict(ctx *context.APIContext) { + log.Info("QueryModelListForPredict by api.") + routerRepo.QueryModelListForPredict(ctx.Context) +} + +func QueryModelFileForPredict(ctx *context.APIContext) { + log.Info("QueryModelFileForPredict by api.") + routerRepo.QueryModelFileForPredict(ctx.Context) +} + +func CreateModelConvert(ctx *context.APIContext) { + log.Info("CreateModelConvert by api.") + routerRepo.SaveModelConvert(ctx.Context) +} + +func ShowModelConvertPage(ctx *context.APIContext) { + log.Info("ShowModelConvertPage by api.") + modelResult, count, err := routerRepo.GetModelConvertPageData(ctx.Context) + if err == nil { + mapInterface := make(map[string]interface{}) + mapInterface["data"] = modelResult + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) + } else { + mapInterface := make(map[string]interface{}) + mapInterface["data"] = nil + mapInterface["count"] = 0 + ctx.JSON(http.StatusOK, mapInterface) + } + +} diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 27ff0091f..959f4bc50 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -691,6 +691,26 @@ func ShowModelConvertPageInfo(ctx *context.Context) { ctx.NotFound(ctx.Req.URL.RequestURI(), nil) return } + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } + modelResult, count, err := GetModelConvertPageData(ctx) + if err == nil { + pager := context.NewPagination(int(count), page, pageSize, 5) + ctx.Data["Page"] = pager + ctx.Data["Tasks"] = modelResult + ctx.Data["MODEL_CONVERT_COUNT"] = count + } else { + ctx.ServerError("Query data error.", err) + } +} + +func GetModelConvertPageData(ctx *context.Context) ([]*models.AiModelConvert, int64, error) { page := ctx.QueryInt("page") if page <= 0 { page = 1 @@ -709,10 +729,8 @@ func ShowModelConvertPageInfo(ctx *context.Context) { }) if err != nil { log.Info("query db error." + err.Error()) - ctx.ServerError("Cloudbrain", err) - return + return nil, 0, err } - ctx.Data["MODEL_CONVERT_COUNT"] = count userIds := make([]int64, len(modelResult)) for i, model := range modelResult { model.IsCanOper = isOper(ctx, model.UserId) @@ -727,10 +745,7 @@ func ShowModelConvertPageInfo(ctx *context.Context) { model.UserRelAvatarLink = value.RelAvatarLink() } } - pager := context.NewPagination(int(count), page, pageSize, 5) - ctx.Data["Page"] = pager - ctx.Data["Tasks"] = modelResult - + return modelResult, count, nil } func ModelConvertDownloadModel(ctx *context.Context) { From ea48719bd528c978f14a5f984789f7974fa4f115 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 26 Oct 2022 15:22:34 +0800 Subject: [PATCH 032/187] #3032 update specification query api --- models/resource_specification.go | 18 +++++++++--------- routers/api/v1/repo/spec.go | 8 ++++---- routers/response/api_response.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 routers/response/api_response.go diff --git a/models/resource_specification.go b/models/resource_specification.go index f7a7dccbd..d69cd7208 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -209,15 +209,15 @@ func (s *Specification) ToShow() *SpecificationShow { } type SpecificationShow struct { - ID int64 - AccCardsNum int - AccCardType string - CpuCores int - MemGiB float32 - GPUMemGiB float32 - ShareMemGiB float32 - ComputeResource string - UnitPrice int + ID int64 `json:"id"` + AccCardsNum int `json:"acc_cards_num"` + AccCardType string `json:"acc_card_type"` + CpuCores int `json:"cpu_cores"` + MemGiB float32 `json:"mem_gi_b"` + GPUMemGiB float32 `json:"gpu_mem_gi_b"` + ShareMemGiB float32 `json:"share_mem_gi_b"` + ComputeResource string `json:"compute_resource"` + UnitPrice int `json:"unit_price"` } func InsertResourceSpecification(r ResourceSpecification) (int64, error) { diff --git a/routers/api/v1/repo/spec.go b/routers/api/v1/repo/spec.go index 1fac207c9..a8aa28d98 100644 --- a/routers/api/v1/repo/spec.go +++ b/routers/api/v1/repo/spec.go @@ -15,7 +15,7 @@ func GetResourceSpec(ctx *context.APIContext) { aiCenterCode := ctx.Query("center") if jobType == "" || computeResource == "" || cluster == "" { log.Info("GetResourceSpec api.param error") - ctx.JSON(200, response.ResponseBizError(response.PARAM_ERROR)) + ctx.JSON(200, response.OuterBizError(response.PARAM_ERROR)) return } specs, err := resource.FindAvailableSpecs4Show(ctx.User.ID, models.FindSpecsOptions{ @@ -26,11 +26,11 @@ func GetResourceSpec(ctx *context.APIContext) { }) if err != nil { log.Error("GetResourceSpec api error. %v", err) - ctx.JSON(200, response.ResponseError(err)) + ctx.JSON(200, response.OuterServerError(err.Error())) return } specMap := make(map[string]interface{}, 0) - specMap["Specs"] = specs - ctx.JSON(200, response.SuccessWithData(specMap)) + specMap["specs"] = specs + ctx.JSON(200, response.OuterSuccessWithData(specMap)) } diff --git a/routers/response/api_response.go b/routers/response/api_response.go new file mode 100644 index 000000000..5cc6ff78e --- /dev/null +++ b/routers/response/api_response.go @@ -0,0 +1,30 @@ +package response + +type AiforgeOuterResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data interface{} `json:"data"` +} + +func OuterSuccess() *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: RESPONSE_CODE_SUCCESS, Msg: RESPONSE_MSG_SUCCESS} +} + +func OuterError(code int, msg string) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: code, Msg: msg} +} + +func OuterServerError(msg string) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: RESPONSE_CODE_ERROR_DEFAULT, Msg: msg} +} + +func OuterBizError(err *BizError) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: err.Code, Msg: err.Err} +} + +func OuterSuccessWithData(data interface{}) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: RESPONSE_CODE_SUCCESS, Msg: RESPONSE_MSG_SUCCESS, Data: data} +} +func OuterErrorWithData(code int, msg string, data interface{}) *AiforgeOuterResponse { + return &AiforgeOuterResponse{Code: code, Msg: msg, Data: data} +} From 331d615412e4a2c63b00eea8054b5a7897eb3981 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 26 Oct 2022 16:19:41 +0800 Subject: [PATCH 033/187] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/attachment.go | 42 ++++++------ models/dataset.go | 40 ++++++------ models/repo.go | 120 +++++++++++++++++----------------- modules/structs/attachment.go | 47 +++++++++++++ modules/structs/cloudbrain.go | 40 ++++++++++++ 5 files changed, 188 insertions(+), 101 deletions(-) diff --git a/models/attachment.go b/models/attachment.go index 5ebb6f0f3..2b5fa8efc 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -33,27 +33,27 @@ const ( // Attachment represent a attachment of issue/comment/release. type Attachment struct { - ID int64 `xorm:"pk autoincr" json:"id"` - UUID string `xorm:"uuid UNIQUE" json:"uuid"` - IssueID int64 `xorm:"INDEX" json:"issueId"` - DatasetID int64 `xorm:"INDEX DEFAULT 0" json:"datasetId"` - ReleaseID int64 `xorm:"INDEX" json:"releaseId"` - UploaderID int64 `xorm:"INDEX DEFAULT 0" json:"uploaderId"` // Notice: will be zero before this column added - CommentID int64 `json:"commentId"` - Name string `json:"name"` - Description string `xorm:"TEXT" json:"description"` - DownloadCount int64 `xorm:"DEFAULT 0" json:"downloadCount"` - UseNumber int64 `xorm:"DEFAULT 0" json:"useNumber"` - Size int64 `xorm:"DEFAULT 0" json:"size"` - IsPrivate bool `xorm:"DEFAULT false" json:"isPrivate"` - DecompressState int32 `xorm:"DEFAULT 0" json:"decompressState"` - Type int `xorm:"DEFAULT 0" json:"type"` - CreatedUnix timeutil.TimeStamp `xorm:"created" json:"createdUnix"` - - FileChunk *FileChunk `xorm:"-" json:"fileChunk"` - CanDel bool `xorm:"-" json:"canDel"` - Uploader *User `xorm:"-" json:"uploader"` - Md5 string `xorm:"-" json:"md5"` + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + IssueID int64 `xorm:"INDEX"` + DatasetID int64 `xorm:"INDEX DEFAULT 0"` + ReleaseID int64 `xorm:"INDEX"` + UploaderID int64 `xorm:"INDEX DEFAULT 0"` // Notice: will be zero before this column added + CommentID int64 + Name string + Description string `xorm:"TEXT"` + DownloadCount int64 `xorm:"DEFAULT 0"` + UseNumber int64 `xorm:"DEFAULT 0"` + Size int64 `xorm:"DEFAULT 0"` + IsPrivate bool `xorm:"DEFAULT false"` + DecompressState int32 `xorm:"DEFAULT 0"` + Type int `xorm:"DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + + FileChunk *FileChunk `xorm:"-"` + CanDel bool `xorm:"-"` + Uploader *User `xorm:"-"` + Md5 string `xorm:"-"` } type AttachmentUsername struct { diff --git a/models/dataset.go b/models/dataset.go index f3c2adbd5..972503641 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -21,26 +21,26 @@ const ( ) type Dataset struct { - ID int64 `xorm:"pk autoincr" json:"id"` - Title string `xorm:"INDEX NOT NULL" json:"title"` - Status int32 `xorm:"INDEX" json:"status"` // normal_private: 0, pulbic: 1, is_delete: 2 - Category string `json:"category"` - Description string `xorm:"TEXT" json:"description"` - DownloadTimes int64 `json:"downloadTimes"` - UseCount int64 `xorm:"DEFAULT 0" json:"useCount"` - NumStars int `xorm:"INDEX NOT NULL DEFAULT 0" json:"numStars"` - Recommend bool `xorm:"INDEX NOT NULL DEFAULT false" json:"recommend"` - License string `json:"license"` - Task string `json:"task"` - ReleaseID int64 `xorm:"INDEX" json:"releaseId"` - UserID int64 `xorm:"INDEX" json:"userId"` - RepoID int64 `xorm:"INDEX" json:"repoId"` - Repo *Repository `xorm:"-" json:"repo"` - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"createdUnix"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"updatedUnix"` - - User *User `xorm:"-" json:"user"` - Attachments []*Attachment `xorm:"-" json:"attachments"` + ID int64 `xorm:"pk autoincr"` + Title string `xorm:"INDEX NOT NULL""` + Status int32 `xorm:"INDEX""` // normal_private: 0, pulbic: 1, is_delete: 2 + Category string + Description string `xorm:"TEXT"` + DownloadTimes int64 + UseCount int64 `xorm:"DEFAULT 0"` + NumStars int `xorm:"INDEX NOT NULL DEFAULT 0"` + Recommend bool `xorm:"INDEX NOT NULL DEFAULT false"` + License string + Task string + ReleaseID int64 `xorm:"INDEX"` + UserID int64 `xorm:"INDEX"` + RepoID int64 `xorm:"INDEX"` + Repo *Repository `xorm:"-"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + + User *User `xorm:"-"` + Attachments []*Attachment `xorm:"-"` } type DatasetWithStar struct { diff --git a/models/repo.go b/models/repo.go index ccb0d680c..333492cee 100755 --- a/models/repo.go +++ b/models/repo.go @@ -162,79 +162,79 @@ const ( // Repository represents a git repository. type Repository struct { - ID int64 `xorm:"pk autoincr" json:"id"` - OwnerID int64 `xorm:"UNIQUE(s) index" json:"ownerId"` - OwnerName string `json:"ownerName"` - Owner *User `xorm:"-" json:"owner"` - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL" json:"lowerName"` - Name string `xorm:"INDEX NOT NULL" json:"name"` - Description string `xorm:"TEXT" json:"description"` - Website string `xorm:"VARCHAR(2048)" json:"website"` - OriginalServiceType api.GitServiceType `xorm:"index" json:"originalServiceType"` - OriginalURL string `xorm:"VARCHAR(2048)" json:"originalUrl"` - DefaultBranch string `json:"defaultBranch"` - CreatorID int64 `xorm:"INDEX NOT NULL DEFAULT 0" json:"creatorId"` - Creator *User `xorm:"-" json:"creator"` - NumWatches int `json:"numWatches"` - NumStars int `json:"numStars"` - NumForks int `json:"numForks"` - NumIssues int `json:"numIssues"` - NumClosedIssues int `json:"numClosedIssues"` - NumOpenIssues int `xorm:"-" json:"numOpenIssues"` - NumPulls int `json:"numPulls"` - NumClosedPulls int `json:"numClosedPulls"` - NumOpenPulls int `xorm:"-" json:"numOpenPulls"` - NumMilestones int `xorm:"NOT NULL DEFAULT 0" json:"numMilestones"` - NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0" json:"numClosedMilestones"` - NumOpenMilestones int `xorm:"-" json:"numOpenMilestones"` - NumCommit int64 `xorm:"NOT NULL DEFAULT 0" json:"numCommit"` - RepoType RepoType `xorm:"NOT NULL DEFAULT 0" json:"repoType"` - - IsPrivate bool `xorm:"INDEX" json:"isPrivate"` - IsEmpty bool `xorm:"INDEX" json:"isEmpty"` - IsArchived bool `xorm:"INDEX" json:"isArchived"` - IsMirror bool `xorm:"INDEX" json:"isMirror"` + ID int64 `xorm:"pk autoincr"` + OwnerID int64 `xorm:"UNIQUE(s) index"` + OwnerName string + Owner *User `xorm:"-"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string `xorm:"INDEX NOT NULL"` + Description string `xorm:"TEXT"` + Website string `xorm:"VARCHAR(2048)"` + OriginalServiceType api.GitServiceType `xorm:"index"` + OriginalURL string `xorm:"VARCHAR(2048)"` + DefaultBranch string + CreatorID int64 `xorm:"INDEX NOT NULL DEFAULT 0"` + Creator *User `xorm:"-"` + NumWatches int + NumStars int + NumForks int + NumIssues int + NumClosedIssues int + NumOpenIssues int `xorm:"-"` + NumPulls int + NumClosedPulls int + NumOpenPulls int `xorm:"-"` + NumMilestones int `xorm:"NOT NULL DEFAULT 0"` + NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"` + NumOpenMilestones int `xorm:"-"` + NumCommit int64 `xorm:"NOT NULL DEFAULT 0"` + RepoType RepoType `xorm:"NOT NULL DEFAULT 0"` + + IsPrivate bool `xorm:"INDEX"` + IsEmpty bool `xorm:"INDEX"` + IsArchived bool `xorm:"INDEX"` + IsMirror bool `xorm:"INDEX"` *Mirror `xorm:"-"` - Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0" json:"status"` + Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` RenderingMetas map[string]string `xorm:"-"` - Units []*RepoUnit `xorm:"-" json:"units"` - PrimaryLanguage *LanguageStat `xorm:"-" json:"primaryLanguage"` - - IsFork bool `xorm:"INDEX NOT NULL DEFAULT false" json:"isFork"` - ForkID int64 `xorm:"INDEX" json:"forkId"` - BaseRepo *Repository `xorm:"-" json:"baseRepo"` - IsTemplate bool `xorm:"INDEX NOT NULL DEFAULT false" json:"isTemplate"` - TemplateID int64 `xorm:"INDEX" json:"templateId"` - TemplateRepo *Repository `xorm:"-" json:"templateRepo"` - Size int64 `xorm:"NOT NULL DEFAULT 0" json:"size"` - CodeIndexerStatus *RepoIndexerStatus `xorm:"-" json:"codeIndexerStatus"` - StatsIndexerStatus *RepoIndexerStatus `xorm:"-" json:"statsIndexerStatus"` - IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true" json:"isFsckEnabled"` - CloseIssuesViaCommitInAnyBranch bool `xorm:"NOT NULL DEFAULT false" json:"closeIssuesViaCommitInAnyBranch"` - Topics []string `xorm:"TEXT JSON" json:"topics"` + Units []*RepoUnit `xorm:"-"` + PrimaryLanguage *LanguageStat `xorm:"-"` + + IsFork bool `xorm:"INDEX NOT NULL DEFAULT false"` + ForkID int64 `xorm:"INDEX"` + BaseRepo *Repository `xorm:"-"` + IsTemplate bool `xorm:"INDEX NOT NULL DEFAULT false"` + TemplateID int64 `xorm:"INDEX"` + TemplateRepo *Repository `xorm:"-"` + Size int64 `xorm:"NOT NULL DEFAULT 0"` + CodeIndexerStatus *RepoIndexerStatus `xorm:"-"` + StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` + IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` + CloseIssuesViaCommitInAnyBranch bool `xorm:"NOT NULL DEFAULT false"` + Topics []string `xorm:"TEXT JSON"` // Avatar: ID(10-20)-md5(32) - must fit into 64 symbols - Avatar string `xorm:"VARCHAR(64)" json:"avatar"` + Avatar string `xorm:"VARCHAR(64)"` //blockchain - ContractAddress string `xorm:"INDEX" json:"contractAddress"` - Balance string `xorm:"NOT NULL DEFAULT '0'" json:"balance"` - BlockChainStatus RepoBlockChainStatus `xorm:"NOT NULL DEFAULT 0" json:"blockChainStatus"` + ContractAddress string `xorm:"INDEX"` + Balance string `xorm:"NOT NULL DEFAULT '0'"` + BlockChainStatus RepoBlockChainStatus `xorm:"NOT NULL DEFAULT 0"` // git clone and git pull total count - CloneCnt int64 `xorm:"NOT NULL DEFAULT 0" json:"clone_cnt" json:"cloneCnt"` + CloneCnt int64 `xorm:"NOT NULL DEFAULT 0" json:"clone_cnt"` // only git clone total count - GitCloneCnt int64 `xorm:"NOT NULL DEFAULT 0" json:"git_clone_cnt" json:"gitCloneCnt"` + GitCloneCnt int64 `xorm:"NOT NULL DEFAULT 0" json:"git_clone_cnt"` - CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"createdUnix"` - UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"updatedUnix"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Hot int64 `xorm:"-" json:"hot"` - Active int64 `xorm:"-" json:"active"` - Alias string `xorm:"INDEX" json:"alias"` - LowerAlias string `xorm:"INDEX" json:"lowerAlias"` + Hot int64 `xorm:"-"` + Active int64 `xorm:"-"` + Alias string `xorm:"INDEX"` + LowerAlias string `xorm:"INDEX"` } type RepositoryShow struct { diff --git a/modules/structs/attachment.go b/modules/structs/attachment.go index 0f3c2ed6a..3ccc2742a 100755 --- a/modules/structs/attachment.go +++ b/modules/structs/attachment.go @@ -6,6 +6,8 @@ package structs // import "code.gitea.io/gitea/modules/structs" import ( "time" + + "code.gitea.io/gitea/modules/timeutil" ) // Attachment a generic attachment @@ -27,3 +29,48 @@ type Attachment struct { type EditAttachmentOptions struct { Name string `json:"name"` } + +type Dataset struct { + ID int64 `json:"id"` + Title string `json:"title"` + Status int32 `json:"status"` + Category string `json:"category"` + Description string `json:"description"` + DownloadTimes int64 `json:"downloadTimes"` + UseCount int64 `json:"useCount"` + NumStars int `json:"numStars"` + Recommend bool `json:"recommend"` + License string `json:"license"` + Task string `json:"task"` + ReleaseID int64 `json:"releaseId"` + UserID int64 `json:"userId"` + RepoID int64 `json:"repoId"` + Repo *RepositoryShow `json:"repo"` + CreatedUnix timeutil.TimeStamp `json:"createdUnix"` + UpdatedUnix timeutil.TimeStamp `json:"updatedUnix"` + + Attachments []*AttachmentShow `json:"attachments"` +} + +type RepositoryShow struct { + OwnerName string `json:"ownerName"` + Name string `json:"name"` +} + +type AttachmentShow struct { + ID int64 `json:"id"` + UUID string `json:"uuid"` + DatasetID int64 `json:"datasetId"` + ReleaseID int64 `json:"releaseId"` + UploaderID int64 `json:"uploaderId"` + CommentID int64 `json:"commentId"` + Name string `json:"name"` + Description string `json:"description"` + DownloadCount int64 `json:"downloadCount"` + UseNumber int64 `json:"useNumber"` + Size int64 `json:"size"` + IsPrivate bool `json:"isPrivate"` + DecompressState int32 `json:"decompressState"` + Type int `json:"type"` + CreatedUnix timeutil.TimeStamp `json:"createdUnix"` +} diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index 0d2576e7e..e0409b340 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -1,5 +1,10 @@ package structs +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/timeutil" +) + type CreateGrampusTrainJobOption struct { DisplayJobName string `json:"display_job_name" binding:"Required"` JobName string `json:"job_name" binding:"Required" ` @@ -40,3 +45,38 @@ type CreateTrainJobOption struct { PreTrainModelUrl string `json:"pre_train_model_url"` SpecId int64 `json:"spec_id" binding:"Required"` } + +type Cloudbrain struct { + ID int64 `json:"id"` + JobID string `json:"job_id"` + JobType string `json:"job_type"` + Type int `json:"type"` + DisplayJobName string `json:"display_job_name"` + Status string `json:"status"` + CreatedUnix timeutil.TimeStamp `json:"created_unix"` + RepoID int64 `json:"repo_id"` + Duration int64 `json:"duration"` //运行时长 单位秒 + TrainJobDuration string `json:"train_job_duration"` + ImageID string `json:"image_id"` //grampus image_id + Image string `json:"image"` + Uuid string `json:"uuid"` //数据集id + DatasetName string `json:"dataset_name"` + ComputeResource string `json:"compute_resource"` //计算资源,例如npu + AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name + BranchName string `json:"branch_name"` //分支名称 + Parameters string `json:"parameters"` //传给modelarts的param参数 + BootFile string `json:"boot_file"` //启动文件 + Description string `json:"description"` //描述 + ModelName string `json:"model_name"` //模型名称 + ModelVersion string `json:"model_version"` //模型版本 + CkptName string `json:"ckpt_name"` //权重文件名称 + + VersionCount int `json:"-"` //任务的当前版本数量,不包括删除的 + IsLatestVersion string `json:"-"` //是否是最新版本,1是,0否 + CommitID string `json:"-"` //提交的仓库代码id + PreVersionName string `json:"-"` //父版本名称 + StartTime timeutil.TimeStamp `json:"start_time"` + EndTime timeutil.TimeStamp `json:"end_time"` + + Spec *models.Specification `json:"spec"` +} From a2332f6219487b2f7908d9c9cf8b06cb3893f637 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 27 Oct 2022 09:19:54 +0800 Subject: [PATCH 034/187] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/resource_specification.go | 26 ----- modules/convert/cloudbrain.go | 103 ++++++++++++++++++ modules/structs/attachment.go | 66 ++++++----- modules/structs/cloudbrain.go | 72 ++++++------ routers/api/v1/api.go | 29 ++++- routers/api/v1/repo/attachments.go | 25 +++++ routers/api/v1/repo/cloudbrain.go | 4 +- routers/api/v1/repo/datasets.go | 13 ++- .../resource/resource_specification.go | 21 ++-- 9 files changed, 248 insertions(+), 111 deletions(-) create mode 100644 modules/convert/cloudbrain.go create mode 100644 routers/api/v1/repo/attachments.go diff --git a/models/resource_specification.go b/models/resource_specification.go index d69cd7208..2da8d015d 100644 --- a/models/resource_specification.go +++ b/models/resource_specification.go @@ -194,32 +194,6 @@ func (Specification) TableName() string { return "resource_specification" } -func (s *Specification) ToShow() *SpecificationShow { - return &SpecificationShow{ - ID: s.ID, - AccCardsNum: s.AccCardsNum, - AccCardType: s.AccCardType, - CpuCores: s.CpuCores, - MemGiB: s.MemGiB, - GPUMemGiB: s.GPUMemGiB, - ShareMemGiB: s.ShareMemGiB, - ComputeResource: s.ComputeResource, - UnitPrice: s.UnitPrice, - } -} - -type SpecificationShow struct { - ID int64 `json:"id"` - AccCardsNum int `json:"acc_cards_num"` - AccCardType string `json:"acc_card_type"` - CpuCores int `json:"cpu_cores"` - MemGiB float32 `json:"mem_gi_b"` - GPUMemGiB float32 `json:"gpu_mem_gi_b"` - ShareMemGiB float32 `json:"share_mem_gi_b"` - ComputeResource string `json:"compute_resource"` - UnitPrice int `json:"unit_price"` -} - func InsertResourceSpecification(r ResourceSpecification) (int64, error) { return x.Insert(&r) } diff --git a/modules/convert/cloudbrain.go b/modules/convert/cloudbrain.go new file mode 100644 index 000000000..7df38ee05 --- /dev/null +++ b/modules/convert/cloudbrain.go @@ -0,0 +1,103 @@ +package convert + +import ( + "code.gitea.io/gitea/models" + api "code.gitea.io/gitea/modules/structs" +) + +func ToCloudBrain(task *models.Cloudbrain) *api.Cloudbrain { + return &api.Cloudbrain{ + ID: task.ID, + JobID: task.JobID, + JobType: task.JobType, + Type: task.Type, + DisplayJobName: task.DisplayJobName, + Status: task.Status, + CreatedUnix: int64(task.CreatedUnix), + RepoID: task.RepoID, + Duration: task.Duration, + TrainJobDuration: task.TrainJobDuration, + ImageID: task.ImageID, + Image: task.Image, + Uuid: task.Uuid, + DatasetName: task.DatasetName, + ComputeResource: task.ComputeResource, + AiCenter: task.AiCenter, + BranchName: task.BranchName, + Parameters: task.Parameters, + BootFile: task.BootFile, + Description: task.Description, + ModelName: task.ModelName, + + ModelVersion: task.ModelVersion, + CkptName: task.CkptName, + + StartTime: int64(task.StartTime), + EndTime: int64(task.EndTime), + + Spec: ToSpecification(task.Spec), + } +} +func ToAttachment(attachment *models.Attachment) *api.AttachmentShow { + return &api.AttachmentShow{ + ID: attachment.ID, + UUID: attachment.UUID, + DatasetID: attachment.DatasetID, + ReleaseID: attachment.ReleaseID, + UploaderID: attachment.UploaderID, + CommentID: attachment.CommentID, + Name: attachment.Name, + Description: attachment.Description, + DownloadCount: attachment.DownloadCount, + UseNumber: attachment.UseNumber, + Size: attachment.Size, + IsPrivate: attachment.IsPrivate, + DecompressState: attachment.DecompressState, + Type: attachment.Type, + CreatedUnix: int64(attachment.CreatedUnix), + } +} + +func ToDataset(dataset *models.Dataset) *api.Dataset { + var convertAttachments []*api.AttachmentShow + for _, attachment := range dataset.Attachments { + convertAttachments = append(convertAttachments, ToAttachment(attachment)) + } + return &api.Dataset{ + ID: dataset.ID, + Title: dataset.Title, + Status: dataset.Status, + Category: dataset.Category, + Description: dataset.Description, + DownloadTimes: dataset.DownloadTimes, + UseCount: dataset.UseCount, + NumStars: dataset.NumStars, + Recommend: dataset.Recommend, + License: dataset.License, + Task: dataset.Task, + ReleaseID: dataset.ReleaseID, + UserID: dataset.UserID, + RepoID: dataset.RepoID, + Repo: &api.RepositoryShow{ + OwnerName: dataset.Repo.OwnerName, + Name: dataset.Repo.Name, + }, + CreatedUnix: int64(dataset.CreatedUnix), + UpdatedUnix: int64(dataset.UpdatedUnix), + Attachments: convertAttachments, + } +} + +func ToSpecification(s *models.Specification) *api.SpecificationShow { + return &api.SpecificationShow{ + ID: s.ID, + AccCardsNum: s.AccCardsNum, + AccCardType: s.AccCardType, + CpuCores: s.CpuCores, + MemGiB: s.MemGiB, + GPUMemGiB: s.GPUMemGiB, + ShareMemGiB: s.ShareMemGiB, + ComputeResource: s.ComputeResource, + UnitPrice: s.UnitPrice, + } +} diff --git a/modules/structs/attachment.go b/modules/structs/attachment.go index 3ccc2742a..4a0f8feec 100755 --- a/modules/structs/attachment.go +++ b/modules/structs/attachment.go @@ -6,8 +6,6 @@ package structs // import "code.gitea.io/gitea/modules/structs" import ( "time" - - "code.gitea.io/gitea/modules/timeutil" ) // Attachment a generic attachment @@ -31,23 +29,23 @@ type EditAttachmentOptions struct { } type Dataset struct { - ID int64 `json:"id"` - Title string `json:"title"` - Status int32 `json:"status"` - Category string `json:"category"` - Description string `json:"description"` - DownloadTimes int64 `json:"downloadTimes"` - UseCount int64 `json:"useCount"` - NumStars int `json:"numStars"` - Recommend bool `json:"recommend"` - License string `json:"license"` - Task string `json:"task"` - ReleaseID int64 `json:"releaseId"` - UserID int64 `json:"userId"` - RepoID int64 `json:"repoId"` - Repo *RepositoryShow `json:"repo"` - CreatedUnix timeutil.TimeStamp `json:"createdUnix"` - UpdatedUnix timeutil.TimeStamp `json:"updatedUnix"` + ID int64 `json:"id"` + Title string `json:"title"` + Status int32 `json:"status"` + Category string `json:"category"` + Description string `json:"description"` + DownloadTimes int64 `json:"downloadTimes"` + UseCount int64 `json:"useCount"` + NumStars int `json:"numStars"` + Recommend bool `json:"recommend"` + License string `json:"license"` + Task string `json:"task"` + ReleaseID int64 `json:"releaseId"` + UserID int64 `json:"userId"` + RepoID int64 `json:"repoId"` + Repo *RepositoryShow `json:"repo"` + CreatedUnix int64 `json:"createdUnix"` + UpdatedUnix int64 `json:"updatedUnix"` Attachments []*AttachmentShow `json:"attachments"` } @@ -58,19 +56,19 @@ type RepositoryShow struct { } type AttachmentShow struct { - ID int64 `json:"id"` - UUID string `json:"uuid"` - DatasetID int64 `json:"datasetId"` - ReleaseID int64 `json:"releaseId"` - UploaderID int64 `json:"uploaderId"` - CommentID int64 `json:"commentId"` - Name string `json:"name"` - Description string `json:"description"` - DownloadCount int64 `json:"downloadCount"` - UseNumber int64 `json:"useNumber"` - Size int64 `json:"size"` - IsPrivate bool `json:"isPrivate"` - DecompressState int32 `json:"decompressState"` - Type int `json:"type"` - CreatedUnix timeutil.TimeStamp `json:"createdUnix"` + ID int64 `json:"id"` + UUID string `json:"uuid"` + DatasetID int64 `json:"datasetId"` + ReleaseID int64 `json:"releaseId"` + UploaderID int64 `json:"uploaderId"` + CommentID int64 `json:"commentId"` + Name string `json:"name"` + Description string `json:"description"` + DownloadCount int64 `json:"downloadCount"` + UseNumber int64 `json:"useNumber"` + Size int64 `json:"size"` + IsPrivate bool `json:"isPrivate"` + DecompressState int32 `json:"decompressState"` + Type int `json:"type"` + CreatedUnix int64 `json:"createdUnix"` } diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index e0409b340..b29d7e722 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -1,10 +1,5 @@ package structs -import ( - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/timeutil" -) - type CreateGrampusTrainJobOption struct { DisplayJobName string `json:"display_job_name" binding:"Required"` JobName string `json:"job_name" binding:"Required" ` @@ -47,36 +42,43 @@ type CreateTrainJobOption struct { } type Cloudbrain struct { - ID int64 `json:"id"` - JobID string `json:"job_id"` - JobType string `json:"job_type"` - Type int `json:"type"` - DisplayJobName string `json:"display_job_name"` - Status string `json:"status"` - CreatedUnix timeutil.TimeStamp `json:"created_unix"` - RepoID int64 `json:"repo_id"` - Duration int64 `json:"duration"` //运行时长 单位秒 - TrainJobDuration string `json:"train_job_duration"` - ImageID string `json:"image_id"` //grampus image_id - Image string `json:"image"` - Uuid string `json:"uuid"` //数据集id - DatasetName string `json:"dataset_name"` - ComputeResource string `json:"compute_resource"` //计算资源,例如npu - AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name - BranchName string `json:"branch_name"` //分支名称 - Parameters string `json:"parameters"` //传给modelarts的param参数 - BootFile string `json:"boot_file"` //启动文件 - Description string `json:"description"` //描述 - ModelName string `json:"model_name"` //模型名称 - ModelVersion string `json:"model_version"` //模型版本 - CkptName string `json:"ckpt_name"` //权重文件名称 + ID int64 `json:"id"` + JobID string `json:"job_id"` + JobType string `json:"job_type"` + Type int `json:"type"` + DisplayJobName string `json:"display_job_name"` + Status string `json:"status"` + CreatedUnix int64 `json:"created_unix"` + RepoID int64 `json:"repo_id"` + Duration int64 `json:"duration"` //运行时长 单位秒 + TrainJobDuration string `json:"train_job_duration"` + ImageID string `json:"image_id"` //grampus image_id + Image string `json:"image"` + Uuid string `json:"uuid"` //数据集id + DatasetName string `json:"dataset_name"` + ComputeResource string `json:"compute_resource"` //计算资源,例如npu + AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name + BranchName string `json:"branch_name"` //分支名称 + Parameters string `json:"parameters"` //传给modelarts的param参数 + BootFile string `json:"boot_file"` //启动文件 + Description string `json:"description"` //描述 + ModelName string `json:"model_name"` //模型名称 + ModelVersion string `json:"model_version"` //模型版本 + CkptName string `json:"ckpt_name"` //权重文件名称 + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` - VersionCount int `json:"-"` //任务的当前版本数量,不包括删除的 - IsLatestVersion string `json:"-"` //是否是最新版本,1是,0否 - CommitID string `json:"-"` //提交的仓库代码id - PreVersionName string `json:"-"` //父版本名称 - StartTime timeutil.TimeStamp `json:"start_time"` - EndTime timeutil.TimeStamp `json:"end_time"` + Spec *SpecificationShow `json:"spec"` +} - Spec *models.Specification `json:"spec"` +type SpecificationShow struct { + ID int64 `json:"id"` + AccCardsNum int `json:"acc_cards_num"` + AccCardType string `json:"acc_card_type"` + CpuCores int `json:"cpu_cores"` + MemGiB float32 `json:"mem_gi_b"` + GPUMemGiB float32 `json:"gpu_mem_gi_b"` + ShareMemGiB float32 `json:"share_mem_gi_b"` + ComputeResource string `json:"compute_resource"` + UnitPrice int `json:"unit_price"` } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 857d810f8..9f8ddd5b5 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -242,6 +242,15 @@ func reqRepoWriter(unitTypes ...models.UnitType) macaron.Handler { } } +func reqWeChat() macaron.Handler { + return func(ctx *context.Context) { + if setting.WechatAuthSwitch && ctx.User.WechatOpenId == "" { + ctx.JSON(http.StatusForbidden, models.BaseErrorMessageApi("settings.no_wechat_bind")) + return + } + } +} + // reqRepoReader user should have specific read permission or be a repo admin or a site admin func reqRepoReader(unitType models.UnitType) macaron.Handler { return func(ctx *context.Context) { @@ -526,6 +535,16 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqToken()) + m.Group("/attachments", func() { + + m.Get("/:uuid", repo.GetAttachment) + m.Get("/get_chunks", repo.GetSuccessChunks) + m.Get("/new_multipart", repo.NewMultipart) + m.Get("/get_multipart_url", repo.GetMultipartUploadUrl) + m.Post("/complete_multipart", repo.CompleteMultipart) + + }, reqToken()) + // Notifications m.Group("/notifications", func() { m.Combo(""). @@ -982,15 +1001,17 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqRepoReader(models.UnitTypeModelManage)) m.Group("/cloudbrain", func() { m.Group("/train-job", func() { - m.Post("/create", bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) + m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) + + m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat, bind(api.CreateTrainJobOption{}), repo.CreateCloudBrain) }) m.Group("/inference-job", func() { - m.Post("/create", bind(api.CreateTrainJobOption{}), repo.CreateCloudBrainInferenceTask) - + m.Post("/create", reqRepoWriter(models.UnitTypeCloudBrain), reqWeChat, bind(api.CreateTrainJobOption{}), repo.CreateCloudBrainInferenceTask) + m.Get("/:jobid", reqRepoReader(models.UnitTypeCloudBrain), repo.CloudBrainShow) }) - }, reqRepoReader(models.UnitTypeCloudBrain)) + }, reqToken()) m.Group("/modelarts", func() { m.Group("/notebook", func() { //m.Get("/:jobid", repo.GetModelArtsNotebook) diff --git a/routers/api/v1/repo/attachments.go b/routers/api/v1/repo/attachments.go new file mode 100644 index 000000000..cb36ba2ee --- /dev/null +++ b/routers/api/v1/repo/attachments.go @@ -0,0 +1,25 @@ +package repo + +import ( + "code.gitea.io/gitea/modules/context" + routeRepo "code.gitea.io/gitea/routers/repo" +) + +func GetSuccessChunks(ctx *context.APIContext) { + routeRepo.GetSuccessChunks(ctx.Context) +} + +func NewMultipart(ctx *context.APIContext) { + routeRepo.NewMultipart(ctx.Context) +} +func GetMultipartUploadUrl(ctx *context.APIContext) { + routeRepo.GetMultipartUploadUrl(ctx.Context) +} + +func CompleteMultipart(ctx *context.APIContext) { + routeRepo.CompleteMultipart(ctx.Context) + +} +func GetAttachment(ctx *context.APIContext) { + routeRepo.GetAttachment(ctx.Context) +} diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 380de0ddb..7092a1f02 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -16,6 +16,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/convert" + "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" api "code.gitea.io/gitea/modules/structs" @@ -71,7 +73,7 @@ func CloudBrainShow(ctx *context.APIContext) { task.Image = task.EngineName } - ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: task}) + ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: convert.ToCloudBrain(task)}) } diff --git a/routers/api/v1/repo/datasets.go b/routers/api/v1/repo/datasets.go index 0539a22e9..0223cf740 100644 --- a/routers/api/v1/repo/datasets.go +++ b/routers/api/v1/repo/datasets.go @@ -4,6 +4,10 @@ import ( "fmt" "strings" + "code.gitea.io/gitea/modules/convert" + + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" @@ -88,15 +92,20 @@ func datasetMultiple(ctx *context.APIContext, opts *models.SearchDatasetOptions) ctx.JSON(200, map[string]interface{}{ "code": 1, "message": err.Error(), - "data": []*models.Dataset{}, + "data": []*api.Dataset{}, "count": 0, }) return } + var convertDatasets []*api.Dataset + for _, dataset := range datasets { + convertDatasets = append(convertDatasets, convert.ToDataset(dataset)) + } + ctx.JSON(200, map[string]interface{}{ "code": 0, "message": "", - "data": datasets, + "data": convertDatasets, "count": count, }) } diff --git a/services/cloudbrain/resource/resource_specification.go b/services/cloudbrain/resource/resource_specification.go index c0b20b21a..93abb2923 100644 --- a/services/cloudbrain/resource/resource_specification.go +++ b/services/cloudbrain/resource/resource_specification.go @@ -1,20 +1,23 @@ package resource import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/grampus" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/routers/response" "code.gitea.io/gitea/services/admin/operate_log" - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" - "time" ) func AddResourceSpecification(doerId int64, req models.ResourceSpecificationReq) error { @@ -210,14 +213,14 @@ func FindAvailableSpecs(userId int64, opts models.FindSpecsOptions) ([]*models.S return specs, err } -func FindAvailableSpecs4Show(userId int64, opts models.FindSpecsOptions) ([]*models.SpecificationShow, error) { +func FindAvailableSpecs4Show(userId int64, opts models.FindSpecsOptions) ([]*api.SpecificationShow, error) { specs, err := FindAvailableSpecs(userId, opts) if err != nil { return nil, err } - result := make([]*models.SpecificationShow, len(specs)) + result := make([]*api.SpecificationShow, len(specs)) for i, v := range specs { - result[i] = v.ToShow() + result[i] = convert.ToSpecification(v) } return result, nil } From 6cea45e40a7fc9cf3aad5c57d2120d4e40982f25 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 28 Oct 2022 09:17:03 +0800 Subject: [PATCH 035/187] =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E4=BF=AE=E6=94=B9=E6=88=90=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E7=9A=84=E6=96=B9=E5=BC=8F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/repo/modelmanage.go | 25 ++++++++++++++++++++++++- routers/repo/ai_model_manage.go | 12 +++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index b32726997..b404558a1 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -8,6 +8,15 @@ import ( routerRepo "code.gitea.io/gitea/routers/repo" ) +type FileInfo struct { + FileName string `json:"fileName"` + ModTime string `json:"modTime"` + IsDir bool `json:"isDir"` + Size int64 `json:"size"` + ParenDir string `json:"parenDir"` + UUID string `json:"uuid"` +} + func CreateNewModel(ctx *context.APIContext) { log.Info("CreateNewModel by api.") routerRepo.SaveModel(ctx.Context) @@ -35,7 +44,21 @@ func QueryModelListForPredict(ctx *context.APIContext) { func QueryModelFileForPredict(ctx *context.APIContext) { log.Info("QueryModelFileForPredict by api.") - routerRepo.QueryModelFileForPredict(ctx.Context) + id := ctx.Query("id") + result := routerRepo.QueryModelFileByID(id) + re := make([]FileInfo, len(result)) + for _, file := range result { + tmpFile := FileInfo{ + FileName: file.FileName, + ModTime: file.ModTime, + IsDir: file.IsDir, + Size: file.Size, + ParenDir: file.ParenDir, + UUID: file.UUID, + } + re = append(re, tmpFile) + } + ctx.JSON(http.StatusOK, re) } func CreateModelConvert(ctx *context.APIContext) { diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 6385cc674..45c21d719 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -910,23 +910,25 @@ func QueryModelFileForPredict(ctx *context.Context) { if id == "" { id = ctx.Query("ID") } + ctx.JSON(http.StatusOK, QueryModelFileByID(id)) +} + +func QueryModelFileByID(id string) []storage.FileInfo { model, err := models.QueryModelById(id) if err == nil { if model.Type == models.TypeCloudBrainTwo { prefix := model.Path[len(setting.Bucket)+1:] fileinfos, _ := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix) - ctx.JSON(http.StatusOK, fileinfos) + return fileinfos } else if model.Type == models.TypeCloudBrainOne { prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:] fileinfos, _ := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, prefix) - ctx.JSON(http.StatusOK, fileinfos) + return fileinfos } } else { log.Error("no such model!", err.Error()) - ctx.ServerError("no such model:", err) - return } - + return nil } func QueryOneLevelModelFile(ctx *context.Context) { From 187eec79b2f8cdc961944f1fb20f0b405832f510 Mon Sep 17 00:00:00 2001 From: zouap Date: Mon, 31 Oct 2022 09:45:41 +0800 Subject: [PATCH 036/187] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/api.go | 1 + routers/api/v1/repo/modelmanage.go | 5 +++ routers/repo/ai_model_manage.go | 49 ++++++++++++++++++++---------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 9f8ddd5b5..5ecbf3a5f 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -990,6 +990,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/show_model_api", repo.ShowModelManageApi) m.Delete("/delete_model", repo.DeleteModel) m.Get("/downloadall", repo.DownloadModel) + m.Get("/query_model_byId", repo.QueryModelById) m.Get("/query_model_for_predict", repo.QueryModelListForPredict) m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) m.Post("/create_model_convert", repo.CreateModelConvert) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index b404558a1..253d51e00 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -37,6 +37,11 @@ func DownloadModel(ctx *context.APIContext) { routerRepo.DownloadMultiModelFile(ctx.Context) } +func QueryModelById(ctx *context.APIContext) { + log.Info("QueryModelById by api.") + routerRepo.QueryModelById(ctx.Context) +} + func QueryModelListForPredict(ctx *context.APIContext) { log.Info("QueryModelListForPredict by api.") routerRepo.QueryModelListForPredict(ctx.Context) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 45c21d719..ca90ff432 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -33,13 +33,13 @@ const ( STATUS_ERROR = 2 ) -func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) error { +func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) (string, error) { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) if err != nil { aiTask, err = models.GetRepoCloudBrainByJobID(ctx.Repo.Repository.ID, jobId) if err != nil { log.Info("query task error." + err.Error()) - return err + return "", err } else { log.Info("query gpu train task.") } @@ -55,7 +55,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio if len(aimodels) > 0 { for _, model := range aimodels { if model.Version == version { - return errors.New(ctx.Tr("repo.model.manage.create_error")) + return "", errors.New(ctx.Tr("repo.model.manage.create_error")) } if model.New == MODEL_LATEST { lastNewModelId = model.ID @@ -113,7 +113,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio err = models.SaveModelToDb(model) if err != nil { - return err + return "", err } if len(lastNewModelId) > 0 { //udpate status and version count @@ -136,7 +136,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio log.Info("save model end.") notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, id, name, models.ActionCreateNewModelTask) - return nil + return id, nil } func asyncToCopyModel(aiTask *models.Cloudbrain, id string, modelSelectedFile string) { @@ -206,29 +206,33 @@ func SaveModel(ctx *context.Context) { engine := ctx.QueryInt("engine") modelSelectedFile := ctx.Query("modelSelectedFile") log.Info("engine=" + fmt.Sprint(engine) + " modelSelectedFile=" + modelSelectedFile) - + re := map[string]string{ + "code": "-1", + } if JobId == "" || VersionName == "" { - ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) + re["msg"] = "JobId or VersionName is null." + ctx.JSON(200, re) return } if modelSelectedFile == "" { - ctx.Error(500, fmt.Sprintf("Not selected model file.")) + re["msg"] = "Not selected model file." + ctx.JSON(200, re) return } - if name == "" || version == "" { - ctx.Error(500, fmt.Sprintf("name or version is null.")) + re["msg"] = "name or version is null." + ctx.JSON(200, re) return } - - err := saveModelByParameters(JobId, VersionName, name, version, label, description, engine, ctx) - + id, err := saveModelByParameters(JobId, VersionName, name, version, label, description, engine, ctx) if err != nil { log.Info("save model error." + err.Error()) - ctx.Error(500, fmt.Sprintf("save model error. %v", err)) - return + re["msg"] = err.Error() + } else { + re["code"] = "0" + re["id"] = id } - ctx.Status(200) + ctx.JSON(200, re) log.Info("save model end.") } @@ -632,6 +636,19 @@ func ShowModelInfo(ctx *context.Context) { ctx.HTML(200, tplModelInfo) } +func QueryModelById(ctx *context.Context) { + id := ctx.Query("id") + model, err := models.QueryModelById(id) + if err == nil { + model.IsCanOper = isOper(ctx, model.UserId) + model.IsCanDelete = isCanDelete(ctx, model.UserId) + removeIpInfo(model) + ctx.JSON(http.StatusOK, model) + } else { + ctx.JSON(http.StatusNotFound, nil) + } +} + func ShowSingleModel(ctx *context.Context) { name := ctx.Query("name") From a4873ff88a40081e6fc3ec20d9cf1e589ad28104 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Mon, 31 Oct 2022 15:30:05 +0800 Subject: [PATCH 037/187] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/modelarts/modelarts.go | 16 +- routers/api/v1/repo/cloudbrain.go | 10 +- routers/repo/modelarts.go | 2 +- services/cloudbrain/cloudbrainTask/train.go | 421 +++++++++++++++++++- 4 files changed, 427 insertions(+), 22 deletions(-) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 4158174cb..e94dd4ad9 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -348,7 +348,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc return nil } -func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error) { +func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId string, err error) { createTime := timeutil.TimeStampNow() var jobResult *models.CreateTrainJobResult var createErr error @@ -408,17 +408,17 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error }) if errTemp != nil { log.Error("InsertCloudbrainTemp failed: %v", errTemp.Error()) - return errTemp + return "", errTemp } } - return createErr + return "", createErr } - jobId := strconv.FormatInt(jobResult.JobID, 10) + jobID := strconv.FormatInt(jobResult.JobID, 10) createErr = models.CreateCloudbrain(&models.Cloudbrain{ Status: TransTrainJobStatus(jobResult.Status), UserID: ctx.User.ID, RepoID: ctx.Repo.Repository.ID, - JobID: jobId, + JobID: jobID, JobName: req.JobName, DisplayJobName: req.DisplayJobName, JobType: string(models.JobTypeTrain), @@ -456,10 +456,10 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error if createErr != nil { log.Error("CreateCloudbrain(%s) failed:%v", req.DisplayJobName, createErr.Error()) - return createErr + return "", createErr } - notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobId, req.DisplayJobName, models.ActionCreateTrainTask) - return nil + notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, jobID, req.DisplayJobName, models.ActionCreateTrainTask) + return jobID, nil } func GenerateModelConvertTrainJob(req *GenerateTrainJobReq) (*models.CreateTrainJobResult, error) { diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 7092a1f02..649e33956 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -78,11 +78,17 @@ func CloudBrainShow(ctx *context.APIContext) { } func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) { + if option.Type == cloudbrainTask.TaskTypeCloudbrainOne { + cloudbrainTask.CloudbrainOneTrainJobCreate(ctx.Context, option) + } + if option.Type == cloudbrainTask.TaskTypeModelArts { + cloudbrainTask.ModelArtsTrainJobNpuCreate(ctx.Context, option) + } - if option.Type == 2 { + if option.Type == cloudbrainTask.TaskTypeGrampusGPU { cloudbrainTask.GrampusTrainJobGpuCreate(ctx.Context, option) } - if option.Type == 3 { + if option.Type == cloudbrainTask.TaskTypeGrampusNPU { cloudbrainTask.GrampusTrainJobNpuCreate(ctx.Context, option) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index d187b211a..e4e73f716 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1230,7 +1230,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } - err = modelarts.GenerateTrainJob(ctx, req) + _, err = modelarts.GenerateTrainJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) trainJobNewDataPrepare(ctx) diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index 6f836557c..5e2cce1ff 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -10,6 +10,7 @@ import ( "os" "path" "regexp" + "strconv" "strings" "code.gitea.io/gitea/modules/timeutil" @@ -39,6 +40,277 @@ import ( var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`) +const TaskTypeCloudbrainOne = 0 +const TaskTypeModelArts = 1 +const TaskTypeGrampusGPU = 2 +const TaskTypeGrampusNPU = 3 + +func CloudbrainOneTrainJobCreate(ctx *context.Context, option api.CreateTrainJobOption) { + + displayJobName := option.DisplayJobName + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + image := strings.TrimSpace(option.Image) + uuids := option.Attachment + jobType := string(models.JobTypeTrain) + + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + branchName := option.BranchName + repo := ctx.Repo.Repository + + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) + defer lock.UnLock() + spec, datasetInfos, datasetNames, err := checkParameters(ctx, option, lock, repo) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + command, err := getTrainJobCommand(option) + if err != nil { + log.Error("getTrainJobCommand failed: %v", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + errStr := loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ModelMountPath) + if errStr != "" { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr(errStr))) + return + } + + commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName) + + req := cloudbrain.GenerateCloudBrainTaskReq{ + Ctx: ctx, + DisplayJobName: displayJobName, + JobName: jobName, + Image: image, + Command: command, + Uuids: uuids, + DatasetNames: datasetNames, + DatasetInfos: datasetInfos, + CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), + ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), + BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), + Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), + BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), + JobType: jobType, + Description: option.Description, + BranchName: branchName, + BootFile: option.BootFile, + Params: option.Params, + CommitID: commitID, + BenchmarkTypeID: 0, + BenchmarkChildTypeID: 0, + ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), + Spec: spec, + } + + if option.ModelName != "" { //使用预训练模型训练 + req.ModelName = option.ModelName + req.LabelName = option.LabelName + req.CkptName = option.CkptName + req.ModelVersion = option.ModelVersion + req.PreTrainModelPath = setting.Attachment.Minio.RealPath + option.PreTrainModelUrl + req.PreTrainModelUrl = option.PreTrainModelUrl + + } + + jobId, err := cloudbrain.GenerateTask(req) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: jobId, + }) +} +func ModelArtsTrainJobNpuCreate(ctx *context.Context, option api.CreateTrainJobOption) { + VersionOutputPath := modelarts.GetOutputPathByCount(modelarts.TotalVersionCount) + displayJobName := option.DisplayJobName + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + uuid := option.Attachment + description := option.Description + workServerNumber := option.WorkServerNumber + engineID, _ := strconv.Atoi(option.ImageID) + bootFile := strings.TrimSpace(option.BootFile) + params := option.Params + repo := ctx.Repo.Repository + codeLocalPath := setting.JobPath + jobName + modelarts.CodePath + codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath + VersionOutputPath + "/" + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" + branchName := option.BranchName + isLatestVersion := modelarts.IsLatestVersion + VersionCount := modelarts.VersionCountOne + EngineName := option.Image + + errStr := checkMultiNode(ctx.User.ID, option.WorkServerNumber) + if errStr != "" { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr(errStr))) + return + } + + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) + defer lock.UnLock() + + spec, _, _, err := checkParameters(ctx, option, lock, repo) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + //todo: del the codeLocalPath + _, err = ioutil.ReadDir(codeLocalPath) + if err == nil { + os.RemoveAll(codeLocalPath) + } + + gitRepo, _ := git.OpenRepository(repo.RepoPath()) + commitID, _ := gitRepo.GetBranchCommitID(branchName) + + if err := downloadCode(repo, codeLocalPath, branchName); err != nil { + log.Error("downloadCode failed, server timed out: %s (%v)", repo.FullName(), err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + + //todo: upload code (send to file_server todo this work?) + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { + log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Failed to obsMkdir_output")) + return + } + + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { + log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Failed to obsMkdir_log")) + return + } + + parentDir := VersionOutputPath + "/" + if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { + // if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { + log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + + var parameters models.Parameters + param := make([]models.Parameter, 0) + existDeviceTarget := false + if len(params) != 0 { + err := json.Unmarshal([]byte(params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", params, err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("运行参数错误")) + return + } + + for _, parameter := range parameters.Parameter { + if parameter.Label == modelarts.DeviceTarget { + existDeviceTarget = true + } + if parameter.Label != modelarts.TrainUrl && parameter.Label != modelarts.DataUrl { + param = append(param, models.Parameter{ + Label: parameter.Label, + Value: parameter.Value, + }) + } + } + } + if !existDeviceTarget { + param = append(param, models.Parameter{ + Label: modelarts.DeviceTarget, + Value: modelarts.Ascend, + }) + } + datasUrlList, dataUrl, datasetNames, isMultiDataset, err := getDatasUrlListByUUIDS(uuid) + if err != nil { + log.Error("Failed to getDatasUrlListByUUIDS: %v", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Failed to getDatasUrlListByUUIDS:"+err.Error())) + return + } + dataPath := dataUrl + jsondatas, err := json.Marshal(datasUrlList) + if err != nil { + log.Error("Failed to Marshal: %v", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("json error:"+err.Error())) + return + } + if isMultiDataset { + param = append(param, models.Parameter{ + Label: modelarts.MultiDataUrl, + Value: string(jsondatas), + }) + } + if option.ModelName != "" { //使用预训练模型训练 + ckptUrl := "/" + option.PreTrainModelUrl + option.CkptName + param = append(param, models.Parameter{ + Label: modelarts.CkptUrl, + Value: "s3:/" + ckptUrl, + }) + } + + req := &modelarts.GenerateTrainJobReq{ + JobName: jobName, + DisplayJobName: displayJobName, + DataUrl: dataPath, + Description: description, + CodeObsPath: codeObsPath, + BootFileUrl: codeObsPath + bootFile, + BootFile: bootFile, + TrainUrl: outputObsPath, + WorkServerNumber: workServerNumber, + EngineID: int64(engineID), + LogUrl: logObsPath, + PoolID: getPoolId(), + Uuid: uuid, + Parameters: param, + CommitID: commitID, + IsLatestVersion: isLatestVersion, + BranchName: branchName, + Params: option.Params, + EngineName: EngineName, + VersionCount: VersionCount, + TotalVersionCount: modelarts.TotalVersionCount, + DatasetName: datasetNames, + Spec: spec, + } + if option.ModelName != "" { //使用预训练模型训练 + req.ModelName = option.ModelName + req.LabelName = option.LabelName + req.CkptName = option.CkptName + req.ModelVersion = option.ModelVersion + req.PreTrainModelUrl = option.PreTrainModelUrl + + } + + userCommand, userImageUrl := getUserCommand(engineID, req) + req.UserCommand = userCommand + req.UserImageUrl = userImageUrl + + //将params转换Parameters.Parameter,出错时返回给前端 + var Parameters modelarts.Parameters + if err := json.Unmarshal([]byte(params), &Parameters); err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("json.Unmarshal failed:"+err.Error())) + return + } + + jobId, err := modelarts.GenerateTrainJob(ctx, req) + if err != nil { + log.Error("GenerateTrainJob failed:%v", err.Error()) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: jobId, + }) + +} + func GrampusTrainJobGpuCreate(ctx *context.Context, option api.CreateTrainJobOption) { displayJobName := option.DisplayJobName @@ -179,13 +451,19 @@ func checkParameters(ctx *context.Context, option api.CreateTrainJobOption, lock } computeResource := models.GPUResource - if option.Type == 3 { + if isNpuTask(option) { computeResource = models.NPUResource } + //check count limit - count, err := GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeC2Net, string(models.JobTypeTrain), computeResource) + taskType := option.Type + if isC2NetTask(option) { + taskType = 2 + } + + count, err := GetNotFinalStatusTaskCount(ctx.User.ID, taskType, string(models.JobTypeTrain), computeResource) if err != nil { - log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"]) + log.Error("GetCountByUserID failed:%v", err, ctx.Data["MsgID"]) return nil, nil, "", fmt.Errorf("system error") } else { if count >= 1 { @@ -195,7 +473,7 @@ func checkParameters(ctx *context.Context, option api.CreateTrainJobOption, lock } //check param - if err := grampusParamCheckCreateTrainJob(option.BootFile, option.BranchName); err != nil { + if err := paramCheckCreateTrainJob(option.BootFile, option.BranchName); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err, ctx.Data["MsgID"]) return nil, nil, "", err } @@ -216,13 +494,26 @@ func checkParameters(ctx *context.Context, option api.CreateTrainJobOption, lock //check specification computeType := models.GPU - if option.Type == 3 { + + if isNpuTask(option) { computeType = models.NPU } + cluster := models.OpenICluster + if isC2NetTask(option) { + cluster = models.C2NetCluster + } + aiCenterCode := "" + if option.Type == TaskTypeCloudbrainOne { + aiCenterCode = models.AICenterOfCloudBrainOne + } else if option.Type == TaskTypeModelArts { + aiCenterCode = models.AICenterOfCloudBrainTwo + } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, option.SpecId, models.FindSpecsOptions{ JobType: models.JobTypeTrain, ComputeResource: computeType, - Cluster: models.C2NetCluster, + Cluster: cluster, + AiCenterCode: aiCenterCode, }) if err != nil || spec == nil { return nil, nil, "", fmt.Errorf("Resource specification is not available.") @@ -234,14 +525,26 @@ func checkParameters(ctx *context.Context, option api.CreateTrainJobOption, lock } //check dataset - datasetInfos, datasetNames, err := models.GetDatasetInfo(option.Attachment, computeType) - if err != nil { - log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - return nil, nil, "", fmt.Errorf(ctx.Tr("cloudbrain.error.dataset_select")) + var datasetInfos map[string]models.DatasetInfo + var datasetNames string + if option.Type != TaskTypeModelArts { + datasetInfos, datasetNames, err = models.GetDatasetInfo(option.Attachment, computeType) + if err != nil { + log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + return nil, nil, "", fmt.Errorf(ctx.Tr("cloudbrain.error.dataset_select")) + } } return spec, datasetInfos, datasetNames, err } +func isNpuTask(option api.CreateTrainJobOption) bool { + return option.Type == TaskTypeModelArts || option.Type == TaskTypeGrampusNPU +} + +func isC2NetTask(option api.CreateTrainJobOption) bool { + return option.Type == TaskTypeGrampusGPU || option.Type == TaskTypeGrampusNPU +} + func GrampusTrainJobNpuCreate(ctx *context.Context, option api.CreateTrainJobOption) { displayJobName := option.DisplayJobName @@ -412,7 +715,7 @@ func uploadCodeToObs(codePath, jobName, parentDir string) error { return nil } -func grampusParamCheckCreateTrainJob(bootFile string, branchName string) error { +func paramCheckCreateTrainJob(bootFile string, branchName string) error { if !strings.HasSuffix(strings.TrimSpace(bootFile), ".py") { log.Error("the boot file(%s) must be a python file", bootFile) return errors.New("启动文件必须是python文件") @@ -792,3 +1095,99 @@ func SyncTaskStatus(task *models.Cloudbrain) error { return nil } + +func getTrainJobCommand(option api.CreateTrainJobOption) (string, error) { + var command string + bootFile := strings.TrimSpace(option.BootFile) + params := option.Params + + if !strings.HasSuffix(bootFile, ".py") { + log.Error("bootFile(%s) format error", bootFile) + return command, errors.New("bootFile format error") + } + + var parameters models.Parameters + var param string + if len(params) != 0 { + err := json.Unmarshal([]byte(params), ¶meters) + if err != nil { + log.Error("Failed to Unmarshal params: %s (%v)", params, err) + return command, err + } + + for _, parameter := range parameters.Parameter { + param += " --" + parameter.Label + "=" + parameter.Value + } + } + if option.CkptName != "" { + param += " --ckpt_url" + "=" + "/pretrainmodel/" + option.CkptName + } + + command += "python /code/" + bootFile + param + " > " + cloudbrain.ModelMountPath + "/" + option.DisplayJobName + "-" + cloudbrain.LogFile + + return command, nil +} + +func checkMultiNode(userId int64, serverNum int) string { + if serverNum == 1 { + return "" + } + modelarts.InitMultiNode() + var isServerNumValid = false + if modelarts.MultiNodeConfig != nil { + for _, info := range modelarts.MultiNodeConfig.Info { + if isInOrg, _ := models.IsOrganizationMemberByOrgName(info.Org, userId); isInOrg { + if isInNodes(info.Node, serverNum) { + isServerNumValid = true + break + } + + } + } + } + if isServerNumValid { + return "" + } else { + return "repo.modelarts.no_node_right" + } +} + +func isInNodes(nodes []int, num int) bool { + for _, node := range nodes { + if node == num { + return true + } + } + return false + +} + +func getUserCommand(engineId int, req *modelarts.GenerateTrainJobReq) (string, string) { + userImageUrl := "" + userCommand := "" + if engineId < 0 { + tmpCodeObsPath := strings.Trim(req.CodeObsPath, "/") + tmpCodeObsPaths := strings.Split(tmpCodeObsPath, "/") + lastCodeDir := "code" + if len(tmpCodeObsPaths) > 0 { + lastCodeDir = tmpCodeObsPaths[len(tmpCodeObsPaths)-1] + } + userCommand = "/bin/bash /home/work/run_train.sh 's3://" + req.CodeObsPath + "' '" + lastCodeDir + "/" + req.BootFile + "' '/tmp/log/train.log' --'data_url'='s3://" + req.DataUrl + "' --'train_url'='s3://" + req.TrainUrl + "'" + var versionInfos modelarts.VersionInfo + if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + log.Info("json parse err." + err.Error()) + } else { + for _, engine := range versionInfos.Version { + if engine.ID == engineId { + userImageUrl = engine.Url + break + } + } + } + for _, param := range req.Parameters { + userCommand += " --'" + param.Label + "'='" + param.Value + "'" + } + return userCommand, userImageUrl + } + return userCommand, userImageUrl +} From 670c13abe6a0ac07cec1c8f83fd8abf19846cb51 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 1 Nov 2022 16:20:27 +0800 Subject: [PATCH 038/187] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/convert/cloudbrain.go | 8 +++++ modules/structs/tagger.go | 7 ++++ routers/api/v1/repo/mlops.go | 63 +++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 modules/structs/tagger.go create mode 100644 routers/api/v1/repo/mlops.go diff --git a/modules/convert/cloudbrain.go b/modules/convert/cloudbrain.go index 7df38ee05..1487f468e 100644 --- a/modules/convert/cloudbrain.go +++ b/modules/convert/cloudbrain.go @@ -101,3 +101,11 @@ func ToSpecification(s *models.Specification) *api.SpecificationShow { UnitPrice: s.UnitPrice, } } + +func ToTagger(user *models.User) *api.Tagger { + return &api.Tagger{ + Name: user.Name, + RelAvatarURL: user.RelAvatarLink(), + Email: user.Email, + } +} diff --git a/modules/structs/tagger.go b/modules/structs/tagger.go new file mode 100644 index 000000000..8933c8c5c --- /dev/null +++ b/modules/structs/tagger.go @@ -0,0 +1,7 @@ +package structs + +type Tagger struct { + Name string `json:"name"` + Email string `json:"email"` + RelAvatarURL string `json:"relAvatarURL"` +} diff --git a/routers/api/v1/repo/mlops.go b/routers/api/v1/repo/mlops.go new file mode 100644 index 000000000..3392ebfec --- /dev/null +++ b/routers/api/v1/repo/mlops.go @@ -0,0 +1,63 @@ +package repo + +import ( + "net/http" + + "code.gitea.io/gitea/models" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/convert" + "code.gitea.io/gitea/modules/log" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/routers/api/v1/utils" +) + +//标注任务可分配人员 +func ListTagger(ctx *context.APIContext) { + + taggers := make([]*api.Tagger, 0) + userRemember := make(map[string]string) + collaborators, err := ctx.Repo.Repository.GetCollaborators(utils.GetListOptions(ctx)) + if err != nil { + log.Warn("ListCollaborators", err) + ctx.JSON(http.StatusOK, taggers) + return + } + for _, collaborator := range collaborators { + taggers = append(taggers, convert.ToTagger(collaborator.User)) + userRemember[collaborator.User.Name] = "" + } + + teams, err := ctx.Repo.Repository.GetRepoTeams() + if err != nil { + log.Warn("ListTeams", err) + ctx.JSON(http.StatusOK, taggers) + return + } + + for _, team := range teams { + for _, user := range team.Members { + if _, ok := userRemember[user.Name]; !ok { + taggers = append(taggers, convert.ToTagger(user)) + } + } + } + ctx.JSON(http.StatusOK, taggers) + +} +func GetRight(ctx *context.APIContext) { + right := "none" + + if ctx.IsUserRepoReaderSpecific(models.UnitTypeCode) { + right = "read" + } + + if ctx.IsUserRepoWriter([]models.UnitType{models.UnitTypeCode}) || ctx.IsUserRepoAdmin() { + right = "write" + } + + ctx.JSON(http.StatusOK, map[string]string{ + "right": right, + }) + +} From 92146da8213e5d7b57276ef1b4b32239acab613c Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 1 Nov 2022 16:45:11 +0800 Subject: [PATCH 039/187] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E5=90=8E=E7=AB=AF?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 8 ++++++++ routers/routes/routes.go | 2 ++ 2 files changed, 10 insertions(+) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index f2b0fc6d1..3545b090b 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -189,6 +189,14 @@ func SaveNewNameModel(ctx *context.Context) { log.Info("save model end.") } +func SaveLocalModel(ctx *context.Context) { + +} + +func UploadLocalModel(ctx *context.Context) { + +} + func SaveModel(ctx *context.Context) { if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { ctx.Error(403, ctx.Tr("repo.model_noright")) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index fd8b274e6..588a00c18 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1233,6 +1233,8 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/modelmanage", func() { m.Post("/create_model", repo.SaveModel) + m.Post("/upload_local_model", repo.UploadLocalModel) + m.Post("/create_local_model", repo.SaveLocalModel) m.Post("/create_model_convert", reqWechatBind, reqRepoModelManageWriter, repo.SaveModelConvert) m.Post("/create_new_model", repo.SaveNewNameModel) m.Delete("/delete_model", repo.DeleteModel) From 7efc877097a6299523972f8ace0fc9df656a9dbf Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 1 Nov 2022 17:16:28 +0800 Subject: [PATCH 040/187] =?UTF-8?q?=E5=A2=9E=E5=8A=A06.8=E7=9A=84=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/api/v1/api.go | 3 ++ routers/api/v1/repo/modelmanage.go | 41 ++++++++++++++++------- routers/repo/ai_model_manage.go | 52 +++++++++++++++++++++++------- 3 files changed, 72 insertions(+), 24 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 5ecbf3a5f..6e2c74b0b 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -993,6 +993,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_model_byId", repo.QueryModelById) m.Get("/query_model_for_predict", repo.QueryModelListForPredict) m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) + m.Get("/query_train_job", repo.QueryTrainJobList) + m.Get("/query_train_model", repo.QueryTrainModelList) + m.Get("/query_train_job_version", repo.QueryTrainJobVersionList) m.Post("/create_model_convert", repo.CreateModelConvert) m.Get("/show_model_convert_page") m.Get("/:id", repo.GetCloudbrainModelConvertTask) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index 253d51e00..2c1fd9f01 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -5,6 +5,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" routerRepo "code.gitea.io/gitea/routers/repo" ) @@ -47,22 +48,38 @@ func QueryModelListForPredict(ctx *context.APIContext) { routerRepo.QueryModelListForPredict(ctx.Context) } +func QueryTrainModelList(ctx *context.APIContext) { + result, err := routerRepo.QueryTrainModelFileById(ctx.Context) + if err != nil { + log.Info("query error." + err.Error()) + } + re := convertFileFormat(result) + ctx.JSON(http.StatusOK, re) +} + +func convertFileFormat(result []storage.FileInfo) []FileInfo { + re := make([]FileInfo, 0) + if result != nil { + for _, file := range result { + tmpFile := FileInfo{ + FileName: file.FileName, + ModTime: file.ModTime, + IsDir: file.IsDir, + Size: file.Size, + ParenDir: file.ParenDir, + UUID: file.UUID, + } + re = append(re, tmpFile) + } + } + return re +} + func QueryModelFileForPredict(ctx *context.APIContext) { log.Info("QueryModelFileForPredict by api.") id := ctx.Query("id") result := routerRepo.QueryModelFileByID(id) - re := make([]FileInfo, len(result)) - for _, file := range result { - tmpFile := FileInfo{ - FileName: file.FileName, - ModTime: file.ModTime, - IsDir: file.IsDir, - Size: file.Size, - ParenDir: file.ParenDir, - UUID: file.UUID, - } - re = append(re, tmpFile) - } + re := convertFileFormat(result) ctx.JSON(http.StatusOK, re) } diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index ca90ff432..99f032572 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -528,23 +528,33 @@ func QueryTrainJobList(ctx *context.Context) { } -func QueryTrainModelList(ctx *context.Context) { - log.Info("query train job list. start.") - jobName := ctx.Query("jobName") - taskType := ctx.QueryInt("type") - VersionName := ctx.Query("versionName") - if VersionName == "" { - VersionName = ctx.Query("VersionName") +func QueryTrainModelFileById(ctx *context.Context) ([]storage.FileInfo, error) { + JobID := ctx.Query("jobId") + VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) + if err == nil { + if count == 1 { + task := VersionListTasks[0] + jobName := task.JobName + taskType := task.Type + VersionName := task.VersionName + modelDbResult, err := getModelFromObjectSave(jobName, taskType, VersionName) + return modelDbResult, err + } } + log.Info("get TypeCloudBrainTwo TrainJobListModel failed:", err) + return nil, errors.New("Not found task.") +} + +func getModelFromObjectSave(jobName string, taskType int, VersionName string) ([]storage.FileInfo, error) { if taskType == models.TypeCloudBrainTwo { objectkey := path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionName) + "/" modelDbResult, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, objectkey) log.Info("bucket=" + setting.Bucket + " objectkey=" + objectkey) if err != nil { log.Info("get TypeCloudBrainTwo TrainJobListModel failed:", err) + return nil, err } else { - ctx.JSON(200, modelDbResult) - return + return modelDbResult, nil } } else if taskType == models.TypeCloudBrainOne { modelSrcPrefix := setting.CBCodePathPrefix + jobName + "/model/" @@ -552,12 +562,30 @@ func QueryTrainModelList(ctx *context.Context) { modelDbResult, err := storage.GetAllObjectByBucketAndPrefixMinio(bucketName, modelSrcPrefix) if err != nil { log.Info("get TypeCloudBrainOne TrainJobListModel failed:", err) + return nil, err } else { - ctx.JSON(200, modelDbResult) - return + return modelDbResult, nil } } - ctx.JSON(200, "") + return nil, errors.New("Not support.") +} + +func QueryTrainModelList(ctx *context.Context) { + log.Info("query train job list. start.") + jobName := ctx.Query("jobName") + taskType := ctx.QueryInt("type") + VersionName := ctx.Query("versionName") + if VersionName == "" { + VersionName = ctx.Query("VersionName") + } + modelDbResult, err := getModelFromObjectSave(jobName, taskType, VersionName) + if err != nil { + log.Info("get TypeCloudBrainTwo TrainJobListModel failed:", err) + ctx.JSON(200, "") + } else { + ctx.JSON(200, modelDbResult) + return + } } func DownloadSingleModelFile(ctx *context.Context) { From 7d310b184aae0f039d863c82579fcb067e18f69f Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 1 Nov 2022 17:54:14 +0800 Subject: [PATCH 041/187] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E5=8F=8A=E6=A8=A1=E5=9E=8B=E8=BD=AC=E6=8D=A2=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E5=86=99=E7=BB=9F=E4=B8=80=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 116 ++++++------- routers/api/v1/api.go | 12 ++ routers/api/v1/repo/modelmanage.go | 106 ++++++++++++ routers/repo/ai_model_convert.go | 61 ++++--- routers/repo/ai_model_manage.go | 162 ++++++++++++------ templates/repo/cloudbrain/inference/new.tmpl | 6 +- templates/repo/cloudbrain/trainjob/show.tmpl | 36 ++-- templates/repo/grampus/trainjob/show.tmpl | 41 ++--- .../repo/modelarts/inferencejob/new.tmpl | 6 +- templates/repo/modelarts/trainjob/show.tmpl | 45 ++--- templates/repo/modelmanage/convertIndex.tmpl | 74 ++++---- templates/repo/modelmanage/index.tmpl | 54 +++--- templates/repo/modelmanage/showinfo.tmpl | 140 ++++++++------- web_src/js/components/Model.vue | 156 ++++++++--------- web_src/js/features/cloudbrainShow.js | 8 +- 15 files changed, 609 insertions(+), 414 deletions(-) create mode 100644 routers/api/v1/repo/modelmanage.go diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index d9adda2dc..0d754b0ba 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -12,67 +12,67 @@ import ( ) type AiModelManage struct { - ID string `xorm:"pk"` - Name string `xorm:"INDEX NOT NULL"` - Version string `xorm:"NOT NULL"` - VersionCount int `xorm:"NOT NULL DEFAULT 0"` - New int `xorm:"NOT NULL"` - Type int `xorm:"NOT NULL"` - Size int64 `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Label string `xorm:"varchar(1000)"` - Path string `xorm:"varchar(400) NOT NULL"` - DownloadCount int `xorm:"NOT NULL DEFAULT 0"` - Engine int64 `xorm:"NOT NULL DEFAULT 0"` - Status int `xorm:"NOT NULL DEFAULT 0"` - StatusDesc string `xorm:"varchar(500)"` - Accuracy string `xorm:"varchar(1000)"` - AttachmentId string `xorm:"NULL"` - RepoId int64 `xorm:"INDEX NULL"` - CodeBranch string `xorm:"varchar(400) NULL"` - CodeCommitID string `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - UserName string - UserRelAvatarLink string - TrainTaskInfo string `xorm:"text NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` - IsCanOper bool - IsCanDelete bool + ID string `xorm:"pk" json:"id"` + Name string `xorm:"INDEX NOT NULL" json:"name"` + Version string `xorm:"NOT NULL" json:"version"` + VersionCount int `xorm:"NOT NULL DEFAULT 0" json:"versionCount"` + New int `xorm:"NOT NULL" json:"new"` + Type int `xorm:"NOT NULL" json:"type"` + Size int64 `xorm:"NOT NULL" json:"size"` + Description string `xorm:"varchar(2000)" json:"description"` + Label string `xorm:"varchar(1000)" json:"label"` + Path string `xorm:"varchar(400) NOT NULL" json:"path"` + DownloadCount int `xorm:"NOT NULL DEFAULT 0" json:"downloadCount"` + Engine int64 `xorm:"NOT NULL DEFAULT 0" json:"engine"` + Status int `xorm:"NOT NULL DEFAULT 0" json:"status"` + StatusDesc string `xorm:"varchar(500)" json:"statusDesc"` + Accuracy string `xorm:"varchar(1000)" json:"accuracy"` + AttachmentId string `xorm:"NULL" json:"attachmentId"` + RepoId int64 `xorm:"INDEX NULL" json:"repoId"` + CodeBranch string `xorm:"varchar(400) NULL" json:"codeBranch"` + CodeCommitID string `xorm:"NULL" json:"codeCommitID"` + UserId int64 `xorm:"NOT NULL" json:"userId"` + UserName string `json:"userName"` + UserRelAvatarLink string `json:"userRelAvatarLink"` + TrainTaskInfo string `xorm:"text NULL" json:"trainTaskInfo"` + CreatedUnix timeutil.TimeStamp `xorm:"created" json:"createdUnix"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated" json:"updatedUnix"` + IsCanOper bool `json:"isCanOper"` + IsCanDelete bool `json:"isCanDelete"` } type AiModelConvert struct { - ID string `xorm:"pk"` - Name string `xorm:"INDEX NOT NULL"` - Status string `xorm:"NULL"` - StatusResult string `xorm:"NULL"` - SrcEngine int `xorm:"NOT NULL DEFAULT 0"` - RepoId int64 `xorm:"INDEX NULL"` - ModelId string `xorm:"NOT NULL"` - ModelName string `xorm:"NULL"` - ModelVersion string `xorm:"NOT NULL"` - ModelPath string `xorm:"NULL"` - DestFormat int `xorm:"NOT NULL DEFAULT 0"` - NetOutputFormat int `xorm:"NULL"` - UserId int64 `xorm:"NOT NULL"` - CloudBrainTaskId string `xorm:"NULL"` - ModelArtsVersionId string `xorm:"NULL"` - ContainerID string - ContainerIp string - RunTime int64 `xorm:"NULL"` - TrainJobDuration string - InputShape string `xorm:"varchar(2000)"` - InputDataFormat string `xorm:"NOT NULL"` - Description string `xorm:"varchar(2000)"` - Path string `xorm:"varchar(400) NOT NULL"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` - UpdatedUnix timeutil.TimeStamp `xorm:"updated"` - StartTime timeutil.TimeStamp - EndTime timeutil.TimeStamp - UserName string - UserRelAvatarLink string - IsCanOper bool - IsCanDelete bool + ID string `xorm:"pk" json:"id"` + Name string `xorm:"INDEX NOT NULL" json:"name"` + Status string `xorm:"NULL" json:"status"` + StatusResult string `xorm:"NULL" json:"statusResult"` + SrcEngine int `xorm:"NOT NULL DEFAULT 0" json:"srcEngine"` + RepoId int64 `xorm:"INDEX NULL" json:"repoId"` + ModelId string `xorm:"NOT NULL" json:"modelId"` + ModelName string `xorm:"NULL" json:"modelName"` + ModelVersion string `xorm:"NOT NULL" json:"modelVersion"` + ModelPath string `xorm:"NULL" json:"modelPath"` + DestFormat int `xorm:"NOT NULL DEFAULT 0" json:"destFormat"` + NetOutputFormat int `xorm:"NULL" json:"netOutputFormat"` + UserId int64 `xorm:"NOT NULL" json:"userId"` + CloudBrainTaskId string `xorm:"NULL" json:"cloudBrainTaskId"` + ModelArtsVersionId string `xorm:"NULL" json:"modelArtsVersionId"` + ContainerID string `json:"containerID"` + ContainerIp string `json:"containerIp"` + RunTime int64 `xorm:"NULL" json:"runTime"` + TrainJobDuration string `json:"trainJobDuration"` + InputShape string `xorm:"varchar(2000)" json:"inputShape"` + InputDataFormat string `xorm:"NOT NULL" json:"inputDataFormat"` + Description string `xorm:"varchar(2000)" json:"description"` + Path string `xorm:"varchar(400) NOT NULL" json:"path"` + CreatedUnix timeutil.TimeStamp `xorm:"created" json:"createdUnix"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated" json:"updatedUnix"` + StartTime timeutil.TimeStamp `json:"startTime"` + EndTime timeutil.TimeStamp `json:"endTime"` + UserName string `json:"userName"` + UserRelAvatarLink string `json:"userRelAvatarLink"` + IsCanOper bool `json:"isCanOper"` + IsCanDelete bool `json:"isCanDelete"` } type AiModelQueryOptions struct { diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 69de79c10..87d5b37f0 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -953,6 +953,18 @@ func RegisterRoutes(m *macaron.Macaron) { }) }, reqRepoReader(models.UnitTypeCloudBrain)) m.Group("/modelmanage", func() { + m.Post("/create_new_model", repo.CreateNewModel) + m.Get("/show_model_api", repo.ShowModelManageApi) + m.Delete("/delete_model", repo.DeleteModel) + m.Get("/downloadall", repo.DownloadModel) + m.Get("/query_model_byId", repo.QueryModelById) + m.Get("/query_model_for_predict", repo.QueryModelListForPredict) + m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) + m.Get("/query_train_job", repo.QueryTrainJobList) + m.Get("/query_train_model", repo.QueryTrainModelList) + m.Get("/query_train_job_version", repo.QueryTrainJobVersionList) + m.Post("/create_model_convert", repo.CreateModelConvert) + m.Get("/show_model_convert_page") m.Get("/:id", repo.GetCloudbrainModelConvertTask) m.Get("/:id/log", repo.CloudbrainForModelConvertGetLog) m.Get("/:id/modelartlog", repo.TrainJobForModelConvertGetLog) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go new file mode 100644 index 000000000..2c1fd9f01 --- /dev/null +++ b/routers/api/v1/repo/modelmanage.go @@ -0,0 +1,106 @@ +package repo + +import ( + "net/http" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" + routerRepo "code.gitea.io/gitea/routers/repo" +) + +type FileInfo struct { + FileName string `json:"fileName"` + ModTime string `json:"modTime"` + IsDir bool `json:"isDir"` + Size int64 `json:"size"` + ParenDir string `json:"parenDir"` + UUID string `json:"uuid"` +} + +func CreateNewModel(ctx *context.APIContext) { + log.Info("CreateNewModel by api.") + routerRepo.SaveModel(ctx.Context) +} + +func ShowModelManageApi(ctx *context.APIContext) { + log.Info("ShowModelManageApi by api.") + routerRepo.ShowModelPageInfo(ctx.Context) +} + +func DeleteModel(ctx *context.APIContext) { + log.Info("DeleteModel by api.") + routerRepo.DeleteModel(ctx.Context) +} + +func DownloadModel(ctx *context.APIContext) { + log.Info("DownloadModel by api.") + routerRepo.DownloadMultiModelFile(ctx.Context) +} + +func QueryModelById(ctx *context.APIContext) { + log.Info("QueryModelById by api.") + routerRepo.QueryModelById(ctx.Context) +} + +func QueryModelListForPredict(ctx *context.APIContext) { + log.Info("QueryModelListForPredict by api.") + routerRepo.QueryModelListForPredict(ctx.Context) +} + +func QueryTrainModelList(ctx *context.APIContext) { + result, err := routerRepo.QueryTrainModelFileById(ctx.Context) + if err != nil { + log.Info("query error." + err.Error()) + } + re := convertFileFormat(result) + ctx.JSON(http.StatusOK, re) +} + +func convertFileFormat(result []storage.FileInfo) []FileInfo { + re := make([]FileInfo, 0) + if result != nil { + for _, file := range result { + tmpFile := FileInfo{ + FileName: file.FileName, + ModTime: file.ModTime, + IsDir: file.IsDir, + Size: file.Size, + ParenDir: file.ParenDir, + UUID: file.UUID, + } + re = append(re, tmpFile) + } + } + return re +} + +func QueryModelFileForPredict(ctx *context.APIContext) { + log.Info("QueryModelFileForPredict by api.") + id := ctx.Query("id") + result := routerRepo.QueryModelFileByID(id) + re := convertFileFormat(result) + ctx.JSON(http.StatusOK, re) +} + +func CreateModelConvert(ctx *context.APIContext) { + log.Info("CreateModelConvert by api.") + routerRepo.SaveModelConvert(ctx.Context) +} + +func ShowModelConvertPage(ctx *context.APIContext) { + log.Info("ShowModelConvertPage by api.") + modelResult, count, err := routerRepo.GetModelConvertPageData(ctx.Context) + if err == nil { + mapInterface := make(map[string]interface{}) + mapInterface["data"] = modelResult + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) + } else { + mapInterface := make(map[string]interface{}) + mapInterface["data"] = nil + mapInterface["count"] = 0 + ctx.JSON(http.StatusOK, mapInterface) + } + +} diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index bd6a01072..560ace8fd 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -74,27 +74,27 @@ func SaveModelConvert(ctx *context.Context) { log.Info("save model convert start.") if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.no_operate_right"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.no_operate_right"), }) return } name := ctx.Query("name") desc := ctx.Query("desc") modelId := ctx.Query("modelId") - modelPath := ctx.Query("ModelFile") - SrcEngine := ctx.QueryInt("SrcEngine") + modelPath := ctx.Query("modelFile") + SrcEngine := ctx.QueryInt("srcEngine") InputShape := ctx.Query("inputshape") InputDataFormat := ctx.Query("inputdataformat") - DestFormat := ctx.QueryInt("DestFormat") - NetOutputFormat := ctx.QueryInt("NetOutputFormat") + DestFormat := ctx.QueryInt("destFormat") + NetOutputFormat := ctx.QueryInt("netOutputFormat") task, err := models.QueryModelById(modelId) if err != nil { log.Error("no such model!", err.Error()) ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.model_not_exist"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.model_not_exist"), }) return } @@ -105,8 +105,8 @@ func SaveModelConvert(ctx *context.Context) { if convert.Name == name { log.Info("convert.Name=" + name + " convert.id=" + convert.ID) ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.create_error1"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.create_error1"), }) return } @@ -119,8 +119,8 @@ func SaveModelConvert(ctx *context.Context) { if isRunningTask(convert.Status) { log.Info("convert.Status=" + convert.Status + " convert.id=" + convert.ID) ctx.JSON(200, map[string]string{ - "result_code": "1", - "message": ctx.Tr("repo.modelconvert.manage.create_error2"), + "code": "1", + "msg": ctx.Tr("repo.modelconvert.manage.create_error2"), }) return } @@ -150,7 +150,7 @@ func SaveModelConvert(ctx *context.Context) { go goCreateTask(modelConvert, ctx, task) ctx.JSON(200, map[string]string{ - "result_code": "0", + "code": "0", }) } @@ -604,11 +604,11 @@ func StopModelConvert(ctx *context.Context) { } func ShowModelConvertInfo(ctx *context.Context) { - ctx.Data["ID"] = ctx.Query("ID") + ctx.Data["ID"] = ctx.Query("id") ctx.Data["isModelManage"] = true ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage) - job, err := models.QueryModelConvertById(ctx.Query("ID")) + job, err := models.QueryModelConvertById(ctx.Query("id")) if err == nil { if job.TrainJobDuration == "" { job.TrainJobDuration = "00:00:00" @@ -707,6 +707,26 @@ func ShowModelConvertPageInfo(ctx *context.Context) { ctx.NotFound(ctx.Req.URL.RequestURI(), nil) return } + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } + modelResult, count, err := GetModelConvertPageData(ctx) + if err == nil { + pager := context.NewPagination(int(count), page, pageSize, 5) + ctx.Data["Page"] = pager + ctx.Data["Tasks"] = modelResult + ctx.Data["MODEL_CONVERT_COUNT"] = count + } else { + ctx.ServerError("Query data error.", err) + } +} + +func GetModelConvertPageData(ctx *context.Context) ([]*models.AiModelConvert, int64, error) { page := ctx.QueryInt("page") if page <= 0 { page = 1 @@ -725,10 +745,8 @@ func ShowModelConvertPageInfo(ctx *context.Context) { }) if err != nil { log.Info("query db error." + err.Error()) - ctx.ServerError("Cloudbrain", err) - return + return nil, 0, err } - ctx.Data["MODEL_CONVERT_COUNT"] = count userIds := make([]int64, len(modelResult)) for i, model := range modelResult { model.IsCanOper = isOper(ctx, model.UserId) @@ -743,10 +761,7 @@ func ShowModelConvertPageInfo(ctx *context.Context) { model.UserRelAvatarLink = value.RelAvatarLink() } } - pager := context.NewPagination(int(count), page, pageSize, 5) - ctx.Data["Page"] = pager - ctx.Data["Tasks"] = modelResult - + return modelResult, count, nil } func ModelConvertDownloadModel(ctx *context.Context) { @@ -757,7 +772,7 @@ func ModelConvertDownloadModel(ctx *context.Context) { ctx.ServerError("Not found task.", err) return } - AllDownload := ctx.QueryBool("AllDownload") + AllDownload := ctx.QueryBool("allDownload") if AllDownload { if job.IsGpuTrainTask() { path := setting.CBCodePathPrefix + job.ID + "/model/" diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 3545b090b..a82930909 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -17,7 +17,6 @@ import ( "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/services/cloudbrain/resource" uuid "github.com/satori/go.uuid" ) @@ -34,13 +33,13 @@ const ( STATUS_ERROR = 2 ) -func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) error { +func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, engine int, ctx *context.Context) (string, error) { aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) if err != nil { aiTask, err = models.GetRepoCloudBrainByJobID(ctx.Repo.Repository.ID, jobId) if err != nil { log.Info("query task error." + err.Error()) - return err + return "", err } else { log.Info("query gpu train task.") } @@ -56,7 +55,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio if len(aimodels) > 0 { for _, model := range aimodels { if model.Version == version { - return errors.New(ctx.Tr("repo.model.manage.create_error")) + return "", errors.New(ctx.Tr("repo.model.manage.create_error")) } if model.New == MODEL_LATEST { lastNewModelId = model.ID @@ -111,7 +110,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio err = models.SaveModelToDb(model) if err != nil { - return err + return "", err } if len(lastNewModelId) > 0 { //udpate status and version count @@ -134,7 +133,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio log.Info("save model end.") notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, id, name, models.ActionCreateNewModelTask) - return nil + return id, nil } func asyncToCopyModel(aiTask *models.Cloudbrain, id string, modelSelectedFile string) { @@ -173,7 +172,7 @@ func SaveNewNameModel(ctx *context.Context) { ctx.Error(403, ctx.Tr("repo.model_noright")) return } - name := ctx.Query("Name") + name := ctx.Query("name") if name == "" { ctx.Error(500, fmt.Sprintf("name or version is null.")) return @@ -203,38 +202,42 @@ func SaveModel(ctx *context.Context) { return } log.Info("save model start.") - JobId := ctx.Query("JobId") - VersionName := ctx.Query("VersionName") - name := ctx.Query("Name") - version := ctx.Query("Version") - label := ctx.Query("Label") - description := ctx.Query("Description") - engine := ctx.QueryInt("Engine") + JobId := ctx.Query("jobId") + VersionName := ctx.Query("versionName") + name := ctx.Query("name") + version := ctx.Query("version") + label := ctx.Query("label") + description := ctx.Query("description") + engine := ctx.QueryInt("engine") modelSelectedFile := ctx.Query("modelSelectedFile") log.Info("engine=" + fmt.Sprint(engine) + " modelSelectedFile=" + modelSelectedFile) - + re := map[string]string{ + "code": "-1", + } if JobId == "" || VersionName == "" { - ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) + re["msg"] = "JobId or VersionName is null." + ctx.JSON(200, re) return } if modelSelectedFile == "" { - ctx.Error(500, fmt.Sprintf("Not selected model file.")) + re["msg"] = "Not selected model file." + ctx.JSON(200, re) return } - if name == "" || version == "" { - ctx.Error(500, fmt.Sprintf("name or version is null.")) + re["msg"] = "name or version is null." + ctx.JSON(200, re) return } - - err := saveModelByParameters(JobId, VersionName, name, version, label, description, engine, ctx) - + id, err := saveModelByParameters(JobId, VersionName, name, version, label, description, engine, ctx) if err != nil { log.Info("save model error." + err.Error()) - ctx.Error(500, fmt.Sprintf("save model error. %v", err)) - return + re["msg"] = err.Error() + } else { + re["code"] = "0" + re["id"] = id } - ctx.Status(200) + ctx.JSON(200, re) log.Info("save model end.") } @@ -299,13 +302,17 @@ func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir func DeleteModel(ctx *context.Context) { log.Info("delete model start.") - id := ctx.Query("ID") + id := ctx.Query("id") err := deleteModelByID(ctx, id) if err != nil { - ctx.JSON(500, err.Error()) + re := map[string]string{ + "code": "-1", + } + re["msg"] = err.Error() + ctx.JSON(200, re) } else { ctx.JSON(200, map[string]string{ - "result_code": "0", + "code": "0", }) } } @@ -362,7 +369,7 @@ func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, in func DownloadMultiModelFile(ctx *context.Context) { log.Info("DownloadMultiModelFile start.") - id := ctx.Query("ID") + id := ctx.Query("id") log.Info("id=" + id) task, err := models.QueryModelById(id) if err != nil { @@ -495,7 +502,10 @@ func downloadFromCloudBrainTwo(path string, task *models.AiModelManage, ctx *con func QueryTrainJobVersionList(ctx *context.Context) { log.Info("query train job version list. start.") - JobID := ctx.Query("JobID") + JobID := ctx.Query("jobId") + if JobID == "" { + JobID = ctx.Query("JobId") + } VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) @@ -523,20 +533,33 @@ func QueryTrainJobList(ctx *context.Context) { } -func QueryTrainModelList(ctx *context.Context) { - log.Info("query train job list. start.") - jobName := ctx.Query("jobName") - taskType := ctx.QueryInt("type") - VersionName := ctx.Query("VersionName") +func QueryTrainModelFileById(ctx *context.Context) ([]storage.FileInfo, error) { + JobID := ctx.Query("jobId") + VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) + if err == nil { + if count == 1 { + task := VersionListTasks[0] + jobName := task.JobName + taskType := task.Type + VersionName := task.VersionName + modelDbResult, err := getModelFromObjectSave(jobName, taskType, VersionName) + return modelDbResult, err + } + } + log.Info("get TypeCloudBrainTwo TrainJobListModel failed:", err) + return nil, errors.New("Not found task.") +} + +func getModelFromObjectSave(jobName string, taskType int, VersionName string) ([]storage.FileInfo, error) { if taskType == models.TypeCloudBrainTwo { objectkey := path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionName) + "/" modelDbResult, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, objectkey) log.Info("bucket=" + setting.Bucket + " objectkey=" + objectkey) if err != nil { log.Info("get TypeCloudBrainTwo TrainJobListModel failed:", err) + return nil, err } else { - ctx.JSON(200, modelDbResult) - return + return modelDbResult, nil } } else if taskType == models.TypeCloudBrainOne { modelSrcPrefix := setting.CBCodePathPrefix + jobName + "/model/" @@ -544,12 +567,30 @@ func QueryTrainModelList(ctx *context.Context) { modelDbResult, err := storage.GetAllObjectByBucketAndPrefixMinio(bucketName, modelSrcPrefix) if err != nil { log.Info("get TypeCloudBrainOne TrainJobListModel failed:", err) + return nil, err } else { - ctx.JSON(200, modelDbResult) - return + return modelDbResult, nil } } - ctx.JSON(200, "") + return nil, errors.New("Not support.") +} + +func QueryTrainModelList(ctx *context.Context) { + log.Info("query train job list. start.") + jobName := ctx.Query("jobName") + taskType := ctx.QueryInt("type") + VersionName := ctx.Query("versionName") + if VersionName == "" { + VersionName = ctx.Query("VersionName") + } + modelDbResult, err := getModelFromObjectSave(jobName, taskType, VersionName) + if err != nil { + log.Info("get TypeCloudBrainTwo TrainJobListModel failed:", err) + ctx.JSON(200, "") + } else { + ctx.JSON(200, modelDbResult) + return + } } func DownloadSingleModelFile(ctx *context.Context) { @@ -620,7 +661,7 @@ func DownloadSingleModelFile(ctx *context.Context) { } func ShowModelInfo(ctx *context.Context) { - ctx.Data["ID"] = ctx.Query("ID") + ctx.Data["ID"] = ctx.Query("id") ctx.Data["name"] = ctx.Query("name") ctx.Data["isModelManage"] = true ctx.Data["ModelManageAccess"] = ctx.Repo.CanWrite(models.UnitTypeModelManage) @@ -628,6 +669,19 @@ func ShowModelInfo(ctx *context.Context) { ctx.HTML(200, tplModelInfo) } +func QueryModelById(ctx *context.Context) { + id := ctx.Query("id") + model, err := models.QueryModelById(id) + if err == nil { + model.IsCanOper = isOper(ctx, model.UserId) + model.IsCanDelete = isCanDelete(ctx, model.UserId) + removeIpInfo(model) + ctx.JSON(http.StatusOK, model) + } else { + ctx.JSON(http.StatusNotFound, nil) + } +} + func ShowSingleModel(ctx *context.Context) { name := ctx.Query("name") @@ -836,8 +890,8 @@ func ModifyModel(id string, description string) error { func ModifyModelInfo(ctx *context.Context) { log.Info("modify model start.") - id := ctx.Query("ID") - description := ctx.Query("Description") + id := ctx.Query("id") + description := ctx.Query("description") task, err := models.QueryModelById(id) if err != nil { @@ -902,28 +956,36 @@ func QueryModelListForPredict(ctx *context.Context) { } func QueryModelFileForPredict(ctx *context.Context) { - id := ctx.Query("ID") + id := ctx.Query("id") + if id == "" { + id = ctx.Query("ID") + } + ctx.JSON(http.StatusOK, QueryModelFileByID(id)) +} + +func QueryModelFileByID(id string) []storage.FileInfo { model, err := models.QueryModelById(id) if err == nil { if model.Type == models.TypeCloudBrainTwo { prefix := model.Path[len(setting.Bucket)+1:] fileinfos, _ := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, prefix) - ctx.JSON(http.StatusOK, fileinfos) + return fileinfos } else if model.Type == models.TypeCloudBrainOne { prefix := model.Path[len(setting.Attachment.Minio.Bucket)+1:] fileinfos, _ := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, prefix) - ctx.JSON(http.StatusOK, fileinfos) + return fileinfos } } else { log.Error("no such model!", err.Error()) - ctx.ServerError("no such model:", err) - return } - + return nil } func QueryOneLevelModelFile(ctx *context.Context) { - id := ctx.Query("ID") + id := ctx.Query("id") + if id == "" { + id = ctx.Query("ID") + } parentDir := ctx.Query("parentDir") model, err := models.QueryModelById(id) if err != nil { diff --git a/templates/repo/cloudbrain/inference/new.tmpl b/templates/repo/cloudbrain/inference/new.tmpl index df737f995..f2b389393 100644 --- a/templates/repo/cloudbrain/inference/new.tmpl +++ b/templates/repo/cloudbrain/inference/new.tmpl @@ -331,9 +331,7 @@ $('#model_name_version').empty() let html = '' nameMap[value].forEach(element => { - let {TrainTaskInfo} = element - TrainTaskInfo = JSON.parse(TrainTaskInfo) - html += `
${element.Version}
` + html += `
${element.version}
` }); $('#model_name_version').append(html) $("#select_model_version").removeClass("loading") @@ -387,7 +385,7 @@ } function loadCheckpointList(value){ return new Promise((resolve,reject)=>{ - $.get(`${RepoLink}/modelmanage/query_modelfile_for_predict`,{ID:value}, (data) => { + $.get(`${RepoLink}/modelmanage/query_modelfile_for_predict`,{id:value}, (data) => { resolve(data) }) }) diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index 44ac57319..2e4d5689b 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -633,24 +633,24 @@
- - + +
-
- +
@@ -364,9 +364,9 @@ $("#task_name").removeClass("error") } - data['desc']= $('#Description').val() - data['modelId'] = $('#ModelVersion').val() - data['SrcEngine'] = $('#SrcEngine').val(); + data['desc']= $('#description').val() + data['modelId'] = $('#modelVersion').val() + data['srcEngine'] = $('#srcEngine').val(); data['inputshape']= $('#inputshape').val(); if(inputshapeNotValid(data['inputshape'])){ @@ -379,10 +379,10 @@ } data['inputdataformat']= $('#inputdataformat').val(); - data['DestFormat'] = $('#DestFormat').val(); - data['NetOutputFormat']= $('#NetOutputFormat').val(); - data['ModelFile'] = $('#ModelFile').val(); - if(data['ModelFile']==""){ + data['destFormat'] = $('#destFormat').val(); + data['netOutputFormat']= $('#netOutputFormat').val(); + data['modelFile'] = $('#modelFile').val(); + if(data['modelFile']==""){ $('.ui.error.message').text("{{.i18n.Tr "repo.modelconvert.modelfileempty"}}") $('.ui.error.message').css('display','block') $("#ModelFile_Div").addClass("error") @@ -392,11 +392,11 @@ } $.post(`${repolink}/modelmanage/create_model_convert`,data,(result) => { console.log("result=" + result); - if(result.result_code ==0){ + if(result.code ==0){ $('.ui.modal.second').modal('hide'); window.location.reload(); }else{ - $('.ui.error.message').text(result.message) + $('.ui.error.message').text(result.msg) $('.ui.error.message').css('display','block') } }) @@ -456,7 +456,7 @@ $('#choice_version').dropdown({ onChange:function(value){ console.log("model version:" + value); - $('#choice_version input[name="ModelVersion"]').val(value) + $('#choice_version input[name="modelVersion"]').val(value) loadModelFile(value); } }) @@ -464,26 +464,26 @@ $('#choice_file').dropdown({ onChange:function(value){ console.log("model file:" + value); - $('#choice_file input[name="ModelFile"]').val(value) + $('#choice_file input[name="modelFile"]').val(value) } }) }) function srcEngineChanged(){ - var ele = window.document.getElementById("SrcEngine"); + var ele = window.document.getElementById("srcEngine"); var index=ele.selectedIndex; var options=ele.options; var option = options[index]; - console.log("SrcEngine value=" + option); + console.log("srcEngine value=" + option); let destFormatHtml = ""; let netOutputFormatHtml = ""; if(option==null || option =="undefined" || option.value == 0){ destFormatHtml += "" netOutputFormatHtml += ""; } - $('#DestFormat').html(destFormatHtml); - $('#NetOutputFormat').html(netOutputFormatHtml); + $('#destFormat').html(destFormatHtml); + $('#netOutputFormat').html(netOutputFormatHtml); } function loadModelList(){ @@ -509,7 +509,7 @@ if(modelId ==null || modelId ==""){ console.log("modelId is null"); }else{ - $.get(`${repolink}/modelmanage/query_modelfile_for_predict?ID=${modelId}`, (data) => { + $.get(`${repolink}/modelmanage/query_modelfile_for_predict?id=${modelId}`, (data) => { const n_length = data.length let file_html='' let firstFileName ='' @@ -526,7 +526,7 @@ } $("#model-file").append(file_html) $('#choice_file .default.text').text(firstFileName) - $('#choice_file input[name="ModelFile"]').val(firstFileName) + $('#choice_file input[name="modelFile"]').val(firstFileName) }) } @@ -550,19 +550,19 @@ n_length = versionList.length let train_html='' for (let i=0;i${versionList[i].Version}
` + train_html += `
${versionList[i].version}
` train_html += '
' } $("#model-version").append(train_html) - $('#choice_version .default.text').text(versionList[0].Version) - $('#choice_version input[name="ModelVersion"]').val(versionList[0].ID) - loadModelFile(versionList[0].ID); + $('#choice_version .default.text').text(versionList[0].version) + $('#choice_version input[name="modelVersion"]').val(versionList[0].id) + loadModelFile(versionList[0].id); } setEngineValue(value); } function setEngineValue(value){ - $('#SrcEngine').dropdown('clear'); + $('#srcEngine').dropdown('clear'); console.log("setEngineValue value=" + value); let html = "" html +=""; @@ -580,13 +580,13 @@ let nameMap = modelData.nameMap let versionList = nameMap[modelName] if(versionList != null && versionList.length >0){ - if(versionList[0].Engine == engineOption){ + if(versionList[0].engine == engineOption){ return "selected=\"selected\""; }else{ - if((versionList[0].Engine==122 || versionList[0].Engine==37) && engineOption==2){ + if((versionList[0].engine==122 || versionList[0].engine==37) && engineOption==2){ return "selected=\"selected\""; } - if((versionList[0].Engine==121 || versionList[0].Engine==38) && engineOption==1){ + if((versionList[0].engine==121 || versionList[0].engine==38) && engineOption==1){ return "selected=\"selected\""; } } diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 062e1908f..2699bed01 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -138,20 +138,20 @@
- +
- +   - {{template "base/footer" .}} + + + diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 2cced090f..7f21e4053 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -77,7 +77,7 @@ 导入本地模型 导入线上模型 + href="{{.RepoLink}}/modelmanage/create_online_model_tmpl">导入线上模型
{{if eq $.MODEL_COUNT 0}} diff --git a/web_src/vuepages/pages/modelmanage/common/modelmanage-common-detail.vue b/web_src/vuepages/pages/modelmanage/common/modelmanage-common-detail.vue index 997722807..1aff6d33f 100644 --- a/web_src/vuepages/pages/modelmanage/common/modelmanage-common-detail.vue +++ b/web_src/vuepages/pages/modelmanage/common/modelmanage-common-detail.vue @@ -21,8 +21,8 @@
可用集群:
-
- {{ state.type == 0 ? 'CPU/GPU' : 'NPU' }} +
+ {{ state.typeStr }}
@@ -32,10 +32,22 @@
{{ state.modelSize }}
-
+
描述:
-
-
{{ state.description }}
+
+
+ {{ state.description }} + +
+
+ + + +
@@ -52,10 +64,22 @@
{{ state.createTime }}
-
+
标签:
-
-
{{ state.label }}
+
+
+ {{ state.label }} + +
+
+ + + +
@@ -66,13 +90,13 @@
训练任务:
-
{{ state.displayJobName }}
+
代码分支:
-
{{ state.branchName }}
+
@@ -192,6 +216,10 @@ export default { label: '', description: '', }, + editDescr: '', + isEidtDescr: false, + editLabel: '', + isEidtLabel: false, engineList: MODEL_ENGINES, curVersion: '', modelList: [], @@ -205,6 +233,7 @@ export default { this.modelType = data.modelType; this.canOperate = data.isCanOper; this.state.type = data.type; + this.state.typeStr = data.type == 0 ? 'CPU/GPU' : data.type == 1 ? 'NPU' : ''; this.state.id = data.id; this.state.name = data.name; this.state.version = data.version; @@ -212,7 +241,9 @@ export default { this.state.engineName = getListValueWithKey(MODEL_ENGINES, data.engine.toString()); this.state.modelSize = transFileSize(data.size); this.state.label = data.label || '--'; + this.state._label = data.label; this.state.description = data.description || '--'; + this.state._description = data.description; this.state.createTime = formatDate(new Date(data.createdUnix * 1000), 'yyyy-MM-dd HH:mm:ss'); const trainTaskInfo = data.trainTaskInfo ? JSON.parse(data.trainTaskInfo) : ''; @@ -228,14 +259,27 @@ export default { if (trainTaskInfo) { const parameters = trainTaskInfo.Parameters ? JSON.parse(trainTaskInfo.Parameters).parameter : []; const parametersStr = parameters.map((item) => { return item.label + '=' + item.value }).join('; '); + const taskType = trainTaskInfo.Type; + let taskUrl = location.href.split('modelmanage')[0]; + if (taskType == 0) { + taskUrl = taskUrl + 'cloudbrain/train-job/' + trainTaskInfo.JobID; + } else if (taskType == 1) { + taskUrl = taskUrl + 'modelarts/train-job/' + trainTaskInfo.JobID; + } else if (taskType == 2) { + taskUrl = taskUrl + 'grampus/train-job/' + trainTaskInfo.JobID; + } + const versionName = trainTaskInfo.VersionName; + const versionHtml = versionName ? `${versionName}` : ''; + const codeCommitID = data.codeCommitID; + const codeCommitIDHtml = codeCommitID ? `${codeCommitID}` : ''; Object.assign(this.state, { - displayJobName: trainTaskInfo.DisplayJobName, - branchName: trainTaskInfo.BranchName, + displayJobName: `${trainTaskInfo.DisplayJobName}${versionHtml}`, + branchName: `${trainTaskInfo.BranchName}${codeCommitIDHtml}`, bootFile: trainTaskInfo.BootFile, datasetName: trainTaskInfo.DatasetName, parameters: parametersStr || '--', workServerNumber: trainTaskInfo.WorkServerNumber || '--', - specStr: trainTaskInfo.Spec ? trainTaskInfo.Spec.toString() : '--' + specStr: trainTaskInfo.FlavorName || '--', }); } this.curVersion = version; @@ -256,6 +300,43 @@ export default { goUploadPage() { window.location.href = `${this.repo}/modelmanage/create_local_model_tmpl2?type=0&name=${this.state.name}&id=${this.state.id}`; }, + submitEidt(type) { + const obj = { + repo: this.repo, + type: this.state.type, + id: this.state.id, + name: this.state.name, + version: this.state.version, + engine: this.state.engine, + label: type == 'label' ? this.editLabel : this.state.label, + description: type == 'descr' ? this.editDescr : this.state.description, + }; + modifyModel(obj).then(res => { + res = res.data; + if (res && res.code == '0') { + if (type == 'label') { + this.state.label = this.editLabel; + this.state._label = this.editLabel; + this.isEidtLabel = false; + } else if (type == 'descr') { + this.state.description = this.editDescr; + this.state._description = this.editDescr; + this.isEidtDescr = false; + } + } else { + this.$message({ + type: 'error', + message: '信息修改失败', + }); + } + }).catch(err => { + console.log(err); + this.$message({ + type: 'error', + message: '信息修改失败', + }); + }); + }, deleteFile(file) { this.$confirm('请确认是否删除当前模型文件?', this.$t('tips'), { confirmButtonText: this.$t('confirm1'), @@ -350,8 +431,12 @@ export default { .row { display: flex; height: 32px; - align-items: center; margin-bottom: 4px; + align-items: center; + + &.edit-row { + height: unset; + } .tit { width: 160px; @@ -365,14 +450,30 @@ export default { position: relative; height: 20px; + &.edit-val { + height: unset; + } + .txt-wrap { position: absolute; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; width: 100%; + + /deep/.append-txt { + margin-left: 6px; + background-color: gainsboro; + padding: 2px; + border-radius: 2px; + font-size: 12px; + } } } + + .txt-edit { + padding-right: 20px; + } } } } @@ -466,9 +567,13 @@ export default { } } -/deep/ .el-input__inner { - font-weight: 600; +.el-select { + /deep/ .el-input__inner { + font-weight: 600; + } +} +/deep/ .el-input__inner { &:focus { border-color: #85b7d9; } From 208a99809fb830d1a29e0074bd713623f3c313c7 Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 9 Nov 2022 15:36:01 +0800 Subject: [PATCH 118/187] Revert "#3157" This reverts commit 3aca1dc0749ccbe1af32063a2a621fe721f6bb01. --- routers/api/v1/repo/cloudbrain.go | 23 ++++------------------- routers/api/v1/repo/modelarts.go | 22 +++++----------------- routers/repo/grampus.go | 12 ++---------- 3 files changed, 11 insertions(+), 46 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 9da1251e4..7022dc011 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -607,24 +607,6 @@ func CloudbrainGetLog(ctx *context.APIContext) { lines := ctx.QueryInt("lines") baseLine := ctx.Query("base_line") order := ctx.Query("order") - - //Logs can only be downloaded if the file exists - //and the current user is an administrator or the creator of the task - if !job.IsUserHasRight(ctx.User) { - re := map[string]interface{}{ - "JobID": ID, - "LogFileName": "", - "StartLine": 0, - "EndLine": 0, - "Content": "", - "Lines": 0, - "CanLogDownload": false, - "StartTime": job.StartTime, - } - ctx.JSON(http.StatusOK, re) - return - } - var result map[string]interface{} resultPath := "/model" if job.JobType == string(models.JobTypeInference) || job.JobType == string(models.JobTypeModelSafety) { @@ -668,7 +650,10 @@ func CloudbrainGetLog(ctx *context.APIContext) { content = content + ctx.Data["existStr"].(string) } logFileName := result["FileName"] - canLogDownload := logFileName != nil && logFileName != "" + + //Logs can only be downloaded if the file exists + //and the current user is an administrator or the creator of the task + canLogDownload := logFileName != nil && logFileName != "" && job.IsUserHasRight(ctx.User) re := map[string]interface{}{ "JobID": ID, diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 00cdea0ef..16e4997a3 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -274,21 +274,6 @@ func TrainJobGetLog(ctx *context.APIContext) { log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) return } - - if !task.IsUserHasRight(ctx.User) { - ctx.JSON(http.StatusOK, map[string]interface{}{ - "JobID": jobID, - "LogFileName": "", - "StartLine": 0, - "EndLine": 0, - "Content": "", - "Lines": 0, - "CanLogDownload": false, - "StartTime": task.StartTime, - }) - return - } - resultLogFile, result, err := trainJobGetLogContent(jobID, task.VersionID, baseLine, order, lines_int) if err != nil { log.Error("trainJobGetLog(%s) failed:%v", jobID, err.Error()) @@ -305,12 +290,15 @@ func TrainJobGetLog(ctx *context.APIContext) { "EndLine": result.EndLine, "Content": result.Content, "Lines": result.Lines, - "CanLogDownload": canLogDownload(task), + "CanLogDownload": canLogDownload(ctx.User, task), "StartTime": task.StartTime, }) } -func canLogDownload(task *models.Cloudbrain) bool { +func canLogDownload(user *models.User, task *models.Cloudbrain) bool { + if task == nil || !task.IsUserHasRight(user) { + return false + } prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, modelarts.LogPath, task.VersionName), "/") + "/job" _, err := storage.GetObsLogFileName(prefix) if err != nil { diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index a2a021a43..e16d39a00 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -935,25 +935,17 @@ func GrampusGetLog(ctx *context.Context) { return } - if !job.IsUserHasRight(ctx.User) { - ctx.JSON(http.StatusOK, map[string]interface{}{ - "JobName": job.JobName, - "Content": "", - "CanLogDownload": false, - }) - return - } - content, err := grampus.GetTrainJobLog(job.JobID) if err != nil { log.Error("GetTrainJobLog failed: %v", err, ctx.Data["MsgID"]) ctx.ServerError(err.Error(), err) return } + canLogDownload := err == nil && job.IsUserHasRight(ctx.User) ctx.JSON(http.StatusOK, map[string]interface{}{ "JobName": job.JobName, "Content": content, - "CanLogDownload": true, + "CanLogDownload": canLogDownload, }) return From ce24e32f900caba3844bbe2fc37bdf21dc4fc44a Mon Sep 17 00:00:00 2001 From: chenyifan01 Date: Wed, 9 Nov 2022 17:07:11 +0800 Subject: [PATCH 119/187] #3157 fix bug --- routers/repo/grampus.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index e16d39a00..aa3b7ef78 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -938,7 +938,11 @@ func GrampusGetLog(ctx *context.Context) { content, err := grampus.GetTrainJobLog(job.JobID) if err != nil { log.Error("GetTrainJobLog failed: %v", err, ctx.Data["MsgID"]) - ctx.ServerError(err.Error(), err) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobName": job.JobName, + "Content": "", + "CanLogDownload": false, + }) return } canLogDownload := err == nil && job.IsUserHasRight(ctx.User) From bedb6e63eaf32a527aa7ad2f37820c82a6d7c427 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 10 Nov 2022 10:52:42 +0800 Subject: [PATCH 120/187] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20'services/cloudbra?= =?UTF-8?q?in/cloudbrainTask/sync=5Fstatus.go'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/cloudbrainTask/sync_status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/cloudbrain/cloudbrainTask/sync_status.go b/services/cloudbrain/cloudbrainTask/sync_status.go index 7153a7ec0..67dc4d3b7 100644 --- a/services/cloudbrain/cloudbrainTask/sync_status.go +++ b/services/cloudbrain/cloudbrainTask/sync_status.go @@ -14,7 +14,7 @@ import ( var noteBookOKMap = make(map[int64]int, 20) //if a task notebook url can get two times, the notebook can browser. -const successfulCount = 2 +const successfulCount = 3 func SyncCloudBrainOneStatus(task *models.Cloudbrain) (*models.Cloudbrain, error) { jobResult, err := cloudbrain.GetJob(task.JobID) From 9a07c96e53580f5d3e0b1611c0f33cdbdc932804 Mon Sep 17 00:00:00 2001 From: chenshihai Date: Thu, 10 Nov 2022 13:08:22 +0800 Subject: [PATCH 121/187] update --- options/locale/locale_en-US.ini | 4 +- options/locale/locale_zh-CN.ini | 4 +- templates/repo/modelmanage/create_online.tmpl | 24 ++- templates/repo/modelmanage/index.tmpl | 7 +- web_src/js/components/Model.vue | 4 +- web_src/js/components/basic/editDialog.vue | 2 +- web_src/js/features/i18nVue.js | 6 + web_src/js/index.js | 12 +- web_src/vuepages/langs/config/en-US.js | 60 +++++++ web_src/vuepages/langs/config/zh-CN.js | 60 +++++++ .../common/modelmanage-common-detail.vue | 90 ++++++----- .../local/modelmanage-local-create-1.vue | 55 ++++--- .../local/modelmanage-local-create-2.vue | 152 ++++++++++-------- 13 files changed, 331 insertions(+), 149 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 955690800..7eac45622 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1266,11 +1266,13 @@ model.manage.model_accuracy = Model Accuracy model.convert=Model Transformation model.list=Model List model.manage.create_new_convert_task=Create Model Transformation Task - +model.manage.import_local_model=Import Local Model +model.manage.import_online_model=Import Online Model model.manage.notcreatemodel=No model has been created model.manage.init1=Code version: You have not initialized the code repository, please model.manage.init2=initialized first ; model.manage.createtrainjob_tip=Training task: you haven't created a training task, please create it first +model.manage.createmodel_tip=You can import local model or online model. Import online model should create model.manage.createtrainjob=Training task. model.manage.delete=Delete Model model.manage.delete_confirm=Are you sure to delete this model? Once this model is deleted, it cannot be restored. diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 63ff87345..afcc50436 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1281,11 +1281,13 @@ model.manage.model_accuracy = 模型精度 model.convert=模型转换任务 model.list=模型列表 model.manage.create_new_convert_task=创建模型转换任务 - +model.manage.import_local_model=导入本地模型 +model.manage.import_online_model=导入线上模型 model.manage.notcreatemodel=未创建过模型 model.manage.init1=代码版本:您还没有初始化代码仓库,请先 model.manage.init2=创建代码版本; model.manage.createtrainjob_tip=训练任务:您还没创建过训练任务,请先创建 +model.manage.createmodel_tip=您可以导入本地模型或者导入线上模型。导入线上模型需先 model.manage.createtrainjob=训练任务。 model.manage.delete=删除模型 model.manage.delete_confirm=你确认删除该模型么?此模型一旦删除不可恢复。 diff --git a/templates/repo/modelmanage/create_online.tmpl b/templates/repo/modelmanage/create_online.tmpl index db3ca6cf9..f3334a98a 100644 --- a/templates/repo/modelmanage/create_online.tmpl +++ b/templates/repo/modelmanage/create_online.tmpl @@ -56,7 +56,7 @@
-

+

{{.i18n.Tr "repo.model.manage.import_online_model"}}

@@ -95,7 +95,7 @@
-
+ -
-
- -
+
-
- +
+ +
@@ -253,8 +251,7 @@ $('.ui.modal.second') .modal({ centered: false, - onShow: function () { - $('#model_header').text({{.i18n.Tr "repo.model.manage.import_new_model"}}) + onShow: function () { $('input[name="version"]').addClass('model_disabled') $('.ui.dimmer').css({ "background-color": "rgb(136, 136, 136,0.7)" }) $("#job-name").empty() @@ -277,7 +274,6 @@ .modal('show') } */ - $('#model_header').text({{.i18n.Tr "repo.model.manage.import_new_model"}}) $('input[name="version"]').addClass('model_disabled') $('.ui.dimmer').css({ "background-color": "rgb(136, 136, 136,0.7)" }) $("#job-name").empty() diff --git a/templates/repo/modelmanage/index.tmpl b/templates/repo/modelmanage/index.tmpl index 7f21e4053..1befa48ee 100644 --- a/templates/repo/modelmanage/index.tmpl +++ b/templates/repo/modelmanage/index.tmpl @@ -75,9 +75,9 @@
导入本地模型 + href="{{.RepoLink}}/modelmanage/create_local_model_tmpl1">{{$.i18n.Tr "repo.model.manage.import_local_model"}} 导入线上模型 + href="{{.RepoLink}}/modelmanage/create_online_model_tmpl">{{$.i18n.Tr "repo.model.manage.import_online_model"}}
{{if eq $.MODEL_COUNT 0}} @@ -85,6 +85,7 @@
{{$.i18n.Tr "repo.model.manage.notcreatemodel"}}
+ +
{{$.i18n.Tr "repo.model.manage.createmodel_tip"}} {{$.i18n.Tr "repo.model.manage.createtrainjob"}}
{{$.i18n.Tr "repo.platform_instructions1"}} {{$.i18n.Tr "repo.platform_instructions2"}} {{$.i18n.Tr "repo.platform_instructions3"}}
diff --git a/web_src/js/components/Model.vue b/web_src/js/components/Model.vue index 62fca9b91..2a179878b 100644 --- a/web_src/js/components/Model.vue +++ b/web_src/js/components/Model.vue @@ -23,7 +23,7 @@
- {{ scope.row.modelType == '1' ? '本地' : '线上' }} + {{ scope.row.modelType == '1' ? i18n.local : i18n.online }} {{ '修改' }}{{ i18n.modify }} - +
diff --git a/web_src/js/features/i18nVue.js b/web_src/js/features/i18nVue.js index f6d5a6411..2983b3dc1 100644 --- a/web_src/js/features/i18nVue.js +++ b/web_src/js/features/i18nVue.js @@ -105,6 +105,9 @@ export const i18nVue = { file_sync_fail:"文件同步失败", no_file_to_download:"没有文件可以下载", task_not_finished:"任务还未结束,稍后再来看看", + local:"本地", + online:"线上", + modify:"修改", }, US: { computer_vision: "computer vision", @@ -216,5 +219,8 @@ export const i18nVue = { file_sync_fail:"File synchronization failed", no_file_to_download:"No files can be downloaded", task_not_finished:"Task not finished yet, please wait", + local:"Local", + online:"Online", + modify:"Modify", }, }; diff --git a/web_src/js/index.js b/web_src/js/index.js index a781261a9..5b40d829e 100755 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -6,8 +6,10 @@ import "./publicpath.js"; import "./polyfills.js"; import "./features/letteravatar.js"; import Vue from "vue"; -import ElementUI from "element-ui"; import "element-ui/lib/theme-chalk/index.css"; +import localeEn from 'element-ui/lib/locale/lang/en'; +import localeZh from 'element-ui/lib/locale/lang/zh-CN'; +import ElementUI from "element-ui"; import axios from "axios"; import qs from "qs"; import Cookies from "js-cookie"; @@ -54,15 +56,19 @@ import { Message } from "element-ui"; import { i18nVue } from "./features/i18nVue.js"; import './features/ad.js'; -Vue.use(ElementUI); Vue.prototype.$axios = axios; Vue.prototype.$Cookies = Cookies; Vue.prototype.qs = qs; Vue.prototype.$message = Message; Vue.prototype.$locale = i18nVue; -window.i18n = i18nVue[document.querySelector('html').getAttribute('lang') == 'zh-CN' ? 'CN' : 'US']; +const lang = document.querySelector('html').getAttribute('lang'); +window.i18n = i18nVue[lang == 'zh-CN' ? 'CN' : 'US']; const { AppSubUrl, StaticUrlPrefix, csrf } = window.config; +Vue.use(ElementUI, { + locale: lang === 'zh-CN' ? localeZh : localeEn, +}); + Object.defineProperty(Vue.prototype, "$echarts", { value: echarts, }); diff --git a/web_src/vuepages/langs/config/en-US.js b/web_src/vuepages/langs/config/en-US.js index bcd1cfe43..1c160ee1d 100644 --- a/web_src/vuepages/langs/config/en-US.js +++ b/web_src/vuepages/langs/config/en-US.js @@ -193,6 +193,66 @@ const en = { dataDesensitizationModelDesc:'Use AI technology to desensitize the face and license plate number in the picture. For more information about this model, please visit the project', limitFilesUpload:'Only jpg/jpeg/png files can be uploaded', limitSizeUpload:'The size of the uploaded file cannot exceed 20M!', + modelManage: { + modelManage: 'Model management', + modelName: 'Model name', + useCluster: 'Available clusters', + local: 'Local', + online: 'Online', + createModel: 'Create Model', + importLocalModel: 'Import Lacal Model', + importOnlineModel: 'Import Online Model', + modifyModelInfo: 'Modify model information', + addModelFiles: 'Add model files', + uploadModelFiles: 'Upload model files', + pleaseInputModelName: 'Please input model name', + version: 'Version', + modelEngine: 'Model engine', + modelLabel: 'Model label', + modelLabelInputTips: 'Input labels, multiple labels are separated by spaces', + modelDescr: 'Model description', + modelDescrInputTips: 'The description should not exceed 256 characters', + confirm: 'Confirm', + cancel: 'Cancel', + modelCreateFailed: 'Model create failed', + modelModifyFailed: 'Model modify failed', + fileUpload: 'File upload', + upload: 'Upload', + uploadStatus: 'Upload status', + modelFileUploadDefaultTips: 'Click to add files or drag files here directly', + modelFileUploadErrTips: 'Up to 10 files can be uploaded at a time, and the total file size of the model does not exceed 200GB', + fileIstoBig: 'File is to big', + removeFile: 'Rmove file', + uploadSuccess: 'upload success', + uploadFailed: 'upload failed', + calcFileMd5: 'Calculating file MD5...', + uploading: 'Uploading...', + fileHasAlreadyInTheModel: 'This file has already in the model: ', + basicInfo: 'Basic information', + modelSize: 'Model size', + descr: 'Description', + createTime: 'Create Time', + label: 'Label', + trainTaskInfo: 'Train task information', + trainTask: 'Train task', + codeBranch: 'Code branch', + bootFile: 'Boot file', + trainDataset: 'Train dataset', + specInfo: 'Specifications', + workServerNumber: 'Amount of compute node', + runParameters: 'Run parameters', + seeMore: 'See more', + collapseDetails: 'Collapse details', + modelFilesList: 'Mode files list', + fileName: 'File name', + fileSize: 'File size', + updateTime: 'Upate Time', + operate: 'Operation', + delete: 'Delete', + infoModificationFailed: 'Information modify failed', + deleteModelFileConfirmTips: 'Are you sure you want to delete the current model file?', + modelFileDeleteFailed: 'Model file delete failed', + }, } export default en; diff --git a/web_src/vuepages/langs/config/zh-CN.js b/web_src/vuepages/langs/config/zh-CN.js index 5b8e800d8..2f87c7e15 100644 --- a/web_src/vuepages/langs/config/zh-CN.js +++ b/web_src/vuepages/langs/config/zh-CN.js @@ -193,6 +193,66 @@ const zh = { dataDesensitizationModelDesc:'利用人工智能AI技术,把图片中的人脸、车牌号码进行脱敏处理。该模型更多信息请访问项目', limitFilesUpload:'只能上传 jpg/jpeg/png 格式的文件', limitSizeUpload:'上传文件大小不能超过 20M !', + modelManage: { + modelManage: '模型管理', + modelName: '模型名称', + useCluster: '可用集群', + local: '本地', + online: '线上', + createModel: '创建模型', + importLocalModel: '导入本地模型', + importOnlineModel: '导入线上模型', + modifyModelInfo: '修改模型信息', + addModelFiles: '增加模型文件', + uploadModelFiles: '上传模型文件', + pleaseInputModelName: '请输入模型名称', + version: '版本', + modelEngine: '模型框架', + modelLabel: '模型标签', + modelLabelInputTips: '输入标签,多个标签用空格区分', + modelDescr: '模型描述', + modelDescrInputTips: '描述字数不超过255个字符', + confirm: '确定', + cancel: '取消', + modelCreateFailed: '模型创建失败', + modelModifyFailed: '模型修改失败', + fileUpload: '文件上传', + upload: '上传', + uploadStatus: '上传状态', + modelFileUploadDefaultTips: '点击添加文件或直接拖拽文件到此处', + modelFileUploadErrTips: '单次最多上传10个文件,模型总文件大小不超过200G', + fileIstoBig: '文件太大', + removeFile: '移除文件', + uploadSuccess: '上传成功', + uploadFailed: '上传失败', + calcFileMd5: '计算文件MD5...', + uploading: '上传中...', + fileHasAlreadyInTheModel: '该文件已上传在模型:', + basicInfo: '基本信息', + modelSize: '模型大小', + descr: '描述', + createTime: '创建时间', + label: '标签', + trainTaskInfo: '训练相关信息', + trainTask: '训练任务', + codeBranch: '代码分支', + bootFile: '启动文件', + trainDataset: '训练数据集', + specInfo: '规格', + workServerNumber: '计算节点', + runParameters: '运行参数', + seeMore: '查看更多信息', + collapseDetails: '折叠详细信息', + modelFilesList: '模型文件列表', + fileName: '文件名称', + fileSize: '文件大小', + updateTime: '更新时间', + operate: '操作', + delete: '删除', + infoModificationFailed: '信息修改失败', + deleteModelFileConfirmTips: '请确认是否删除当前模型文件?', + modelFileDeleteFailed: '模型文件删除失败', + }, } export default zh; diff --git a/web_src/vuepages/pages/modelmanage/common/modelmanage-common-detail.vue b/web_src/vuepages/pages/modelmanage/common/modelmanage-common-detail.vue index 1aff6d33f..442db17be 100644 --- a/web_src/vuepages/pages/modelmanage/common/modelmanage-common-detail.vue +++ b/web_src/vuepages/pages/modelmanage/common/modelmanage-common-detail.vue @@ -2,7 +2,7 @@
@@ -15,11 +15,11 @@
-
基本信息:
+
{{ $t('modelManage.basicInfo') }}:
-
可用集群:
+
{{ $t('modelManage.useCluster') }}:
{{ state.typeStr }} @@ -27,13 +27,13 @@
-
模型大小:
+
{{ $t('modelManage.modelSize') }}:
{{ state.modelSize }}
-
描述:
+
{{ $t('modelManage.descr') }}:
@@ -42,7 +42,8 @@ class="el-icon-edit" @click="editDescr = state._description; isEidtDescr = true;">
- +
-
模型框架:
+
{{ $t('modelManage.modelEngine') }}:
{{ state.engineName }}
-
创建时间:
+
{{ $t('modelManage.createTime') }}:
{{ state.createTime }}
-
标签:
+
{{ $t('modelManage.label') }}:
@@ -74,7 +75,8 @@ class="el-icon-edit" @click="editLabel = state._label; isEidtLabel = true;">
- +
-
训练相关信息:
+
{{ $t('modelManage.trainTaskInfo') }}:
-
训练任务:
+
{{ $t('modelManage.trainTask') }}:
-
代码分支:
+
{{ $t('modelManage.codeBranch') }}:
-
启动文件:
+
{{ $t('modelManage.bootFile') }}:
{{ state.bootFile }}
-
训练数据集:
+
{{ $t('modelManage.trainDataset') }}:
{{ state.datasetName }}
@@ -114,19 +116,19 @@
-
规格:
+
{{ $t('modelManage.specInfo') }}:
{{ state.specStr }}
-
计算节点:
+
{{ $t('modelManage.workServerNumber') }}:
{{ state.workServerNumber }}
-
运行参数:
+
{{ $t('modelManage.runParameters') }}:
{{ state.parameters }}
@@ -138,27 +140,28 @@
- {{ isExpanded ? '折叠详细信息' : '查看更多信息' }} + {{ isExpanded ? $t('modelManage.collapseDetails') : $t('modelManage.seeMore') }}
-
模型文件列表:
+
{{ $t('modelManage.modelFilesList') }}:
{{ state.version }} /
- 上传模型文件 + {{ $t('modelManage.uploadModelFiles') }}
- + - + - + - + @@ -198,6 +201,7 @@ import { MODEL_ENGINES } from '~/const'; import { formatDate } from 'element-ui/lib/utils/date-util'; const REPO_NAME = location.pathname.split('/')[2]; +const MAX_LABEL_COUNT = 5; export default { data() { @@ -298,7 +302,18 @@ export default { }) }, goUploadPage() { - window.location.href = `${this.repo}/modelmanage/create_local_model_tmpl2?type=0&name=${this.state.name}&id=${this.state.id}`; + window.location.href = `${this.repo}/modelmanage/create_local_model_tmpl2?type=1&name=${this.state.name}&id=${this.state.id}`; + }, + backToModelListPage() { + const list = window.location.href.split('/'); + list.pop(); + list.push('show_model'); + window.location.href = list.join('/'); + }, + labelInput() { + const hasEndSpace = this.editLabel[this.editLabel.length - 1] == ' '; + const list = this.editLabel.trim().split(' ').filter(label => label != ''); + this.editLabel = list.slice(0, MAX_LABEL_COUNT).join(' ') + (hasEndSpace && list.length < MAX_LABEL_COUNT ? ' ' : ''); }, submitEidt(type) { const obj = { @@ -326,19 +341,19 @@ export default { } else { this.$message({ type: 'error', - message: '信息修改失败', + message: this.$t('modelManage.infoModificationFailed'), }); } }).catch(err => { console.log(err); this.$message({ type: 'error', - message: '信息修改失败', + message: this.$t('modelManage.infoModificationFailed'), }); }); }, deleteFile(file) { - this.$confirm('请确认是否删除当前模型文件?', this.$t('tips'), { + this.$confirm(this.$t('modelManage.deleteModelFileConfirmTips'), this.$t('tips'), { confirmButtonText: this.$t('confirm1'), cancelButtonText: this.$t('cancel'), type: 'warning', @@ -355,14 +370,14 @@ export default { } else { this.$message({ type: 'error', - message: '模型文件删除失败', + message: this.$t('modelManage.modelFileDeleteFailed'), }); } }).catch(err => { console.log(err); this.$message({ type: 'error', - message: '模型文件删除失败', + message: this.$t('modelManage.modelFileDeleteFailed'), }); }); }).catch(() => { }); @@ -387,10 +402,10 @@ export default { }).catch(err => { this.loading = false; console.log(err); - this.cancel(); + this.backToModelListPage(); }); } else { - this.cancel(); + this.backToModelListPage(); } }, beforeDestroy() { @@ -414,6 +429,7 @@ export default { font-weight: 550; font-size: 14px; color: rgb(16, 16, 16); + margin-bottom: 10px; } .detail-info { diff --git a/web_src/vuepages/pages/modelmanage/local/modelmanage-local-create-1.vue b/web_src/vuepages/pages/modelmanage/local/modelmanage-local-create-1.vue index 2a1895c16..3c3b35c58 100644 --- a/web_src/vuepages/pages/modelmanage/local/modelmanage-local-create-1.vue +++ b/web_src/vuepages/pages/modelmanage/local/modelmanage-local-create-1.vue @@ -1,23 +1,24 @@