diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 20e643884..44b669181 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -30,7 +30,6 @@ const ( JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeBrainScore JobType = "BRAINSCORE" JobTypeTrain JobType = "TRAIN" - JobVersionName JobType = "V0001" ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 ModelArtsCreating ModelArtsJobStatus = "CREATING" //创建中 @@ -53,64 +52,46 @@ type Cloudbrain struct { ID int64 `xorm:"pk autoincr"` JobID string `xorm:"INDEX NOT NULL"` JobType string `xorm:"INDEX NOT NULL DEFAULT 'DEBUG'"` - JobName string `xorm:"INDEX"` - Status string `xorm:"INDEX"` - UserID int64 `xorm:"INDEX"` - RepoID int64 `xorm:"INDEX"` - SubTaskName string `xorm:"INDEX"` + JobName string + Status string + UserID int64 + RepoID int64 + SubTaskName string ContainerID string ContainerIp string CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Duration int64 `xorm:"INDEX duration"` + Duration int64 TrainJobDuration string DeletedAt time.Time `xorm:"deleted"` CanDebug bool `xorm:"-"` CanDel bool `xorm:"-"` - Type int `xorm:"INDEX DEFAULT 0"` - - VersionID int64 `xorm:"INDEX DEFAULT 0"` - VersionName string `xorm:"INDEX"` - Uuid string - DatasetName string - VersionCount int `xorm:"INDEX DEFAULT 1"` - IsLatestVersion string - CommitID string - FatherVersionName string - ComputeResource string - EngineID int64 - - TrainUrl string - BranchName string - Parameters string - BootFile string - DataUrl string - LogUrl string - PreVersionId int64 - FlavorCode string - Description string - WorkServerNumber int - FlavorName string - EngineName string - - User *User `xorm:"-"` - Repo *Repository `xorm:"-"` -} + Type int -type TrainjobConfigDetail struct { - ID int64 `xorm:"pk autoincr"` - JobID string `xorm:"INDEX"` - JobName string `xorm:"INDEX"` - ResourcePools string `xorm:"INDEX"` - EngineVersions int `xorm:"INDEX"` - FlavorInfos string `xorm:"INDEX"` - TrainUrl string `xorm:"INDEX"` - BootFile string `xorm:"INDEX"` - Uuid string `xorm:"INDEX"` - DatasetName string `xorm:"INDEX"` - Params string `xorm:"INDEX"` - BranchName string `xorm:"INDEX"` - VersionName string `xorm:"INDEX"` + VersionID int64 //版本id + VersionName string `xorm:"INDEX"` //当前版本 + Uuid string //数据集id + DatasetName string + VersionCount int //任务的当前版本数量，不包括删除的 + IsLatestVersion string //是否是最新版本，1是，0否 + CommitID string //提交的仓库代码id + PreVersionName string //父版本名称 + ComputeResource string //计算资源，例如npu + EngineID int64 //引擎id + + TrainUrl string //输出的obs路径 + BranchName string //分支名称 + Parameters string //传给modelarts的param参数 + BootFile string //启动文件 + DataUrl string //数据集的obs路径 + LogUrl string //日志输出的obs路径 + PreVersionId int64 //父版本的版本id + FlavorCode string //modelarts上的规格id + Description string //描述 + WorkServerNumber int //节点数 + FlavorName string //规格名称 + EngineName string //引擎名称 + TotalVersionCount int //任务的所有版本数量，包括删除的 User *User `xorm:"-"` Repo *Repository `xorm:"-"` @@ -621,20 +602,14 @@ type Config struct { BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下 Parameter []Parameter `json:"parameter"` DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL - //DatasetID string `json:"dataset_id"` - //DataVersionID string `json:"dataset_version_id"` - //DataSource []DataSource `json:"data_source"` - //SpecID int64 `json:"spec_id"` - EngineID int64 `json:"engine_id"` - //ModelID int64 `json:"model_id"` - TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL - LogUrl string `json:"log_url"` + EngineID int64 `json:"engine_id"` + TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL + LogUrl string `json:"log_url"` //UserImageUrl string `json:"user_image_url"` //UserCommand string `json:"user_command"` - CreateVersion bool `json:"create_version"` - //Volumes []Volumes `json:"volumes"` - Flavor Flavor `json:"flavor"` - PoolID string `json:"pool_id"` + CreateVersion bool `json:"create_version"` + Flavor Flavor `json:"flavor"` + PoolID string `json:"pool_id"` } type CreateTrainJobVersionParams struct { @@ -648,20 +623,12 @@ type TrainJobVersionConfig struct { BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下 Parameter []Parameter `json:"parameter"` DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL - //DatasetID string `json:"dataset_id"` - //DataVersionID string `json:"dataset_version_id"` - //DataSource []DataSource `json:"data_source"` - //SpecID int64 `json:"spec_id"` - EngineID int64 `json:"engine_id"` - //ModelID int64 `json:"model_id"` - TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL - LogUrl string `json:"log_url"` - //UserImageUrl string `json:"user_image_url"` - //UserCommand string `json:"user_command"` - //Volumes []Volumes `json:"volumes"` - Flavor Flavor `json:"flavor"` - PoolID string `json:"pool_id"` - PreVersionId int64 `json:"pre_version_id"` + EngineID int64 `json:"engine_id"` + TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL + LogUrl string `json:"log_url"` + Flavor Flavor `json:"flavor"` + PoolID string `json:"pool_id"` + PreVersionId int64 `json:"pre_version_id"` } type CreateConfigParams struct { @@ -672,20 +639,11 @@ type CreateConfigParams struct { BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下 Parameter []Parameter `json:"parameter"` DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL - //DatasetID string `json:"dataset_id"` - //DataVersionID string `json:"dataset_version_id"` - //DataSource []DataSource `json:"data_source"` - //SpecID int64 `json:"spec_id"` - EngineID int64 `json:"engine_id"` - //ModelID int64 `json:"model_id"` - TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL - LogUrl string `json:"log_url"` - //UserImageUrl string `json:"user_image_url"` - //UserCommand string `json:"user_command"` - //CreateVersion bool `json:"create_version"` - //Volumes []Volumes `json:"volumes"` - Flavor Flavor `json:"flavor"` - PoolID string `json:"pool_id"` + EngineID int64 `json:"engine_id"` + TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL + LogUrl string `json:"log_url"` + Flavor Flavor `json:"flavor"` + PoolID string `json:"pool_id"` } type Parameter struct { @@ -799,18 +757,10 @@ type GetConfigResult struct { BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下 Parameter []Parameter `json:"parameter"` DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL - //DatasetID string `json:"dataset_id"` - //DataVersionID string `json:"dataset_version_id"` - //DataSource []DataSource `json:"data_source"` - //SpecID int64 `json:"spec_id"` - EngineID int64 `json:"engine_id"` - //ModelID int64 `json:"model_id"` - TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL - LogUrl string `json:"log_url"` - //UserImageUrl string `json:"user_image_url"` - //UserCommand string `json:"user_command"` - //CreateVersion bool `json:"create_version"` - //Volumes []Volumes `json:"volumes"` + EngineID int64 `json:"engine_id"` + TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL + LogUrl string `json:"log_url"` + Flavor Flavor `json:"flavor"` PoolID string `json:"pool_id"` } @@ -841,26 +791,18 @@ type GetTrainJobResult struct { BootFileUrl string `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下 Parameter []Parameter `json:"parameter"` DataUrl string `json:"data_url"` //训练作业需要的数据集OBS路径URL - //DatasetID string `json:"dataset_id"` - //DataVersionID string `json:"dataset_version_id"` - //DataSource []DataSource `json:"data_source"` - //SpecID int64 `json:"spec_id"` - EngineID int64 `json:"engine_id"` - EngineName string `json:"engine_name"` - EngineVersion string `json:"engine_version"` - //ModelID int64 `json:"model_id"` - TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL - LogUrl string `json:"log_url"` - //UserImageUrl string `json:"user_image_url"` - //UserCommand string `json:"user_command"` - //Volumes []Volumes `json:"volumes"` - Flavor Flavor `json:"flavor"` - PoolID string `json:"pool_id"` - PoolName string `json:"pool_name"` - NasMountPath string `json:"nas_mount_path"` - NasShareAddr string `json:"nas_share_addr"` - DatasetName string - ModelMetricList string `json:"model_metric_list"` //列表里包含f1_score，recall，precision，accuracy，若有的话 + EngineID int64 `json:"engine_id"` + EngineName string `json:"engine_name"` + EngineVersion string `json:"engine_version"` + TrainUrl string `json:"train_url"` //训练作业的输出文件OBS路径URL + LogUrl string `json:"log_url"` + Flavor Flavor `json:"flavor"` + PoolID string `json:"pool_id"` + PoolName string `json:"pool_name"` + NasMountPath string `json:"nas_mount_path"` + NasShareAddr string `json:"nas_share_addr"` + DatasetName string + ModelMetricList string `json:"model_metric_list"` //列表里包含f1_score，recall，precision，accuracy，若有的话 } type GetTrainJobLogResult struct { @@ -931,17 +873,6 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { ) } - // switch opts.JobStatus { - // case JobWaiting: - // cond.And(builder.Eq{"cloudbrain.status": int(JobWaiting)}) - // case JobFailed: - // cond.And(builder.Eq{"cloudbrain.status": int(JobFailed)}) - // case JobStopped: - // cond.And(builder.Eq{"cloudbrain.status": int(JobStopped)}) - // case JobSucceeded: - // cond.And(builder.Eq{"cloudbrain.status": int(JobSucceeded)}) - // } - if len(opts.CloudbrainIDs) > 0 { cond = cond.And(builder.In("cloudbrain.id", opts.CloudbrainIDs)) } @@ -968,7 +899,6 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } - sess.Close() return cloudbrains, count, nil } @@ -1034,7 +964,6 @@ func CloudbrainsVersionList(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int, e Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } - sess.Close() return cloudbrains, int(count), nil } @@ -1046,13 +975,6 @@ func CreateCloudbrain(cloudbrain *Cloudbrain) (err error) { return nil } -func CreateTrainjobConfigDetail(trainjobConfigDetail *TrainjobConfigDetail) (err error) { - if _, err = x.Insert(trainjobConfigDetail); err != nil { - return err - } - return nil -} - func getRepoCloudBrain(cb *Cloudbrain) (*Cloudbrain, error) { has, err := x.Get(cb) if err != nil { @@ -1068,11 +990,6 @@ func GetRepoCloudBrainByJobID(repoID int64, jobID string) (*Cloudbrain, error) { return getRepoCloudBrain(cb) } -func GetRepoCloudBrainByJobIDAndVersionName(repoID int64, jobID string, versionName string) (*Cloudbrain, error) { - cb := &Cloudbrain{JobID: jobID, RepoID: repoID, VersionName: versionName} - return getRepoCloudBrain(cb) -} - func GetCloudbrainByJobID(jobID string) (*Cloudbrain, error) { cb := &Cloudbrain{JobID: jobID} return getRepoCloudBrain(cb) @@ -1112,9 +1029,9 @@ func SetTrainJobStatusByJobID(jobID string, status string, duration int64, train return } -func SetVersionCountAndLatestVersionByJobIDAndVersionName(jobID string, versionName string, versionCount int, isLatestVersion string) (err error) { - cb := &Cloudbrain{JobID: jobID, VersionName: versionName, VersionCount: versionCount, IsLatestVersion: isLatestVersion} - _, err = x.Cols("version_Count", "is_latest_version").Where("cloudbrain.job_id=? AND cloudbrain.version_name=?", jobID, versionName).Update(cb) +func SetVersionCountAndLatestVersion(jobID string, versionName string, versionCount int, isLatestVersion string, totalVersionCount int) (err error) { + cb := &Cloudbrain{JobID: jobID, VersionName: versionName, VersionCount: versionCount, IsLatestVersion: isLatestVersion, TotalVersionCount: totalVersionCount} + _, err = x.Cols("version_Count", "is_latest_version", "total_version_count").Where("cloudbrain.job_id=? AND cloudbrain.version_name=?", jobID, versionName).Update(cb) return } @@ -1129,16 +1046,16 @@ func updateJob(e Engine, job *Cloudbrain) error { return err } -// func UpdateTrainJob(job *CloudbrainInfo) error { -// return updateTrainJob(x, job) -// } +func UpdateTrainJobVersion(job *Cloudbrain) error { + return updateJobTrainVersion(x, job) +} -// func updateTrainJob(e Engine, job *CloudbrainInfo) error { -// var sess *xorm.Session -// sess = e.Where("job_id = ?", job.Cloudbrain.JobID) -// _, err := sess.Cols("status", "container_id", "container_ip").Update(job) -// return err -// } +func updateJobTrainVersion(e Engine, job *Cloudbrain) error { + var sess *xorm.Session + sess = e.Where("job_id = ? AND version_name=?", job.JobID, job.VersionName) + _, err := sess.Cols("status", "train_job_duration").Update(job) + return err +} func DeleteJob(job *Cloudbrain) error { return deleteJob(x, job) diff --git a/models/models.go b/models/models.go index 6631a9124..4858ec104 100755 --- a/models/models.go +++ b/models/models.go @@ -134,7 +134,6 @@ func init() { new(BlockChain), new(RecommendOrg), new(AiModelManage), - new(TrainjobConfigDetail), ) tablesStatistic = append(tablesStatistic, diff --git a/models/repo_activity_custom.go b/models/repo_activity_custom.go index 04f8f7ae1..7f343bdc6 100644 --- a/models/repo_activity_custom.go +++ b/models/repo_activity_custom.go @@ -14,7 +14,6 @@ type ContributorWithUserId struct { UserId int64 IsAdmin bool RelAvatarLink string - Email string } func GetRepoKPIStats(repo *Repository) (*git.RepoKPIStats, error) { @@ -127,12 +126,12 @@ func getRepoKPIStats(repoPath string, wikiPath string) (*git.RepoKPIStats, error } -func GetTop10Contributor(repoPath string) ([]ContributorWithUserId, error) { +func GetTop10Contributor(repoPath string) ([]*ContributorWithUserId, error) { contributors, err := git.GetContributors(repoPath) if err != nil { - return make([]ContributorWithUserId, 0), err + return make([]*ContributorWithUserId, 0), err } - contributorDistinctDict := make(map[string]ContributorWithUserId, 0) + contributorDistinctDict := make(map[string]*ContributorWithUserId, 0) if contributors != nil { for _, contributor := range contributors { if strings.Compare(contributor.Email, "") == 0 { @@ -144,12 +143,15 @@ func GetTop10Contributor(repoPath string) ([]ContributorWithUserId, error) { value, ok := contributorDistinctDict[user.Email] if !ok { - contributorDistinctDict[user.Email] = ContributorWithUserId{ - contributor, + contributorDistinctDict[user.Email] = &ContributorWithUserId{ + git.Contributor{ + contributor.CommitCnt, + user.Name, + user.Email, + }, user.ID, user.IsAdmin, user.RelAvatarLink(), - user.Email, } } else { @@ -159,12 +161,11 @@ func GetTop10Contributor(repoPath string) ([]ContributorWithUserId, error) { } else { value, ok := contributorDistinctDict[contributor.Email] if !ok { - contributorDistinctDict[contributor.Email] = ContributorWithUserId{ + contributorDistinctDict[contributor.Email] = &ContributorWithUserId{ contributor, -1, false, "", - contributor.Email, } } else { value.CommitCnt += contributor.CommitCnt @@ -173,7 +174,7 @@ func GetTop10Contributor(repoPath string) ([]ContributorWithUserId, error) { } } - v := make([]ContributorWithUserId, 0, len(contributorDistinctDict)) + v := make([]*ContributorWithUserId, 0, len(contributorDistinctDict)) for _, value := range contributorDistinctDict { v = append(v, value) } @@ -188,7 +189,7 @@ func GetTop10Contributor(repoPath string) ([]ContributorWithUserId, error) { return v[0:10], nil } } - return make([]ContributorWithUserId, 0), nil + return make([]*ContributorWithUserId, 0), nil } func setKeyContributerDict(contributorDistinctDict map[string]int, email string, keyContributorsDict map[string]struct{}) { diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 99ca262f1..a64c317a8 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -2,6 +2,7 @@ package modelarts import ( "encoding/json" + "fmt" "path" "strconv" @@ -35,24 +36,24 @@ const ( // "{\"code\":\"modelarts.bm.910.arm.public.4\",\"value\":\"Ascend : 4 * Ascend 910 CPU：96 核 1024GiB\"}," + // "{\"code\":\"modelarts.bm.910.arm.public.1\",\"value\":\"Ascend : 1 * Ascend 910 CPU：24 核 256GiB\"}" + // "]}" - CodePath = "/code/" - OutputPath = "/output/" - LogPath = "/log/" - JobPath = "/job/" - OrderDesc = "desc" //向下查询 - OrderAsc = "asc" //向上查询 - Lines = 20 - TrainUrl = "train_url" - DataUrl = "data_url" - PerPage = 10 - IsLatestVersion = "1" - NotLatestVersion = "0" - ComputeResource = "NPU" - InitFatherVersionName = "V0001" - VersionCount = 1 - - SortByCreateTime = "create_time" - ConfigTypeCustom = "custom" + CodePath = "/code/" + OutputPath = "/output/" + LogPath = "/log/" + JobPath = "/job/" + OrderDesc = "desc" //向下查询 + OrderAsc = "asc" //向上查询 + Lines = 500 + TrainUrl = "train_url" + DataUrl = "data_url" + PerPage = 10 + IsLatestVersion = "1" + NotLatestVersion = "0" + ComputeResource = "NPU" + VersionCount = 1 + + SortByCreateTime = "create_time" + ConfigTypeCustom = "custom" + TotalVersionCount = 1 ) var ( @@ -79,33 +80,37 @@ type GenerateTrainJobReq struct { IsLatestVersion string Params string BranchName string - FatherVersionName string + PreVersionId int64 + PreVersionName string FlavorName string VersionCount int EngineName string + TotalVersionCount int } type GenerateTrainJobVersionReq struct { - JobName string - Uuid string - Description string - CodeObsPath string - BootFile string - BootFileUrl string - DataUrl string - TrainUrl string - FlavorCode string - LogUrl string - PoolID string - WorkServerNumber int - EngineID int64 - Parameters []models.Parameter - Params string - PreVersionId int64 - CommitID string - BranchName string - FlavorName string - EngineName string + JobName string + Uuid string + Description string + CodeObsPath string + BootFile string + BootFileUrl string + DataUrl string + TrainUrl string + FlavorCode string + LogUrl string + PoolID string + WorkServerNumber int + EngineID int64 + Parameters []models.Parameter + Params string + PreVersionId int64 + CommitID string + BranchName string + FlavorName string + EngineName string + PreVersionName string + TotalVersionCount int } type VersionInfo struct { @@ -270,7 +275,6 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error IsLatestVersion: req.IsLatestVersion, ComputeResource: ComputeResource, EngineID: req.EngineID, - FatherVersionName: req.FatherVersionName, TrainUrl: req.TrainUrl, BranchName: req.BranchName, Parameters: req.Params, @@ -283,6 +287,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error FlavorName: req.FlavorName, EngineName: req.EngineName, VersionCount: req.VersionCount, + TotalVersionCount: req.TotalVersionCount, }) if err != nil { @@ -293,7 +298,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error return nil } -func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobVersionReq, jobId string, fatherVersionName string) (err error) { +func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, jobId string) (err error) { jobResult, err := createTrainJobVersion(models.CreateTrainJobVersionParams{ Description: req.Description, Config: models.TrainJobVersionConfig{ @@ -323,6 +328,19 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobVersionR return err } + repo := ctx.Repo.Repository + VersionTaskList, VersionListCount, err := models.CloudbrainsVersionList(&models.CloudbrainsOptions{ + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, + JobType: string(models.JobTypeTrain), + JobID: strconv.FormatInt(jobResult.JobID, 10), + }) + if err != nil { + ctx.ServerError("Cloudbrain", err) + return err + } + //将当前版本的isLatestVersion设置为"1"和任务数量更新,任务数量包括当前版本数VersionCount和历史创建的总版本数TotalVersionCount + err = models.CreateCloudbrain(&models.Cloudbrain{ Status: TransTrainJobStatus(jobResult.Status), UserID: ctx.User.ID, @@ -336,7 +354,8 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobVersionR Uuid: req.Uuid, DatasetName: attach.Name, CommitID: req.CommitID, - FatherVersionName: fatherVersionName, + IsLatestVersion: req.IsLatestVersion, + PreVersionName: req.PreVersionName, ComputeResource: ComputeResource, EngineID: req.EngineID, TrainUrl: req.TrainUrl, @@ -351,48 +370,18 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobVersionR WorkServerNumber: req.WorkServerNumber, FlavorName: req.FlavorName, EngineName: req.EngineName, + TotalVersionCount: VersionTaskList[0].TotalVersionCount + 1, + VersionCount: VersionListCount + 1, }) if err != nil { log.Error("CreateCloudbrain(%s) failed:%v", req.JobName, err.Error()) return err } - repo := ctx.Repo.Repository - page := ctx.QueryInt("page") - if page <= 0 { - page = 1 - } - _, VersionListCount, err := models.CloudbrainsVersionList(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: setting.UI.IssuePagingNum, - }, - RepoID: repo.ID, - Type: models.TypeCloudBrainTwo, - JobType: string(models.JobTypeTrain), - JobID: strconv.FormatInt(jobResult.JobID, 10), - }) - if err != nil { - ctx.ServerError("Cloudbrain", err) - return err - } - //将训练任务的上一版本的isLatestVersion设置为"0" - latestTask, err := models.GetCloudbrainByJobIDAndIsLatestVersion(strconv.FormatInt(jobResult.JobID, 10), IsLatestVersion) + err = models.SetVersionCountAndLatestVersion(strconv.FormatInt(jobResult.JobID, 10), VersionTaskList[0].VersionName, VersionCount, NotLatestVersion, TotalVersionCount) if err != nil { - ctx.ServerError("GetCloudbrainByJobIDAndIsLatestVersion faild:", err) - return err - } - err = models.SetVersionCountAndLatestVersionByJobIDAndVersionName(strconv.FormatInt(jobResult.JobID, 10), latestTask.VersionName, VersionListCount, NotLatestVersion) - if err != nil { - ctx.ServerError("UpdateJobVersionCount failed", err) - return err - } - - //将当前版本的isLatestVersion设置为"1"和任务数量更新 - err = models.SetVersionCountAndLatestVersionByJobIDAndVersionName(strconv.FormatInt(jobResult.JobID, 10), jobResult.VersionName, VersionListCount, IsLatestVersion) - if err != nil { - ctx.ServerError("UpdateJobVersionCount failed", err) + ctx.ServerError("Update IsLatestVersion failed", err) return err } @@ -449,6 +438,10 @@ func TransTrainJobStatus(status int) string { default: return strconv.Itoa(status) } +} - return "" +func GetVersionOutputPathByTotalVersionCount(TotalVersionCount int) (VersionOutputPath string) { + talVersionCountToString := fmt.Sprintf("%04d", TotalVersionCount) + VersionOutputPath = "V" + talVersionCountToString + return VersionOutputPath } diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go index c967c0eda..2cc9e34be 100755 --- a/modules/modelarts/resty.go +++ b/modules/modelarts/resty.go @@ -366,6 +366,16 @@ sendjob: return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) } log.Error("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + BootFileErrorMsg := "Invalid OBS path '" + createJobParams.Config.BootFileUrl + "'." + DataSetErrorMsg := "Invalid OBS path '" + createJobParams.Config.DataUrl + "'." + if temp.ErrorMsg == BootFileErrorMsg { + log.Error("启动文件错误！createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("启动文件错误！") + } + if temp.ErrorMsg == DataSetErrorMsg { + log.Error("数据集错误！createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("数据集错误！") + } return &result, fmt.Errorf("createTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) } @@ -411,7 +421,16 @@ sendjob: log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) } - log.Error("createTrainJobVersion failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + BootFileErrorMsg := "Invalid OBS path '" + createJobVersionParams.Config.BootFileUrl + "'." + DataSetErrorMsg := "Invalid OBS path '" + createJobVersionParams.Config.DataUrl + "'." + if temp.ErrorMsg == BootFileErrorMsg { + log.Error("启动文件错误！createTrainJobVersion failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("启动文件错误！") + } + if temp.ErrorMsg == DataSetErrorMsg { + log.Error("数据集错误！createTrainJobVersion failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("数据集错误！") + } return &result, fmt.Errorf("createTrainJobVersion failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) } @@ -814,3 +833,44 @@ sendjob: return &result, nil } + +func DelTrainJobVersion(jobID string, versionID string) (*models.TrainJobResult, error) { + checkSetting() + client := getRestyClient() + var result models.TrainJobResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetAuthToken(TOKEN). + SetResult(&result). + Delete(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID) + + if err != nil { + return &result, fmt.Errorf("resty DelTrainJobVersion: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + if res.StatusCode() != http.StatusOK { + var temp models.ErrorResult + if err = json.Unmarshal([]byte(res.String()), &temp); err != nil { + log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error()) + } + log.Error("DelTrainJob failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + return &result, fmt.Errorf("删除训练作业版本失败(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg) + } + + if !result.IsSuccess { + log.Error("DelTrainJob(%s) failed", jobID) + return &result, fmt.Errorf("删除训练作业版本失败:%s", result.ErrorMsg) + } + + return &result, nil +} diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 7a065636d..31a19806e 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -431,62 +431,6 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { } } -func GetVersionObsListObject(jobName, parentDir string) ([]FileInfo, error) { - input := &obs.ListObjectsInput{} - input.Bucket = setting.Bucket - input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") - strPrefix := strings.Split(input.Prefix, "/") - output, err := ObsCli.ListObjects(input) - fileInfos := make([]FileInfo, 0) - if err == nil { - for _, val := range output.Contents { - str1 := strings.Split(val.Key, "/") - var isDir bool - var fileName, nextParentDir string - if strings.HasSuffix(val.Key, "/") { - //dirs in next level dir - if len(str1)-len(strPrefix) > 2 { - continue - } - fileName = str1[len(str1)-2] - isDir = true - if parentDir == "" { - nextParentDir = fileName - } else { - nextParentDir = parentDir + "/" + fileName - } - - if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath { - continue - } - } else { - //files in next level dir - if len(str1)-len(strPrefix) > 1 { - continue - } - fileName = str1[len(str1)-1] - isDir = false - nextParentDir = parentDir - } - - fileInfo := FileInfo{ - ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), - FileName: fileName, - Size: val.Size, - IsDir: isDir, - ParenDir: nextParentDir, - } - fileInfos = append(fileInfos, fileInfo) - } - return fileInfos, err - } else { - if obsError, ok := err.(obs.ObsError); ok { - log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) - } - return nil, err - } -} - func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} diff --git a/modules/templates/helper.go b/modules/templates/helper.go index c399cc289..da278ba32 100755 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -92,6 +92,7 @@ func NewFuncMap() []template.FuncMap { "Str2html": Str2html, "TimeSince": timeutil.TimeSince, "TimeSinceUnix": timeutil.TimeSinceUnix, + "TimeSinceUnix1": timeutil.TimeSinceUnix1, "RawTimeSince": timeutil.RawTimeSince, "FileSize": base.FileSize, "PrettyNumber": base.PrettyNumber, @@ -340,6 +341,7 @@ func NewTextFuncMap() []texttmpl.FuncMap { }, "TimeSince": timeutil.TimeSince, "TimeSinceUnix": timeutil.TimeSinceUnix, + "TimeSinceUnix1": timeutil.TimeSinceUnix1, "RawTimeSince": timeutil.RawTimeSince, "DateFmtLong": func(t time.Time) string { return t.Format(time.RFC1123Z) diff --git a/modules/timeutil/since.go b/modules/timeutil/since.go index e6c29c19f..a7854ed91 100755 --- a/modules/timeutil/since.go +++ b/modules/timeutil/since.go @@ -162,3 +162,8 @@ func htmlTimeSinceUnix(then, now TimeStamp, lang string) template.HTML { then.FormatInLocation(GetTimeFormat(lang), setting.DefaultUILocation), timeSinceUnix(int64(then), int64(now), lang))) } +func TimeSinceUnix1(then TimeStamp) string { + format := time.Unix(int64(then), 0).Format("2006-01-02 15:04:05") + return format + +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e09a18fbf..440147d58 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -823,7 +823,11 @@ modelarts.train_job.new_train=New Train Task modelarts.train_job.config=Configuration information modelarts.train_job.new=New train Task modelarts.train_job.new_place=The description should not exceed 256 characters - +modelarts.modify=Modify +modelarts.current_version=Current version +modelarts.parent_version=Parent Version +modelarts.run_version=Run Version +modelarts.train_job.compute_node=Compute Node modelarts.train_job.basic_info=Basic Info @@ -845,6 +849,7 @@ modelarts.train_job.start_file=Start File modelarts.train_job.boot_file_helper=The startup file is the entry file that your program executes, and it must be a file ending in .py modelarts.train_job.dataset=Dataset modelarts.code_version = Code Version +modelarts.parents_version = Parents Version modelarts.train_job.run_parameter=Run Parameter modelarts.train_job.add_run_parameter=Add Run Parameter modelarts.train_job.parameter_name=Parameter Name diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index b57acc683..c2e0aa891 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -816,9 +816,11 @@ total_count_get_error=查询总页数失败。 last_update_time_error=查询最新更新时间失败。 get_repo_stat_error=查询当前仓库的统计信息失败。 get_repo_info_error=查询当前仓库信息失败。 -generate_statistic_file_error=生成文件失败。 -repo_stat_inspect=项目分析 -all=所有 + +modelarts.status=状态 +modelarts.createtime=创建时间 +modelarts.version_nums=版本数 +modelarts.computing_resources=计算资源 modelarts.notebook=调试任务 modelarts.train_job=训练任务 modelarts.train_job.new_debug=新建调试任务 @@ -826,6 +828,10 @@ modelarts.train_job.new_train=新建训练任务 modelarts.train_job.config=配置信息 modelarts.train_job.new=新建训练任务 modelarts.train_job.new_place=描述字数不超过256个字符 +modelarts.modify=修改 +modelarts.current_version=当前版本 +modelarts.parent_version=父版本 +modelarts.run_version=运行版本 @@ -845,10 +851,14 @@ modelarts.train_job.frames=常用框架 modelarts.train_job.algorithm_origin=算法来源 modelarts.train_job.AI_driver=AI引擎 modelarts.train_job.start_file=启动文件 -modelarts.train_job.boot_file_helper=启动文件是您程序执行的入口文件，必须是以.py结尾的文件。 +modelarts.train_job.boot_file_helper=启动文件是您程序执行的入口文件，必须是以.py结尾的文件。比如train.py、main.py、example/train.py、case/main.py。 modelarts.train_job.boot_file_place=填写启动文件路径，默认为train.py modelarts.train_job.dataset=数据集 -modelarts.code_version=代码版本 +modelarts.code_version=代码分支 +modelarts.parents_version=基于版本 +modelarts.train_job.compute_node=计算节点 +modelarts.train_job.train_dataset=训练数据集 + modelarts.train_job.run_parameter=运行参数 modelarts.train_job.add_run_parameter=增加运行参数 modelarts.train_job.parameter_name=参数名 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index f2ff19217..b20713bca 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -874,13 +874,12 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Group("/train-job", func() { m.Group("/:jobid", func() { - // m.Get("", repo.GetModelArtsTrainJob) m.Get("", repo.GetModelArtsTrainJobVersion) - // m.Get("/log", repo.TrainJobGetLog) m.Get("/log", repo.TrainJobGetLog) - // m.Group("/:version-name", func() { - // m.Get("", repo.GetModelArtsTrainJobVersion) - // }) + m.Post("/del_version", repo.DelTrainJobVersion) + m.Post("/stop_version", repo.StopTrainJobVersion) + m.Get("/model_list", repo.ModelList) + m.Get("/model_download", repo.ModelDownload) }) }) }, reqRepoReader(models.UnitTypeCloudBrain)) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 181d97983..c53e62efa 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -8,11 +8,13 @@ package repo import ( "net/http" "strconv" + "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/storage" ) func GetModelArtsNotebook(ctx *context.APIContext) { @@ -87,8 +89,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { jobID := ctx.Params(":jobid") versionName := ctx.Query("version_name") - repoID := ctx.Repo.Repository.ID - job, err := models.GetRepoCloudBrainByJobIDAndVersionName(repoID, jobID, versionName) + job, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) if err != nil { ctx.NotFound(err) return @@ -102,7 +103,15 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { job.Status = modelarts.TransTrainJobStatus(result.IntStatus) job.Duration = result.Duration job.TrainJobDuration = result.TrainJobDuration - err = models.UpdateJob(job) + + if result.Duration != 0 { + job.TrainJobDuration = addZero(result.Duration/3600000) + ":" + addZero(result.Duration%3600000/60000) + ":" + addZero(result.Duration%60000/1000) + + } else { + job.TrainJobDuration = "00:00:00" + } + + err = models.UpdateTrainJobVersion(job) if err != nil { log.Error("UpdateJob failed:", err) } @@ -110,23 +119,35 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { ctx.JSON(http.StatusOK, map[string]interface{}{ "JobID": jobID, "JobStatus": job.Status, - "JobDuration": job.Duration, + "JobDuration": job.TrainJobDuration, }) } +func addZero(t int64) (m string) { + if t < 10 { + m = "0" + strconv.FormatInt(t, 10) + return m + } else { + return strconv.FormatInt(t, 10) + } +} + func TrainJobGetLog(ctx *context.APIContext) { var ( err error ) - log.Info("test") - var jobID = ctx.Params(":jobid") var versionName = ctx.Query("version_name") - var logFileName = ctx.Query("file_name") + // var logFileName = ctx.Query("file_name") var baseLine = ctx.Query("base_line") var order = ctx.Query("order") + var lines = ctx.Query("lines") + lines_int, err := strconv.Atoi(lines) + if err != nil { + log.Error("change lines(%d) string to int failed", lines_int) + } if order != modelarts.OrderDesc && order != modelarts.OrderAsc { log.Error("order(%s) check failed", order) @@ -136,29 +157,192 @@ func TrainJobGetLog(ctx *context.APIContext) { return } - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + resultLogFile, result, err := trainJobGetLogContent(jobID, versionName, baseLine, order, lines_int) if err != nil { - log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", jobID, err.Error()) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err_msg": "GetCloudbrainByJobIDAndVersionName failed", - }) + log.Error("trainJobGetLog(%s) failed:%v", jobID, err.Error()) + // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) return } - result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines) + ctx.Data["log_file_name"] = resultLogFile.LogFileList[0] + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "LogFileName": resultLogFile.LogFileList[0], + "StartLine": result.StartLine, + "EndLine": result.EndLine, + "Content": result.Content, + "Lines": result.Lines, + }) +} + +func trainJobGetLogContent(jobID string, versionName string, baseLine string, order string, lines int) (*models.GetTrainJobLogFileNamesResult, *models.GetTrainJobLogResult, error) { + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) + return nil, nil, err + } + + resultLogFile, err := modelarts.GetTrainJobLogFileNames(jobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("GetTrainJobLogFileNames(%s) failed:%v", jobID, err.Error()) + return nil, nil, err + } + + result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, resultLogFile.LogFileList[0], order, lines) if err != nil { log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error()) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err_msg": "GetTrainJobLog failed", - }) + return nil, nil, err + } + + return resultLogFile, result, err +} + +func DelTrainJobVersion(ctx *context.APIContext) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + var versionName = ctx.Query("version_name") + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + ctx.NotFound(err) + return + } + + //删除modelarts上的记录 + _, err = modelarts.DelTrainJobVersion(jobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("DelTrainJobVersion(%s) failed:%v", task.JobName, err.Error()) + ctx.NotFound(err) + return + } + + //删除数据库记录 + err = models.DeleteJob(task) + if err != nil { + ctx.ServerError("DeleteJob failed", err) + ctx.NotFound(err) + return + } + + //获取删除后的版本数量 + repo := ctx.Repo.Repository + VersionTaskList, VersionListCount, err := models.CloudbrainsVersionList(&models.CloudbrainsOptions{ + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, + JobType: string(models.JobTypeTrain), + JobID: jobID, + }) + if err != nil { + ctx.ServerError("get VersionListCount faild", err) return } + // 判断当前删掉的任务是否是最新版本，若是，将排序后的TotalVersionCount置为删掉的最新版本的TotalVersionCount，若不是，按时间排序后的版本列表的第一个版本设置为最新版本，TotalVersionCount不变 + if task.IsLatestVersion == modelarts.IsLatestVersion { + err = models.SetVersionCountAndLatestVersion(jobID, VersionTaskList[0].Cloudbrain.VersionName, VersionListCount, modelarts.IsLatestVersion, task.TotalVersionCount) + if err != nil { + ctx.ServerError("UpdateJobVersionCount failed", err) + return + } + } else { + err = models.SetVersionCountAndLatestVersion(jobID, VersionTaskList[0].VersionName, VersionListCount, modelarts.IsLatestVersion, VersionTaskList[0].Cloudbrain.TotalVersionCount) + if err != nil { + ctx.ServerError("UpdateJobVersionCount failed", err) + return + } + } + ctx.JSON(http.StatusOK, map[string]interface{}{ - "JobID": jobID, - "StartLine": result.StartLine, - "EndLine": result.EndLine, - "Content": result.Content, - "Lines": result.Lines, + "JobID": jobID, + "VersionName": versionName, + "StatusOK": 0, + }) +} + +func StopTrainJobVersion(ctx *context.APIContext) { + var ( + err error + ) + var jobID = ctx.Params(":jobid") + var versionName = ctx.Query("version_name") + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + return + } + + _, err = modelarts.StopTrainJob(jobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("StopTrainJob(%s) failed:%v", task.JobName, err.Error()) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "VersionName": versionName, + "StatusOK": 0, + }) +} + +func ModelList(ctx *context.APIContext) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + var versionName = ctx.Query("version_name") + parentDir := ctx.Query("parentDir") + dirArray := strings.Split(parentDir, "/") + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + return + } + VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount) + parentDir = VersionOutputPath + "/" + parentDir + models, err := storage.GetObsListObject(task.JobName, parentDir) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + ctx.ServerError("GetObsListObject:", err) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "VersionName": versionName, + "StatusOK": 0, + "Path": dirArray, + "Dirs": models, + "task": task, + "PageIsCloudBrain": true, }) } + +func ModelDownload(ctx *context.APIContext) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + versionName := ctx.Query("version_name") + parentDir := ctx.Query("parent_dir") + fileName := ctx.Query("file_name") + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + return + } + VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount) + parentDir = VersionOutputPath + "/" + parentDir + url, err := storage.GetObsCreateSignedUrl(task.JobName, parentDir, fileName) + if err != nil { + log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrl", err) + return + } + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) +} diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 519077af2..8f7f03533 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -362,7 +362,7 @@ func generateRadarSql(beginTime time.Time, endTime time.Time, repoId int64) stri } func generateTargetSql(beginTime time.Time, endTime time.Time, repoId int64) string { - sql := "SELECT date, num_visits,num_downloads,num_commits FROM repo_statistic" + + sql := "SELECT date, num_visits,num_downloads_added as num_downloads,num_commits_added as num_commits FROM repo_statistic" + " where repo_id=" + strconv.FormatInt(repoId, 10) + " and created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " order by created_unix desc" diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 0a20cf576..a996524d2 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -34,7 +34,6 @@ const ( tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index" tplModelArtsTrainJobNew base.TplName = "repo/modelarts/trainjob/new" tplModelArtsTrainJobShow base.TplName = "repo/modelarts/trainjob/show" - tplModelArtsTrainJobShowModels base.TplName = "repo/modelarts/trainjob/models/index" tplModelArtsTrainJobVersionNew base.TplName = "repo/modelarts/trainjob/version_new" ) @@ -463,7 +462,7 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { ctx.Data["dataset_name"] = task.DatasetName ctx.Data["work_server_number"] = task.WorkServerNumber ctx.Data["flavor_name"] = task.FlavorName - ctx.Data["engine_name"] = task.FlavorName + ctx.Data["engine_name"] = task.EngineName ctx.Data["uuid"] = task.Uuid ctx.Data["flavor_code"] = task.FlavorCode ctx.Data["engine_id"] = task.EngineID @@ -480,6 +479,7 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) { ctx.Data["PageIsTrainJob"] = true + VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(modelarts.TotalVersionCount) jobName := form.JobName uuid := form.Attachment description := form.Description @@ -493,8 +493,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) repo := ctx.Repo.Repository codeLocalPath := setting.JobPath + jobName + modelarts.CodePath codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath - outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath - logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" branch_name := form.BranchName isLatestVersion := modelarts.IsLatestVersion @@ -527,7 +527,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) if err := git.Clone(repo.RepoPath(), codeLocalPath, git.CloneRepoOptions{ Branch: branch_name, }); err != nil { - log.Error("创建任务失败，任务名称已存在！: %s (%v)", repo.FullName(), err) + log.Error("创建任务失败，服务器超时！: %s (%v)", repo.FullName(), err) trainJobNewDataPrepare(ctx) ctx.Data["bootFile"] = form.BootFile @@ -536,21 +536,19 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) ctx.Data["params"] = form.Params ctx.Data["branch_name"] = branch_name trainJobNewDataPrepare(ctx) - // ctx.RenderWithErr("Failed to clone repository", tplModelArtsTrainJobNew, &form) - ctx.RenderWithErr("创建任务失败，任务名称已存在！", tplModelArtsTrainJobNew, &form) - // ctx.RenderWithErr(err, tplModelArtsTrainJobNew, &form) + ctx.RenderWithErr("创建任务失败，服务器超时！", tplModelArtsTrainJobNew, &form) return } //todo: upload code (send to file_server todo this work?) - if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) trainJobNewDataPrepare(ctx) ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobNew, &form) return } - if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) trainJobNewDataPrepare(ctx) ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobNew, &form) @@ -647,10 +645,10 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) IsLatestVersion: isLatestVersion, BranchName: branch_name, Params: form.Params, - FatherVersionName: modelarts.InitFatherVersionName, FlavorName: FlavorName, EngineName: EngineName, VersionCount: VersionCount, + TotalVersionCount: modelarts.TotalVersionCount, } err = modelarts.GenerateTrainJob(ctx, req) @@ -665,42 +663,19 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form) return } - // // 保存openi创建训练任务界面的参数 - // err = models.CreateTrainjobConfigDetail(&models.TrainjobConfigDetail{ - - // JobName: req.JobName, - // JobID: strconv.FormatInt(jobResult.JobID, 10), - // VersionName: jobResult.VersionName, - // ResourcePools: form.PoolID, - // EngineVersions: form.EngineID, - // FlavorInfos: form.Flavor, - // TrainUrl: outputObsPath, - // BootFile: form.BootFile, - // Uuid: form.Attachment, - // DatasetName: attach.Name, - // Params: form.Params, - // BranchName: branch_name, - // }) - - // if err != nil { - // log.Error("CreateTrainjobConfigDetail failed:%v", err.Error()) - // trainJobNewVersionDataPrepare(ctx) - // ctx.Data["bootFile"] = form.BootFile - // ctx.Data["uuid"] = form.Attachment - // ctx.Data["datasetName"] = attach.Name - // ctx.Data["params"] = form.Params - // ctx.Data["branch_name"] = branch_name - // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) - // return - // } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") } func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) { ctx.Data["PageIsTrainJob"] = true var jobID = ctx.Params(":jobid") - // var versionName = ctx.Params(":version-name") - var versionName = ctx.Query("version_name") + + latestTask, err := models.GetCloudbrainByJobIDAndIsLatestVersion(jobID, modelarts.IsLatestVersion) + if err != nil { + ctx.ServerError("GetCloudbrainByJobIDAndIsLatestVersion faild:", err) + return + } + VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(latestTask.TotalVersionCount + 1) jobName := form.JobName uuid := form.Attachment @@ -715,13 +690,14 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ repo := ctx.Repo.Repository codeLocalPath := setting.JobPath + jobName + modelarts.CodePath codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath - outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath - logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/" + logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/" dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" branch_name := form.BranchName - fatherVersionName := versionName + PreVersionName := form.VersionName FlavorName := form.FlavorName EngineName := form.EngineName + isLatestVersion := modelarts.IsLatestVersion if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) @@ -755,21 +731,19 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ ctx.Data["datasetName"] = attach.Name ctx.Data["params"] = form.Params ctx.Data["branch_name"] = branch_name - // ctx.RenderWithErr("Failed to clone repository", tplModelArtsTrainJobNew, &form) ctx.RenderWithErr("创建任务失败，任务名称已存在！", tplModelArtsTrainJobVersionNew, &form) - // ctx.RenderWithErr(err, tplModelArtsTrainJobNew, &form) return } //todo: upload code (send to file_server todo this work?) - if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) trainJobNewVersionDataPrepare(ctx) ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobVersionNew, &form) return } - if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath); err != nil { + if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) trainJobNewVersionDataPrepare(ctx) ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobVersionNew, &form) @@ -853,34 +827,39 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ return } - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, fatherVersionName) + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, PreVersionName) if err != nil { log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", jobID, err.Error()) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) return } - req := &modelarts.GenerateTrainJobVersionReq{ - JobName: task.JobName, - DataUrl: dataPath, - Description: description, - CodeObsPath: codeObsPath, - BootFileUrl: codeObsPath + bootFile, - BootFile: bootFile, - TrainUrl: outputObsPath, - FlavorCode: flavorCode, - WorkServerNumber: workServerNumber, - EngineID: int64(engineID), - LogUrl: logObsPath, - PoolID: poolID, - Uuid: uuid, - Params: form.Params, - PreVersionId: task.VersionID, - CommitID: commitID, - BranchName: branch_name, - FlavorName: FlavorName, - EngineName: EngineName, - } - err = modelarts.GenerateTrainJobVersion(ctx, req, jobID, fatherVersionName) + req := &modelarts.GenerateTrainJobReq{ + JobName: task.JobName, + DataUrl: dataPath, + Description: description, + CodeObsPath: codeObsPath, + BootFileUrl: codeObsPath + bootFile, + BootFile: bootFile, + TrainUrl: outputObsPath, + FlavorCode: flavorCode, + WorkServerNumber: workServerNumber, + IsLatestVersion: isLatestVersion, + EngineID: int64(engineID), + LogUrl: logObsPath, + PoolID: poolID, + Uuid: uuid, + Params: form.Params, + Parameters: parameters.Parameter, + PreVersionId: task.VersionID, + CommitID: commitID, + BranchName: branch_name, + FlavorName: FlavorName, + EngineName: EngineName, + PreVersionName: PreVersionName, + TotalVersionCount: latestTask.TotalVersionCount + 1, + } + + err = modelarts.GenerateTrainJobVersion(ctx, req, jobID) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) trainJobNewVersionDataPrepare(ctx) @@ -891,36 +870,8 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) return } - // 保存openi创建训练任务界面的参数 - // err = models.CreateTrainjobConfigDetail(&models.TrainjobConfigDetail{ - - // JobName: req.JobName, - // JobID: strconv.FormatInt(jobResult.JobID, 10), - // VersionName: jobResult.VersionName, - // ResourcePools: form.PoolID, - // EngineVersions: form.EngineID, - // FlavorInfos: form.Flavor, - // TrainUrl: outputObsPath, - // BootFile: form.BootFile, - // Uuid: form.Attachment, - // DatasetName: attach.Name, - // Params: form.Params, - // BranchName: branch_name, - // }) - - // if err != nil { - // log.Error("CreateTrainjobConfigDetail failed:%v", err.Error()) - // trainJobNewVersionDataPrepare(ctx) - // ctx.Data["bootFile"] = form.BootFile - // ctx.Data["uuid"] = form.Attachment - // ctx.Data["datasetName"] = attach.Name - // ctx.Data["params"] = form.Params - // ctx.Data["branch_name"] = branch_name - // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) - // return - // } - // ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") - ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") + // ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) } // readDir reads the directory named by dirname and returns @@ -1014,11 +965,6 @@ func TrainJobShow(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true var jobID = ctx.Params(":jobid") - task, err := models.GetCloudbrainByJobID(jobID) - if err != nil { - ctx.ServerError("GetCloudbrainByJobID faild", err) - return - } repo := ctx.Repo.Repository page := ctx.QueryInt("page") @@ -1035,75 +981,43 @@ func TrainJobShow(ctx *context.Context) { JobType: string(models.JobTypeTrain), JobID: jobID, }) - if err != nil { - ctx.ServerError("Cloudbrain", err) - return - } - - if err != nil { - log.Error("GetCloudbrainByJobID(%s) failed:%v", jobID, err.Error()) - ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) - return - } - // attach, err := models.GetAttachmentByUUID(task.Uuid) - // if err != nil { - // log.Error("GetAttachmentByUUID(%s) failed:%v", jobID, err.Error()) - // ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) - // return - // } - - result, err := modelarts.GetTrainJob(jobID, strconv.FormatInt(task.VersionID, 10)) if err != nil { - log.Error("GetJob(%s) failed:%v", jobID, err.Error()) + log.Error("GetVersionListTasks(%s) failed:%v", jobID, err.Error()) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) return } + //将运行参数转化为epoch_size = 3, device_target = Ascend的格式 + for i, _ := range VersionListTasks { - if result != nil { - result.CreateTime = time.Unix(int64(result.LongCreateTime/1000), 0).Format("2006-01-02 15:04:05") - if result.Duration != 0 { - result.TrainJobDuration = addZero(result.Duration/3600000) + ":" + addZero(result.Duration%3600000/60000) + ":" + addZero(result.Duration%60000/1000) + var parameters models.Parameters - } else { - result.TrainJobDuration = "00:00:00" - } - result.Status = modelarts.TransTrainJobStatus(result.IntStatus) - err = models.SetTrainJobStatusByJobID(jobID, result.Status, result.Duration, string(result.TrainJobDuration)) + err := json.Unmarshal([]byte(VersionListTasks[i].Parameters), ¶meters) if err != nil { - ctx.ServerError("UpdateJob failed", err) + log.Error("Failed to Unmarshal Parameters: %s (%v)", VersionListTasks[i].Parameters, err) + trainJobNewDataPrepare(ctx) return } - result.DatasetName = task.DatasetName - } - - resultLogFile, resultLog, err := trainJobGetLog(jobID) - if err != nil { - log.Error("trainJobGetLog(%s) failed:%v", jobID, err.Error()) - ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) - return + if len(parameters.Parameter) > 0 { + paramTemp := "" + for _, Parameter := range parameters.Parameter { + param := Parameter.Label + " = " + Parameter.Value + ", " + paramTemp = paramTemp + param + } + VersionListTasks[i].Parameters = paramTemp[:len(paramTemp)-2] + } else { + VersionListTasks[i].Parameters = "" + } } - ctx.Data["log_file_name"] = resultLogFile.LogFileList[0] - ctx.Data["log"] = resultLog - ctx.Data["task"] = task ctx.Data["jobID"] = jobID - ctx.Data["result"] = result + ctx.Data["jobName"] = VersionListTasks[0].JobName ctx.Data["version_list_task"] = VersionListTasks ctx.Data["version_list_count"] = VersionListCount ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) } -func addZero(t int64) (m string) { - if t < 10 { - m = "0" + strconv.FormatInt(t, 10) - return m - } else { - return strconv.FormatInt(t, 10) - } -} - func TrainJobGetLog(ctx *context.Context) { ctx.Data["PageIsTrainJob"] = true @@ -1160,24 +1074,34 @@ func trainJobGetLog(jobID string) (*models.GetTrainJobLogFileNamesResult, *model func TrainJobDel(ctx *context.Context) { var jobID = ctx.Params(":jobid") - task, err := models.GetCloudbrainByJobID(jobID) + repo := ctx.Repo.Repository + + VersionListTasks, _, err := models.CloudbrainsVersionList(&models.CloudbrainsOptions{ + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, + JobType: string(models.JobTypeTrain), + JobID: jobID, + }) if err != nil { - log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) - ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) + ctx.ServerError("get VersionListTasks failed", err) return } + //删除modelarts上的任务记录 _, err = modelarts.DelTrainJob(jobID) if err != nil { - log.Error("DelTrainJob(%s) failed:%v", task.JobName, err.Error()) + log.Error("DelTrainJob(%s) failed:%v", jobID, err.Error()) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) return } - err = models.DeleteJob(task) - if err != nil { - ctx.ServerError("DeleteJob failed", err) - return + //删除数据库Cloudbrain表的记录 + for _, task := range VersionListTasks { + err = models.DeleteJob(&task.Cloudbrain) + if err != nil { + ctx.ServerError("DeleteJob failed", err) + return + } } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") @@ -1202,54 +1126,6 @@ func TrainJobStop(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") } -func TrainJobVersionDel(ctx *context.Context) { - var jobID = ctx.Params(":jobid") - var versionName = ctx.Query(":versionName") - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - if err != nil { - log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) - ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) - return - } - - _, err = modelarts.DelTrainJob(jobID) - if err != nil { - log.Error("DelTrainJob(%s) failed:%v", task.JobName, err.Error()) - ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobShow, nil) - return - } - - err = models.DeleteJob(task) - if err != nil { - ctx.ServerError("DeleteJob failed", err) - return - } - - // ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") - ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) -} - -func TrainJobVersionStop(ctx *context.Context) { - var jobID = ctx.Params(":jobid") - var versionName = ctx.Query(":versionName") - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - if err != nil { - log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) - ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) - return - } - - _, err = modelarts.StopTrainJob(jobID, strconv.FormatInt(task.VersionID, 10)) - if err != nil { - log.Error("StopTrainJob(%s) failed:%v", task.JobName, err.Error()) - ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobIndex, nil) - return - } - - // ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job") - ctx.HTML(http.StatusOK, tplModelArtsTrainJobShow) -} - func canUserCreateTrainJob(uid int64) (bool, error) { org, err := models.GetOrgByName(setting.AllowedOrg) if err != nil { @@ -1313,74 +1189,3 @@ func getConfigList(perPage, page int, sortBy, order, searchContent, configType s return list, nil } - -func TrainJobShowModels(ctx *context.Context) { - ctx.Data["PageIsCloudBrain"] = true - - jobID := ctx.Params(":jobid") - parentDir := ctx.Query("parentDir") - - log.Info("parentDir=" + parentDir) - - dirArray := strings.Split(parentDir, "/") - task, err := models.GetCloudbrainByJobID(jobID) - if err != nil { - log.Error("no such job!", ctx.Data["msgID"]) - ctx.ServerError("no such job:", err) - return - } - - models, err := storage.GetObsListObject(task.JobName, parentDir) - if err != nil { - log.Info("get TrainJobListModel failed:", err) - ctx.ServerError("GetObsListObject:", err) - return - } - - ctx.Data["Path"] = dirArray - ctx.Data["Dirs"] = models - ctx.Data["task"] = task - ctx.Data["JobID"] = jobID - ctx.HTML(200, tplModelArtsTrainJobShowModels) -} - -func TrainJobVersionShowModels(ctx *context.Context) { - ctx.Data["PageIsCloudBrain"] = true - - jobID := ctx.Params(":jobid") - parentDir := ctx.Query("parentDir") - versionName := ctx.Query("version_name") - dirArray := strings.Split(parentDir, "/") - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - if err != nil { - log.Error("no such job!", ctx.Data["msgID"]) - ctx.ServerError("no such job:", err) - return - } - parentDir = versionName - models, err := storage.GetVersionObsListObject(task.JobName, parentDir) - if err != nil { - log.Info("get TrainJobListModel failed:", err) - ctx.ServerError("GetVersionObsListObject:", err) - return - } - - ctx.Data["Path"] = dirArray - ctx.Data["Dirs"] = models - ctx.Data["task"] = task - ctx.Data["JobID"] = jobID - ctx.HTML(200, tplModelArtsTrainJobShowModels) -} - -func TrainJobDownloadModel(ctx *context.Context) { - parentDir := ctx.Query("parentDir") - fileName := ctx.Query("fileName") - jobName := ctx.Query("jobName") - url, err := storage.GetObsCreateSignedUrl(jobName, parentDir, fileName) - if err != nil { - log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) - ctx.ServerError("GetObsCreateSignedUrl", err) - return - } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) -} diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 26ca3ae8f..8d7f72ce0 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -83,10 +83,9 @@ func QueryUserStaticDataPage(ctx *context.Context) { if IsReturnFile { //writer exec file. xlsx := excelize.NewFile() - xlsx.DeleteSheet("Sheet1") sheetName := ctx.Tr("user.static.sheetname") index := xlsx.NewSheet(sheetName) - + xlsx.DeleteSheet("Sheet1") dataHeader := map[string]string{ "A1": ctx.Tr("user.static.id"), "B1": ctx.Tr("user.static.name"), diff --git a/routers/routes/routes.go b/routers/routes/routes.go index d4384bb60..5a066c92f 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1000,23 +1000,12 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow) m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) - m.Get("/log", reqRepoCloudBrainReader, repo.TrainJobGetLog) - m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobShowModels) - m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobDownloadModel) - m.Get("/version_models", reqRepoCloudBrainReader, repo.TrainJobVersionShowModels) - // m.Group("/:version-name", func() { m.Get("/create_version", reqRepoCloudBrainReader, repo.TrainJobNewVersion) m.Post("/create_version", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) - // }) - m.Post("/stop_version", reqRepoCloudBrainWriter, repo.TrainJobVersionStop) - m.Post("/del_version", reqRepoCloudBrainWriter, repo.TrainJobVersionDel) }) m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) - // m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNewVersion) - // m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) - m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList) }) }, context.RepoRef()) diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl index 842f629c9..8cec8f5d2 100755 --- a/templates/repo/cloudbrain/show.tmpl +++ b/templates/repo/cloudbrain/show.tmpl @@ -6,7 +6,19 @@ {{template "base/alert" .}}

- 返回 +
+ + {{.i18n.Tr "repo.cloudbrain"}} + +
/
+ + {{$.i18n.Tr "repo.modelarts.notebook"}} + +
/
+ {{with .task}} +
{{.JobName}}
+ {{end}} +

diff --git a/templates/repo/modelarts/notebook/show.tmpl b/templates/repo/modelarts/notebook/show.tmpl index 3f914b56d..cac87df79 100755 --- a/templates/repo/modelarts/notebook/show.tmpl +++ b/templates/repo/modelarts/notebook/show.tmpl @@ -6,7 +6,19 @@ {{template "base/alert" .}}

- 返回 +
+ + {{.i18n.Tr "repo.cloudbrain"}} + +
/
+ + {{$.i18n.Tr "repo.modelarts.notebook"}} + +
/
+ {{with .task}} +
{{.JobName}}
+ {{end}} +

diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 4a8664082..6c989adef 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -333,7 +333,7 @@
- + {{.Status}}
@@ -381,12 +381,12 @@ {{end}}
-
- + +
{{$.CsrfTokenHtml}} @@ -442,6 +442,8 @@ {{template "base/footer" .}} \ No newline at end of file diff --git a/templates/repo/modelarts/trainjob/version_new.tmpl b/templates/repo/modelarts/trainjob/version_new.tmpl index 46fc3f678..442f10a1a 100644 --- a/templates/repo/modelarts/trainjob/version_new.tmpl +++ b/templates/repo/modelarts/trainjob/version_new.tmpl @@ -156,15 +156,24 @@ {{.CsrfTokenHtml}} + + +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

+
+ + + +
+
- +
@@ -198,7 +207,7 @@
-
+
{{if .flavor_name}} @@ -331,9 +340,12 @@