update

models/cloudbrain.go

@@ -59,8 +59,8 @@ type Cloudbrain struct {
 	SubTaskName      string
 	ContainerID      string
 	ContainerIp      string
-	CreatedUnix      timeutil.TimeStamp
-	UpdatedUnix      timeutil.TimeStamp
+	CreatedUnix      timeutil.TimeStamp `xorm:"INDEX created"`
+	UpdatedUnix      timeutil.TimeStamp `xorm:"INDEX updated"`
 	Duration         int64
 	TrainJobDuration string
 	DeletedAt        time.Time `xorm:"deleted"`

modules/modelarts/modelarts.go

@@ -2,6 +2,7 @@ package modelarts
 
 import (
 	"encoding/json"
+	"fmt"
 	"path"
 	"strconv"
 
@@ -258,22 +259,21 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error
 	}
 
 	err = models.CreateCloudbrain(&models.Cloudbrain{
-		Status:          TransTrainJobStatus(jobResult.Status),
-		UserID:          ctx.User.ID,
-		RepoID:          ctx.Repo.Repository.ID,
-		JobID:           strconv.FormatInt(jobResult.JobID, 10),
-		JobName:         req.JobName,
-		JobType:         string(models.JobTypeTrain),
-		Type:            models.TypeCloudBrainTwo,
-		VersionID:       jobResult.VersionID,
-		VersionName:     jobResult.VersionName,
-		Uuid:            req.Uuid,
-		DatasetName:     attach.Name,
-		CommitID:        req.CommitID,
-		IsLatestVersion: req.IsLatestVersion,
-		ComputeResource: ComputeResource,
-		EngineID:        req.EngineID,
-		// FatherVersionName: req.FatherVersionName,
+		Status:            TransTrainJobStatus(jobResult.Status),
+		UserID:            ctx.User.ID,
+		RepoID:            ctx.Repo.Repository.ID,
+		JobID:             strconv.FormatInt(jobResult.JobID, 10),
+		JobName:           req.JobName,
+		JobType:           string(models.JobTypeTrain),
+		Type:              models.TypeCloudBrainTwo,
+		VersionID:         jobResult.VersionID,
+		VersionName:       jobResult.VersionName,
+		Uuid:              req.Uuid,
+		DatasetName:       attach.Name,
+		CommitID:          req.CommitID,
+		IsLatestVersion:   req.IsLatestVersion,
+		ComputeResource:   ComputeResource,
+		EngineID:          req.EngineID,
 		TrainUrl:          req.TrainUrl,
 		BranchName:        req.BranchName,
 		Parameters:        req.Params,
@@ -340,7 +340,7 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobVersionR
 		Uuid:              req.Uuid,
 		DatasetName:       attach.Name,
 		CommitID:          req.CommitID,
-		FatherVersionName: req.FatherVersionName,
+		PreVersionName:    req.PreVersionName,
 		ComputeResource:   ComputeResource,
 		EngineID:          req.EngineID,
 		TrainUrl:          req.TrainUrl,
@@ -454,6 +454,10 @@ func TransTrainJobStatus(status int) string {
 	default:
 		return strconv.Itoa(status)
 	}
+}
 
-	return ""
+func GetVersionOutputPathByTotalVersionCount(TotalVersionCount int) (VersionOutputPath string) {
+	talVersionCountToString := fmt.Sprintf("%04d", TotalVersionCount)
+	VersionOutputPath = "V" + talVersionCountToString
+	return VersionOutputPath
 }

modules/storage/obs.go

@@ -235,65 +235,6 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) {
 	}
 }
 
-func GetObsListObjectVersion(jobName, parentDir string, VersionOutputPath string) ([]FileInfo, error) {
-	input := &obs.ListObjectsInput{}
-	input.Bucket = setting.Bucket
-	input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionOutputPath, parentDir), "/")
-	strPrefix := strings.Split(input.Prefix, "/")
-	output, err := ObsCli.ListObjects(input)
-	fileInfos := make([]FileInfo, 0)
-	if err == nil {
-		for _, val := range output.Contents {
-			str1 := strings.Split(val.Key, "/")
-			var isDir bool
-			var fileName, nextParentDir string
-			if strings.HasSuffix(val.Key, "/") {
-				//dirs in next level dir
-				if len(str1)-len(strPrefix) > 2 {
-					continue
-				}
-				fileName = str1[len(str1)-2]
-				isDir = true
-				if parentDir == "" {
-					nextParentDir = fileName
-				} else {
-					nextParentDir = parentDir + "/" + fileName
-				}
-
-				if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath {
-					continue
-				}
-			} else {
-				//files in next level dir
-				if len(str1)-len(strPrefix) > 1 {
-					continue
-				}
-				fileName = str1[len(str1)-1]
-				isDir = false
-				nextParentDir = parentDir
-			}
-
-			fileInfo := FileInfo{
-				ModTime:  val.LastModified.Local().Format("2006-01-02 15:04:05"),
-				FileName: fileName,
-				Size:     val.Size,
-				IsDir:    isDir,
-				ParenDir: nextParentDir,
-			}
-			fileInfos = append(fileInfos, fileInfo)
-		}
-		sort.Slice(fileInfos, func(i, j int) bool {
-			return fileInfos[i].ModTime > fileInfos[j].ModTime
-		})
-		return fileInfos, err
-	} else {
-		if obsError, ok := err.(obs.ObsError); ok {
-			log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
-		}
-		return nil, err
-	}
-}
-
 func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) {
 
 	input := &obs.CreateSignedUrlInput{}

routers/api/v1/repo/modelarts.go

@@ -314,8 +314,9 @@ func ModelList(ctx *context.APIContext) {
 		log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
 		return
 	}
-	VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount)
-	models, err := storage.GetObsListObjectVersion(task.JobName, parentDir, VersionOutputPath)
+	VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount)
+	parentDir = VersionOutputPath + "/" + parentDir
+	models, err := storage.GetObsListObject(task.JobName, parentDir)
 	if err != nil {
 		log.Info("get TrainJobListModel failed:", err)
 		ctx.ServerError("GetObsListObject:", err)
@@ -347,9 +348,9 @@ func ModelDownload(ctx *context.APIContext) {
 		log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
 		return
 	}
-	VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount)
-
-	url, err := storage.GetObsCreateVersionSignedUrl(task.JobName, parentDir, fileName, VersionOutputPath)
+	VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(task.TotalVersionCount)
+	parentDir = VersionOutputPath + "/" + parentDir
+	url, err := storage.GetObsCreateSignedUrl(task.JobName, parentDir, fileName)
 	if err != nil {
 		log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
 		ctx.ServerError("GetObsCreateSignedUrl", err)

routers/repo/modelarts.go

@@ -3,7 +3,6 @@ package repo
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -289,17 +288,6 @@ func TrainJobIndex(ctx *context.Context) {
 		return
 	}
 
-	// for i, task := range tasks {
-	// 	result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10))
-	// 	if err != nil {
-	// 		log.Error("GetJob(%s) failed:%v", task.JobID, err.Error())
-	// 		return
-	// 	}
-	// 	// tasks[i].Status = modelarts.TransTrainJobStatus(result.Status)
-	// 	tasks[i].Status = result.Status
-	// 	tasks[i].Duration = result.Duration
-	// }
-
 	pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)
 	pager.SetDefaultParams(ctx)
 	ctx.Data["Page"] = pager
@@ -492,8 +480,9 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error {
 
 func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) {
 	ctx.Data["PageIsTrainJob"] = true
-	StringTotalVersionCount := fmt.Sprintf("%04d", modelarts.TotalVersionCount)
-	VersionOutputPath := "V" + StringTotalVersionCount
+	VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(modelarts.TotalVersionCount)
+	// StringTotalVersionCount := fmt.Sprintf("%04d", modelarts.TotalVersionCount)
+	// VersionOutputPath := "V" + StringTotalVersionCount
 	jobName := form.JobName
 	uuid := form.Attachment
 	description := form.Description
@@ -644,25 +633,24 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 	}
 
 	req := &modelarts.GenerateTrainJobReq{
-		JobName:          jobName,
-		DataUrl:          dataPath,
-		Description:      description,
-		CodeObsPath:      codeObsPath,
-		BootFileUrl:      codeObsPath + bootFile,
-		BootFile:         bootFile,
-		TrainUrl:         outputObsPath,
-		FlavorCode:       flavorCode,
-		WorkServerNumber: workServerNumber,
-		EngineID:         int64(engineID),
-		LogUrl:           logObsPath,
-		PoolID:           poolID,
-		Uuid:             uuid,
-		Parameters:       parameters.Parameter,
-		CommitID:         commitID,
-		IsLatestVersion:  isLatestVersion,
-		BranchName:       branch_name,
-		Params:           form.Params,
-		// FatherVersionName: InitVersionName,
+		JobName:           jobName,
+		DataUrl:           dataPath,
+		Description:       description,
+		CodeObsPath:       codeObsPath,
+		BootFileUrl:       codeObsPath + bootFile,
+		BootFile:          bootFile,
+		TrainUrl:          outputObsPath,
+		FlavorCode:        flavorCode,
+		WorkServerNumber:  workServerNumber,
+		EngineID:          int64(engineID),
+		LogUrl:            logObsPath,
+		PoolID:            poolID,
+		Uuid:              uuid,
+		Parameters:        parameters.Parameter,
+		CommitID:          commitID,
+		IsLatestVersion:   isLatestVersion,
+		BranchName:        branch_name,
+		Params:            form.Params,
 		FlavorName:        FlavorName,
 		EngineName:        EngineName,
 		VersionCount:      VersionCount,
@@ -693,8 +681,9 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 		ctx.ServerError("GetCloudbrainByJobIDAndIsLatestVersion faild:", err)
 		return
 	}
-	StringTotalVersionCount := fmt.Sprintf("%04d", latestTask.TotalVersionCount+1)
-	VersionOutputPath := "V" + StringTotalVersionCount
+	// StringTotalVersionCount := fmt.Sprintf("%04d", latestTask.TotalVersionCount+1)
+	// VersionOutputPath := "V" + StringTotalVersionCount
+	VersionOutputPath := modelarts.GetVersionOutputPathByTotalVersionCount(latestTask.TotalVersionCount + 1)
 
 	jobName := form.JobName
 	uuid := form.Attachment
@@ -1245,37 +1234,6 @@ func TrainJobShowModels(ctx *context.Context) {
 	ctx.HTML(200, tplModelArtsTrainJobShowModels)
 }
 
-func TrainJobVersionShowModels(ctx *context.Context) {
-	ctx.Data["PageIsCloudBrain"] = true
-
-	jobID := ctx.Params(":jobid")
-	parentDir := ctx.Query("parentDir")
-	versionName := ctx.Query("version_name")
-	// dirArray := strings.Split(parentDir, "/")
-	task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
-	if err != nil {
-		log.Error("no such job!", ctx.Data["msgID"])
-		ctx.ServerError("no such job:", err)
-		return
-	}
-	// parentDir = versionName
-	VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount)
-	dirArray := strings.Split(VersionOutputPath, "/")
-	models, err := storage.GetObsListObjectVersion(task.JobName, parentDir, VersionOutputPath)
-	if err != nil {
-		log.Info("get TrainJobListModel failed:", err)
-		ctx.ServerError("GetVersionObsListObject:", err)
-		return
-	}
-
-	ctx.Data["Path"] = dirArray
-	// ctx.Data["Path"] = VersionOutputPath
-	ctx.Data["Dirs"] = models
-	ctx.Data["task"] = task
-	ctx.Data["JobID"] = jobID
-	ctx.HTML(200, tplModelArtsTrainJobShowModels)
-}
-
 func TrainJobDownloadModel(ctx *context.Context) {
 	parentDir := ctx.Query("parentDir")
 	fileName := ctx.Query("fileName")
@@ -1288,26 +1246,3 @@ func TrainJobDownloadModel(ctx *context.Context) {
 	}
 	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 }
-
-func TrainJobVersionDownloadModel(ctx *context.Context) {
-	var jobID = ctx.Params(":jobid")
-
-	parentDir := ctx.Query("parentDir")
-	fileName := ctx.Query("fileName")
-	versionName := ctx.Query("version_name")
-
-	task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
-	if err != nil {
-		log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
-		return
-	}
-	VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount)
-
-	url, err := storage.GetObsCreateVersionSignedUrl(task.JobName, parentDir, fileName, VersionOutputPath)
-	if err != nil {
-		log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
-		ctx.ServerError("GetObsCreateSignedUrl", err)
-		return
-	}
-	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
-}

routers/routes/routes.go

@@ -989,8 +989,6 @@ func RegisterRoutes(m *macaron.Macaron) {
 					m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow)
 					m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop)
 					m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel)
-					m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobVersionShowModels)
-					m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobVersionDownloadModel)
 					m.Get("/create_version", reqRepoCloudBrainReader, repo.TrainJobNewVersion)
 					m.Post("/create_version", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion)
 				})

templates/repo/modelarts/trainjob/show.tmpl

@@ -194,9 +194,9 @@ td, th {
                                 </div>
                                 <div class="ac-display-inblock title_text acc-margin-bottom">
                                     
-                                    <span class="cti-mgRight-sm">2021/11/08 19:35:19</span>
+                                    <span class="cti-mgRight-sm">{{.Cloudbrain.CreatedUnix}}</span>   
                                     <span class="cti-mgRight-sm"> 当前版本：{{.VersionName}}</span>
-                                    <span class="cti-mgRight-sm">  父版本：{{.FatherVersionName}}</span>
+                                    <span class="cti-mgRight-sm">  父版本：{{.PreVersionName}}</span>
                                     <span class="cti-mgRight-sm ac-text-normal title_text">状态：
                                         <span id="{{.VersionName}}-status-span"><i id="icon" style="vertical-align: middle;" class="{{.Status}}"></i><span id="text" style="margin-left: 0.4em;font-size: 12px;">{{.Status}}</span></span>
                                     </span>