generate command

modules/grampus/grampus.go

@@ -10,17 +10,11 @@ import (
 )
 
 const (
-	//notebook
-	storageTypeOBS     = "obs"
-	autoStopDuration   = 4 * 60 * 60
-	autoStopDurationMs = 4 * 60 * 60 * 1000
-
-	DataSetMountPath = "/home/ma-user/work"
-	NotebookEnv      = "Python3"
-	NotebookType     = "Ascend"
-	FlavorInfo       = "Ascend: 1*Ascend 910 CPU: 24 核 96GiB （modelarts.kat1.xlarge）"
+	storageTypeOBS = "obs"
 
+	WorkPath         = "/home/ma-user/work"
 	CodePath         = "/code/"
+	DatasetPath      = "/dataset"
 	OutputPath       = "/output/"
 	ResultPath       = "/result/"
 	LogPath          = "/log/"
@@ -45,6 +39,9 @@ const (
 
 	ProcessorTypeNPU = "npu.huawei.com/NPU"
 	ProcessorTypeGPU = "nvidia.com/gpu"
+
+	CommandPrepareScript        = "pwd;cd /tmp;wget https://git.openi.org.cn/lewis/script_for_grampus/archive/master.zip;unzip master.zip;cd script_for_grampus;"
+	ScriptSyncObsCodeAndDataset = "sync_obs_code_and_dataset.py"
 )
 
 var (

modules/util/path.go

@@ -31,3 +31,13 @@ func GetDirectorySize(path string) (int64, error) {
 	})
 	return size, err
 }
+
+// check whether the path is dir
+func IsDir(path string) bool {
+	s, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+
+	return s.IsDir()
+}

routers/repo/grampus.go

@@ -9,6 +9,7 @@ import (
 	"code.gitea.io/gitea/modules/util"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -273,6 +274,15 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		}
 	}
 
+	//check dataset
+	attachment, err := models.GetAttachmentByUUID(uuid)
+	if err != nil {
+		log.Error("GetAttachmentByUUID failed:", err.Error(), ctx.Data["MsgID"])
+		grampusTrainJobNpuNewDataPrepare(ctx)
+		ctx.RenderWithErr("dataset is not exist", tplGrampusTrainJobNPUNew, &form)
+		return
+	}
+
 	//prepare code and out path
 	_, err = ioutil.ReadDir(codeLocalPath)
 	if err == nil {
@@ -310,7 +320,8 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 
 	//prepare command
 	//todo: download code, download dataset, unzip dataset, exec code, upload model
-	command, err := generateCommand(grampus.ProcessorTypeNPU, codeObsPath, dataPath, params, "")
+	command, err := generateCommand(grampus.ProcessorTypeNPU, "obs:/"+codeObsPath, "obs:/"+dataPath, params, "", attachment.Name)
+	log.Info(command)
 	var parameters models.Parameters
 	param := make([]models.Parameter, 0)
 	if len(params) != 0 {
@@ -360,6 +371,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		EngineName:        engineName,
 		VersionCount:      versionCount,
 		TotalVersionCount: modelarts.TotalVersionCount,
+		DatasetName:       attachment.Name,
 	}
 
 	err = grampus.GenerateTrainJob(ctx, req)
@@ -474,11 +486,6 @@ func GrampusTrainJobShow(ctx *context.Context) {
 		return
 	}
 
-	attachment, err := models.GetAttachmentByUUID(task.Uuid)
-	if err == nil {
-		task.DatasetName = attachment.Name
-	}
-
 	if task.DeletedAt.IsZero() { //normal record
 		result, err := grampus.GetJob(task.JobID)
 		if err != nil {
@@ -524,6 +531,8 @@ func GrampusTrainJobShow(ctx *context.Context) {
 				paramTemp = paramTemp + param
 			}
 			task.Parameters = paramTemp[:len(paramTemp)-2]
+		} else {
+			task.Parameters = ""
 		}
 	}
 
@@ -558,12 +567,36 @@ func GrampusGetLog(ctx *context.Context) {
 	return
 }
 
-func generateCommand(processorType, codePath, dataPath, params, outputPath string) (string, error) {
+func generateCommand(processorType, codeObsPath, dataObsPath, params, outputPath, datasetName string) (string, error) {
 	var command string
-	//download code
-	//download dataset
+
+	command += grampus.CommandPrepareScript
+	//download code & dataset
+	if processorType == grampus.ProcessorTypeNPU {
+		commandDownload := "python " + grampus.ScriptSyncObsCodeAndDataset + " --access_key=" + setting.AccessKeyID + " --secret_key=" + setting.SecretAccessKey + " --project_id=" + setting.ProjectID + " --region_name=" + setting.Location + " --code_obs_dir=" + codeObsPath + " --data_obs_dir=" + dataObsPath + " --dataset_name=" + datasetName + ";"
+		command += commandDownload
+	} else if processorType == grampus.ProcessorTypeGPU {
+
+	}
+
 	//unzip dataset
 	//exec code
 	//upload models
 	return command, nil
 }
+
+func generateCommandObsDownloadFile(srcObsFile, dstLocalDir string) (string, error) {
+	var command string
+
+	command = "python;"
+	command += "from modelarts.session import Session \n"
+	command += fmt.Sprintf("session = Session(access_key='%s',secret_key='%s', project_id='%s', region_name='%s') \n", setting.AccessKeyID, setting.SecretAccessKey, setting.ProjectID, setting.Location)
+
+	if util.IsDir(srcObsFile) {
+		command += fmt.Sprintf("session.obs.download_dir(src_obs_dir=\"%s\", dst_local_dir=\"%s\") \n", srcObsFile, dstLocalDir)
+	} else {
+		command += fmt.Sprintf("session.obs.download_file(src_obs_file=\"%s\", dst_local_dir=\"%s\") \n", srcObsFile, dstLocalDir)
+	}
+
+	return command, nil
+}