gen command

modules/grampus/grampus.go

@@ -18,7 +18,7 @@ const (
 	OutputPath       = "/output/"
 	ResultPath       = "/result/"
 	LogPath          = "/log/"
-	JobPath          = "/job/"
+	JobPath          = "job/"
 	OrderDesc        = "desc" //向下查询
 	OrderAsc         = "asc"  //向上查询
 	Lines            = 500
@@ -40,8 +40,8 @@ const (
 	ProcessorTypeNPU = "npu.huawei.com/NPU"
 	ProcessorTypeGPU = "nvidia.com/gpu"
 
-	CommandPrepareScript        = "pwd;cd /tmp;wget https://git.openi.org.cn/lewis/script_for_grampus/archive/master.zip;unzip master.zip;cd script_for_grampus;"
-	ScriptSyncObsCodeAndDataset = "sync_obs_code_and_dataset.py"
+	CommandPrepareScript = "pwd;cd /tmp;wget https://git.openi.org.cn/lewis/script_for_grampus/archive/master.zip;unzip master.zip;cd script_for_grampus;chmod 777 sync_for_arm;"
+	CodeArchiveName      = "master.zip"
 )
 
 var (

routers/repo/grampus.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/unknwon/com"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -224,8 +225,8 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 	params := form.Params
 	repo := ctx.Repo.Repository
 	codeLocalPath := setting.JobPath + jobName + modelarts.CodePath
-	codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath
-	dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
+	codeObsPath := grampus.JobPath + jobName + modelarts.CodePath
+	dataObsPath := setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + "/"
 	branchName := form.BranchName
 	isLatestVersion := modelarts.IsLatestVersion
 	flavorName := form.FlavorName
@@ -289,8 +290,8 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		os.RemoveAll(codeLocalPath)
 	}
 
-	if err := downloadCode(repo, codeLocalPath, branchName); err != nil {
-		log.Error("downloadCode failed, server timed out: %s (%v)", repo.FullName(), err)
+	if err := downloadZipCode(ctx, codeLocalPath, branchName); err != nil {
+		log.Error("downloadZipCode failed, server timed out: %s (%v)", repo.FullName(), err)
 		grampusTrainJobNpuNewDataPrepare(ctx)
 		ctx.RenderWithErr("Create task failed, server timed out", tplGrampusTrainJobNPUNew, &form)
 		return
@@ -320,33 +321,10 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 
 	//prepare command
 	//todo: download code, download dataset, unzip dataset, exec code, upload model
-	command, err := generateCommand(grampus.ProcessorTypeNPU, "obs:/"+codeObsPath, "obs:/"+dataPath, params, "", attachment.Name)
+	command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", dataObsPath+attachment.Name, bootFile, params, "", attachment.Name)
 	log.Info(command)
-	var parameters models.Parameters
-	param := make([]models.Parameter, 0)
-	if len(params) != 0 {
-		err := json.Unmarshal([]byte(params), &parameters)
-		if err != nil {
-			log.Error("Failed to Unmarshal params: %s (%v)", params, err)
-			grampusTrainJobNpuNewDataPrepare(ctx)
-			ctx.RenderWithErr("运行参数错误", tplGrampusTrainJobNPUNew, &form)
-			return
-		}
 
-		for _, parameter := range parameters.Parameter {
-			param = append(param, models.Parameter{
-				Label: parameter.Label,
-				Value: parameter.Value,
-			})
-		}
-	}
-	param = append(param, models.Parameter{
-		Label: modelarts.DeviceTarget,
-		Value: modelarts.Ascend,
-	})
-
-	gitRepo, _ := git.OpenRepository(repo.RepoPath())
-	commitID, _ := gitRepo.GetBranchCommitID(branchName)
+	commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)
 
 	req := &grampus.GenerateTrainJobReq{
 		JobName:           jobName,
@@ -356,7 +334,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		ResourceSpecId:    form.FlavorID,
 		ImageUrl:          "",
 		ImageId:           form.ImageID,
-		DataUrl:           dataPath,
+		DataUrl:           dataObsPath,
 		Description:       description,
 		CodeObsPath:       codeObsPath,
 		BootFileUrl:       codeObsPath + bootFile,
@@ -567,21 +545,47 @@ func GrampusGetLog(ctx *context.Context) {
 	return
 }
 
-func generateCommand(processorType, codeObsPath, dataObsPath, params, outputPath, datasetName string) (string, error) {
+func generateCommand(repoName, processorType, codeObsPath, dataObsPath, bootFile, paramSrc, outputPath, datasetName string) (string, error) {
 	var command string
 
 	command += grampus.CommandPrepareScript
 	//download code & dataset
 	if processorType == grampus.ProcessorTypeNPU {
-		commandDownload := "python " + grampus.ScriptSyncObsCodeAndDataset + " --access_key=" + setting.AccessKeyID + " --secret_key=" + setting.SecretAccessKey + " --project_id=" + setting.ProjectID + " --region_name=" + setting.Location + " --code_obs_dir=" + codeObsPath + " --data_obs_dir=" + dataObsPath + " --dataset_name=" + datasetName + ";"
+		commandDownload := "./sync_for_arm " + setting.Bucket + " " + codeObsPath + " " + grampus.CodeArchiveName + " " + dataObsPath + " " + datasetName + ";"
 		command += commandDownload
 	} else if processorType == grampus.ProcessorTypeGPU {
 
 	}
 
-	//unzip dataset
+	//unzip code & dataset
+	commandUnzip := "cd dataset;unzip " + datasetName + ";cd ../code;unzip master.zip;"
+	command += commandUnzip
+
 	//exec code
+	var parameters models.Parameters
+	var paramCode string
+	param := make([]models.Parameter, 0)
+	if len(paramSrc) != 0 {
+		err := json.Unmarshal([]byte(paramSrc), &parameters)
+		if err != nil {
+			log.Error("Failed to Unmarshal params: %s (%v)", paramSrc, err)
+			return command, err
+		}
+
+		for _, parameter := range parameters.Parameter {
+			param = append(param, models.Parameter{
+				Label: parameter.Label,
+				Value: parameter.Value,
+			})
+			paramCode += " --" + parameter.Label + "=" + parameter.Value
+		}
+	}
+
+	commandCode := "cd " + repoName + ";python " + bootFile + paramCode
+	command += commandCode
+
 	//upload models
+
 	return command, nil
 }
 
@@ -600,3 +604,48 @@ func generateCommandObsDownloadFile(srcObsFile, dstLocalDir string) (string, err
 
 	return command, nil
 }
+
+func downloadZipCode(ctx *context.Context, codePath, branchName string) error {
+	archiveType := git.ZIP
+	archivePath := codePath
+
+	if !com.IsDir(archivePath) {
+		if err := os.MkdirAll(archivePath, os.ModePerm); err != nil {
+			log.Error("MkdirAll failed:" + err.Error())
+			return err
+		}
+	}
+
+	// Get corresponding commit.
+	var (
+		commit *git.Commit
+		err    error
+	)
+
+	gitRepo := ctx.Repo.GitRepo
+	if err != nil {
+		log.Error("OpenRepository failed:" + err.Error())
+		return err
+	}
+
+	if gitRepo.IsBranchExist(branchName) {
+		commit, err = gitRepo.GetBranchCommit(branchName)
+		if err != nil {
+			log.Error("GetBranchCommit failed:" + err.Error())
+			return err
+		}
+	}
+
+	archivePath = path.Join(archivePath, grampus.CodeArchiveName)
+	if !com.IsFile(archivePath) {
+		if err := commit.CreateArchive(archivePath, git.CreateArchiveOpts{
+			Format: archiveType,
+			Prefix: setting.Repository.PrefixArchiveFiles,
+		}); err != nil {
+			log.Error("CreateArchive failed:" + err.Error())
+			return err
+		}
+	}
+
+	return nil
+}