Merge remote-tracking branch 'origin/V20221116' into fix-csh

modules/storage/obs.go

@@ -367,51 +367,58 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative
 	if !strings.HasSuffix(input.Prefix, "/") {
 		input.Prefix += "/"
 	}
-	output, err := ObsCli.ListObjects(input)
 	fileInfos := make([]FileInfo, 0)
 	prefixLen := len(input.Prefix)
 	fileMap := make(map[string]bool, 0)
-	if err == nil {
-		for _, val := range output.Contents {
-			log.Info("val key=" + val.Key)
-			var isDir bool
-			var fileName string
-			if val.Key == input.Prefix {
-				continue
-			}
-			fileName = val.Key[prefixLen:]
-			log.Info("fileName =" + fileName)
-			files := strings.Split(fileName, "/")
-			if fileMap[files[0]] {
-				continue
-			} else {
-				fileMap[files[0]] = true
+	index := 1
+	for {
+		output, err := ObsCli.ListObjects(input)
+		if err == nil {
+			log.Info("Page:%d\n", index)
+			index++
+			for _, val := range output.Contents {
+				var isDir bool
+				var fileName string
+				if val.Key == input.Prefix {
+					continue
+				}
+				fileName = val.Key[prefixLen:]
+				files := strings.Split(fileName, "/")
+				if fileMap[files[0]] {
+					continue
+				} else {
+					fileMap[files[0]] = true
+				}
+				ParenDir := relativePath
+				fileName = files[0]
+				if len(files) > 1 {
+					isDir = true
+					ParenDir += fileName + "/"
+				} else {
+					isDir = false
+				}
+				fileInfo := FileInfo{
+					ModTime:  val.LastModified.Local().Format("2006-01-02 15:04:05"),
+					FileName: fileName,
+					Size:     val.Size,
+					IsDir:    isDir,
+					ParenDir: ParenDir,
+				}
+				fileInfos = append(fileInfos, fileInfo)
 			}
-			ParenDir := relativePath
-			fileName = files[0]
-			if len(files) > 1 {
-				isDir = true
-				ParenDir += fileName + "/"
+			if output.IsTruncated {
+				input.Marker = output.NextMarker
 			} else {
-				isDir = false
+				break
 			}
-			fileInfo := FileInfo{
-				ModTime:  val.LastModified.Local().Format("2006-01-02 15:04:05"),
-				FileName: fileName,
-				Size:     val.Size,
-				IsDir:    isDir,
-				ParenDir: ParenDir,
+		} else {
+			if obsError, ok := err.(obs.ObsError); ok {
+				log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
 			}
-			fileInfos = append(fileInfos, fileInfo)
-		}
-		return fileInfos, err
-	} else {
-		if obsError, ok := err.(obs.ObsError); ok {
-			log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
+			return nil, err
 		}
-		return nil, err
 	}
-
+	return fileInfos, nil
 }
 
 func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) {

options/locale/locale_zh-CN.ini

@@ -268,7 +268,7 @@ page_dev_yunlao_desc4=开发者可以根据使用需求，自由选择相应计
 page_dev_yunlao_desc5=如果您的模型需要更多的计算资源，也可以单独申请。
 page_dev_yunlao_apply=单独申请
 c2net_title=中国算力网(C²NET)
-c2net_desc=广泛接入全国各地智算中心、超算中心与大数据中心等，为用户提供免费算力资源
+c2net_desc=广泛接入全国各地智算中心、超算中心与大数据中心等，为用户提供普惠算力资源
 c2net_center=中心
 search=搜索
 search_repo=项目
@@ -287,7 +287,7 @@ explore_AI = 探索更好的AI，来这里发现更有意思的
 datasets = 数据集
 repositories = 项目
 use_plt__fuction = 使用本平台提供的AI协作功能，如：托管代码、共享数据、调试算法或训练模型，请先
-provide_resoure = 平台目前免费提供CPU、GPU、NPU的算力资源，可进行多种类型的AI任务。
+provide_resoure = 平台目前提供CPU、GPU、NPU的普惠算力资源，可进行多种类型的AI任务。
 create_pro = 创建项目
 activity = 活动
 no_events = 还没有与您相关的活动

routers/admin/cloudbrains.go

@@ -70,7 +70,7 @@ func CloudBrains(ctx *context.Context) {
 
 	keyword := strings.Trim(ctx.Query("q"), " ")
 
-	ciTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{
+	ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
 		ListOptions: models.ListOptions{
 			Page:     page,
 			PageSize: setting.UI.IssuePagingNum,
@@ -84,6 +84,7 @@ func CloudBrains(ctx *context.Context) {
 		IsLatestVersion: modelarts.IsLatestVersion,
 		ComputeResource: listType,
 		Type:            models.TypeCloudBrainAll,
+		AiCenter:        aiCenter,
 		Cluster:         cluster,
 	})
 	if err != nil {
@@ -92,24 +93,26 @@ func CloudBrains(ctx *context.Context) {
 	}
 
 	models.LoadSpecs4CloudbrainInfo(ciTasks)
-	tasks := []*models.CloudbrainInfo{}
-
-	for _, task := range ciTasks {
-		if aiCenter == "" || aiCenter == task.Cloudbrain.Spec.AiCenterCode {
-			task.CanDebug = true
-			task.CanDel = true
-			task.Cloudbrain.ComputeResource = task.ComputeResource
-			tasks = append(tasks, task)
+
+	for i, task := range ciTasks {
+		ciTasks[i].CanDebug = true
+		ciTasks[i].CanDel = true
+		ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource
+		if ciTasks[i].Cloudbrain.Spec != nil {
+			if ciTasks[i].Cloudbrain.Type == models.TypeC2Net {
+				ciTasks[i].Cloudbrain.Spec.Cluster = models.C2NetCluster
+			} else {
+				ciTasks[i].Cloudbrain.Spec.Cluster = models.OpenICluster
+			}
 		}
 	}
 
-	count := int64(len(tasks))
 	pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum))
 	pager.SetDefaultParams(ctx)
 	pager.AddParam(ctx, "listType", "ListType")
 	ctx.Data["Page"] = pager
 	ctx.Data["PageIsCloudBrain"] = true
-	ctx.Data["Tasks"] = tasks
+	ctx.Data["Tasks"] = ciTasks
 	ctx.Data["CanCreate"] = true
 	ctx.Data["Keyword"] = keyword
 

routers/api/v1/repo/modelarts.go

@@ -182,7 +182,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) {
 		}
 		if oldStatus != job.Status {
 			notification.NotifyChangeCloudbrainStatus(job, oldStatus)
-			if models.IsTrainJobTerminal(job.Status) {
+			if models.IsTrainJobTerminal(job.Status) && job.ComputeResource == models.NPUResource {
 				if len(result.JobInfo.Tasks[0].CenterID) == 1 {
 					urchin.GetBackNpuModel(job.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(job.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID))
 				}
@@ -470,7 +470,11 @@ func ModelList(ctx *context.APIContext) {
 	status := models.StorageScheduleSucceed
 	var fileInfos []storage.FileInfo
 	if task.ComputeResource == models.NPUResource {
-		fileInfos, err = storage.GetObsListObject(task.JobName, "output/", parentDir, versionName)
+		prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName), "/")
+		if !strings.HasSuffix(prefix, "/") {
+			prefix += "/"
+		}
+		fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir)
 		if err != nil {
 			log.Info("get TrainJobListModel failed:", err)
 			ctx.ServerError("GetObsListObject:", err)
@@ -484,6 +488,10 @@ func ModelList(ctx *context.APIContext) {
 				if models.IsTrainJobTerminal(task.Status) {
 					if task.Status == models.GrampusStatusStopped {
 						status = models.StorageNoFile
+					} else if task.Status == models.GrampusStatusFailed {
+						if task.AiCenter == "" {
+							status = models.StorageNoFile
+						}
 					} else {
 						record, _ := models.GetScheduleRecordByCloudbrainID(task.ID)
 						if record != nil {

routers/repo/cloudbrain.go

@@ -757,7 +757,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo
 	if ctx.Written() {
 		return
 	}
-    if task.Status==string(models.JobWaiting) || task.Status==string(models.JobRunning) {
+	if task.Status == string(models.JobWaiting) || task.Status == string(models.JobRunning) {
 		result, err := cloudbrain.GetJob(task.JobID)
 		if err != nil {
 			log.Info("error:" + err.Error())
@@ -1944,7 +1944,7 @@ func SyncCloudbrainStatus() {
 				task.CorrectCreateUnix()
 				if oldStatus != task.Status {
 					notification.NotifyChangeCloudbrainStatus(task, oldStatus)
-					if models.IsTrainJobTerminal(task.Status) {
+					if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource {
 						if len(result.JobInfo.Tasks[0].CenterID) == 1 {
 							urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID))
 						}

routers/repo/grampus.go

@@ -879,7 +879,7 @@ func GrampusTrainJobShow(ctx *context.Context) {
 				task.CorrectCreateUnix()
 				if oldStatus != task.Status {
 					notification.NotifyChangeCloudbrainStatus(task, oldStatus)
-					if models.IsTrainJobTerminal(task.Status) {
+					if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource {
 						if len(result.JobInfo.Tasks[0].CenterID) == 1 {
 							urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID))
 						}

routers/user/home.go

@@ -812,7 +812,7 @@ func Cloudbrains(ctx *context.Context) {
 	for i, _ := range repos {
 		repoIDList = append(repoIDList, repos[i].ID)
 	}
-	ciTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{
+	ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
 		ListOptions: models.ListOptions{
 			Page:     page,
 			PageSize: setting.UI.IssuePagingNum,
@@ -828,6 +828,7 @@ func Cloudbrains(ctx *context.Context) {
 		RepoIDList:      repoIDList,
 		ComputeResource: listType,
 		Type:            models.TypeCloudBrainAll,
+		AiCenter:        aiCenter,
 		Cluster:         cluster,
 	})
 	if err != nil {
@@ -835,22 +836,24 @@ func Cloudbrains(ctx *context.Context) {
 		return
 	}
 	models.LoadSpecs4CloudbrainInfo(ciTasks)
-	tasks := []*models.CloudbrainInfo{}
-	for _, task := range ciTasks {
-		if aiCenter == "" || aiCenter == task.Cloudbrain.Spec.AiCenterCode {
-			task.CanDebug = true
-			task.CanDel = true
-			task.Cloudbrain.ComputeResource = task.ComputeResource
-			tasks = append(tasks, task)
+	for i, _ := range ciTasks {
+		ciTasks[i].CanDebug = true
+		ciTasks[i].CanDel = true
+		ciTasks[i].Cloudbrain.ComputeResource = ciTasks[i].ComputeResource
+		if ciTasks[i].Cloudbrain.Spec != nil {
+			if ciTasks[i].Cloudbrain.Type == models.TypeC2Net {
+				ciTasks[i].Cloudbrain.Spec.Cluster = models.C2NetCluster
+			} else {
+				ciTasks[i].Cloudbrain.Spec.Cluster = models.OpenICluster
+			}
 		}
 	}
-	count := int64(len(tasks))
 	pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum))
 	pager.SetDefaultParams(ctx)
 	pager.AddParam(ctx, "listType", "ListType")
 	ctx.Data["Page"] = pager
 	ctx.Data["PageIsUserCloudBrain"] = true
-	ctx.Data["Tasks"] = tasks
+	ctx.Data["Tasks"] = ciTasks
 	ctx.Data["CanCreate"] = true
 	ctx.Data["Keyword"] = keyword
 

templates/admin/cloudbrain/search.tmpl

@@ -71,33 +71,36 @@
         document.addEventListener('DOMContentLoaded', function() {
             $.ajax({
                 type: "GET",
-                url: "/admin/resources/queue/centers",
+                url: "/api/v1/cloudbrain/get_center_info",
                 dataType: "json",
                 data: {},
                 success: function (res) {
-                    if (res && res.Code === 0) {
-                        var data = res.Data;
-                        var aiCenterSelEl = $('#aiCenter-sel');
-                        var itemEl = aiCenterSelEl.find('.menu .item').eq(0);
-                        var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter');
-                        var selectAiCenterName = '';
-                        for (var i = 0, iLen = data.length; i < iLen; i++) {
-                            var dataI = data[i];
-                            var itemClone = itemEl.clone();
-                            var oHref = itemClone.attr('href');
-                            var oId = itemClone.attr('id');
-                            itemClone.attr('data-value', dataI.AiCenterCode);
-                            itemClone.removeAttr('id');
-                            itemClone.attr('href', oHref + dataI.AiCenterCode);                        
-                            itemClone.text(dataI.AiCenterName);
-                            aiCenterSelEl.find('.menu').append(itemClone);
-                            if (selectAiCenterCode === dataI.AiCenterCode) {
-                                selectAiCenterName = dataI.AiCenterName;
-                            }
+                    var data = res || [];
+                    var aiCenterSelEl = $('#aiCenter-sel');
+                    var itemEl = aiCenterSelEl.find('.menu .item').eq(0);
+                    var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter');
+                    var selectAiCenterName = '';
+                    var lang = document.querySelector('html').getAttribute('lang') || 'en-US';
+                    var except = ['', 'more'];
+                    for (var i = 0, iLen = data.length; i < iLen; i++) {
+                        var dataI = data[i];
+                        var aiCenterCode = dataI.name;
+                        if (except.indexOf(aiCenterCode) >= 0) continue;
+                        var aiCenterName = lang === 'en-US' ? dataI.content_en : dataI.content;
+                        var itemClone = itemEl.clone();
+                        var oHref = itemClone.attr('href');
+                        var oId = itemClone.attr('id');
+                        itemClone.attr('data-value', aiCenterCode);
+                        itemClone.removeAttr('id');
+                        itemClone.attr('href', oHref + aiCenterCode);                        
+                        itemClone.text(aiCenterName);
+                        aiCenterSelEl.find('.menu').append(itemClone);
+                        if (selectAiCenterCode === aiCenterCode) {
+                            selectAiCenterName = aiCenterName;
                         }
-                        selectAiCenterCode && aiCenterSelEl.dropdown('set selected', selectAiCenterCode);
-                        selectAiCenterName && aiCenterSelEl.dropdown('set text', selectAiCenterName);
                     }
+                    selectAiCenterCode && aiCenterSelEl.dropdown('set selected', selectAiCenterCode);
+                    selectAiCenterName && aiCenterSelEl.dropdown('set text', selectAiCenterName);
                 },
                 error: function (err) {
                     console.log(err);

templates/admin/cloudbrain/search_dashboard.tmpl

@@ -85,33 +85,36 @@
         document.addEventListener('DOMContentLoaded', function() {
             $.ajax({
                 type: "GET",
-                url: "/admin/resources/queue/centers",
+                url: "/api/v1/cloudbrain/get_center_info",
                 dataType: "json",
                 data: {},
                 success: function (res) {
-                    if (res && res.Code === 0) {
-                        var data = res.Data;
-                        var aiCenterSelEl = $('#aiCenter-sel');
-                        var itemEl = aiCenterSelEl.find('.menu .item').eq(0);
-                        var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter');
-                        var selectAiCenterName = '';
-                        for (var i = 0, iLen = data.length; i < iLen; i++) {
-                            var dataI = data[i];
-                            var itemClone = itemEl.clone();
-                            var oHref = itemClone.attr('href');
-                            var oId = itemClone.attr('id');
-                            itemClone.attr('data-value', dataI.AiCenterCode);
-                            itemClone.removeAttr('id');
-                            itemClone.attr('href', oHref + dataI.AiCenterCode);                        
-                            itemClone.text(dataI.AiCenterName);
-                            aiCenterSelEl.find('.menu').append(itemClone);
-                            if (selectAiCenterCode === dataI.AiCenterCode) {
-                                selectAiCenterName = dataI.AiCenterName;
-                            }
+                    var data = res || [];
+                    var aiCenterSelEl = $('#aiCenter-sel');
+                    var itemEl = aiCenterSelEl.find('.menu .item').eq(0);
+                    var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter');
+                    var selectAiCenterName = '';
+                    var lang = document.querySelector('html').getAttribute('lang') || 'en-US';
+                    var except = ['', 'more'];
+                    for (var i = 0, iLen = data.length; i < iLen; i++) {
+                        var dataI = data[i];
+                        var aiCenterCode = dataI.name;
+                        if (except.indexOf(aiCenterCode) >= 0) continue;
+                        var aiCenterName = lang === 'en-US' ? dataI.content_en : dataI.content;
+                        var itemClone = itemEl.clone();
+                        var oHref = itemClone.attr('href');
+                        var oId = itemClone.attr('id');
+                        itemClone.attr('data-value', aiCenterCode);
+                        itemClone.removeAttr('id');
+                        itemClone.attr('href', oHref + aiCenterCode);                        
+                        itemClone.text(aiCenterName);
+                        aiCenterSelEl.find('.menu').append(itemClone);
+                        if (selectAiCenterCode === aiCenterCode) {
+                            selectAiCenterName = aiCenterName;
                         }
-                        selectAiCenterCode && aiCenterSelEl.dropdown('set selected', selectAiCenterCode);
-                        selectAiCenterName && aiCenterSelEl.dropdown('set text', selectAiCenterName);
                     }
+                    selectAiCenterCode && aiCenterSelEl.dropdown('set selected', selectAiCenterCode);
+                    selectAiCenterName && aiCenterSelEl.dropdown('set text', selectAiCenterName);
                 },
                 error: function (err) {
                     console.log(err);
@@ -119,4 +122,4 @@
             });
         });
     })();
-</script>
+</script>