diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/AdminApplication.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/AdminApplication.java index d4abdf7..c9d7bcf 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/AdminApplication.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/AdminApplication.java @@ -19,6 +19,7 @@ package org.dubhe.admin; import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; /** @@ -27,6 +28,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; */ @SpringBootApplication(scanBasePackages = "org.dubhe") @MapperScan(basePackages = {"org.dubhe.**.dao"}) +@EnableScheduling public class AdminApplication { public static void main(String[] args) { SpringApplication.run(AdminApplication.class, args); diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsCreateDTO.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsCreateDTO.java index 7c6f44e..a0783a2 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsCreateDTO.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsCreateDTO.java @@ -42,10 +42,10 @@ public class ResourceSpecsCreateDTO implements Serializable { @Pattern(regexp = StringConstant.REGEXP_SPECS, message = "规格名称支持字母、数字、汉字、英文横杠、下划线和空白字符") private String specsName; - @ApiModelProperty(value = "所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving)", required = true) + @ApiModelProperty(value = "所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving, 4：dubhe-tadl)", required = true) @NotNull(message = "所属业务场景不能为空") @Min(value = MagicNumConstant.ZERO, message = "所属业务场景错误") - @Max(value = MagicNumConstant.THREE, message = "所属业务场景错误") + @Max(value = MagicNumConstant.FOUR, message = "所属业务场景错误") private Integer module; @ApiModelProperty(value = "CPU数量,单位：核", required = true) diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsQueryDTO.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsQueryDTO.java index ff7c247..d706fe9 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsQueryDTO.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsQueryDTO.java @@ -37,6 +37,9 @@ public class ResourceSpecsQueryDTO extends PageQueryBase implements Serializable private static final long serialVersionUID = 1L; + @ApiModelProperty(value = "多GPU，true：GPU数大于1核，false:GPU数等于1核") + private Boolean multiGpu; + @ApiModelProperty("规格名称") @Length(max = MagicNumConstant.THIRTY_TWO, message = "规格名称错误") private String specsName; @@ -44,8 +47,8 @@ public class ResourceSpecsQueryDTO extends PageQueryBase implements Serializable @ApiModelProperty("规格类型(0为CPU, 1为GPU)") private Boolean resourcesPoolType; - @ApiModelProperty("所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving)") + @ApiModelProperty("所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving,4：dubhe-tadl)") @Min(value = MagicNumConstant.ZERO, message = "所属业务场景错误") - @Max(value = MagicNumConstant.THREE, message = "所属业务场景错误") + @Max(value = MagicNumConstant.FOUR, message = "所属业务场景错误") private Integer module; } \ No newline at end of file diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsUpdateDTO.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsUpdateDTO.java index 3662bb1..df20b37 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsUpdateDTO.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsUpdateDTO.java @@ -49,10 +49,10 @@ public class ResourceSpecsUpdateDTO implements Serializable { @Pattern(regexp = StringConstant.REGEXP_SPECS, message = "规格名称支持字母、数字、汉字、英文横杠、下划线和空白字符") private String specsName; - @ApiModelProperty(value = "所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving)", required = true) + @ApiModelProperty(value = "所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving, 4：dubhe-tadl)", required = true) @NotNull(message = "所属业务场景不能为空") @Min(value = MagicNumConstant.ZERO, message = "所属业务场景错误") - @Max(value = MagicNumConstant.THREE, message = "所属业务场景错误") + @Max(value = MagicNumConstant.FOUR, message = "所属业务场景错误") private Integer module; @ApiModelProperty(value = "CPU数量,单位：核") diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/ResourceSpecsController.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/ResourceSpecsController.java index b445977..a00bc78 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/ResourceSpecsController.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/ResourceSpecsController.java @@ -29,6 +29,7 @@ import org.dubhe.biz.base.vo.DataResponseBody; import org.dubhe.biz.base.vo.QueryResourceSpecsVO; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; import javax.validation.Valid; @@ -47,16 +48,23 @@ public class ResourceSpecsController { @ApiOperation("查询资源规格") @GetMapping - public DataResponseBody getResourceSpecs(ResourceSpecsQueryDTO resourceSpecsQueryDTO) { + public DataResponseBody getResourceSpecs(@Validated ResourceSpecsQueryDTO resourceSpecsQueryDTO) { return new DataResponseBody(resourceSpecsService.getResourceSpecs(resourceSpecsQueryDTO)); } - @ApiOperation("查询资源规格(远程调用)") + @ApiOperation("查询资源规格(训练远程调用)") @GetMapping("/queryResourceSpecs") - public DataResponseBody queryResourceSpecs(QueryResourceSpecsDTO queryResourceSpecsDTO) { + public DataResponseBody queryResourceSpecs(@Validated QueryResourceSpecsDTO queryResourceSpecsDTO) { return new DataResponseBody(resourceSpecsService.queryResourceSpecs(queryResourceSpecsDTO)); } + + @ApiOperation("查询资源规格(tadl远程调用)") + @GetMapping("/queryTadlResourceSpecs") + public DataResponseBody queryTadlResourceSpecs(Long id) { + return new DataResponseBody(resourceSpecsService.queryTadlResourceSpecs(id)); + } + @ApiOperation("新增资源规格") @PostMapping @PreAuthorize(Permissions.SPECS_CREATE) diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/ResourceSpecsService.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/ResourceSpecsService.java index 998d387..e95aa82 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/ResourceSpecsService.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/ResourceSpecsService.java @@ -62,4 +62,11 @@ public interface ResourceSpecsService { * @return QueryResourceSpecsVO 资源规格返回结果实体类 */ QueryResourceSpecsVO queryResourceSpecs(QueryResourceSpecsDTO queryResourceSpecsDTO); + + /** + * 查询资源规格 + * @param id 资源规格id + * @return QueryResourceSpecsVO 资源规格返回结果实体类 + */ + QueryResourceSpecsVO queryTadlResourceSpecs(Long id); } \ No newline at end of file diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/RecycleTaskServiceImpl.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/RecycleTaskServiceImpl.java index c6d3acd..02a35f4 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/RecycleTaskServiceImpl.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/RecycleTaskServiceImpl.java @@ -329,7 +329,7 @@ public class RecycleTaskServiceImpl implements RecycleTaskService { } String emptyDir = recycleFileTmpPath + randomPath + File.separator; LogUtil.debug(LogEnum.GARBAGE_RECYCLE, "recycle task sourcePath:{},emptyDir:{}", sourcePath, emptyDir); - Process process = Runtime.getRuntime().exec(new String[]{"/bin/sh", "-c", String.format(ShellFileStoreApiImpl.DEL_COMMAND, userName, ip, emptyDir, emptyDir, sourcePath, emptyDir, sourcePath)}); + Process process = Runtime.getRuntime().exec(new String[]{"/bin/sh", "-c", String.format(ShellFileStoreApiImpl.DEL_COMMAND, emptyDir, emptyDir, sourcePath, emptyDir, sourcePath)}); return processRecycle(process); } else { LogUtil.error(LogEnum.GARBAGE_RECYCLE, "file recycle is failed! sourcePath:{}", sourcePath); @@ -460,7 +460,7 @@ public class RecycleTaskServiceImpl implements RecycleTaskService { String delRealPath = fileStoreApi.formatPath(sourcePath + File.separator + fileName + File.separator + directoryName); delRealPath = delRealPath.endsWith(File.separator) ? delRealPath : delRealPath + File.separator; String emptyDir = invalidFileTmpPath + directoryName + File.separator; - Process process = Runtime.getRuntime().exec(new String[]{"/bin/sh", "-c", String.format(ShellFileStoreApiImpl.DEL_COMMAND, userName, ip, emptyDir, emptyDir, delRealPath, emptyDir, delRealPath)}); + Process process = Runtime.getRuntime().exec(new String[]{"/bin/sh", "-c", String.format(ShellFileStoreApiImpl.DEL_COMMAND, emptyDir, emptyDir, delRealPath, emptyDir, delRealPath)}); Integer deleteStatus = process.waitFor(); LogUtil.info(LogEnum.GARBAGE_RECYCLE, "recycle resources path:{},recycle status:{}", delRealPath, deleteStatus); } catch (Exception e) { diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/ResourceSpecsServiceImpl.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/ResourceSpecsServiceImpl.java index 4f3dc05..0f4ee1b 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/ResourceSpecsServiceImpl.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/ResourceSpecsServiceImpl.java @@ -27,6 +27,7 @@ import org.dubhe.admin.domain.dto.ResourceSpecsUpdateDTO; import org.dubhe.admin.domain.entity.ResourceSpecs; import org.dubhe.admin.domain.vo.ResourceSpecsQueryVO; import org.dubhe.admin.service.ResourceSpecsService; +import org.dubhe.biz.base.constant.MagicNumConstant; import org.dubhe.biz.base.constant.StringConstant; import org.dubhe.biz.base.context.UserContext; import org.dubhe.biz.base.dto.QueryResourceSpecsDTO; @@ -72,6 +73,13 @@ public class ResourceSpecsServiceImpl implements ResourceSpecsService { queryResourceSpecsWrapper.like(resourceSpecsQueryDTO.getSpecsName() != null, "specs_name", resourceSpecsQueryDTO.getSpecsName()) .eq(resourceSpecsQueryDTO.getResourcesPoolType() != null, "resources_pool_type", resourceSpecsQueryDTO.getResourcesPoolType()) .eq(resourceSpecsQueryDTO.getModule() != null, "module", resourceSpecsQueryDTO.getModule()); + if (resourceSpecsQueryDTO.getMultiGpu() != null) { + if (resourceSpecsQueryDTO.getMultiGpu()) { + queryResourceSpecsWrapper.gt("gpu_num", MagicNumConstant.ONE); + } else { + queryResourceSpecsWrapper.eq("gpu_num", MagicNumConstant.ONE); + } + } if (StringConstant.SORT_ASC.equals(resourceSpecsQueryDTO.getOrder())) { queryResourceSpecsWrapper.orderByAsc(StringUtils.humpToLine(sort)); } else { @@ -206,4 +214,23 @@ public class ResourceSpecsServiceImpl implements ResourceSpecsService { BeanUtils.copyProperties(resourceSpecs, queryResourceSpecsVO); return queryResourceSpecsVO; } + + /** + * 查询资源规格 + * @param id 资源规格id + * @return QueryResourceSpecsVO 资源规格返回结果实体类 + */ + @Override + public QueryResourceSpecsVO queryTadlResourceSpecs(Long id) { + LogUtil.info(LogEnum.BIZ_SYS,"Query resource specification information with resource id:{}",id); + ResourceSpecs resourceSpecs = resourceSpecsMapper.selectById(id); + LogUtil.info(LogEnum.BIZ_SYS,"Obtain resource specification information:{} ",resourceSpecs); + if (resourceSpecs == null) { + throw new BusinessException("资源规格不存在或已被删除"); + } + QueryResourceSpecsVO queryResourceSpecsVO = new QueryResourceSpecsVO(); + BeanUtils.copyProperties(resourceSpecs, queryResourceSpecsVO); + LogUtil.info(LogEnum.BIZ_SYS,"Return resource specification information :{} ",queryResourceSpecsVO); + return queryResourceSpecsVO; + } } \ No newline at end of file diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/ApplicationNameConst.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/ApplicationNameConst.java index a04c803..eb0543e 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/ApplicationNameConst.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/ApplicationNameConst.java @@ -81,6 +81,10 @@ public class ApplicationNameConst { */ public final static String SERVER_DATA_DCM = "dubhe-data-dcm"; + /** + * TADL + */ + public final static String SERVER_TADL = "dubhe-tadl"; /** * k8s */ diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/NumberConstant.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/NumberConstant.java index 53306ea..99d06d2 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/NumberConstant.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/NumberConstant.java @@ -32,6 +32,8 @@ public class NumberConstant { public final static int NUMBER_6 = 6; public final static int NUMBER_8 = 8; public final static int NUMBER_10 = 10; + public final static int NUMBER_12 = 12; + public final static int NUMBER_24 = 24; public final static int NUMBER_30 = 30; public final static int NUMBER_32 = 32; public final static int NUMBER_50 = 50; diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java index 5d35700..55d3b02 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java @@ -53,6 +53,19 @@ public final class StringConstant { * 整数匹配 */ public static final Pattern PATTERN_NUM = Pattern.compile("^[-\\+]?[\\d]*$"); + /** + * 数字匹配 + */ + public static final String NUMBER ="(\\d+)"; + /** + * 整数匹配 + */ + public static final Pattern PATTERN_NUMBER = Pattern.compile("(\\d+)"); + /** + * 小数匹配 + */ + public static final Pattern PATTERN_DECIMAL = Pattern.compile("(\\d+\\.\\d+)"); + /** diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/SymbolConstant.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/SymbolConstant.java index db633fd..2faef75 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/SymbolConstant.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/SymbolConstant.java @@ -27,6 +27,7 @@ public class SymbolConstant { public static final String COLON = ":"; public static final String LINEBREAK = "\n"; public static final String BLANK = ""; + public static final String SPACE = " "; public static final String QUESTION = "?"; public static final String ZERO = "0"; public static final String DOT = "."; diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/QueryResourceSpecsDTO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/QueryResourceSpecsDTO.java index 08e48de..160559b 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/QueryResourceSpecsDTO.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/QueryResourceSpecsDTO.java @@ -45,10 +45,10 @@ public class QueryResourceSpecsDTO implements Serializable { private String specsName; /** - * 所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving) + * 所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving，4：dubhe-tadl) */ @NotNull(message = "所属业务场景不能为空") @Min(value = MagicNumConstant.ZERO, message = "所属业务场景错误") - @Max(value = MagicNumConstant.THREE, message = "所属业务场景错误") + @Max(value = MagicNumConstant.FOUR, message = "所属业务场景错误") private Integer module; } \ No newline at end of file diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BizEnum.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BizEnum.java index 00b7d78..c931273 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BizEnum.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BizEnum.java @@ -61,7 +61,10 @@ public enum BizEnum { * 专业版终端 */ TERMINAL("专业版终端", "terminal", 7), - ; + /** + * TADL + */ + TADL("TADL服务", "tadl", 8); /** * 业务模块名称 diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/PtModelUtil.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/PtModelUtil.java index 8a521a6..77dbe55 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/PtModelUtil.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/PtModelUtil.java @@ -54,6 +54,8 @@ public class PtModelUtil { public static final int MODEL_OPTIMIZATION = 2; + public static final int AUTOMATIC_MACHINE_LEARNING = 4; + public static final int RANDOM_LENGTH = 4; } \ No newline at end of file diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/QueryResourceSpecsVO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/QueryResourceSpecsVO.java index bac1c42..76782fb 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/QueryResourceSpecsVO.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/QueryResourceSpecsVO.java @@ -48,7 +48,7 @@ public class QueryResourceSpecsVO implements Serializable { private Boolean resourcesPoolType; /** - *所属业务场景 + *所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving，4：dubhe-tadl) */ private Integer module; diff --git a/dubhe-server/common-biz/db/src/main/java/org/dubhe/biz/db/base/BaseLogQuery.java b/dubhe-server/common-biz/db/src/main/java/org/dubhe/biz/db/base/BaseLogQuery.java new file mode 100644 index 0000000..66ac107 --- /dev/null +++ b/dubhe-server/common-biz/db/src/main/java/org/dubhe/biz/db/base/BaseLogQuery.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.biz.db.base; + +import lombok.Data; +import lombok.experimental.Accessors; + +/** + * @description 日志基类 + * @date 2021-03-11 + */ +@Data +@Accessors(chain = true) +public class BaseLogQuery { + + private Integer startLine; + + private Integer lines; +} diff --git a/dubhe-server/common-biz/file/pom.xml b/dubhe-server/common-biz/file/pom.xml index 4f411e3..83422e8 100644 --- a/dubhe-server/common-biz/file/pom.xml +++ b/dubhe-server/common-biz/file/pom.xml @@ -72,6 +72,16 @@ org.apache.poi poi-ooxml + + junit + junit + 4.13.1 + + + io.minio + minio + 7.0.2 + diff --git a/dubhe-server/common-biz/file/src/main/java/org/dubhe/biz/file/api/impl/ShellFileStoreApiImpl.java b/dubhe-server/common-biz/file/src/main/java/org/dubhe/biz/file/api/impl/ShellFileStoreApiImpl.java index 74fc431..14954d1 100644 --- a/dubhe-server/common-biz/file/src/main/java/org/dubhe/biz/file/api/impl/ShellFileStoreApiImpl.java +++ b/dubhe-server/common-biz/file/src/main/java/org/dubhe/biz/file/api/impl/ShellFileStoreApiImpl.java @@ -62,7 +62,7 @@ public class ShellFileStoreApiImpl implements FileStoreApi { * 删除服务器无效文件(大文件) * 示例：rsync --delete-before -d /空目录 /需要回收的源目录 */ - public static final String DEL_COMMAND = "ssh %s@%s \"mkdir -p %s; rsync --delete-before -d %s %s; rmdir %s %s\""; + public static final String DEL_COMMAND = "mkdir -p %s; rsync --delete-before -d %s %s; rmdir %s %s"; /** * 拷贝文件并重命名 diff --git a/dubhe-server/common-biz/file/src/main/java/org/dubhe/biz/file/utils/MinioUtil.java b/dubhe-server/common-biz/file/src/main/java/org/dubhe/biz/file/utils/MinioUtil.java index c73e6ca..34ef613 100644 --- a/dubhe-server/common-biz/file/src/main/java/org/dubhe/biz/file/utils/MinioUtil.java +++ b/dubhe-server/common-biz/file/src/main/java/org/dubhe/biz/file/utils/MinioUtil.java @@ -18,6 +18,7 @@ package org.dubhe.biz.file.utils; import cn.hutool.core.io.IoUtil; +import com.alibaba.fastjson.JSONObject; import io.minio.CopyConditions; import io.minio.MinioClient; import io.minio.PutObjectOptions; @@ -40,6 +41,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * @description Minio工具类 @@ -280,4 +283,33 @@ public class MinioUtil { } } + /** + * 生成给HTTP PUT请求用的presigned URLs。浏览器/移动端的客户端可以用这个URL进行上传， + * 即使其所在的存储桶是私有的。这个presigned URL可以设置一个失效时间，默认值是7天 + * + * @param bucketName 存储桶名称 + * @param objectNames 存储桶里的对象名称 + * @param expires 失效时间（以秒为单位），默认是7天，不得大于七天 + * @return String + */ + public JSONObject getEncryptedPutUrls(String bucketName,String objectNames, Integer expires) { + List filePaths = JSONObject.parseObject(objectNames, List.class); + List urls = new ArrayList<>(); + filePaths.stream().forEach(filePath->{ + if (StringUtils.isEmpty(filePath)) { + throw new BusinessException("filePath cannot be empty"); + } + try { + urls.add(client.presignedPutObject(bucketName, filePath, expires)); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_DATASET, e.getMessage()); + throw new BusinessException("MinIO an error occurred, please contact the administrator"); + } + }); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("preUrls",urls); + jsonObject.put("bucketName", bucketName); + return jsonObject; + } + } diff --git a/dubhe-server/common-biz/log/src/main/java/org/dubhe/biz/log/enums/LogEnum.java b/dubhe-server/common-biz/log/src/main/java/org/dubhe/biz/log/enums/LogEnum.java index 6106118..e0d3e40 100644 --- a/dubhe-server/common-biz/log/src/main/java/org/dubhe/biz/log/enums/LogEnum.java +++ b/dubhe-server/common-biz/log/src/main/java/org/dubhe/biz/log/enums/LogEnum.java @@ -79,7 +79,9 @@ public enum LogEnum { //云端Serving SERVING, //专业版终端 - TERMINAL; + TERMINAL, + //tadl + TADL; /** * 判断日志类型不能为空 diff --git a/dubhe-server/common-biz/redis/pom.xml b/dubhe-server/common-biz/redis/pom.xml index 60960da..d672e59 100644 --- a/dubhe-server/common-biz/redis/pom.xml +++ b/dubhe-server/common-biz/redis/pom.xml @@ -24,6 +24,11 @@ com.liferay com.fasterxml.jackson.databind + + com.amazonaws + aws-java-sdk + 1.12.35 + diff --git a/dubhe-server/common-biz/redis/src/main/java/org/dubhe/biz/redis/utils/RedisUtils.java b/dubhe-server/common-biz/redis/src/main/java/org/dubhe/biz/redis/utils/RedisUtils.java index 16e8d09..500b02c 100644 --- a/dubhe-server/common-biz/redis/src/main/java/org/dubhe/biz/redis/utils/RedisUtils.java +++ b/dubhe-server/common-biz/redis/src/main/java/org/dubhe/biz/redis/utils/RedisUtils.java @@ -24,10 +24,7 @@ import org.dubhe.biz.log.utils.LogUtil; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.core.Cursor; -import org.springframework.data.redis.core.RedisConnectionUtils; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.data.redis.core.ScanOptions; +import org.springframework.data.redis.core.*; import org.springframework.data.redis.core.script.DefaultRedisScript; import org.springframework.data.redis.core.script.RedisScript; import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; @@ -670,6 +667,40 @@ public class RedisUtils { return zRangeByScorePop( key,0, max,0,1); } + /** + * 根据键获取score值为 min 到 max 之间的所有 member 和 score + * @param key 健 + * @param min score最小值 + * @param max score最大值 + * @return + */ + public Set> zRangeByScoreWithScores(String key, Long min, Long max){ + try { + return redisTemplate.opsForZSet().rangeWithScores(key, min, max); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_DATASET, "RedisUtils rangeWithScores key {} error:{}", key, e.getMessage(), e); + return null; + } + } + + /** + * 根据 key 和 member 移除元素 + * @param key + * @param member + * @return + */ + public Boolean zRem(String key,Object member){ + try{ + if (StringUtils.isEmpty(key) || null == member){ + return false; + } + redisTemplate.opsForZSet().remove(key,member); + return true; + }catch (Exception e){ + LogUtil.error(LogEnum.REDIS, "RedisUtils zrem key {} member {} error:{}", key, member, e); + return false; + } + } // ===============================list================================= diff --git a/dubhe-server/common-cloud/auth-config/src/main/java/org/dubhe/cloud/authconfig/service/impl/OAuth2UserContextServiceImpl.java b/dubhe-server/common-cloud/auth-config/src/main/java/org/dubhe/cloud/authconfig/service/impl/OAuth2UserContextServiceImpl.java index 390f319..13297f7 100644 --- a/dubhe-server/common-cloud/auth-config/src/main/java/org/dubhe/cloud/authconfig/service/impl/OAuth2UserContextServiceImpl.java +++ b/dubhe-server/common-cloud/auth-config/src/main/java/org/dubhe/cloud/authconfig/service/impl/OAuth2UserContextServiceImpl.java @@ -32,12 +32,12 @@ public class OAuth2UserContextServiceImpl implements UserContextService { @Override public UserContext getCurUser() { JwtUserDTO jwtUserDTO = JwtUtils.getCurUser(); - return jwtUserDTO == null?null:jwtUserDTO.getUser(); + return jwtUserDTO == null ? null : jwtUserDTO.getUser(); } @Override public Long getCurUserId() { UserContext userContext = getCurUser(); - return userContext == null?null:userContext.getId(); + return userContext == null ? null : userContext.getId(); } } diff --git a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml index 85610a8..f507bd9 100644 --- a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml +++ b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml @@ -6,7 +6,7 @@ spring: context-path: /nacos config: namespace: dubhe-server-cloud-dev - server-addr: 10.105.1.132:8848 + server-addr: 127.0.0.1:8848 discovery: namespace: dubhe-server-cloud-dev - server-addr: 10.105.1.132:8848 + server-addr: 127.0.0.1:8848 diff --git a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-open-dev.yml b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-open-dev.yml new file mode 100644 index 0000000..f70f2c4 --- /dev/null +++ b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-open-dev.yml @@ -0,0 +1,12 @@ +spring: + cloud: + nacos: + username: nacos + password: Tianshu + context-path: /nacos + config: + namespace: dubhe-server-cloud-open-dev + server-addr: 10.105.1.132:8848 + discovery: + namespace: dubhe-server-cloud-open-dev + server-addr: 10.105.1.132:8848 \ No newline at end of file diff --git a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-opendev.yaml b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-opendev.yaml new file mode 100644 index 0000000..b53c1a1 --- /dev/null +++ b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-opendev.yaml @@ -0,0 +1,12 @@ +spring: + cloud: + nacos: + username: nacos + password: Tianshu + context-path: /nacos + config: + namespace: dubhe-server-cloud-open-dev + server-addr: 10.105.1.132:8848 + discovery: + namespace: dubhe-server-cloud-open-dev + server-addr: 10.105.1.132:8848 diff --git a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-pre.yml b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-pre.yml index b8cc6ee..180217e 100644 --- a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-pre.yml +++ b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-pre.yml @@ -3,7 +3,7 @@ spring: nacos: config: namespace: dubhe-server-cloud-pre - server-addr: 10.105.1.133:8848 + server-addr: 127.0.0.1:8848 discovery: namespace: dubhe-server-cloud-pre - server-addr: 10.105.1.133:8848 + server-addr: 127.0.0.1:8848 diff --git a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-test.yml b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-test.yml index 9352755..3c02c72 100644 --- a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-test.yml +++ b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-test.yml @@ -6,7 +6,7 @@ spring: context-path: /nacos config: namespace: dubhe-server-cloud-test - server-addr: 10.105.1.132:8848 + server-addr: 127.0.0.1:8848 discovery: namespace: dubhe-server-cloud-test - server-addr: 10.105.1.132:8848 + server-addr: 127.0.0.1:8848 diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/docker/utils/DockerCallbackTool.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/docker/utils/DockerCallbackTool.java index e583a57..68c829e 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/docker/utils/DockerCallbackTool.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/docker/utils/DockerCallbackTool.java @@ -74,6 +74,7 @@ public class DockerCallbackTool { LogUtil.info(LogEnum.TERMINAL, "{} sendPushCallback {} count {} status：{}", url, dockerPushCallbackDTO,count,httpResponse.getStatus()); //重试 if (HttpStatus.HTTP_OK != httpResponse.getStatus() && count > MagicNumConstant.ZERO){ + Thread.sleep(MagicNumConstant.ONE_THOUSAND); sendPushCallback(dockerPushCallbackDTO,url,--count); } }catch (Exception e){ diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/LogMonitoringApi.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/LogMonitoringApi.java index 20d9bd0..91a62ad 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/LogMonitoringApi.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/LogMonitoringApi.java @@ -75,4 +75,31 @@ public interface LogMonitoringApi { */ long searchLogCountByPodName(LogMonitoringBO logMonitoringBo); + + /** + * 日志查询方法 + * + * @param logMonitoringBo 日志查询bo + * @return LogMonitoringVO 日志查询结果类 + */ + LogMonitoringVO searchLog(LogMonitoringBO logMonitoringBo); + + /** + * 添加 TADL 服务日志到 Elasticsearch + * + * @param experimentId Experiment ID + * @param log 日志 + * @return boolean 日志添加是否成功 + */ + boolean addTadlLogsToEs(long experimentId, String log); + + /** + * TADL 服务日志查询方法 + * + * @param from 日志查询起始值，初始值为1，表示从第一条日志记录开始查询 + * @param size 日志查询记录数 + * @param experimentId TADL Experiment ID + * @return LogMonitoringVO 日志查询结果类 + */ + LogMonitoringVO searchTadlLogById(int from, int size, long experimentId); } diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/LogMonitoringApiImpl.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/LogMonitoringApiImpl.java index a61b2d8..8e9a2d7 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/LogMonitoringApiImpl.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/LogMonitoringApiImpl.java @@ -18,12 +18,14 @@ package org.dubhe.k8s.api.impl; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; import io.fabric8.kubernetes.api.model.DoneablePod; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.dsl.PodResource; import org.dubhe.biz.base.constant.MagicNumConstant; +import org.dubhe.biz.base.enums.BizEnum; import org.dubhe.biz.base.utils.StringUtils; import org.dubhe.biz.base.utils.TimeTransferUtil; import org.dubhe.biz.log.enums.LogEnum; @@ -39,6 +41,10 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; @@ -48,12 +54,9 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; - -import java.io.IOException; import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; - import static org.dubhe.biz.base.constant.MagicNumConstant.ZERO; import static org.dubhe.biz.base.constant.MagicNumConstant.*; import static org.dubhe.biz.base.constant.SymbolConstant.*; @@ -78,6 +81,14 @@ public class LogMonitoringApiImpl implements LogMonitoringApi { private KubernetesClient kubernetesClient; private static final String INDEX_NAME = "kubelogs"; + private static final String TADL_INDEX_NAME = "tadllogs"; + private static final String INDEX_SHARDS_NUMBER = "index.number_of_shards"; + private static final String INDEX_REPLICAS_NUMBER = "index.number_of_replicas"; + private static final String TYPE = "type"; + private static final String TEXT = "text"; + private static final String DATE = "date"; + private static final String PROPERTIES = "properties"; + private static final String EXPERIMENT_ID = "experimentId"; private static final String POD_NAME_KEY = "kubernetes.pod_name.keyword"; private static final String POD_NAME = "kubernetes.pod_name"; private static final String NAMESPACE_KEY = "kubernetes.namespace_name.keyword"; @@ -86,6 +97,12 @@ public class LogMonitoringApiImpl implements LogMonitoringApi { private static final String MESSAGE = "log"; private static final String LOG_PREFIX = "[Dubhe Service Log] "; private static final String INDEX_FORMAT = "yyyy.MM.dd"; + private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.sss"; + private static final String LOG_FORMAT = "[Dubhe Service Log]-[%s]-%s"; + private static final String KUBERNETES_KEY = "kubernetes"; + private static final String KUBERNETES_POD_NAME_KEY = "pod_name"; + private static final String BUSINESS_KEY = "kubernetes.labels.platform/business.keyword"; + private static final String BUSINESS_GROUP_ID_KEY = "kubernetes.labels.platform/business-group-id.keyword"; public LogMonitoringApiImpl(K8sUtils k8sUtils) { this.kubernetesClient = k8sUtils.getClient(); @@ -151,7 +168,7 @@ public class LogMonitoringApiImpl implements LogMonitoringApi { /**通过restHighLevelClient发送http的请求批量创建文档**/ restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT); - } catch (IOException e) { + } catch (Exception e) { LogUtil.error(LogEnum.BIZ_K8S, "LogMonitoringApi.addLogsToEs error:{}", e); return false; } @@ -169,7 +186,7 @@ public class LogMonitoringApiImpl implements LogMonitoringApi { @Override public LogMonitoringVO searchLogByResName(int from, int size, LogMonitoringBO logMonitoringBo) { List logList = new ArrayList<>(); - LogMonitoringVO logMonitoringResult = new LogMonitoringVO(ZERO_LONG, logList); + LogMonitoringVO logMonitoringResult = new LogMonitoringVO(ZERO, logList); String namespace = logMonitoringBo.getNamespace(); String resourceName = logMonitoringBo.getResourceName(); if (StringUtils.isBlank(resourceName) || StringUtils.isBlank(namespace)) { @@ -195,7 +212,7 @@ public class LogMonitoringApiImpl implements LogMonitoringApi { } logMonitoringResult.setLogs(logList); - logMonitoringResult.setTotalLogs(Long.valueOf(logList.size())); + logMonitoringResult.setTotalLogs(logList.size()); return logMonitoringResult; } @@ -212,7 +229,23 @@ public class LogMonitoringApiImpl implements LogMonitoringApi { LogMonitoringVO logMonitoringResult = new LogMonitoringVO(); List logs = searchLogInfoByEs(from, size, logMonitoringBo); logMonitoringResult.setLogs(logs); - logMonitoringResult.setTotalLogs(Long.valueOf(logs.size())); + logMonitoringResult.setTotalLogs(logs.size()); + return logMonitoringResult; + + } + + /** + * 日志查询方法 + * + * @param logMonitoringBo 日志查询bo + * @return LogMonitoringVO 日志查询结果类 + */ + @Override + public LogMonitoringVO searchLog(LogMonitoringBO logMonitoringBo) { + LogMonitoringVO logMonitoringResult = new LogMonitoringVO(); + List logs = searchLogInfoByEs(logMonitoringBo); + logMonitoringResult.setLogs(logs); + logMonitoringResult.setTotalLogs(logs.size()); return logMonitoringResult; } @@ -235,6 +268,127 @@ public class LogMonitoringApiImpl implements LogMonitoringApi { } } + /** + * 添加 TADL 服务日志到 Elasticsearch + * + * @param experimentId 日志查询起始值，初始值为1，表示从第一条日志记录开始查询 + * @param log 日志 + * @return boolean 日志添加是否成功 + */ + @Override + public boolean addTadlLogsToEs(long experimentId, String log) { + + + Date date = new Date(); + String timestamp = TimeTransferUtil.dateTransferToUtc(date); + BulkRequest bulkRequest = new BulkRequest(); + try { + /**查询索引是否存在, 不存在则创建**/ + GetIndexRequest getIndexRequest = new GetIndexRequest(TADL_INDEX_NAME); + boolean exists = restHighLevelClient.indices().exists(getIndexRequest, RequestOptions.DEFAULT); + if (!exists){ + CreateIndexRequest createIndexRequest = new CreateIndexRequest(TADL_INDEX_NAME); + createIndexRequest.settings(Settings.builder() + .put(INDEX_SHARDS_NUMBER, 3) + .put(INDEX_REPLICAS_NUMBER, 2) + ); + Map timestampMapping = new HashMap<>(); + timestampMapping.put(TYPE, DATE); + Map logMapping = new HashMap<>(); + logMapping.put(TYPE, TEXT); + Map experimentIdMapping = new HashMap<>(); + experimentIdMapping.put(TYPE, TEXT); + Map properties = new HashMap<>(); + properties.put(TIMESTAMP,timestampMapping); + properties.put(EXPERIMENT_ID,experimentIdMapping); + properties.put(MESSAGE,logMapping); + Map mapping = new HashMap<>(); + mapping.put(PROPERTIES, properties); + createIndexRequest.mapping(mapping); + CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT); + + } + LinkedHashMap jsonMap = new LinkedHashMap() {{ + put(EXPERIMENT_ID, experimentId); + put(MESSAGE, new SimpleDateFormat(TIMESTAMP_FORMAT).format(date) + SPACE + log); + put(TIMESTAMP, timestamp); + }}; + + /**添加索引创建对象到bulkRequest**/ + bulkRequest.add(new IndexRequest(TADL_INDEX_NAME).source(jsonMap)); + + /**通过restHighLevelClient发送http的请求创建文档**/ + restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_K8S, "LogMonitoringApi.addTadlLogsToEs error:{}", e); + return false; + } + return true; + } + + + + /** + * TADL 服务日志查询方法 + * + * @param from 日志查询起始值，初始值为1，表示从第一条日志记录开始查询 + * @param size 日志查询记录数 + * @param experimentId TADL Experiment ID + * @return LogMonitoringVO 日志查询结果类 + */ + @Override + public LogMonitoringVO searchTadlLogById(int from, int size, long experimentId) { + List logList = new ArrayList<>(); + LogMonitoringVO logMonitoringResult = new LogMonitoringVO(ZERO, logList); + /**处理查询范围参数起始值**/ + from = from <= MagicNumConstant.ZERO ? MagicNumConstant.ZERO : --from; + size = size <= MagicNumConstant.ZERO || size > TEN_THOUSAND ? TEN_THOUSAND : size; + /**创建搜索请求对象**/ + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(TADL_INDEX_NAME); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.trackTotalHits(true).from(from).size(size); + /**根据时间戳排序**/ + searchSourceBuilder.sort(TIMESTAMP, SortOrder.ASC); + /**创建布尔查询对象**/ + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + + boolQueryBuilder.filter(QueryBuilders.matchQuery(EXPERIMENT_ID, experimentId)); + + /**设置boolQueryBuilder到searchSourceBuilder**/ + searchSourceBuilder.query(boolQueryBuilder); + + searchRequest = searchRequest.source(searchSourceBuilder); + /**执行搜索**/ + SearchResponse searchResponse; + try { + searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_K8S, "LogMonitoringApiImpl.searchTadlLogById error,param:[experimentId]={}, error:{}", experimentId, e); + return logMonitoringResult; + } + /**获取响应结果**/ + SearchHits hits = searchResponse.getHits(); + + SearchHit[] searchHits = hits.getHits(); + if (searchHits.length == MagicNumConstant.ZERO) { + return logMonitoringResult; + } + + for (SearchHit hit : searchHits) { + /**源文档**/ + Map sourceAsMap = hit.getSourceAsMap(); + /**取出message**/ + String message = (String) sourceAsMap.get(MESSAGE); + message = message.replace(LINEBREAK, BLANK); + /**添加日志信息到集合**/ + logList.add(message); + } + logMonitoringResult.setLogs(logList); + logMonitoringResult.setTotalLogs(logList.size()); + return logMonitoringResult; + } + /** * 得到日志信息String * @@ -256,6 +410,51 @@ public class LogMonitoringApiImpl implements LogMonitoringApi { return null; } + /** + * 从Elasticsearch查询日志 + * + * @param logMonitoringBo 日志查询bo + * @return List 日志集合 + */ + private List searchLogInfoByEs(LogMonitoringBO logMonitoringBo) { + + List logList = new ArrayList<>(); + + SearchRequest searchRequest = buildSearchRequest(logMonitoringBo); + /**执行搜索**/ + SearchResponse searchResponse; + try { + searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_K8S, "LogMonitoringApiImpl.searchLogInfoByEs error,param:[logMonitoringBo]={}, error:{}", JSON.toJSONString(logMonitoringBo), e); + return logList; + } + /**获取响应结果**/ + SearchHits hits = searchResponse.getHits(); + + SearchHit[] searchHits = hits.getHits(); + if (searchHits.length == MagicNumConstant.ZERO) { + return logList; + } + + for (SearchHit hit : searchHits) { + + String esResult = hit.getSourceAsString(); + JSONObject jsonObject = JSON.parseObject(esResult); + String message = jsonObject.getString(MESSAGE); + message = message.replace(LINEBREAK, BLANK); + + String podName = jsonObject.getJSONObject(KUBERNETES_KEY). + getString(KUBERNETES_POD_NAME_KEY); + + /**拼接日志信息**/ + String logString = String.format(LOG_FORMAT, podName, message); + /**添加日志信息到集合**/ + logList.add(logString); + } + return logList; + } + /** * 从Elasticsearch查询日志 @@ -366,4 +565,74 @@ public class LogMonitoringApiImpl implements LogMonitoringApi { return searchRequest.source(searchSourceBuilder); } + /** + * 构建搜索请求对象 + * + * @param logMonitoringBo 日志查询bo + * @return SearchRequest ES搜索请求对象 + */ + private SearchRequest buildSearchRequest(LogMonitoringBO logMonitoringBo) { + + /**创建搜索请求对象**/ + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(INDEX_NAME); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.trackTotalHits(true).from(logMonitoringBo.getFrom()).size(logMonitoringBo.getSize()); + + /**根据时间戳排序**/ + searchSourceBuilder.sort(TIMESTAMP, SortOrder.ASC); + /**过虑源字段**/ + String[] sourceFieldArray = sourceField.split(COMMA); + + searchSourceBuilder.fetchSource(sourceFieldArray, new String[]{}); + + /**创建布尔查询对象**/ + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + + /**添加podName查询条件**/ + Set podNames = logMonitoringBo.getPodNames(); + if (CollectionUtils.isNotEmpty(podNames)) { + boolQueryBuilder.filter(QueryBuilders.termsQuery(POD_NAME_KEY, podNames.toArray(new String[podNames.size()]))); + } + /**添加namespace查询条件**/ + String namespace = logMonitoringBo.getNamespace(); + if (StringUtils.isNotEmpty(namespace)) { + boolQueryBuilder.filter(QueryBuilders.matchQuery(NAMESPACE_KEY, namespace)); + } + /**添加业务查询条件**/ + BizEnum business = logMonitoringBo.getBusiness(); + if (null != business) { + boolQueryBuilder.filter(QueryBuilders.termQuery(BUSINESS_KEY, business.getBizCode())); + } + /**添加实验Id查询条件**/ + String businessGroupId = logMonitoringBo.getBusinessGroupId(); + if (StringUtils.isNotEmpty(businessGroupId)) { + boolQueryBuilder.filter(QueryBuilders.termQuery(BUSINESS_GROUP_ID_KEY, businessGroupId)); + } + /**添加关键字查询条件**/ + String logKeyword = logMonitoringBo.getLogKeyword(); + if (StringUtils.isNotEmpty(logKeyword)) { + boolQueryBuilder.filter(QueryBuilders.matchQuery(MESSAGE, logKeyword).operator(Operator.AND)); + } + /**添加时间范围查询条件**/ + Long beginTimeMillis = logMonitoringBo.getBeginTimeMillis(); + Long endTimeMillis = logMonitoringBo.getEndTimeMillis(); + if (beginTimeMillis != null || endTimeMillis != null){ + beginTimeMillis = beginTimeMillis == null ? ZERO_LONG : beginTimeMillis; + endTimeMillis = endTimeMillis == null ? System.currentTimeMillis() : endTimeMillis; + + /**将毫秒值转换为UTC时间**/ + String beginUtcTime = TimeTransferUtil.dateTransferToUtc(new Date(beginTimeMillis)); + String endUtcTime = TimeTransferUtil.dateTransferToUtc(new Date(endTimeMillis)); + boolQueryBuilder.filter(QueryBuilders.rangeQuery(TIMESTAMP).gte(beginUtcTime).lte(endUtcTime)); + } + + + /**设置boolQueryBuilder到searchSourceBuilder**/ + searchSourceBuilder.query(boolQueryBuilder); + + return searchRequest.source(searchSourceBuilder); + } + + } diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TerminalApiImpl.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TerminalApiImpl.java index 01c5c8d..74ff6af 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TerminalApiImpl.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TerminalApiImpl.java @@ -47,6 +47,7 @@ import org.dubhe.k8s.domain.bo.TerminalBO; import org.dubhe.k8s.domain.vo.PtJupyterDeployVO; import org.dubhe.k8s.domain.vo.TerminalResourceVO; import org.dubhe.k8s.domain.vo.VolumeVO; +import org.dubhe.k8s.enums.ImagePullPolicyEnum; import org.dubhe.k8s.enums.K8sKindEnum; import org.dubhe.k8s.enums.K8sResponseEnum; import org.dubhe.k8s.enums.LackOfResourcesEnum; @@ -132,6 +133,7 @@ public class TerminalApiImpl implements TerminalApi { Map podLabels = LabelUtils.getChildLabels(bo.getResourceName(), deploymentName, K8sKindEnum.DEPLOYMENT.getKind(), bo.getBusinessLabel(), bo.getTaskIdentifyLabel()); //部署deployment + bo.setImagePullPolicy(ImagePullPolicyEnum.ALWAYS.getPolicy()); Deployment deployment = ResourceBuildUtils.buildDeployment(bo, volumeVO, deploymentName); LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", deploymentName, YamlUtils.dumpAsYaml(deployment)); resourceIisolationApi.addIisolationInfo(deployment); diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TrainJobApiImpl.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TrainJobApiImpl.java index 8e61e87..9d22b1a 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TrainJobApiImpl.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TrainJobApiImpl.java @@ -250,8 +250,8 @@ public class TrainJobApiImpl implements TrainJobApi { this.fsMounts = bo.getFsMounts(); businessLabel = bo.getBusinessLabel(); + this.baseLabels = LabelUtils.getBaseLabels(baseName,bo.getBusinessLabel(),bo.getExtraLabelMap()); this.taskIdentifyLabel = bo.getTaskIdentifyLabel(); - this.baseLabels = LabelUtils.getBaseLabels(baseName,bo.getBusinessLabel()); this.volumeMounts = new ArrayList<>(); this.volumes = new ArrayList<>(); @@ -458,7 +458,7 @@ public class TrainJobApiImpl implements TrainJobApi { .withNewTemplate() .withNewMetadata() .withName(jobName) - .addToLabels(LabelUtils.getChildLabels(baseName, jobName, K8sKindEnum.JOB.getKind(),businessLabel, taskIdentifyLabel)) + .addToLabels(LabelUtils.getChildLabels(baseName, jobName, K8sKindEnum.JOB.getKind(),businessLabel,taskIdentifyLabel,baseLabels)) .withNamespace(namespace) .endMetadata() .withNewSpec() @@ -474,7 +474,7 @@ public class TrainJobApiImpl implements TrainJobApi { if (delayCreate == null || delayCreate == MagicNumConstant.ZERO){ resourceIisolationApi.addIisolationInfo(job); LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}", jobName); - job = client.batch().jobs().create(job); + job = client.batch().jobs().inNamespace(namespace).create(job); LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", jobName); } if (delayCreate > MagicNumConstant.ZERO || delayDelete > MagicNumConstant.ZERO){ diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/DeploymentBO.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/DeploymentBO.java index 43a66f6..07133fd 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/DeploymentBO.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/DeploymentBO.java @@ -93,6 +93,14 @@ public class DeploymentBO { */ private Set ports; + /** + * 镜像拉取策略 + * IfNotPresent 默认值 + * Always + * Never + */ + private String imagePullPolicy; + /** * 获取nfs路径 * @return diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/LogMonitoringBO.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/LogMonitoringBO.java index 832261a..3bef045 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/LogMonitoringBO.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/LogMonitoringBO.java @@ -17,6 +17,7 @@ package org.dubhe.k8s.domain.bo; import lombok.Data; import lombok.experimental.Accessors; +import org.dubhe.biz.base.enums.BizEnum; import org.dubhe.k8s.domain.dto.PodLogQueryDTO; import java.util.Set; @@ -56,6 +57,26 @@ public class LogMonitoringBO { **/ private Long endTimeMillis; + /** + * 日志查询起始行 + **/ + private Integer from; + + /** + * 日志查询行数 + **/ + private Integer size; + + /** + * 业务标签,用于标识一个组的业务模块 比如:TRAIN模块的trainId, TADL模块的experimentId + */ + private String businessGroupId; + + /** + * 业务标签,用于标识业务模块 + */ + private BizEnum business; + public LogMonitoringBO(String namespace,String podName){ this.namespace = namespace; this.podName = podName; diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/PtJupyterJobBO.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/PtJupyterJobBO.java index 55e45af..b71dd43 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/PtJupyterJobBO.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/PtJupyterJobBO.java @@ -65,6 +65,8 @@ public class PtJupyterJobBO { private GraphicsCardTypeEnum graphicsCardType; /**业务标签,用于标识业务模块**/ private String businessLabel; + /**额外扩展的标签**/ + private Map extraLabelMap; /**任务身份标签,用于标识任务身份**/ private String taskIdentifyLabel; /**延时创建时间，单位：分钟**/ diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/dto/BaseK8sPodCallbackCreateDTO.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/dto/BaseK8sPodCallbackCreateDTO.java index c283837..ac53965 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/dto/BaseK8sPodCallbackCreateDTO.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/dto/BaseK8sPodCallbackCreateDTO.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; import javax.validation.constraints.NotEmpty; +import java.util.Map; /** * @descripton 统一通用参数实现与校验 @@ -58,6 +59,9 @@ public class BaseK8sPodCallbackCreateDTO { @ApiModelProperty(value = "k8s pod containerStatuses state") private String messages; + @ApiModelProperty(value = "k8s pod lables") + private Map lables; + public BaseK8sPodCallbackCreateDTO(){ } diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/vo/LogMonitoringVO.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/vo/LogMonitoringVO.java index 68ff8a4..88502ed 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/vo/LogMonitoringVO.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/vo/LogMonitoringVO.java @@ -30,7 +30,7 @@ import java.util.List; @Data @AllArgsConstructor public class LogMonitoringVO extends PtBaseResult { - private Long totalLogs; + private Integer totalLogs; private List logs; public LogMonitoringVO() { diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/enums/BusinessLabelServiceNameEnum.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/enums/BusinessLabelServiceNameEnum.java index 275add2..c3de05c 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/enums/BusinessLabelServiceNameEnum.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/enums/BusinessLabelServiceNameEnum.java @@ -52,7 +52,10 @@ public enum BusinessLabelServiceNameEnum { * 专业版终端 */ TERMINAL(BizEnum.TERMINAL.getBizCode(), ApplicationNameConst.TERMINAL), - ; + /** + * TADL + */ + TADL(BizEnum.TADL.getBizCode(), ApplicationNameConst.SERVER_TADL); /** * 业务标签 */ @@ -74,18 +77,19 @@ public enum BusinessLabelServiceNameEnum { this.businessLabel = businessLabel; this.serviceName = serviceName; } - public static String getServiceNameByBusinessLabel(String businessLabel){ + + public static String getServiceNameByBusinessLabel(String businessLabel) { for (BusinessLabelServiceNameEnum businessLabelServiceNameEnum : BusinessLabelServiceNameEnum.values()) { - if (StringUtils.equals(businessLabel, businessLabelServiceNameEnum.getBusinessLabel() )){ + if (StringUtils.equals(businessLabel, businessLabelServiceNameEnum.getBusinessLabel())) { return businessLabelServiceNameEnum.getServiceName(); } } return BLANK; } - public static String getBusinessLabelByServiceName(String serviceName){ + public static String getBusinessLabelByServiceName(String serviceName) { for (BusinessLabelServiceNameEnum businessLabelServiceNameEnum : BusinessLabelServiceNameEnum.values()) { - if (StringUtils.equals(serviceName, businessLabelServiceNameEnum.getServiceName() )){ + if (StringUtils.equals(serviceName, businessLabelServiceNameEnum.getServiceName())) { return businessLabelServiceNameEnum.getBusinessLabel(); } } diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sCallBackTool.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sCallBackTool.java index 6f9db2d..1df466f 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sCallBackTool.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sCallBackTool.java @@ -73,7 +73,7 @@ public class K8sCallBackTool { * k8s 回调路径 */ private static final String K8S_CALLBACK_PATH_DEPLOYMENT = "/api/k8s/callback/deployment/"; - public static final String K8S_CALLBACK_PATH_POD = StringConstant.K8S_CALLBACK_URI+ SymbolConstant.SLASH; + public static final String K8S_CALLBACK_PATH_POD = StringConstant.K8S_CALLBACK_URI + SymbolConstant.SLASH; static { K8S_CALLBACK_PATH = new ArrayList<>(); @@ -113,7 +113,7 @@ public class K8sCallBackTool { */ public boolean validateToken(String token) { String expireTime = AesUtil.decrypt(token, secretKey); - if (StringUtils.isEmpty(expireTime)){ + if (StringUtils.isEmpty(expireTime)) { return false; } String nowTime = DateUtil.format( @@ -141,7 +141,7 @@ public class K8sCallBackTool { * @return String */ public String getPodCallbackUrl(String podLabel) { - return "http://"+BusinessLabelServiceNameEnum.getServiceNameByBusinessLabel(podLabel) + K8S_CALLBACK_PATH_POD + podLabel; + return "http://" + BusinessLabelServiceNameEnum.getServiceNameByBusinessLabel(podLabel) + K8S_CALLBACK_PATH_POD + podLabel; } /** @@ -151,7 +151,7 @@ public class K8sCallBackTool { * @return String */ public String getDeploymentCallbackUrl(String businessLabel) { - return "http://"+BusinessLabelServiceNameEnum.getServiceNameByBusinessLabel(businessLabel) + K8S_CALLBACK_PATH_DEPLOYMENT + businessLabel; + return "http://" + BusinessLabelServiceNameEnum.getServiceNameByBusinessLabel(businessLabel) + K8S_CALLBACK_PATH_DEPLOYMENT + businessLabel; } diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/ResourceBuildUtils.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/ResourceBuildUtils.java index 03e0d9d..885ee70 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/ResourceBuildUtils.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/ResourceBuildUtils.java @@ -63,6 +63,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static org.dubhe.biz.base.constant.MagicNumConstant.ZERO_LONG; + /** * @description 构建 Kubernetes 资源对象 * @date 2020-09-10 @@ -256,6 +258,7 @@ public class ResourceBuildUtils { .withNamespace(bo.getNamespace()) .endMetadata() .withNewSpec() + .withTerminationGracePeriodSeconds(ZERO_LONG) .addToNodeSelector(K8sUtils.gpuSelector(bo.getGpuNum())) .addToContainers(buildContainer(bo, volumeVO, deploymentName)) .addToVolumes(volumeVO.getVolumes().toArray(new Volume[0])) @@ -281,7 +284,7 @@ public class ResourceBuildUtils { Container container = new ContainerBuilder() .withNewName(name) .withNewImage(bo.getImage()) - .withNewImagePullPolicy(ImagePullPolicyEnum.IFNOTPRESENT.getPolicy()) + .withNewImagePullPolicy(StringUtils.isEmpty(bo.getImagePullPolicy())?ImagePullPolicyEnum.IFNOTPRESENT.getPolicy():bo.getImagePullPolicy()) .withVolumeMounts(volumeVO.getVolumeMounts()) .withNewResources().addToLimits(resourcesLimitsMap).endResources() .build(); diff --git a/dubhe-server/common-k8s/src/main/resources/kubeconfig_dev b/dubhe-server/common-k8s/src/main/resources/kubeconfig_dev new file mode 100644 index 0000000..bac8fb6 --- /dev/null +++ b/dubhe-server/common-k8s/src/main/resources/kubeconfig_dev @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: + server: https://127.0.0.1:6443 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: kubernetes-admin + name: kubernetes-admin@kubernetes +current-context: kubernetes-admin@kubernetes +kind: Config +preferences: {} +users: +- name: kubernetes-admin + user: + client-certificate-data: + client-key-data: \ No newline at end of file diff --git a/dubhe-server/common-k8s/src/main/resources/kubeconfig_pre b/dubhe-server/common-k8s/src/main/resources/kubeconfig_pre new file mode 100644 index 0000000..bac8fb6 --- /dev/null +++ b/dubhe-server/common-k8s/src/main/resources/kubeconfig_pre @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: + server: https://127.0.0.1:6443 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: kubernetes-admin + name: kubernetes-admin@kubernetes +current-context: kubernetes-admin@kubernetes +kind: Config +preferences: {} +users: +- name: kubernetes-admin + user: + client-certificate-data: + client-key-data: \ No newline at end of file diff --git a/dubhe-server/common-k8s/src/main/resources/kubeconfig_prod b/dubhe-server/common-k8s/src/main/resources/kubeconfig_prod index edd6705..bac8fb6 100644 --- a/dubhe-server/common-k8s/src/main/resources/kubeconfig_prod +++ b/dubhe-server/common-k8s/src/main/resources/kubeconfig_prod @@ -1,7 +1,7 @@ apiVersion: v1 clusters: - cluster: - certificate-authority-data: + certificate-authority-data: server: https://127.0.0.1:6443 name: kubernetes contexts: @@ -15,5 +15,5 @@ preferences: {} users: - name: kubernetes-admin user: - client-certificate-data: - client-key-data: + client-certificate-data: + client-key-data: \ No newline at end of file diff --git a/dubhe-server/common-k8s/src/main/resources/kubeconfig_test b/dubhe-server/common-k8s/src/main/resources/kubeconfig_test new file mode 100644 index 0000000..bac8fb6 --- /dev/null +++ b/dubhe-server/common-k8s/src/main/resources/kubeconfig_test @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: + server: https://127.0.0.1:6443 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: kubernetes-admin + name: kubernetes-admin@kubernetes +current-context: kubernetes-admin@kubernetes +kind: Config +preferences: {} +users: +- name: kubernetes-admin + user: + client-certificate-data: + client-key-data: \ No newline at end of file diff --git a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/config/RecycleConfig.java b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/config/RecycleConfig.java index b698b81..56cb46a 100644 --- a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/config/RecycleConfig.java +++ b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/config/RecycleConfig.java @@ -68,5 +68,9 @@ public class RecycleConfig { * 回收serving相关文件后，回收文件最大有效时长，以天为单位 */ private Integer servingValid; + /** + * 用户删除tadl算法版本文件后，文件最大有效时长，以天为单位 + */ + private Integer tadlValid; } \ No newline at end of file diff --git a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleModuleEnum.java b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleModuleEnum.java index 11cb250..5fbf13f 100644 --- a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleModuleEnum.java +++ b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleModuleEnum.java @@ -39,7 +39,8 @@ public enum RecycleModuleEnum { BIZ_MODEL(7, "模型管理",SERVER_MODEL), BIZ_DATAMEDICINE(8, "医学影像",SERVER_DATA_DCM), BIZ_MEASURE(9, "度量管理",SERVER_MEASURE), - BIZ_SERVING(10, "云端Serving", SERVER_SERVING); + BIZ_SERVING(10, "云端部署", SERVER_SERVING), + BIZ_TADL(11,"自动机器学习",SERVER_TADL); private Integer value; diff --git a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java index 5358662..6915e28 100644 --- a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java +++ b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java @@ -50,6 +50,15 @@ public enum RecycleResourceEnum { */ BATCH_SERVING_RECYCLE_FILE("batchServingRecycleFile", "云端Serving批量服务文件回收"), + /** + * tadl算法文件回收 + */ + TADL_ALGORITHM_RECYCLE_FILE("tadlAlgorithmRecycleFile", "tadl算法文件回收"), + /** + * tadl实验文件回收 + */ + TADL_EXPERIMENT_RECYCLE_FILE("tadlExperimentRecycleFile","tadl实验文件回收"), + /** * 标签组文件回收 */ diff --git a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/utils/RecycleTool.java b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/utils/RecycleTool.java index 466ff38..cf2a88c 100644 --- a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/utils/RecycleTool.java +++ b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/utils/RecycleTool.java @@ -197,7 +197,7 @@ public class RecycleTool { if (sourcePath.length() > nfsBucket.length()) { String emptyDir = recycleFileTmpPath + randomPath + StrUtil.SLASH; LogUtil.info(LogEnum.GARBAGE_RECYCLE, "recycle task sourcePath:{},emptyDir:{}", sourcePath, emptyDir); - process = Runtime.getRuntime().exec(new String[]{"/bin/sh", "-c", String.format(ShellFileStoreApiImpl.DEL_COMMAND, userName, ip, emptyDir, emptyDir, sourcePath, emptyDir, sourcePath)}); + process = Runtime.getRuntime().exec(new String[]{"/bin/sh", "-c", String.format(ShellFileStoreApiImpl.DEL_COMMAND, emptyDir, emptyDir, sourcePath, emptyDir, sourcePath)}); } return processRecycle(process); diff --git a/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/DataTaskExecuteThread.java b/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/DataTaskExecuteThread.java index c017a3e..324b8d7 100644 --- a/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/DataTaskExecuteThread.java +++ b/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/DataTaskExecuteThread.java @@ -559,7 +559,7 @@ public class DataTaskExecuteThread implements Runnable { for (List el : lists) { List fileIds = csvImportSaveDb(el, dataset); LogUtil.info(LogEnum.BIZ_DATASET, "table import transport to es datasetid:{}", datasetId); - fileService.transportTextToEs(dataset, fileIds); + fileService.transportTextToEs(dataset, fileIds,Boolean.FALSE); } } //------- 导入完成后 更改数据集状态 --------- diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ErrorEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ErrorEnum.java index f0dc878..220653a 100755 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ErrorEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ErrorEnum.java @@ -112,6 +112,7 @@ public enum ErrorEnum implements ErrorCode { DATASET_NOT_ANNOTATION(1718, "数据集暂不支持自动标注"), DATASET_NOT_OPERATIONS_BASE_DATASET(1719, "禁止操作内置的数据集"), DATASET_PUBLISH_REJECT(1720, "文本暂不支持多版本发布"), + DATASET_CHECK_VERSION_ERROR(1721,"目标版本不存在"), /** * 数据集版本校验 diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionFileMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionFileMapper.java index 0f11182..58543db 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionFileMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionFileMapper.java @@ -19,6 +19,7 @@ package org.dubhe.data.dao; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.ibatis.annotations.*; +import org.dubhe.data.domain.bo.FileUploadBO; import org.dubhe.data.domain.dto.DatasetVersionFileDTO; import org.dubhe.data.domain.entity.DataFileAnnotation; import org.dubhe.data.domain.entity.Dataset; @@ -302,4 +303,11 @@ public interface DatasetVersionFileMapper extends BaseMapper * @return Long 版本文件id */ Long getVersionFileIdByFileName(@Param("datasetId")Long datasetId, @Param("fileName")String fileName, @Param("versionName")String versionName); + + /** + * 获取导入文件所需信息 + * @param datasetId 数据集id + * @return List + */ + List getFileUploadContent(@Param("datasetId")Long datasetId, @Param("fileIds")List fileIds); } \ No newline at end of file diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/FileMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/FileMapper.java index a83b534..6535167 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/FileMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/FileMapper.java @@ -164,7 +164,8 @@ public interface FileMapper extends BaseMapper { * @param fileIdsNotToEs 需要同步的文件ID * @return List ES数据同步DTO */ - List selectTextDataNoTransport(@Param("datasetId") Long datasetId,@Param("fileIdsNotToEs")List fileIdsNotToEs); + List selectTextDataNoTransport(@Param("datasetId") Long datasetId,@Param("fileIdsNotToEs")List fileIdsNotToEs, + @Param("ifImport") Boolean ifImport); /** * 更新同步es标志 diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/domain/bo/FileUploadBO.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/domain/bo/FileUploadBO.java new file mode 100644 index 0000000..2188625 --- /dev/null +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/domain/bo/FileUploadBO.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.data.domain.bo; + +import lombok.*; + +import java.io.Serializable; + +@Builder +@Data +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class FileUploadBO implements Serializable { + + String fileUrl; + + String fileName; + + Long fileId; + + Long versionFileId; + + String annPath; +} diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/domain/dto/BatchFileCreateDTO.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/domain/dto/BatchFileCreateDTO.java index 457d1ac..a71198b 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/domain/dto/BatchFileCreateDTO.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/domain/dto/BatchFileCreateDTO.java @@ -43,4 +43,5 @@ public class BatchFileCreateDTO implements Serializable { @NotNull(message = "文件不能为空") private List files; + Boolean ifImport; } diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/rest/FileController.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/rest/FileController.java index 4bdedc2..4e684e5 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/rest/FileController.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/rest/FileController.java @@ -187,6 +187,13 @@ public class FileController { return new DataResponseBody(minioUtil.getEncryptedPutUrl(bucketName, objectName, expiry)); } + @ApiOperation("MinIO生成put请求的上传路径列表") + @PostMapping(value = "/minio/getUrls") + @PreAuthorize(Permissions.DATA) + public DataResponseBody getEncryptedPutUrls(@RequestBody String objectNames) { + return new DataResponseBody(minioUtil.getEncryptedPutUrls(bucketName, objectNames, expiry)); + } + @ApiOperation("获取MinIO相关信息") @GetMapping(value = "/minio/info") public DataResponseBody getMinIOInfo() { diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/rest/LabelGroupController.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/rest/LabelGroupController.java index 132c35a..a899aa7 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/rest/LabelGroupController.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/rest/LabelGroupController.java @@ -53,8 +53,7 @@ public class LabelGroupController { @PostMapping(value = "/labelGroup") @PreAuthorize(Permissions.DATA) public DataResponseBody create(@Validated @RequestBody LabelGroupCreateDTO labelGroupCreateDTO) { - labelGroupService.creatLabelGroup(labelGroupCreateDTO); - return new DataResponseBody(); + return new DataResponseBody(labelGroupService.creatLabelGroup(labelGroupCreateDTO)); } @ApiOperation(value = "标签组分页列表") @@ -103,8 +102,7 @@ public class LabelGroupController { public DataResponseBody importLabelGroup( @RequestParam(value = "file", required = false) MultipartFile file, LabelGroupImportDTO labelGroupImportDTO) { - labelGroupService.importLabelGroup(labelGroupImportDTO, file); - return new DataResponseBody(); + return new DataResponseBody(labelGroupService.importLabelGroup(labelGroupImportDTO, file)); } diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/DatasetVersionFileService.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/DatasetVersionFileService.java index ae69a56..f37a606 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/DatasetVersionFileService.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/DatasetVersionFileService.java @@ -18,6 +18,7 @@ package org.dubhe.data.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.dubhe.data.domain.bo.FileUploadBO; import org.dubhe.data.domain.dto.DatasetVersionFileDTO; import org.dubhe.data.domain.entity.Dataset; import org.dubhe.data.domain.entity.DatasetVersion; @@ -342,4 +343,11 @@ public interface DatasetVersionFileService { * @param versionName 版本名称 */ Long getVersionFileIdByFileName(Long datasetId, String fileName, String versionName); + + /** + * 获取导入文件所需信息 + * @param datasetId 数据集id + * @return List + */ + List getFileUploadContent(Long datasetId, List fileIds); } diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/FileService.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/FileService.java index 8c9f412..96df8fd 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/FileService.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/FileService.java @@ -315,7 +315,7 @@ public interface FileService { * @param dataset 数据集 * @param fileIdsNotToEs 需要同步的文件ID */ - void transportTextToEs(Dataset dataset,List fileIdsNotToEs); + void transportTextToEs(Dataset dataset,List fileIdsNotToEs,Boolean ifImport); /** * 还原es_transport状态 diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/LabelGroupService.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/LabelGroupService.java index ae89bc0..6c49111 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/LabelGroupService.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/LabelGroupService.java @@ -38,7 +38,7 @@ public interface LabelGroupService { * * @param labelGroupCreateDTO 创建标签组DTO */ - void creatLabelGroup(LabelGroupCreateDTO labelGroupCreateDTO); + Long creatLabelGroup(LabelGroupCreateDTO labelGroupCreateDTO); /** * 更新（编辑）标签组 @@ -94,7 +94,7 @@ public interface LabelGroupService { * @param labelGroupImportDTO 标签组导入DTO * @param file 导入文件 */ - void importLabelGroup(LabelGroupImportDTO labelGroupImportDTO, MultipartFile file); + Long importLabelGroup(LabelGroupImportDTO labelGroupImportDTO, MultipartFile file); /** * 标签组复制 diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/impl/DatasetServiceImpl.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/impl/DatasetServiceImpl.java index ae0cd09..239c8d6 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/impl/DatasetServiceImpl.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/impl/DatasetServiceImpl.java @@ -22,6 +22,7 @@ import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -56,6 +57,7 @@ import org.dubhe.data.constant.*; import org.dubhe.data.dao.DatasetMapper; import org.dubhe.data.dao.TaskMapper; import org.dubhe.biz.base.vo.ProgressVO; +import org.dubhe.data.domain.bo.FileUploadBO; import org.dubhe.data.domain.dto.*; import org.dubhe.data.domain.entity.*; import org.dubhe.data.domain.vo.*; @@ -67,6 +69,7 @@ import org.dubhe.data.machine.utils.StateMachineUtil; import org.dubhe.data.pool.BasePool; import org.dubhe.data.service.*; import org.dubhe.data.service.task.DatasetRecycleFile; +import org.dubhe.data.util.GeneratorKeyUtil; import org.dubhe.data.util.ZipUtil; import org.dubhe.recycle.domain.dto.RecycleCreateDTO; import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO; @@ -260,6 +263,9 @@ public class DatasetServiceImpl extends ServiceImpl impl @Resource private MinioUtil minioUtil; + @Autowired + private GeneratorKeyUtil generatorKeyUtil; + /** * 线程池 */ @@ -603,25 +609,23 @@ public class DatasetServiceImpl extends ServiceImpl impl } catch (DuplicateKeyException e) { throw new BusinessException(ErrorEnum.DATASET_NAME_DUPLICATED_ERROR); } - if (!dataset.isImport()) { - //新增数据标签关系 - List