diff --git a/dubhe-server/.gitignore b/dubhe-server/.gitignore index 35a9b43..6862255 100644 --- a/dubhe-server/.gitignore +++ b/dubhe-server/.gitignore @@ -48,4 +48,6 @@ output/ .project .classpath logs/ -*.log \ No newline at end of file +*.log + +kubeconfig_test \ No newline at end of file diff --git a/dubhe-server/README.md b/dubhe-server/README.md index 1f6a8ed..0cd4079 100644 --- a/dubhe-server/README.md +++ b/dubhe-server/README.md @@ -11,8 +11,8 @@ Nacos + Fegin + Gateway + （Spring Security + JWT + OAuth2） 初始化sql位置 /sql -**地址：** 127.0.0.1:3306 -**用户名：** test **密码：** test +**地址：** 10.5.29.66:3306 +**用户名：** test **密码：** zj12345678 ### Nacos diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsCreateDTO.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsCreateDTO.java index a0783a2..55131b9 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsCreateDTO.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsCreateDTO.java @@ -42,10 +42,10 @@ public class ResourceSpecsCreateDTO implements Serializable { @Pattern(regexp = StringConstant.REGEXP_SPECS, message = "规格名称支持字母、数字、汉字、英文横杠、下划线和空白字符") private String specsName; - @ApiModelProperty(value = "所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving, 4：dubhe-tadl)", required = true) + @ApiModelProperty(value = "所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving, 4：dubhe-tadl, 5:dubhe-point-cloud,6:data-annotation,7:atlas)", required = true) @NotNull(message = "所属业务场景不能为空") @Min(value = MagicNumConstant.ZERO, message = "所属业务场景错误") - @Max(value = MagicNumConstant.FOUR, message = "所属业务场景错误") + @Max(value = MagicNumConstant.SEVEN, message = "所属业务场景错误") private Integer module; @ApiModelProperty(value = "CPU数量,单位：核", required = true) @@ -71,4 +71,4 @@ public class ResourceSpecsCreateDTO implements Serializable { @Min(value = MagicNumConstant.ZERO, message = "工作空间的存储配额不能小于0") @Max(value = MagicNumConstant.TWO_BILLION, message = "工作空间的存储配额超限") private Integer workspaceRequest; -} \ No newline at end of file +} diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsQueryDTO.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsQueryDTO.java index d706fe9..ff76702 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsQueryDTO.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsQueryDTO.java @@ -47,8 +47,8 @@ public class ResourceSpecsQueryDTO extends PageQueryBase implements Serializable @ApiModelProperty("规格类型(0为CPU, 1为GPU)") private Boolean resourcesPoolType; - @ApiModelProperty("所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving,4：dubhe-tadl)") + @ApiModelProperty("所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving,4：dubhe-tadl,5：dubhe-point-cloud,6：data-annotation,7：atlas)") @Min(value = MagicNumConstant.ZERO, message = "所属业务场景错误") - @Max(value = MagicNumConstant.FOUR, message = "所属业务场景错误") + @Max(value = MagicNumConstant.SEVEN, message = "所属业务场景错误") private Integer module; -} \ No newline at end of file +} diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsUpdateDTO.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsUpdateDTO.java index df20b37..eca2eae 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsUpdateDTO.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/domain/dto/ResourceSpecsUpdateDTO.java @@ -49,10 +49,10 @@ public class ResourceSpecsUpdateDTO implements Serializable { @Pattern(regexp = StringConstant.REGEXP_SPECS, message = "规格名称支持字母、数字、汉字、英文横杠、下划线和空白字符") private String specsName; - @ApiModelProperty(value = "所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving, 4：dubhe-tadl)", required = true) + @ApiModelProperty(value = "所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving, 4：dubhe-tadl, 5:dubhe-point-cloud,6:data-annotation,7:atlas)", required = true) @NotNull(message = "所属业务场景不能为空") @Min(value = MagicNumConstant.ZERO, message = "所属业务场景错误") - @Max(value = MagicNumConstant.FOUR, message = "所属业务场景错误") + @Max(value = MagicNumConstant.SEVEN, message = "所属业务场景错误") private Integer module; @ApiModelProperty(value = "CPU数量,单位：核") @@ -74,4 +74,4 @@ public class ResourceSpecsUpdateDTO implements Serializable { @Min(value = MagicNumConstant.ZERO, message = "工作空间的存储配额不能小于0") @Max(value = MagicNumConstant.TWO_BILLION, message = "工作空间的存储配额超限") private Integer workspaceRequest; -} \ No newline at end of file +} diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/MinIoController.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/MinIoController.java new file mode 100644 index 0000000..ad05de3 --- /dev/null +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/MinIoController.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.admin.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.dubhe.admin.service.MinIoService; +import org.dubhe.biz.base.vo.DataResponseBody; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * @description minio 服务 + * @date 2022-06-08 + */ +@Api(tags = "系统：minio服务") +@RestController +@RequestMapping("/minio") +public class MinIoController { + + @Autowired + private MinIoService minIoService; + + @ApiOperation("获取MinIO相关信息") + @GetMapping(value = "/info") + public DataResponseBody getMinIOInfo() { + return new DataResponseBody(minIoService.getMinIOInfo()); + } + +} diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/UserController.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/UserController.java index 576db7e..f5ce840 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/UserController.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/rest/UserController.java @@ -30,7 +30,9 @@ import org.dubhe.biz.base.context.UserContext; import org.dubhe.biz.base.dto.UserDTO; import org.dubhe.biz.base.vo.DataResponseBody; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.security.core.Authentication; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; @@ -121,7 +123,7 @@ public class UserController { @ApiOperation("根据用户昵称搜索用户列表") @GetMapping(value = "/findByNickName") - public DataResponseBody> findByNickName(@RequestParam(value = "nickName",required = false) String nickName) { + public DataResponseBody> findByNickName(@RequestParam(value = "nickName", required = false) String nickName) { return new DataResponseBody(userService.findByNickName(nickName)); } @@ -130,4 +132,10 @@ public class UserController { public DataResponseBody> getUserList(@RequestParam(value = "ids") List ids) { return new DataResponseBody(userService.getUserList(ids)); } + + @ApiOperation("见微平台专用") + @GetMapping("/decryptVisUser") + public DataResponseBody encryptVisUser(Authentication authentication) { + return new DataResponseBody<>(userService.encryptUserForVis(authentication)); + } } diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/MinIoService.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/MinIoService.java new file mode 100644 index 0000000..2b96515 --- /dev/null +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/MinIoService.java @@ -0,0 +1,31 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.admin.service; + +import java.util.Map; + +/** + * @description minIo服务 service + * @date 2022-6-8 + **/ +public interface MinIoService { + /** + * 对minio 的账户密码进行加密操作 + * @return Map minio账户密码加密map + */ + Map getMinIOInfo(); +} diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/UserService.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/UserService.java index c7589cc..adb29a5 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/UserService.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/UserService.java @@ -26,6 +26,7 @@ import org.dubhe.biz.base.dto.TeamDTO; import org.dubhe.biz.base.dto.UserDTO; import org.dubhe.biz.base.vo.DataResponseBody; import org.dubhe.cloud.authconfig.service.AdminUserService; +import org.springframework.security.core.Authentication; import javax.servlet.http.HttpServletResponse; import java.io.IOException; @@ -239,4 +240,6 @@ public interface UserService extends AdminUserService, IService { * @return org.dubhe.admin.domain.vo.UserConfigCreateVO 用户配置 VO */ UserConfigCreateVO createOrUpdateUserConfig(UserConfigDTO userConfigDTO); + + String encryptUserForVis(Authentication authentication); } diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/MenuServiceImpl.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/MenuServiceImpl.java index 908ee3a..cf3d636 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/MenuServiceImpl.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/MenuServiceImpl.java @@ -384,6 +384,7 @@ public class MenuServiceImpl implements MenuService { menuVo1.setComponent(menuVo.getComponent()); } else { menuVo1.setPath(menuDTO.getPath()); + menuVo1.setName(menuDTO.getComponentName()); } menuVo.setName(null); menuVo.setMeta(null); diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/MinIoServiceImpl.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/MinIoServiceImpl.java new file mode 100644 index 0000000..ea306d7 --- /dev/null +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/MinIoServiceImpl.java @@ -0,0 +1,70 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.admin.service.impl; + +import org.dubhe.admin.service.MinIoService; +import org.dubhe.biz.base.constant.MagicNumConstant; +import org.dubhe.biz.base.exception.BusinessException; +import org.dubhe.biz.base.utils.RsaEncrypt; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; + +/** + * @description minIo服务 实现类 + * @date 2022-06-08 + */ +@Service +public class MinIoServiceImpl implements MinIoService { + + /** + * minIO公钥 + */ + @Value("${minio.accessKey}") + private String accessKey; + + /** + * minIO私钥 + */ + @Value("${minio.secretKey}") + private String secretKey; + + /** + * 加密字符串 + */ + @Value("${minio.url}") + private String url; + + @Override + public Map getMinIOInfo() { + try { + Map keyPair = RsaEncrypt.genKeyPair(); + String publicKey = RsaEncrypt.getPublicKey(keyPair); + String privateKey = RsaEncrypt.getPrivateKey(keyPair); + return new HashMap(MagicNumConstant.FOUR) {{ + put("url", RsaEncrypt.encrypt(url, publicKey)); + put("accessKey", RsaEncrypt.encrypt(accessKey, publicKey)); + put("secretKey", RsaEncrypt.encrypt(secretKey, publicKey)); + put("privateKey", privateKey); + }}; + } catch (Exception e) { + throw new BusinessException("MinIo info empty"); + } + } +} diff --git a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/UserServiceImpl.java b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/UserServiceImpl.java index babb286..2eaae7e 100644 --- a/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/UserServiceImpl.java +++ b/dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/UserServiceImpl.java @@ -47,6 +47,7 @@ import org.dubhe.admin.service.convert.UserConvert; import org.dubhe.biz.base.constant.AuthConst; import org.dubhe.biz.base.constant.ResponseCode; import org.dubhe.biz.base.constant.UserConstant; +import org.dubhe.biz.base.context.EncryptVisUser; import org.dubhe.biz.base.context.UserContext; import org.dubhe.biz.base.dto.*; import org.dubhe.biz.base.enums.BaseErrorCodeEnum; @@ -55,6 +56,7 @@ import org.dubhe.biz.base.exception.BusinessException; import org.dubhe.biz.base.exception.CaptchaException; import org.dubhe.biz.base.utils.DateUtil; import org.dubhe.biz.base.utils.Md5Util; +import org.dubhe.biz.base.utils.RSAUtil; import org.dubhe.biz.base.utils.RandomUtil; import org.dubhe.biz.base.utils.RsaEncrypt; import org.dubhe.biz.base.vo.DataResponseBody; @@ -73,6 +75,7 @@ import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cglib.beans.BeanMap; +import org.springframework.security.core.Authentication; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -81,6 +84,8 @@ import org.springframework.util.CollectionUtils; import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; import java.util.*; import java.util.stream.Collectors; @@ -109,6 +114,9 @@ public class UserServiceImpl extends ServiceImpl implements Us @Value("${user.config.gpu-limit}") private Integer gpuLimit; + @Value("${vis.public_key}") + private String visPublicKey; + @Autowired private UserMapper userMapper; @@ -783,6 +791,22 @@ public class UserServiceImpl extends ServiceImpl implements Us return userConfigCreateVO; } + @Override + public String encryptUserForVis(Authentication authentication) { + String encodeStr = ""; + try { + JwtUserDTO jwtUser = (JwtUserDTO) authentication.getPrincipal(); + EncryptVisUser visUser = new EncryptVisUser(); + BeanUtils.copyProperties(jwtUser.getUser(), visUser); + encodeStr = RSAUtil.publicEncrypt(JSONObject.toJSONString(visUser), RSAUtil.getPublicKey(visPublicKey)); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } catch (InvalidKeySpecException e) { + e.printStackTrace(); + } + return encodeStr; + } + /** * 校验验证码 diff --git a/dubhe-server/admin/src/main/resources/bootstrap.yml b/dubhe-server/admin/src/main/resources/bootstrap.yml index d5c9584..8c1abf5 100644 --- a/dubhe-server/admin/src/main/resources/bootstrap.yml +++ b/dubhe-server/admin/src/main/resources/bootstrap.yml @@ -11,7 +11,7 @@ spring: config: enabled: true server-addr: 127.0.0.1:8848 - namespace: dubhe-server-cloud-prod + namespace: dubhe-prod shared-configs[0]: data-id: common-biz.yaml group: dubhe @@ -29,9 +29,13 @@ spring: data-id: admin.yaml group: dubhe refresh: true + shared-configs[4]: + data-id: kubeconfig.yaml + group: dubhe + refresh: true discovery: enabled: true - namespace: dubhe-server-cloud-dev + namespace: dubhe-prod group: dubhe server-addr: 127.0.0.1:8848 diff --git a/dubhe-server/admin/src/test/java/org/dubhe/admin/AdminApplicationTests.java b/dubhe-server/admin/src/test/java/org/dubhe/admin/AdminApplicationTests.java index b145a05..0711ac7 100644 --- a/dubhe-server/admin/src/test/java/org/dubhe/admin/AdminApplicationTests.java +++ b/dubhe-server/admin/src/test/java/org/dubhe/admin/AdminApplicationTests.java @@ -1,3 +1,20 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + package org.dubhe.admin; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; diff --git a/dubhe-server/auth/src/main/java/org/dubhe/auth/exception/AuthenticationProviderImpl.java b/dubhe-server/auth/src/main/java/org/dubhe/auth/exception/AuthenticationProviderImpl.java old mode 100644 new mode 100755 diff --git a/dubhe-server/auth/src/main/java/org/dubhe/auth/exception/CustomerOauthException.java b/dubhe-server/auth/src/main/java/org/dubhe/auth/exception/CustomerOauthException.java old mode 100644 new mode 100755 diff --git a/dubhe-server/auth/src/main/java/org/dubhe/auth/exception/CustomerOauthExceptionSerializer.java b/dubhe-server/auth/src/main/java/org/dubhe/auth/exception/CustomerOauthExceptionSerializer.java old mode 100644 new mode 100755 diff --git a/dubhe-server/auth/src/main/java/org/dubhe/auth/exception/CustomerOauthWebResponseExceptionTranslator.java b/dubhe-server/auth/src/main/java/org/dubhe/auth/exception/CustomerOauthWebResponseExceptionTranslator.java old mode 100644 new mode 100755 diff --git a/dubhe-server/auth/src/main/resources/bootstrap.yml b/dubhe-server/auth/src/main/resources/bootstrap.yml index de12431..393e707 100644 --- a/dubhe-server/auth/src/main/resources/bootstrap.yml +++ b/dubhe-server/auth/src/main/resources/bootstrap.yml @@ -12,7 +12,7 @@ spring: config: enabled: true server-addr: 127.0.0.1:8848 - namespace: dubhe-server-cloud-prod + namespace: dubhe-prod shared-configs[0]: data-id: common-biz.yaml group: dubhe @@ -23,7 +23,7 @@ spring: refresh: true discovery: enabled: true - namespace: dubhe-server-cloud-dev + namespace: dubhe-prod group: dubhe server-addr: 127.0.0.1:8848 diff --git a/dubhe-server/auth/src/test/java/org/dubhe/auth/AuthApplicationTests.java b/dubhe-server/auth/src/test/java/org/dubhe/auth/AuthApplicationTests.java index fe4ff09..eeac482 100644 --- a/dubhe-server/auth/src/test/java/org/dubhe/auth/AuthApplicationTests.java +++ b/dubhe-server/auth/src/test/java/org/dubhe/auth/AuthApplicationTests.java @@ -1,3 +1,20 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + package org.dubhe.auth; import org.junit.jupiter.api.Test; diff --git a/dubhe-server/common-biz/base/pom.xml b/dubhe-server/common-biz/base/pom.xml index abc26cc..952e28c 100644 --- a/dubhe-server/common-biz/base/pom.xml +++ b/dubhe-server/common-biz/base/pom.xml @@ -63,6 +63,19 @@ commons-codec commons-codec + + com.google.guava + guava + + + commons-io + commons-io + + + com.alibaba + fastjson + 1.2.70 + diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/ApplicationNameConst.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/ApplicationNameConst.java index eb0543e..555326d 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/ApplicationNameConst.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/ApplicationNameConst.java @@ -94,4 +94,9 @@ public class ApplicationNameConst { * 专业版终端 */ public final static String TERMINAL = "dubhe-terminal"; + + /** + * 点云服务 + */ + public final static String SERVER_POINT_CLOUD ="dubhe-point-cloud"; } diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/MagicNumConstant.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/MagicNumConstant.java index 8f406e8..c9fc56d 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/MagicNumConstant.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/MagicNumConstant.java @@ -37,6 +37,8 @@ public final class MagicNumConstant { public static final int TEN = 10; public static final int ELEVEN = 11; + + public static final int TWELVE = 12; public static final int SIXTEEN = 16; public static final int TWENTY = 20; public static final int TWENTY_TWO = 22; diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/Permissions.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/Permissions.java index d572156..bfec32b 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/Permissions.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/Permissions.java @@ -113,6 +113,7 @@ public final class Permissions { public static final String IMAGE_UPLOAD = "hasAuthority('ROLE_training:image:upload')"; public static final String IMAGE_EDIT = "hasAuthority('ROLE_training:image:edit')"; public static final String IMAGE_DELETE = "hasAuthority('ROLE_training:image:delete')"; + public static final String IMAGE_EDIT_DEFAULT = "hasAuthority('ROLE_training:image:editDefault')"; /** * 度量管理 diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java index 55d3b02..4087d43 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java @@ -127,6 +127,13 @@ public final class StringConstant { public static final String CACHE_TASK_NAME ="task_name"; + public static final String DEFAULT_IMAGE_PROJECT ="dubhe"; + + /** + * python命令行参数格式 + */ + public static final String PYTHON_COMMAND_PATTERN = " --%s=%s"; + private StringConstant() { } } diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/context/EncryptVisUser.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/context/EncryptVisUser.java new file mode 100644 index 0000000..9095ecd --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/context/EncryptVisUser.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.context; + +import lombok.Data; + +import java.time.Instant; + +/** + * @description 见微平台加密用户信息 + * @date 2022-07-28 + */ +@Data +public class EncryptVisUser { + + private Long id; + + private String username; + + private String phone; + + private long timestamp = Instant.now().toEpochMilli(); +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageIdDTO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageIdDTO.java new file mode 100644 index 0000000..bb3f573 --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageIdDTO.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.dto; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +@Data +@Accessors(chain = true) +@EqualsAndHashCode +public class PtImageIdDTO { + + /** + * 镜像ID + */ + @NotNull(message = "镜像ID不能为空") + private Long id; +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageIdsDTO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageIdsDTO.java new file mode 100644 index 0000000..bb8fda9 --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageIdsDTO.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.dto; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotNull; +import java.util.List; + +@Data +@Accessors(chain = true) +@EqualsAndHashCode +public class PtImageIdsDTO { + + /** + * 镜像ID + */ + @NotNull(message = "镜像ID不能为空") + private List ids; +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageQueryUrlDTO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageQueryUrlDTO.java index 9c7f7bc..9d78a92 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageQueryUrlDTO.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtImageQueryUrlDTO.java @@ -20,6 +20,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; +import java.util.List; + /** * @description 查询镜像路径 * @date 2020-12-14 @@ -35,6 +37,8 @@ public class PtImageQueryUrlDTO { private String imageTag; - private Integer projectType; + private Boolean isDefault; + + private List imageTypes; } diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtModelBranchQueryByIdsDTO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtModelBranchQueryByIdsDTO.java new file mode 100644 index 0000000..6029c89 --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/PtModelBranchQueryByIdsDTO.java @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.dto; + +import lombok.Data; + +import javax.validation.constraints.NotEmpty; +import java.io.Serializable; +import java.util.List; + +@Data +public class PtModelBranchQueryByIdsDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * 模型版本ID + */ + @NotEmpty(message = "模型版本ID列表不能为空") + private List ids; +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/QueryResourceSpecsDTO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/QueryResourceSpecsDTO.java index 160559b..344503b 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/QueryResourceSpecsDTO.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/QueryResourceSpecsDTO.java @@ -45,10 +45,10 @@ public class QueryResourceSpecsDTO implements Serializable { private String specsName; /** - * 所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving，4：dubhe-tadl) + * 所属业务场景(0:通用，1：dubhe-notebook，2：dubhe-train，3：dubhe-serving，4：dubhe-tadl, 5:dubhe-point-cloud,6:data-annotation,7:atlas) */ @NotNull(message = "所属业务场景不能为空") @Min(value = MagicNumConstant.ZERO, message = "所属业务场景错误") - @Max(value = MagicNumConstant.FOUR, message = "所属业务场景错误") + @Max(value = MagicNumConstant.SEVEN, message = "所属业务场景错误") private Integer module; -} \ No newline at end of file +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/TrainAlgorithmSelectByNameDTO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/TrainAlgorithmSelectByNameDTO.java new file mode 100644 index 0000000..ce72702 --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/dto/TrainAlgorithmSelectByNameDTO.java @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.dto; + +import lombok.Data; +import org.dubhe.biz.base.constant.MagicNumConstant; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotBlank; +import java.io.Serializable; + + +@Data +public class TrainAlgorithmSelectByNameDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @Length(max = MagicNumConstant.THIRTY_TWO, message = "算法名称有误") + @NotBlank + private String algorithmName; +} \ No newline at end of file diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BizEnum.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BizEnum.java index c931273..5ae5427 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BizEnum.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BizEnum.java @@ -64,7 +64,15 @@ public enum BizEnum { /** * TADL */ - TADL("TADL服务", "tadl", 8); + TADL("TADL服务", "tadl", 8), + /** + * POINT_CLOUD + */ + POINT_CLOUD("点云服务","pointcloud",9), + /** + * DATA + */ + DATA("数据服务","data",10); /** * 业务模块名称 diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/ImageTypeEnum.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/ImageTypeEnum.java index 9d8acfa..87c110a 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/ImageTypeEnum.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/ImageTypeEnum.java @@ -47,7 +47,11 @@ public enum ImageTypeEnum { /** * terminal镜像 */ - TERMINAL("terminal镜像", "terminal", 3) + TERMINAL("terminal镜像", "terminal", 3), + /** + * 点云镜像 + */ + POINT_CLOUD("点云镜像","pointCloud",4) ; /** @@ -59,7 +63,7 @@ public enum ImageTypeEnum { */ private String code; /** - * 镜像项目类型 + * 镜像用途 */ private Integer type; @@ -89,7 +93,7 @@ public enum ImageTypeEnum { /** * 根据type获取code * - * @param type 镜像项目类型 + * @param type 镜像用途 * @return String 镜像项目代码 */ public static String getType(Integer type) { diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/BaseUtils.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/BaseUtils.java new file mode 100644 index 0000000..2fdc90a --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/BaseUtils.java @@ -0,0 +1,79 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.utils; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import javax.servlet.ServletRequest; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; + +/** + * @description + * @date 2022-07-06 + */ +public class BaseUtils { + /** + * 获取request 中的参数，以map形式返回 + * @param request + * @return + */ + public static Map getParamMap(ServletRequest request) { + //Assert.notNull(request,"参数不能为空"); + Map map = Maps.newHashMap(); + Enumeration en = request.getParameterNames(); + while (en.hasMoreElements()) { + String name = en.nextElement(); + String[] values = request.getParameterValues(name); + if (values == null || values.length == 0) { + continue; + } + String value = values[0]; + if (value != null) { + map.put(name, value); + } + } + return map; + } + + /** + * 把数组所有元素，按字母排序，然后按照“参数=参数值”的模式用“&”字符拼接成字符串 + * + * @param param + * 需要签名的参数 + * @return 签名的字符串 + */ + public static String createLinkString(Map param) { + Map params = Maps.newHashMap(); + params.putAll(param); + params.put("dev-name", "VMS"); + params.put("project", "TIANSHU"); + List keys = Lists.newArrayList(params.keySet().iterator()); + Collections.sort(keys); + StringBuilder signStr = new StringBuilder(); + for (String key : keys) { + if (!StringUtils.isNotEmpty(params.get(key).toString())) { + continue; + } + signStr.append(key).append("=").append(params.get(key)).append("&"); + } + return signStr.deleteCharAt(signStr.length() - 1).toString(); + } +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/CommandUtil.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/CommandUtil.java new file mode 100644 index 0000000..b0708bd --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/CommandUtil.java @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.biz.base.utils; + +import com.alibaba.fastjson.JSONObject; + +import static org.dubhe.biz.base.constant.StringConstant.PYTHON_COMMAND_PATTERN; + +/** + * @description 命令行工具类 + * @date 2022-05-16 + */ +public class CommandUtil { + /** + * 构造python运行命令 + * + * @param runCommand + * @param runParams + * @return + */ + public static String buildPythonCommand(String runCommand, JSONObject runParams) { + StringBuilder sb = new StringBuilder(); + sb.append(runCommand); + if (null != runParams && !runParams.isEmpty()) { + runParams.forEach((k, v) -> sb.append(String.format(PYTHON_COMMAND_PATTERN, k, v))); + } + return sb.toString(); + } +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/EncryptUtils.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/EncryptUtils.java new file mode 100644 index 0000000..8750beb --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/EncryptUtils.java @@ -0,0 +1,101 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.utils; + +import com.google.common.collect.Maps; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.DESKeySpec; +import javax.crypto.spec.IvParameterSpec; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** + * @description vms服务加解密工具类 + * @date 2022-07-06 + */ +public class EncryptUtils { + private static final String STR_PARAM = "Passw0rd"; + + private static Cipher cipher; + + private static final IvParameterSpec IV = new IvParameterSpec(STR_PARAM.getBytes(StandardCharsets.UTF_8)); + + private static DESKeySpec getDesKeySpec(String source) throws Exception { + if (source == null || source.length() == 0) { + return null; + } + cipher = Cipher.getInstance("DES/CBC/PKCS5Padding"); + String strKey = "Passw0rd"; + return new DESKeySpec(strKey.getBytes(StandardCharsets.UTF_8)); + } + + /** + * 对称加密 + */ + public static String desEncrypt(String source) throws Exception { + DESKeySpec desKeySpec = getDesKeySpec(source); + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES"); + SecretKey secretKey = keyFactory.generateSecret(desKeySpec); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, IV); + return byte2hex( + cipher.doFinal(source.getBytes(StandardCharsets.UTF_8))).toUpperCase(); + } + + /** + * 对称解密 + */ + public static String desDecrypt(String source) throws Exception { + byte[] src = hex2byte(source.getBytes(StandardCharsets.UTF_8)); + DESKeySpec desKeySpec = getDesKeySpec(source); + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES"); + SecretKey secretKey = keyFactory.generateSecret(desKeySpec); + cipher.init(Cipher.DECRYPT_MODE, secretKey, IV); + byte[] retByte = cipher.doFinal(src); + return new String(retByte); + } + + private static String byte2hex(byte[] inStr) { + String stmp; + StringBuilder out = new StringBuilder(inStr.length * 2); + for (byte b : inStr) { + stmp = Integer.toHexString(b & 0xFF); + if (stmp.length() == 1) { + // 如果是0至F的单位字符串，则添加0 + out.append("0").append(stmp); + } else { + out.append(stmp); + } + } + return out.toString(); + } + + private static byte[] hex2byte(byte[] b) { + int size = 2; + if ((b.length % size) != 0) { + throw new IllegalArgumentException("长度不是偶数"); + } + byte[] b2 = new byte[b.length / 2]; + for (int n = 0; n < b.length; n += size) { + String item = new String(b, n, 2); + b2[n / 2] = (byte) Integer.parseInt(item, 16); + } + return b2; + } +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/MapUtil.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/MapUtil.java new file mode 100644 index 0000000..fdb9031 --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/MapUtil.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + + +package org.dubhe.biz.base.utils; + +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.TypeReference; + +import java.util.HashMap; +import java.util.Map; + +/** + * @description Map工具类 + * @date 2022-05-16 + */ +public class MapUtil { + public static Map convertJsonObject(JSONObject object) { + if (object == null) { + return new HashMap<>(); + } + return JSONObject.parseObject(object.toJSONString(), + new TypeReference>(){}); + } +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/PtModelUtil.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/PtModelUtil.java index 77dbe55..4d07f36 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/PtModelUtil.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/PtModelUtil.java @@ -36,6 +36,8 @@ public class PtModelUtil { public static final int NUMBER_EIGHT = 8; + public static final int NUMBER_THIRTY_TWO = 32; + public static final int NUMBER_ONE_HUNDRED_TWENTY_EIGHT = 128; public static final int NUMBER_TWO_HUNDRED_FIFTY_FIVE = 255; diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/RSAUtil.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/RSAUtil.java new file mode 100644 index 0000000..2f4c48e --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/RSAUtil.java @@ -0,0 +1,178 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.utils; + +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; + +import javax.crypto.Cipher; +import java.io.ByteArrayOutputStream; +import java.security.Key; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.interfaces.RSAPrivateKey; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + + +/** + * @Desc RSA 非对称加密 + * @Author MrAshin + * @Date 2022-07-27 + */ +public class RSAUtil { + + public static final String CHARSET = "UTF-8"; + public static final String RSA_ALGORITHM = "RSA"; + + /** + * 生成公钥和私钥 + * @param keySize + * @return + */ + public static Map createKeys(int keySize) { + //为RSA算法创建一个KeyPairGenerator对象 + KeyPairGenerator kpg; + try { + kpg = KeyPairGenerator.getInstance(RSA_ALGORITHM); + } catch (NoSuchAlgorithmException e) { + throw new IllegalArgumentException("No such algorithm-->[" + RSA_ALGORITHM + "]"); + } + + //初始化KeyPairGenerator对象,密钥长度 + kpg.initialize(keySize); + //生成密匙对 + KeyPair keyPair = kpg.generateKeyPair(); + //得到公钥 + Key publicKey = keyPair.getPublic(); + String publicKeyStr = Base64.encodeBase64URLSafeString(publicKey.getEncoded()); + //得到私钥 + Key privateKey = keyPair.getPrivate(); + String privateKeyStr = Base64.encodeBase64URLSafeString(privateKey.getEncoded()); + Map keyPairMap = new HashMap(); + keyPairMap.put("publicKey", publicKeyStr); + keyPairMap.put("privateKey", privateKeyStr); + + return keyPairMap; + } + + /** + * 得到公钥 + * @param publicKey 密钥字符串（经过base64编码） + * @throws Exception + */ + public static RSAPublicKey getPublicKey(String publicKey) throws NoSuchAlgorithmException, InvalidKeySpecException { + //通过X509编码的Key指令获得公钥对象 + KeyFactory keyFactory = KeyFactory.getInstance(RSA_ALGORITHM); + X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(Base64.decodeBase64(publicKey)); + RSAPublicKey key = (RSAPublicKey) keyFactory.generatePublic(x509KeySpec); + return key; + } + + /** + * 得到私钥 + * @param privateKey 密钥字符串（经过base64编码） + * @throws Exception + */ + public static RSAPrivateKey getPrivateKey(String privateKey) throws NoSuchAlgorithmException, InvalidKeySpecException { + //通过PKCS#8编码的Key指令获得私钥对象 + KeyFactory keyFactory = KeyFactory.getInstance(RSA_ALGORITHM); + PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(Base64.decodeBase64(privateKey)); + RSAPrivateKey key = (RSAPrivateKey) keyFactory.generatePrivate(pkcs8KeySpec); + return key; + } + + /** + * 公钥加密 + * @param data + * @param publicKey + * @return + */ + public static String publicEncrypt(String data, RSAPublicKey publicKey) { + try { + Cipher cipher = Cipher.getInstance(RSA_ALGORITHM); + cipher.init(Cipher.ENCRYPT_MODE, publicKey); + return Base64.encodeBase64URLSafeString(rsaSplitCodec(cipher, Cipher.ENCRYPT_MODE, data.getBytes(CHARSET), publicKey.getModulus().bitLength())); + } catch (Exception e) { + throw new RuntimeException("加密字符串[" + data + "]时遇到异常", e); + } + } + + /** + * 私钥解密 + * @param data + * @param privateKey + * @return + */ + public static String privateDecrypt(String data, RSAPrivateKey privateKey) { + try { + Cipher cipher = Cipher.getInstance(RSA_ALGORITHM); + cipher.init(Cipher.DECRYPT_MODE, privateKey); + return new String(rsaSplitCodec(cipher, Cipher.DECRYPT_MODE, Base64.decodeBase64(data), privateKey.getModulus().bitLength()), CHARSET); + } catch (Exception e) { + throw new RuntimeException("解密字符串[" + data + "]时遇到异常", e); + } + } + + + /** + * 分段加解密 + * @param cipher + * @param opmode + * @param datas + * @param keySize + * @return + */ + private static byte[] rsaSplitCodec(Cipher cipher, int opmode, byte[] datas, int keySize) { + int maxBlock = 0; + if (opmode == Cipher.DECRYPT_MODE) { + maxBlock = keySize / 8; + } else { + maxBlock = keySize / 8 - 11; + } + ByteArrayOutputStream + out = new ByteArrayOutputStream(); + int offSet = 0; + byte[] buff; + int i = 0; + try { + while (datas.length > offSet) { + if (datas.length - offSet > maxBlock) { + buff = cipher.doFinal(datas, offSet, maxBlock); + } else { + buff = cipher.doFinal(datas, offSet, datas.length - offSet); + } + out.write(buff, 0, buff.length); + i++; + offSet = i * maxBlock; + } + } catch (Exception e) { + throw new RuntimeException("加解密阀值为[" + maxBlock + "]的数据时发生异常", e); + } + byte[] resultDatas = out.toByteArray(); + IOUtils.closeQuietly(out); + return resultDatas; + } +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/ResultUtil.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/ResultUtil.java index 8c1d9c4..ee7c7ec 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/ResultUtil.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/utils/ResultUtil.java @@ -57,4 +57,10 @@ public class ResultUtil { throw new BusinessException(String.format(errorMessageTemplate, params)); } } + + public static void isTrue(Boolean object, String errorMessageTemplate, Object... params) { + if (!Boolean.TRUE.equals(object)) { + throw new BusinessException(String.format(errorMessageTemplate, params)); + } + } } \ No newline at end of file diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/DatasetVO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/DatasetVO.java index 14ff331..5eee388 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/DatasetVO.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/DatasetVO.java @@ -157,4 +157,14 @@ public class DatasetVO implements Serializable { */ private Integer fileCount; + /** + * 模板 + */ + private Integer templateType; + + /** + * 所属模块 + */ + private Integer module; + } diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/LabelGroupBaseVO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/LabelGroupBaseVO.java new file mode 100644 index 0000000..1afaebd --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/LabelGroupBaseVO.java @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.vo; + +import lombok.Data; + +/** + * @description + * @date 2022/4/22 + **/ +@Data +public class LabelGroupBaseVO { + /** + * 标签组ID + */ + private Long id; + + /** + * 标签组名称 + */ + private String name; + /** + * 标签组类型：0: private 私有标签组, 1:public 公开标签组 + */ + private Integer type; + + /** + * 标签组描述 + */ + private String remark; + /** + * 标签组类型 + */ + private Integer labelGroupType; +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/ProgressVO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/ProgressVO.java index 02bb09f..b756532 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/ProgressVO.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/ProgressVO.java @@ -47,5 +47,9 @@ public class ProgressVO implements Serializable { private Long finishAutoTrack = MagicNumConstant.ZERO_LONG; @Builder.Default private Long annotationNotDistinguishFile = MagicNumConstant.ZERO_LONG; + @Builder.Default + private Long haveAnnotation = MagicNumConstant.ZERO_LONG; + @Builder.Default + private Long noAnnotation = MagicNumConstant.ZERO_LONG; } \ No newline at end of file diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/PtImageVO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/PtImageVO.java new file mode 100644 index 0000000..15b3bc7 --- /dev/null +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/PtImageVO.java @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.biz.base.vo; + +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; + +@Data +@Accessors(chain = true) +public class PtImageVO implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * 镜像id + */ + private Long id; + + /** + * 镜像名称 + */ + private String name; + + /** + * 镜像地址 + */ + private String imageUrl; + + /** + * 镜像版本 + */ + private String tag; + +} diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/PtModelInfoQueryVO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/PtModelInfoQueryVO.java index 5dd37d2..96b1d3b 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/PtModelInfoQueryVO.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/PtModelInfoQueryVO.java @@ -55,11 +55,6 @@ public class PtModelInfoQueryVO implements Serializable { */ private String modelDescription; - /** - * 模型分类 - */ - private String modelClassName; - /** * 模型地址 */ @@ -90,6 +85,11 @@ public class PtModelInfoQueryVO implements Serializable { */ private Long createUserId; + /** + * 创建人用户名 + */ + private String createUserName; + /** * 修改人ID */ @@ -124,4 +124,19 @@ public class PtModelInfoQueryVO implements Serializable { * 是否能提供服务（true:能，false：否） */ private Boolean servingModel; + + /** + * 炼制模型结构名称 + */ + private String structName; + + /** + * 炼制模型尺寸 + */ + private Integer modelSize; + + /** + * 模型分类(目标检测、目标分类) + */ + private String modelClassName; } diff --git a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/TrainAlgorithmQureyVO.java b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/TrainAlgorithmQureyVO.java index 83eb8c0..fb6b173 100644 --- a/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/TrainAlgorithmQureyVO.java +++ b/dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/vo/TrainAlgorithmQureyVO.java @@ -59,6 +59,11 @@ public class TrainAlgorithmQureyVO implements Serializable { */ private String imageName; + /** + * 环境镜像版本 + */ + private String imageTag; + /** * 代码目录 */ diff --git a/dubhe-server/common-biz/data-permission/src/main/java/org/dubhe/biz/permission/interceptor/PaginationInterceptor.java b/dubhe-server/common-biz/data-permission/src/main/java/org/dubhe/biz/permission/interceptor/PaginationInterceptor.java index f785952..b03f103 100644 --- a/dubhe-server/common-biz/data-permission/src/main/java/org/dubhe/biz/permission/interceptor/PaginationInterceptor.java +++ b/dubhe-server/common-biz/data-permission/src/main/java/org/dubhe/biz/permission/interceptor/PaginationInterceptor.java @@ -87,7 +87,7 @@ public class PaginationInterceptor extends AbstractSqlParserHandler implements I /** * 单页限制 500 条，小于 0 如 -1 不受限制 */ - protected long limit = 500L; + protected long limit = -1L; /** * 数据类型 */ diff --git a/dubhe-server/common-biz/log/src/main/java/org/dubhe/biz/log/enums/LogEnum.java b/dubhe-server/common-biz/log/src/main/java/org/dubhe/biz/log/enums/LogEnum.java index e0d3e40..1f95489 100644 --- a/dubhe-server/common-biz/log/src/main/java/org/dubhe/biz/log/enums/LogEnum.java +++ b/dubhe-server/common-biz/log/src/main/java/org/dubhe/biz/log/enums/LogEnum.java @@ -78,10 +78,14 @@ public enum LogEnum { MEASURE, //云端Serving SERVING, + //serving gateway + SERVING_GATEWAY, //专业版终端 TERMINAL, //tadl - TADL; + TADL, + //点云 + POINT_CLOUD; /** * 判断日志类型不能为空 diff --git a/dubhe-server/common-biz/redis/src/main/java/org/dubhe/biz/redis/utils/RedisUtils.java b/dubhe-server/common-biz/redis/src/main/java/org/dubhe/biz/redis/utils/RedisUtils.java index 500b02c..d36a527 100644 --- a/dubhe-server/common-biz/redis/src/main/java/org/dubhe/biz/redis/utils/RedisUtils.java +++ b/dubhe-server/common-biz/redis/src/main/java/org/dubhe/biz/redis/utils/RedisUtils.java @@ -52,6 +52,14 @@ public class RedisUtils { // =============================common============================ + /** + * 模糊删除 key + */ + public void del(String pattern){ + Set keys = redisTemplate.keys(pattern); + redisTemplate.delete(keys); + } + /** * 指定缓存失效时间 * diff --git a/dubhe-server/common-cloud/auth-config/src/main/java/org/dubhe/cloud/authconfig/exception/handler/CustomerAccessDeniedHandler.java b/dubhe-server/common-cloud/auth-config/src/main/java/org/dubhe/cloud/authconfig/exception/handler/CustomerAccessDeniedHandler.java old mode 100644 new mode 100755 diff --git a/dubhe-server/common-cloud/auth-config/src/main/java/org/dubhe/cloud/authconfig/exception/handler/CustomerTokenExceptionEntryPoint.java b/dubhe-server/common-cloud/auth-config/src/main/java/org/dubhe/cloud/authconfig/exception/handler/CustomerTokenExceptionEntryPoint.java old mode 100644 new mode 100755 diff --git a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml index f507bd9..b5d22ed 100644 --- a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml +++ b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml @@ -5,8 +5,8 @@ spring: password: Tianshu context-path: /nacos config: - namespace: dubhe-server-cloud-dev - server-addr: 127.0.0.1:8848 + namespace: dubhe-prod + server-addr: 10.105.1.132:8848 discovery: - namespace: dubhe-server-cloud-dev - server-addr: 127.0.0.1:8848 + namespace: dubhe-prod + server-addr: 10.105.1.132:8848 diff --git a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-open-dev.yml b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-open-dev.yml index f70f2c4..2881669 100644 --- a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-open-dev.yml +++ b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-open-dev.yml @@ -1,12 +1,12 @@ spring: cloud: nacos: - username: nacos - password: Tianshu + username: root + password: zjlab context-path: /nacos config: namespace: dubhe-server-cloud-open-dev - server-addr: 10.105.1.132:8848 + server-addr: 10.101.12.15:8848 discovery: namespace: dubhe-server-cloud-open-dev - server-addr: 10.105.1.132:8848 \ No newline at end of file + server-addr: 10.101.12.15:8848 diff --git a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-test.yml b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-test.yml index 3c02c72..9352755 100644 --- a/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-test.yml +++ b/dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-test.yml @@ -6,7 +6,7 @@ spring: context-path: /nacos config: namespace: dubhe-server-cloud-test - server-addr: 127.0.0.1:8848 + server-addr: 10.105.1.132:8848 discovery: namespace: dubhe-server-cloud-test - server-addr: 127.0.0.1:8848 + server-addr: 10.105.1.132:8848 diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/ModelServiceApi.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/ModelServiceApi.java new file mode 100644 index 0000000..e4acd7f --- /dev/null +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/ModelServiceApi.java @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.k8s.api; + +import org.dubhe.k8s.domain.PtBaseResult; +import org.dubhe.k8s.domain.bo.ModelServiceBO; +import org.dubhe.k8s.domain.bo.ModelServingBO; +import org.dubhe.k8s.domain.resource.BizDeployment; +import org.dubhe.k8s.domain.vo.ModelServiceVO; +import org.dubhe.k8s.domain.vo.ModelServingVO; + + +public interface ModelServiceApi { + + /** + * 创建 + * @param bo + * @return + */ + ModelServiceVO create(ModelServiceBO bo); + + /** + * 删除 + * @param namespace 命名空间 + * @param resourceName 资源名称 + * @return PtBaseResult 基础结果类 + */ + PtBaseResult delete(String namespace, String resourceName); +} diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/ModelServiceApiImpl.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/ModelServiceApiImpl.java new file mode 100644 index 0000000..82695a9 --- /dev/null +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/ModelServiceApiImpl.java @@ -0,0 +1,207 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.k8s.api.impl; + +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.RandomUtil; +import cn.hutool.core.util.StrUtil; +import com.google.common.collect.Maps; +import io.fabric8.kubernetes.api.model.*; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder; +import io.fabric8.kubernetes.api.model.apps.DeploymentList; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientException; +import org.apache.commons.compress.utils.Lists; +import org.dubhe.biz.base.constant.MagicNumConstant; +import org.dubhe.biz.base.constant.SymbolConstant; +import org.dubhe.biz.file.api.FileStoreApi; +import org.dubhe.biz.log.enums.LogEnum; +import org.dubhe.biz.log.utils.LogUtil; +import org.dubhe.k8s.api.ModelServiceApi; +import org.dubhe.k8s.api.ResourceIisolationApi; +import org.dubhe.k8s.api.ResourceQuotaApi; +import org.dubhe.k8s.api.VolumeApi; +import org.dubhe.k8s.constant.K8sParamConstants; +import org.dubhe.k8s.domain.PtBaseResult; +import org.dubhe.k8s.domain.bo.BuildFsVolumeBO; +import org.dubhe.k8s.domain.bo.ModelServiceBO; +import org.dubhe.k8s.domain.bo.ModelServingBO; +import org.dubhe.k8s.domain.resource.BizDeployment; +import org.dubhe.k8s.domain.vo.ModelServiceVO; +import org.dubhe.k8s.domain.vo.ModelServingVO; +import org.dubhe.k8s.domain.vo.VolumeVO; +import org.dubhe.k8s.enums.*; +import org.dubhe.k8s.utils.BizConvertUtils; +import org.dubhe.k8s.utils.K8sUtils; +import org.dubhe.k8s.utils.LabelUtils; +import org.dubhe.k8s.utils.YamlUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.annotation.Resource; +import java.util.*; + +public class ModelServiceApiImpl implements ModelServiceApi { + + @Autowired + private ResourceQuotaApi resourceQuotaApi; + + @Autowired + private ResourceIisolationApi resourceIisolationApi; + + @Resource(name = "hostFileStoreApiImpl") + private FileStoreApi fileStoreApi; + + @Autowired + private VolumeApi volumeApi; + + private KubernetesClient client; + private K8sUtils k8sUtils; + + + public ModelServiceApiImpl(K8sUtils k8sUtils) { + this.k8sUtils = k8sUtils; + this.client = k8sUtils.getClient(); + } + + + @Override + public ModelServiceVO create(ModelServiceBO bo) { + //资源配额校验 + LimitsOfResourcesEnum limitsOfResources = resourceQuotaApi.reachLimitsOfResources(bo.getNamespace(), bo.getCpuNum(), bo.getMemNum(), bo.getGpuNum()); + if (!LimitsOfResourcesEnum.ADEQUATE.equals(limitsOfResources)) { + return new ModelServiceVO().error(K8sResponseEnum.LACK_OF_RESOURCES.getCode(), limitsOfResources.getMessage()); + } + LogUtil.info(LogEnum.BIZ_K8S, "Params of creating ModelServing--create:{}", bo); + if (!fileStoreApi.createDirs(bo.getDirList().toArray(new String[MagicNumConstant.ZERO]))) { + return new ModelServiceVO().error(K8sResponseEnum.INTERNAL_SERVER_ERROR.getCode(), K8sResponseEnum.INTERNAL_SERVER_ERROR.getMessage()); + } + //存储卷构建 + VolumeVO volumeVO = volumeApi.buildFsVolumes(new BuildFsVolumeBO(bo.getNamespace(), bo.getResourceName(), bo.getFsMounts())); + if (!K8sResponseEnum.SUCCESS.getCode().equals(volumeVO.getCode())) { + return new ModelServiceVO().error(volumeVO.getCode(), volumeVO.getMessage()); + } + + //名称生成 + String deploymentName = StrUtil.format(K8sParamConstants.RESOURCE_NAME_TEMPLATE, bo.getResourceName(), RandomUtil.randomString(MagicNumConstant.EIGHT)); + + //部署deployment + Deployment deployment = buildDeployment(bo, volumeVO, deploymentName); + LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", deploymentName, YamlUtils.dumpAsYaml(deployment)); + resourceIisolationApi.addIisolationInfo(deployment); + Deployment deploymentResult = client.apps().deployments().inNamespace(bo.getNamespace()).create(deployment); + + return new ModelServiceVO(BizConvertUtils.toBizDeployment(deploymentResult)); + } + + /** + * 构建Deployment + * + * @return Deployment + */ + private Deployment buildDeployment(ModelServiceBO bo, VolumeVO volumeVO, String deploymentName) { + Map childLabels = LabelUtils.getChildLabels(bo.getResourceName(), deploymentName, + K8sKindEnum.DEPLOYMENT.getKind(), bo.getBusinessLabel(), bo.getTaskIdentifyLabel()); + LabelSelector labelSelector = new LabelSelector(); + labelSelector.setMatchLabels(childLabels); + return new DeploymentBuilder() + .withNewMetadata() + .withName(deploymentName) + .addToLabels(LabelUtils.getBaseLabels(bo.getResourceName(), bo.getBusinessLabel())) + .withNamespace(bo.getNamespace()) + .endMetadata() + .withNewSpec() + .withReplicas(bo.getReplicas()) + .withSelector(labelSelector) + .withNewTemplate() + .withNewMetadata() + .withName(deploymentName) + .addToLabels(childLabels) + .withNamespace(bo.getNamespace()) + .endMetadata() + .withNewSpec() + .addToNodeSelector(K8sUtils.gpuSelector(bo.getGpuNum())) + .addToContainers(buildContainer(bo, volumeVO, deploymentName)) + .addToVolumes(volumeVO.getVolumes().toArray(new Volume[0])) + .withRestartPolicy(RestartPolicyEnum.ALWAYS.getRestartPolicy()) + .endSpec() + .endTemplate() + .endSpec() + .build(); + } + + /** + * 构建 Container + * @param bo + * @param volumeVO + * @param name + * @return + */ + private Container buildContainer(ModelServiceBO bo, VolumeVO volumeVO, String name) { + Map resourcesLimitsMap = Maps.newHashMap(); + Optional.ofNullable(bo.getCpuNum()).ifPresent(v -> resourcesLimitsMap.put(K8sParamConstants.QUANTITY_CPU_KEY, new Quantity(v.toString(), K8sParamConstants.CPU_UNIT))); + Optional.ofNullable(bo.getGpuNum()).ifPresent(v -> resourcesLimitsMap.put(K8sParamConstants.GPU_RESOURCE_KEY, new Quantity(v.toString()))); + Optional.ofNullable(bo.getMemNum()).ifPresent(v -> resourcesLimitsMap.put(K8sParamConstants.QUANTITY_MEMORY_KEY, new Quantity(v.toString(), K8sParamConstants.MEM_UNIT))); + Container container = new ContainerBuilder() + .withNewName(name) + .withNewImage(bo.getImage()) + .withNewImagePullPolicy(ImagePullPolicyEnum.IFNOTPRESENT.getPolicy()) + .withVolumeMounts(volumeVO.getVolumeMounts()) + .withNewResources().addToLimits(resourcesLimitsMap).endResources() + .build(); + if (bo.getCmdLines() != null) { + container.setCommand(Arrays.asList(ShellCommandEnum.BIN_BANSH.getShell())); + container.setArgs(bo.getCmdLines()); + } + Probe livenessProbe = new Probe(); + ExecAction execAction = new ExecAction(); + List commands = Lists.newArrayList(); + commands.add("test"); + commands.add("-e"); + commands.add("/tmp/.startup"); + execAction.setCommand(commands); + livenessProbe.setExec(execAction); + livenessProbe.setPeriodSeconds(3); + livenessProbe.setSuccessThreshold(1); + livenessProbe.setFailureThreshold(10); +// container.setLivenessProbe(livenessProbe); + return container; + } + + @Override + public PtBaseResult delete(String namespace, String resourceName) { + try { + LogUtil.info(LogEnum.BIZ_K8S, "delete model serving namespace:{} resourceName:{}",namespace,resourceName); + DeploymentList deploymentList = client.apps().deployments().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(resourceName)).list(); + if (deploymentList == null || deploymentList.getItems().size() == 0){ + return new PtBaseResult(); + } + Boolean res = client.extensions().ingresses().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(resourceName)).delete() + && client.services().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(resourceName)).delete() + && client.apps().deployments().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(resourceName)).delete() + && client.secrets().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(resourceName)).delete(); + if (res) { + return new PtBaseResult(); + } else { + return K8sResponseEnum.REPEAT.toPtBaseResult(); + } + } catch (KubernetesClientException e) { + LogUtil.error(LogEnum.BIZ_K8S, "delete error:", e); + return new PtBaseResult(String.valueOf(e.getCode()), e.getMessage()); + } + } +} diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/ModelServingApiImpl.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/ModelServingApiImpl.java index 40058a0..46a2582 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/ModelServingApiImpl.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/ModelServingApiImpl.java @@ -173,6 +173,8 @@ public class ModelServingApiImpl implements ModelServingApi { if (StringUtils.isNotEmpty(buildIngressBO.getMaxUploadSize())) { buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_PROXY_BODY_SIZE_KEY, buildIngressBO.getMaxUploadSize()); } + buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_READ_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN)); + buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_NEXT_UPSTREAM_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN)); if (bo.getHttpPort() != null) { String httpHost = RandomUtil.randomString(MagicNumConstant.SIX) + SymbolConstant.DOT + servingHost; buildIngressBO.addIngressRule(ResourceBuildUtils.buildIngressRule(httpHost, svcName, SymbolConstant.HTTP)); diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/config/K8sConfig.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/config/K8sConfig.java index 5ca236a..b21a2d8 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/config/K8sConfig.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/config/K8sConfig.java @@ -23,38 +23,8 @@ import org.apache.http.HttpHost; import org.apache.http.client.config.RequestConfig; import org.dubhe.biz.log.enums.LogEnum; import org.dubhe.biz.log.utils.LogUtil; -import org.dubhe.k8s.api.DistributeTrainApi; -import org.dubhe.k8s.api.DubheDeploymentApi; -import org.dubhe.k8s.api.JupyterResourceApi; -import org.dubhe.k8s.api.LimitRangeApi; -import org.dubhe.k8s.api.LogMonitoringApi; -import org.dubhe.k8s.api.MetricsApi; -import org.dubhe.k8s.api.ModelOptJobApi; -import org.dubhe.k8s.api.ModelServingApi; -import org.dubhe.k8s.api.NamespaceApi; -import org.dubhe.k8s.api.NativeResourceApi; -import org.dubhe.k8s.api.NodeApi; -import org.dubhe.k8s.api.PersistentVolumeClaimApi; -import org.dubhe.k8s.api.PodApi; -import org.dubhe.k8s.api.ResourceQuotaApi; -import org.dubhe.k8s.api.TerminalApi; -import org.dubhe.k8s.api.TrainJobApi; -import org.dubhe.k8s.api.impl.DistributeTrainApiImpl; -import org.dubhe.k8s.api.impl.DubheDeploymentApiImpl; -import org.dubhe.k8s.api.impl.JupyterResourceApiImpl; -import org.dubhe.k8s.api.impl.LimitRangeApiImpl; -import org.dubhe.k8s.api.impl.LogMonitoringApiImpl; -import org.dubhe.k8s.api.impl.MetricsApiImpl; -import org.dubhe.k8s.api.impl.ModelOptJobApiImpl; -import org.dubhe.k8s.api.impl.ModelServingApiImpl; -import org.dubhe.k8s.api.impl.NamespaceApiImpl; -import org.dubhe.k8s.api.impl.NativeResourceApiImpl; -import org.dubhe.k8s.api.impl.NodeApiImpl; -import org.dubhe.k8s.api.impl.PersistentVolumeClaimApiImpl; -import org.dubhe.k8s.api.impl.PodApiImpl; -import org.dubhe.k8s.api.impl.ResourceQuotaApiImpl; -import org.dubhe.k8s.api.impl.TerminalApiImpl; -import org.dubhe.k8s.api.impl.TrainJobApiImpl; +import org.dubhe.k8s.api.*; +import org.dubhe.k8s.api.impl.*; import org.dubhe.k8s.cache.ResourceCache; import org.dubhe.k8s.properties.ClusterProperties; import org.dubhe.k8s.utils.K8sUtils; @@ -92,6 +62,19 @@ public class K8sConfig { @Value("${k8s.elasticsearch.hostlist}") private String hostlist; + @Value("${clusters[0].cluster.certificate-authority-data}") + private String certificateAuthorityData; + + @Value("${clusters[0].cluster.server}") + private String server; + + @Value("${users[0].user.client-certificate-data}") + private String clientCertificateData; + + @Value("${users[0].user.client-key-data}") + private String clientKeyData; + + @Bean public K8sUtils k8sUtils() throws IOException { LogUtil.debug(LogEnum.BIZ_K8S, "ClusterProperties======{}", JSONObject.toJSONString(clusterProperties)); @@ -105,7 +88,7 @@ public class K8sConfig { if (StrUtil.isEmpty(url) && StrUtil.isEmpty(kubeconfig)) { return null; } - return new K8sUtils(clusterProperties); + return new K8sUtils(clusterProperties,certificateAuthorityData,server,clientCertificateData,clientKeyData); } @Bean @@ -217,4 +200,8 @@ public class K8sConfig { return new TerminalApiImpl(k8sUtils); } + @Bean + public ModelServiceApi modelServiceApi(K8sUtils k8sUtils){ + return new ModelServiceApiImpl(k8sUtils); + } } diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/constant/K8sParamConstants.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/constant/K8sParamConstants.java index 912cd36..9ecad21 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/constant/K8sParamConstants.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/constant/K8sParamConstants.java @@ -67,6 +67,8 @@ public class K8sParamConstants { public static final String INGRESS_CLASS_KEY = "kubernetes.io/ingress.class"; public static final String INGRESS_SSL_REDIRECT_KEY = "nginx.ingress.kubernetes.io/ssl-redirect"; public static final String INGRESS_BACKEND_PROTOCOL_KEY = "nginx.ingress.kubernetes.io/backend-protocol"; + public static final String INGRESS_READ_TIMEOUT_KEY = "nginx.ingress.kubernetes.io/proxy-read-timeout"; + public static final String INGRESS_NEXT_UPSTREAM_TIMEOUT_KEY = "nginx.ingress.kubernetes.io/proxy-next-upstream-timeout"; /** * k8s资源对象名称校验正则表达式 */ diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/ModelServiceBO.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/ModelServiceBO.java new file mode 100644 index 0000000..5769073 --- /dev/null +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/ModelServiceBO.java @@ -0,0 +1,97 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.k8s.domain.bo; + +import cn.hutool.core.collection.CollectionUtil; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.k8s.annotation.K8sValidation; +import org.dubhe.k8s.enums.ValidationTypeEnum; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @description 模型服务部署 BO + */ +@Data +@Accessors(chain = true) +public class ModelServiceBO { + + /** + * 命名空间 + **/ + @K8sValidation(ValidationTypeEnum.K8S_RESOURCE_NAME) + private String namespace; + /** + * 资源名称 + **/ + @K8sValidation(ValidationTypeEnum.K8S_RESOURCE_NAME) + private String resourceName; + /** + * Number of desired pods + */ + private Integer replicas; + /** + * GPU数量 + **/ + private Integer gpuNum; + /** + * 内存数量 单位Mi Gi + **/ + private Integer memNum; + /** + * CPU数量 + **/ + private Integer cpuNum; + /** + * 镜像名称 + **/ + private String image; + /** + * 执行命令 + **/ + private List cmdLines; + /** + * 文件存储服务挂载 key：pod内挂载路径 value：文件存储路径及配置 + **/ + private Map fsMounts; + /** + * 业务标签,用于标识业务模块 + **/ + @K8sValidation(ValidationTypeEnum.K8S_RESOURCE_NAME) + private String businessLabel; + + /** + * 任务身份标签,用于标识任务唯一身份 + **/ + private String taskIdentifyLabel; + + /** + * 获取nfs路径 + * @return + */ + public List getDirList(){ + if (CollectionUtil.isNotEmpty(fsMounts)){ + return fsMounts.values().stream().map(PtMountDirBO::getDir).collect(Collectors.toList()); + } + return new ArrayList<>(); + } +} diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/PrometheusMetricBO.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/PrometheusMetricBO.java index 1c5cb88..77c6cbb 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/PrometheusMetricBO.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/bo/PrometheusMetricBO.java @@ -24,6 +24,7 @@ import org.dubhe.biz.base.functional.StringFormat; import org.dubhe.k8s.domain.vo.GpuTotalMemResultVO; import org.dubhe.k8s.domain.vo.MetricsDataResultValueVO; import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; import java.util.ArrayList; import java.util.HashMap; @@ -63,7 +64,7 @@ public class PrometheusMetricBO { return gpuMemValueMap; } StringFormat memMetricsFormat = (value) -> { - return NumberUtil.isNumber(String.valueOf(value)) ? String.valueOf(Long.valueOf(String.valueOf(value)) / MagicNumConstant.BINARY_TEN_EXP) : String.valueOf(MagicNumConstant.ZERO); + return NumberUtil.isNumber(String.valueOf(value)) ? String.valueOf(Long.valueOf(String.valueOf(value)) * MagicNumConstant.BINARY_TEN_EXP) : String.valueOf(MagicNumConstant.ZERO); }; for (MetricResult result : data.getResult()) { gpuMemValueMap.put(result.getMetric().getAcc_id(), memMetricsFormat.format(result.getValue().get(1).toString())); @@ -81,7 +82,7 @@ public class PrometheusMetricBO { return gpuTotalMemValueVOList; } StringFormat memMetricsFormat = (value) -> { - return NumberUtil.isNumber(String.valueOf(value)) ? String.valueOf(Long.valueOf(String.valueOf(value)) / MagicNumConstant.BINARY_TEN_EXP) : String.valueOf(MagicNumConstant.ZERO); + return NumberUtil.isNumber(String.valueOf(value)) ? String.valueOf(Long.valueOf(String.valueOf(value)) * MagicNumConstant.BINARY_TEN_EXP) : String.valueOf(MagicNumConstant.ZERO); }; for (MetricResult result : data.getResult()) { gpuTotalMemValueVOList.add(new GpuTotalMemResultVO(result.getMetric().getAcc_id(), memMetricsFormat.format(result.getValue().get(1).toString()))); @@ -162,7 +163,7 @@ public class PrometheusMetricBO { return map; } StringFormat memMetricsFormat = (value) -> { - return NumberUtil.isNumber(String.valueOf(value)) ? String.valueOf(Long.valueOf(String.valueOf(value)) / MagicNumConstant.BINARY_TEN_EXP) : String.valueOf(MagicNumConstant.ZERO); + return NumberUtil.isNumber(String.valueOf(value)) ? String.valueOf(Long.valueOf(String.valueOf(value)) * MagicNumConstant.BINARY_TEN_EXP) : String.valueOf(MagicNumConstant.ZERO); }; for (MetricResult result : data.getResult()) { map.put(result.getMetric().getAcc_id(), getFormatValues(result, memMetricsFormat)); @@ -195,7 +196,7 @@ public class PrometheusMetricBO { return list; } StringFormat memMetricsFormat = (value) -> { - return NumberUtil.isNumber(String.valueOf(value)) ? String.valueOf(Long.valueOf(String.valueOf(value)) / MagicNumConstant.BINARY_TEN_EXP) : String.valueOf(MagicNumConstant.ZERO); + return NumberUtil.isNumber(String.valueOf(value)) ? String.valueOf(Long.valueOf(String.valueOf(value)) * MagicNumConstant.BINARY_TEN_EXP) : String.valueOf(MagicNumConstant.ZERO); }; for (MetricResult result : data.getResult()) { list.add(new GpuTotalMemResultVO(result.getMetric().getAcc_id(), getGpuTotalValues(result, memMetricsFormat))); @@ -220,7 +221,16 @@ class MetricResult { @Data class Metric { + private String UUID; private String acc_id; private String pod; + + public String getAcc_id(){ + if (!StringUtils.isEmpty(acc_id)){ + return acc_id; + }else { + return UUID; + } + } } diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/resource/BizContainerStatus.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/resource/BizContainerStatus.java index afbbd73..843bd23 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/resource/BizContainerStatus.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/resource/BizContainerStatus.java @@ -41,4 +41,7 @@ public class BizContainerStatus { @K8sField("containerID") private String containerID; + + @K8sField("ready") + private Boolean ready; } diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/vo/ModelServiceVO.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/vo/ModelServiceVO.java new file mode 100644 index 0000000..c4b7a42 --- /dev/null +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/vo/ModelServiceVO.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.k8s.domain.vo; + + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.Accessors; +import org.dubhe.k8s.domain.PtBaseResult; +import org.dubhe.k8s.domain.resource.BizDeployment; +import org.dubhe.k8s.domain.resource.BizIngress; +import org.dubhe.k8s.domain.resource.BizSecret; +import org.dubhe.k8s.domain.resource.BizService; + +@Data +@NoArgsConstructor +@Accessors(chain = true) +public class ModelServiceVO extends PtBaseResult { + private BizDeployment bizDeployment; + + public ModelServiceVO( BizDeployment bizDeployment){ + this.bizDeployment = bizDeployment; + } +} diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/enums/BusinessLabelServiceNameEnum.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/enums/BusinessLabelServiceNameEnum.java index c3de05c..d18d41d 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/enums/BusinessLabelServiceNameEnum.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/enums/BusinessLabelServiceNameEnum.java @@ -55,7 +55,15 @@ public enum BusinessLabelServiceNameEnum { /** * TADL */ - TADL(BizEnum.TADL.getBizCode(), ApplicationNameConst.SERVER_TADL); + TADL(BizEnum.TADL.getBizCode(), ApplicationNameConst.SERVER_TADL), + /** + * POINT_CLOUD + */ + POINT(BizEnum.POINT_CLOUD.getBizCode(),ApplicationNameConst.SERVER_POINT_CLOUD), + /** + * DATA + */ + DATA(BizEnum.DATA.getBizCode(),ApplicationNameConst.SERVER_DATA); /** * 业务标签 */ diff --git a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sUtils.java b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sUtils.java index 73bab81..abcde76 100644 --- a/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sUtils.java +++ b/dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sUtils.java @@ -82,32 +82,20 @@ public class K8sUtils implements ApplicationContextAware { * @param clusterProperties * @throws IOException */ - public K8sUtils(ClusterProperties clusterProperties) throws IOException{ + public K8sUtils(ClusterProperties clusterProperties, String certificateAuthorityData, String server, String clientCertificateData, String clientKeyData) throws IOException{ String kubeConfig = clusterProperties.getKubeconfig(); - if (StrUtil.isNotBlank(kubeConfig)) { - String kubeConfigFile = getKubeconfigFile(kubeConfig); - //修改环境变量，重新指定kubeconfig读取位置 - System.setProperty(Config.KUBERNETES_KUBECONFIG_FILE, kubeConfigFile); - client = new DefaultKubernetesClient(); - config = client.getConfiguration(); - - } else { - LogUtil.warn(LogEnum.BIZ_K8S, "can't find kubeconfig in classpath, ignoring"); - String k8sUrl = clusterProperties.getUrl(); - if (k8sUrl.startsWith(HTTPS_PREFIX)) { - config = new ConfigBuilder().withMasterUrl(k8sUrl) - .withTrustCerts(true) - .withCaCertData(IOUtils.toString(clusterProperties.getCaCrt().getInputStream(), "UTF-8")) - .withClientCertData(Base64.getEncoder().encodeToString(IOUtils.toByteArray(clusterProperties.getClientCrt().getInputStream()))) - .withClientKeyData(IOUtils.toString(clusterProperties.getClientKey().getInputStream(), "UTF-8")) - .build(); - } else { - config = new ConfigBuilder().withMasterUrl(k8sUrl).build(); - } - LogUtil.info(LogEnum.BIZ_K8S, "config信息为{}", JSON.toJSONString(config)); - client = new DefaultKubernetesClient(config); - LogUtil.info(LogEnum.BIZ_K8S, "client为{}", JSON.toJSONString(client)); - } + + LogUtil.warn(LogEnum.BIZ_K8S, "can't find kubeconfig in classpath, ignoring"); + config = new ConfigBuilder().withMasterUrl(server) + .withCaCertData(certificateAuthorityData) + .withClientCertData(clientCertificateData) + .withClientKeyData(clientKeyData) + .build(); + + LogUtil.info(LogEnum.BIZ_K8S, "config信息为{}", JSON.toJSONString(config)); + client = new DefaultKubernetesClient(config); + LogUtil.info(LogEnum.BIZ_K8S, "client为{}", JSON.toJSONString(client)); + nfs = clusterProperties.getNfs(); host = clusterProperties.getHost(); diff --git a/dubhe-server/common-k8s/src/main/resources/kubeconfig_dev b/dubhe-server/common-k8s/src/main/resources/kubeconfig_dev index bac8fb6..c6d4211 100644 --- a/dubhe-server/common-k8s/src/main/resources/kubeconfig_dev +++ b/dubhe-server/common-k8s/src/main/resources/kubeconfig_dev @@ -1,12 +1,13 @@ apiVersion: v1 clusters: - cluster: - certificate-authority-data: - server: https://127.0.0.1:6443 + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1URXlPVEE0TXpNeE4xb1hEVE14TVRFeU56QTRNek14TjFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTmpCCmxkVWhGQkdqSDNsYXIxTW9wVUgvMW8xNVUwbEhvcGQzZjgyL0tWMlEyOU9BKzhjMkhXYTd3R01QSzhxL1JmSVAKTjlnYXJJd0FqZXFwNHFSMWhEcVhka05WN2NOZUZ2OGdoMUVYSzMyTUxoaFZJdXhkTnF6OW9GVm85RDZqTzQ5awpEOWQrM215eU4zamxsT2wyTWVRMUY0STlHSTFabjFCOG5LQVZWVjFIZTBPTGZjVTIrZG5mU0g4ZFdqV2RPQlBhCkJmbGU4UFRLQy9tOFVmcTQvanArVU92R04yVEhIaXk0V2UyTWFEK0xxWDIzekE0T1FlK3NrdlY4bHlqK0ppVUQKVVRiM2xzazlhTDMxblMxUmpnVStkbVIyR2xHQ1BHWEhNc2Z3MlVCZzljZEM5ZzhKc2dFeTJDa04xMitjeWx3dApDbHFQQ3o0MlNlNEJMNWF1ZGJFQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZCTmFSa0hpTzdvZDd2U3pUUm9qaHFTWStSNXpNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDVnZGRUg0ZEYzUTQrRDlOeHBZdDM4ZG9mb0ExYjlYOWNndU5YYktvbGxHK1lQeWl4NwpBcGlMWkp2WFlsbk0rUU5BYjMyOU84VmNkQUp0aDZ4MjVHSVNUVjFGbXBWWHFlRVIvUFJOZURsaHFybzVwYUlVCkV1YUQzYldxaitWb2p6b1AyVzZMU21RdzQwNUIzV0JKRlJtdm51bW9LYStyUWEySVoralRNMVQ5bDV5bXR2YloKS0lXNGZFZ0N2VEhPSmtRLzBGWDg3dEc0VlVJZ3hHRXZJU2lUUkJYc3hnQXdVK0I5SnN4bEZSS3NHYTJ4QkE2bQprZXR1VFVlZkswTGZRMFlMWVVKSVA5Y0RsNm5QSld5U0NTcWFZUGRoaGxERWxEYUt4d2JZeHdBVkI0ZzV1cWhNCnZrMkN0RVFHOWg0b2ZxZjQ4VDFLVlJVRGo5UzdBL2xtbEZxNgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + server: https://10.5.26.91:6443 name: kubernetes contexts: - context: cluster: kubernetes + namespace: default user: kubernetes-admin name: kubernetes-admin@kubernetes current-context: kubernetes-admin@kubernetes @@ -15,5 +16,5 @@ preferences: {} users: - name: kubernetes-admin user: - client-certificate-data: - client-key-data: \ No newline at end of file + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lJT21TandKODdaUWt3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TVRFeE1qa3dPRE16TVRkYUZ3MHlNakV4TWprd09ETXpNVGhhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdSWTJuYkJZc25KMWxlQ1MKS1lxbCtLSlBoOW5UdkxqanNRczJobEJrcmM0M1o5eUNEUTM5V1RWSkZHdHZzUDVQK1JNdzFOYUY5QlJUbTQ2dQpOQXBYTGt2bTRLZm1mL2taVUh5clpLYTllSnlkbTRWZTNjNWVWeXpidHRVUVgxQUNFOGZIM0VqY21hRHhPZGViCjBHa3dHc2FnVVJiQ1JxSWpZYmQ3V3BkK3F1OGRuUC9TbmJkaURuaU9sNVVoSlhMUEl6TmlLZTZnREt3NUN6VGgKbVVjZnc1azB4WFpwRFBIckwxOG4yMVB0dlRZSEhxYXo0ckx5MWZiWWsyOG54cXZEaHYzUVBXVldRNGhVR25tZQpNRXZlcTRuNU1sRGJOU2tZUHhaSDNUbUhrZldXR1Z2dGI1RDFLeStSbnFISUxNZEIvdHN3YisyaDVBQTE0WXJICjhBbnZ5UUlEQVFBQm8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0h3WURWUjBqQkJnd0ZvQVVFMXBHUWVJN3VoM3U5TE5OR2lPR3BKajVIbk13RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFNdVZaem5pdlhXeDV1dHl5cGppcHJQWkdxeXVYb1J1NVR0eEZwMVJFcHVIb0Z1NHg4VkNUYkplCnNzNmN2VitORmdIa3BLSHlWb2Y0YStRNHlOcGZmMWcrL3d3a05XcHlBQTZBUWdCbW00djBIV1ZVTzYwVVN0VEcKckFhQVUyZ1R6dGYvWll3RFpreFdIUmo0V3ZIRjZRaDhUYUdFVTdaL1dWbC9rOW82MkJGNGRza1pCRVZmcDFvMApMSkorcGxmdXpLOTNwN0t4ME9hSUhHUFRPcnBydEc1ZmR1U2NmRDFUck1nOE1zbmIyRG5XTHVXNzZPUi92cVRBCmVaRFI0Nis5aTllKytQMUh0eDZZMFJNVVpuS1o0MCtIc3dFUjZFZHFINmdJd3Zia2FvSXQydmF2Y2dxdXQ2bWUKZ1VoaVlYelljREhmRllFOUJVWU5UVm1zdmxNOSt1RT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBd1JZMm5iQllzbkoxbGVDU0tZcWwrS0pQaDluVHZMampzUXMyaGxCa3JjNDNaOXlDCkRRMzlXVFZKRkd0dnNQNVArUk13MU5hRjlCUlRtNDZ1TkFwWExrdm00S2ZtZi9rWlVIeXJaS2E5ZUp5ZG00VmUKM2M1ZVZ5emJ0dFVRWDFBQ0U4ZkgzRWpjbWFEeE9kZWIwR2t3R3NhZ1VSYkNScUlqWWJkN1dwZCtxdThkblAvUwpuYmRpRG5pT2w1VWhKWExQSXpOaUtlNmdES3c1Q3pUaG1VY2Z3NWsweFhacERQSHJMMThuMjFQdHZUWUhIcWF6CjRyTHkxZmJZazI4bnhxdkRodjNRUFdWV1E0aFVHbm1lTUV2ZXE0bjVNbERiTlNrWVB4WkgzVG1Ia2ZXV0dWdnQKYjVEMUt5K1JucUhJTE1kQi90c3diKzJoNUFBMTRZckg4QW52eVFJREFRQUJBb0lCQVFDQVA4cmpEbUM3bGo3Mwo0QVAxUjd4WjJ5Vk1MS2p3N3pWOVFOOWtjejJNSEM2ODg4QUYvOGJKWEJWQVZPUUpBY0lJeEhCb3pseUc5SUZjCkQzY3g1YlJtY2MvV1pHODdqUzc5UVBGdWx6bStSUGlDZGlHKzNmWFhuVm9LcEphTldFMG82a2ViejJHSFFWazQKZkRkb1JKWEpFeWtrOHlMelczcDR5ekJKeWRvK21oQmhZWjRCd2xwRVExeURQU0xoTnpSR0NRL0tUTm94T1ZQTQppcVJUdkR2MGhZMTFLdGk1dExMbk1RWnIvaDJRNTliT3VaQVg0NDFiWVJjSzNyU2R0TzFBSGhYVmpSMDlZWFZUCnJwbE44VkhGeE1wZjVaNHFRTW5idk1yb1B0cU5PTzlqWCtrWjQrY2p2UjlQbzNaK1Y5QU5tUHR1bEtMejhFbjYKcUZSYTVTc3BBb0dCQU0vTStvU2t1QVpsT1ZXR1JsdkNlbE0vMHJWUDFrRTdSeHl2NkZDT2EySTlFcTEvUThraQo0N05saHZ2ei9TcmZia040Ymg2Z1RXOUc3WGVpZGtWWEhiaVRmZGlNcXA4YzJ5QTByQVJrWGtZNStycUpZRWlGCmFpOE1jUUVhKzRObzdHOC9xSk5hZTFtVzBGNE5JUTFoRlRMNHMxTnJ5Z0pIaUxDeHhIdVg4cjVUQW9HQkFPM2YKaU9uSm5qY3IrTmltbytGVXYvaHp5SjMzQ2FjNWNjakF5bk0vUGpheGRhOXk4RW1mL2x2eURDQmsranRydlV3NQo4eG1lMHc3UXZJNzlZZkljc2JrYTRnb1dtREIySXZXeGlJRkZCcWwzOUd1eFRYVUpCb2R6MXF4TFJWL2tSekZYCkhIZFM2VTVVTDJINkRQMmo2aVk3NCtWOHhSVlBhQnpPRnQrdDlMM3pBb0dBVjZMVDJGTDdoVDcxWWh0QjJ2cjIKeWRzMXl0K0R0WGtCY1pqQ09nOEdQS1VURm5hQXFQZjgzYkNvcitOalZzeHAzU3lTREhxQWpiaDA5NnBkaExyTAp2d2I4NFBIYUYwWjlKMlR2VXQ3ZFgzS2VTa01iaHhvMUFPaVhVK2NFUVpSamVqdDNwY1ZZRmgxamZQYStoTHozCnlzcFdIbzFyNFhBM3RvVVNIeW1pNXg4Q2dZQlBmZTBXRzU5NDFvUUJlcWEzYllmOXNrdXZwbGVlZjQ3cnhLbWoKRzlaaGxxbzZhSkd1T1A1YW9hZTF4NmpyNUpSTS9leWtlalFUOW1PYzhVWWpENzNBbXlCQXVNNUJHNm1SSzdQdgpaeTdUd1ZkNENiZFZ6aDlWY3J1aHRrSEh5dFNUdzRXcXhwVmE3TDBzNDlBbGp2cDNybm41UGRucXhpK0h1TU5pCkdwTDRTd0tCZ1FET0hRZ1ZGeGV0eU5zVjRkSXd6UThzbVp2ZHF1QzRzRXZTakx4UXd6ZmZadU1aMHFBc1R6VkUKYkE2dk01WlNpQ3A5SUxTd3VKTUc1eGhqMXZUdGVDOHBCSmdsM3BmM1ArS0pXeGZ4S3lYRCtRai9STDN6V2lxcwpFaDd4SFN6UlFCcDJESjhqMWdtakd4em5IbnlDdnBNdFJlQzBzcGp2N0JVZzNqaFR4SVRRdFE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= \ No newline at end of file diff --git a/dubhe-server/common-k8s/src/main/resources/kubeconfig_pre b/dubhe-server/common-k8s/src/main/resources/kubeconfig_pre index bac8fb6..ce15511 100644 --- a/dubhe-server/common-k8s/src/main/resources/kubeconfig_pre +++ b/dubhe-server/common-k8s/src/main/resources/kubeconfig_pre @@ -1,8 +1,8 @@ apiVersion: v1 clusters: - cluster: - certificate-authority-data: - server: https://127.0.0.1:6443 + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EY3hNekEzTlRNek5sb1hEVE14TURjeE1UQTNOVE16Tmxvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTEZlCmp5KzlwTjNybnJpYTlpbDhzOEp0WWZoekVMUU1LbXVmb0Y0VEhjTnFObGtOVDJiL2pTcUUzbGdRYTR2Zm9oL1AKUFVOaWVkZEZvczFoRGNxRnF0QmRpaXJ5QXhSNERPeTJsMWpjSWVpVHJNaHYyd1JZc09ZMG4yanZITFdGdENCaQphY0lEeXFmNmhHM01SV2ZNUEhPVVdDQ0JIUmxLRFlnYkNDZXp5NWxxUEF4QXpmcUpEalUxWkNUVmNNa2Z3R1FDCmFSYVZ4aTFiRHhRSk9Ud2dTTGlJdEJIUGhrdGtOcnY1MU1XZzRSWkVUbWF3azg5Y09oWExpSEg3ZExFZjFvK0oKVE80Q3J1K2pyb3lIRkNvQmo0bGpKZHd4N0ZRS25XZFpEbWZiOFFoZEdyTkRuK21WR3BRNnZSRGN5YzloSVJsMwpyM3U4bkM2TS9mZ29UTHhWaTFFQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFIbDZMYWpscFZLMk9PbG1JYzRVK3A4bTZoVGEKd2NCTW5MZjVJcExwME9CRmZ6M1hvVi9EVUlmTE1LU2k2bzY3a1Y0aFYzNDZMTnErU244elBUZEZkbDdjZUlvbQp2azc3YXE4c2VERk5xbTJMSytSUldmUWZQbFQrSGhiWFVpS1lOUUhmanlXdjh6QzFGaWpBQTBqaGVjWVBaSnU4ClpUb2JaenhPaS8yOGNwSmNOSFFDSW14OG16UkU2c3JTRjFsbE4yaWF0eTQ1cWhEZWwyZHpmSldaeTFzTXVwc2YKaTc4Z25qdlhFU3BYaGoweGZNbEdMNnA3QkVjdUw1YkgyN3Y3djJjelh2WVJnbk0vamhscGVzbVRsNzFiSE1KQQpnSzlpR2pqMHhoZ1Jnb3h2VHNQNVZLSUszTFZVbGxNNXJhNXYxTjdOOXpmQVZybnprRGRoaXNZQzZCUT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + server: https://10.105.0.29:6443 name: kubernetes contexts: - context: @@ -15,5 +15,5 @@ preferences: {} users: - name: kubernetes-admin user: - client-certificate-data: - client-key-data: \ No newline at end of file + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJVjB5b2I4RnFSdGt3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TVRBM01UTXdOelV6TXpaYUZ3MHlNakEzTVRNd056VXpNemhhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXVPN0M2OWNlUjVVbmZuUDIKREZNR1M2RzZ2RkYxLzNxSU85ZVFTR0pZRU15dDhHWmo4QWNlbnMrQTNDS05jRHJ4dk5oQlpvazlPa2w4SnpzRQpZOFhKMTFmZVdPS0Zwb0dYVGtseHBuakcvRWVwbWh1d1JodUZhaTlsZ1BXQnUyei9JOG5meTdjSUhscW1EYW1UCkViaEJGWitSbzlPYjB5dTJ1MklhUWNUTVpPREc5eEdTRURIT1VYTkIzcG5oWGhYRmhZUTZyeUpVMkFzYy9TS0UKa1ZTN3RWY2hoVHZOSHVnWC9YQ1hwZ1dpSXJhMUZNbUNBMGMzY003NDJhNFI4QnhmKzBremNBV0ZqQUIwc2xFdQphcURSK3I2SWQyUWRyVjVBQXUra29rU1MyRnhKNzF6R3VtRnRQdlVyT1UzeXhreWFJbHZ1aFcxZEV5cHl4WjVJCkJvSUNWd0lEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFLKzRlV280b0g0K1RFS3doUU9ZSFJvWXhWcEVxWDBXZmoxSQp5US9Qbm9sTXh5ZmQxTkc4VzdyeW1FdWJZbEtIV3JjOTNKdHJBOTA3N01BRHMrWGcvWVZrbFdpVnF2b2Eva2xOCm9aY1pVc3Q2VEdMNVJ6QmE5NzFheCt3Q09xMXRiSng0ZGtwSDdmWVVWeXVaTUg0MXhxL1hIZEwzSnV3RzFSbU8KQnEvZWdmY3d0QzdhZGthRU1SbWxlRCtzelVBZkQxYjFmd29JNnpyV2ZyV0JnUTUwTmhFUzk0NldLOER1QU84cQovalFXRlVtd3M3Nzg0d3ZPR0JjTGpSeWNvWGpxSDZXeW5hSE9BRVdCakJaMUtyYjh3LzNucnNucTArOTdtbG1oCi9IMkpNQ1lIbDRZQi9Bb2pXUnhpVE50b3BsM0k1Sll4Vlkza1J1ZWF2c0pQNElmNmJ1MD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBdU83QzY5Y2VSNVVuZm5QMkRGTUdTNkc2dkZGMS8zcUlPOWVRU0dKWUVNeXQ4R1pqCjhBY2VucytBM0NLTmNEcnh2TmhCWm9rOU9rbDhKenNFWThYSjExZmVXT0tGcG9HWFRrbHhwbmpHL0VlcG1odXcKUmh1RmFpOWxnUFdCdTJ6L0k4bmZ5N2NJSGxxbURhbVRFYmhCRlorUm85T2IweXUydTJJYVFjVE1aT0RHOXhHUwpFREhPVVhOQjNwbmhYaFhGaFlRNnJ5SlUyQXNjL1NLRWtWUzd0VmNoaFR2Tkh1Z1gvWENYcGdXaUlyYTFGTW1DCkEwYzNjTTc0MmE0UjhCeGYrMGt6Y0FXRmpBQjBzbEV1YXFEUityNklkMlFkclY1QUF1K2tva1NTMkZ4SjcxekcKdW1GdFB2VXJPVTN5eGt5YUlsdnVoVzFkRXlweXhaNUlCb0lDVndJREFRQUJBb0lCQVFDaC9SSmtmdlFaQTcrcQpkbXpwOHJlcS9DbVQxMDhpei9RUlp3c05QSWVqZjRaRTg0dEtyeEhWVGpHem9kaCtuRU12aGNZVHlOY0cvV054CkFiTWdxaG5aTlRDZ2J4dGU5RmpTekdadXlaQ1RYenBpc1NwQTNzNklhcWZneEN3MVBvNW1qT2dwaTFQak1zZ04KWTZKZGZTWVZpTWFMMkVuQU9hUkFrdmdvNy9lUnpBeFowNkhZeEJ5WlM2dzZrRVhpQmJQYThCeUpkd0pIRzRqLwpHSU9yZkhXRWhldkdjcTU3SmF6c3hrTzczZXBYbW5IU21NQWVKcXY0RmF2czBtSUZ4THlRU1NjSmxZZ2xXWlJuCjRmekZVM2tLYmRaak9NdWJpbGY1ODluQUZ4cVpnMzhJblU0bGpzSjRsSFVhU0M1VlZ5UTFvc2pmaGlFeFg1NVMKQmJmSFU3MnhBb0dCQU91T2pHTTlwYi9rQ282ZXhaREZNSlRqbmZMQmtEQlAxNVN6NjByUjY5cjFRQ1hwZk1CWgppT0N1aHZTSnJvRGJQYVVvUXc4QkhvV01BT3d4VU1veXpUZDYwOC8xVWRmY1F2RUtsY0dXSXpGMWJxSTllcFNpCmQ3Q3dSZ2Vxd01rUTF6SHBsYU4rWkY3MEJsZ3RJYkx3Ymd6dWhuSWlZRjVRbi82dWVMQXkzQkhyQW9HQkFNajcKZWoybklVT1JxY2lpeUZwQlVERGUwT01GYWNiYkk2VzA3RUl4cElxRWpLejZPenVmZFBYSFpLVU44QmEwRjlBUgpsSzVBdXd1STErbnFqeTJJOWV3Lzk1QTQ1NGFBYjM0WjRqekUzUm8vS1N0dytQR0xUb0RSeEZneGVhQmYvUkdlClIvZlpTK2h2VHgrYy9pQjRQYU5sOGVpQ2ZFcVFhYS9MbW5WcitncEZBb0dBYnNWN0tWUWROTzdsTkFwZjkrTnoKSkNFaDdyMnRzN3BvTTZxa05Hd2hVTGRTTWtIcGczN1hTbWxvVjJqRG9oNzNqMG91dHNpYzlNcFF5TUdzTDFuUwpmWXVLUGRvc1lhbFg1WWhId21CN0xrLzk5ZGVaWkhvK2ROMkFJU3pnT2Uxc2RURldTQ3N3d2lKWk5YQUx6OTBXCnM0Z2J1MktGRlBVdk9CSU4rVFlCblg4Q2dZRUFyWEdBYTZaSXFaUVNMb2gwV0pkV3llWHY1SXJ1WHVNTW4xdEUKTEZmRkJKa2hBY0lzemVadFBCR29CRnpEM2dQckxPK1BITlhWMVQxeC8zY2h1bzBnbFpJYVpnY0ZudWhGejFBdApFbjVkeE9ITytLTlU0clp5dCs3Ty84RXFra0ZrRndrK1dHRFpCaXpRM3BwUUlOdERiamh6REZGWFM4M1d0eFFCCkp1Wlk4UFVDZ1lBOFlsZnNsQ2VmZktvTVBVK3pKZWF5SUlQNTR0M3NySkFwblNyc1BuNlNBSlB1WjU5ZkYrUTcKNGlvNk0weEw0dE5VckhPQ2ZpODJXSzlRQ29EbDRIYTJGZ0Y3YkNMTTQ1Qld5cVp5THg0b0JRYmFGNE5ybmVuTgpVUVVFbkU2V2ZyN1RFWWVBNngrOG10dXAvcDJzbTd2eWdkYngrRFB1OXYzV1JyZE1DZDBNWUE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/dubhe-server/common-k8s/src/main/resources/kubeconfig_prod b/dubhe-server/common-k8s/src/main/resources/kubeconfig_prod index bac8fb6..ee32ce4 100644 --- a/dubhe-server/common-k8s/src/main/resources/kubeconfig_prod +++ b/dubhe-server/common-k8s/src/main/resources/kubeconfig_prod @@ -1,8 +1,8 @@ apiVersion: v1 clusters: - cluster: - certificate-authority-data: - server: https://127.0.0.1:6443 + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EVXdOekE0TURFME9Gb1hEVE14TURVd05UQTRNREUwT0Zvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTXU2CmRJYWVmOWQzYXByK3lKK3BLY0EyaU5NQXlQVWFsbFgyZHBwelprd282T0R2TW9FYmgrRWFaY0Y1aEdMVnhSWDYKbnhtcUQvVFNCWXdENFNzY1E3c0YzcWtxeWxBLzJXaTg1NTJKbGJQcXFSQm5CaEUwV211ZE9EVXZYYVF0N3BnWApzR1JKcDROcFBsd0tLMUVpZmhsdkJIMmRVWHFjZDRENmZKbWRSSWgrNEpOS3ZOL09Hait4WjNKUG5Cc0pKOUlICms4TWFsc3NuTTYvaFpna0tKVlplc2YvcVorN2I3dXpJSVJteEd3L0RBcmtNaGgvL1VCZUUzVEFsd1lWWisxZWkKU293eldEN3EzYzFhU3NJYkdrbXJWaXNQcVVZTk9sUEplcHJLTVFJRUJKVEdPWUYzSzk0eHNKaEtBMkI0Z0VCRgpOdkJxTzVqZFZ4RkM0SFdxM1VFQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFLV254YlZqTFRWZVVyZDB0Q004SE83ejk2QnQKQTBGS3pmclRSTkhtT0I2WVVSRnVRbWZJdE9GTkY4QnJoYVBCZVNKMFZrNVdNUXZBd1BkdnY2R2l1NU1VNU45TApHV3R2eXhsS0Z5aVkxR25RUy9sWjRjR1JaSE9kMmtMNFY3bVNLQmo3ZFpzcDN0dW42d3BQZWM3dUJ6Z1UvNzdxCjN1b3BGMGVzR21wY3ZFaVhNSlRrZUN1NTNhaTVFVHhSS0Q0V0xxUUFhbDViUlB4b0UwL3Mrams5SGI4b1JuNnQKa0RDaElpQnVjL3RDaHhwTmNFVWt6UUwyRjBHR2hRaVlQTFFQdkgzRml5S2tlWjBiYUNHdFdpODEvZG9lVVJKZQpOM21jU3pvZEM1SUhMc1Zta05HTzh3Y2pocjVZck0yVGxHS01sV1huRVk5QWFMUEl1blI0VmQ4ZU1iWT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + server: https://10.105.0.1:6443 name: kubernetes contexts: - context: @@ -15,5 +15,5 @@ preferences: {} users: - name: kubernetes-admin user: - client-certificate-data: - client-key-data: \ No newline at end of file + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJSDByc3FWMTBudUF3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TVRBMU1EY3dPREF4TkRoYUZ3MHlNakExTURjd09EQXhOVEJhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXUwVlR6ejIvK1VqbEtVRHkKL211bVYrbTBjejg1UURTUjU1aXN5QmNDTWVzbEJsSEYyVEhWQm9FRHhUMEFHQmF5ai8rdEx6c1J4Vjg4Z28vNQpkZFVZL0llNllNVmVlMVZuQ29CL2VvYStkUDBQZUlhZVl3amU3WTJhdFhpMDBOM3EzZ253Q1AvM3FodUpabDBTClQraTJHMktOQlRvbjYvSGUxRVBva0hlcTZaMU5yYm5aTWROTzBWM1VaTExzMXdhS2ZESDJHRStlang1QzU4VEQKSnVGdjN5QkFPWW5CblI0YTBObWJpTFJ2RmN3d1BFR24wamlHazZGY2oyK0RWRlVCQjYzbVFOV3puWGVlaDQzVwpodm9GQnZKUVg0OFZTeng0U2tHVURtaUU1Sktncms0T3dJNk9vR0Fsa0kvbU4wQnl2b3o4UjFFbENMY29jZzJWCkhvN1Rrd0lEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFNSGpQdnNNWDUyMjNzbTZTQTJzYlBzc3NaalJJSUhCSWloUwp6MlBVR0Q2R2NXZ0RRRXBzRWVRRDYzby9vLzVLcndBbHUveGlnVW9VK2dkRTQ0S29PTmM0Z05tdHdZOVhzcnZXCmZSamc3YXh2MGN5czBuSzBCdVJyYjBQVDZ0ZkVqb25yQXUvQ1NuSG9LZERuaVBweUNQSys5amZONGpJR1VobFkKU0Z4Qnh6N3ZGQlUrRlVZbjBYR3BHV3FnQjd6bVRwRllSVXNKOVc5eXJlaWlsZDVJcDdzSGljZG0za0NiYlJWagpmQ1k0eVV5elVSbFV2Q2xrOE8zQktEQkYxeWZIUGErUW1STStabzhEK3V2T2VHVVFqSW92eHdXNVNPM1RkNkE5Clk2WEoxbEpwRmtUWndrMk13N1N0RjhxK3MwMGFnNFB0RGJhKzNoM2FmWG1DWU9FOG5pUT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBdTBWVHp6Mi8rVWpsS1VEeS9tdW1WK20wY3o4NVFEU1I1NWlzeUJjQ01lc2xCbEhGCjJUSFZCb0VEeFQwQUdCYXlqLyt0THpzUnhWODhnby81ZGRVWS9JZTZZTVZlZTFWbkNvQi9lb2ErZFAwUGVJYWUKWXdqZTdZMmF0WGkwME4zcTNnbndDUC8zcWh1SlpsMFNUK2kyRzJLTkJUb242L0hlMUVQb2tIZXE2WjFOcmJuWgpNZE5PMFYzVVpMTHMxd2FLZkRIMkdFK2VqeDVDNThUREp1RnYzeUJBT1luQm5SNGEwTm1iaUxSdkZjd3dQRUduCjBqaUdrNkZjajIrRFZGVUJCNjNtUU5Xem5YZWVoNDNXaHZvRkJ2SlFYNDhWU3p4NFNrR1VEbWlFNUpLZ3JrNE8Kd0k2T29HQWxrSS9tTjBCeXZvejhSMUVsQ0xjb2NnMlZIbzdUa3dJREFRQUJBb0lCQUVYR0pOM1lZZ2lkY2xTVwprSExlNVJGb1VBV0lqdW92TEJXZ092QXFNblVxNlphYkxSNHBoUGR4WmxnOHpDWXRmc1pNT3RpWUo1emtTUVZVClkxdlYxQU56QnF3N25XSlNoWnZTR0swc094WVhtNFlLa2tUUDcwK1BMUTlrTStxR1pKWHFHZmNnZDhSM2toQUQKcVdrQWlhbFdaTGlIM0l2NmlFMktKOEo3ODhBcWFrM1liemJFR0xvUkxKbzZONjZzSm9CcHRUS3BOOENpcjF0bQozR1MvRllZK2ZzcVFudU8vWkNjOUl1ck5scVJkU0ZNdThRT2pRMFp6TkJqMVVEZFBNa24vWEhWN3BIQWF1NkVrCm5lLzlrSUNWMWxMZ2FuK0lLYW1kVElvZk5aVUhsL3k5QksvZ1U0SFRVMGo1Sit5WkV0bXI2VktmeGloQks5aWcKelZRM1R0RUNnWUVBNnh0b0RRd2dFRmNSdEJ3WXZQZWR5L0FZTjlwc0ZjaDhGRFJJUXlxUlhUWUVURlRjaXFNQwpwMHYyaXY2dHdoWUo1QmZVcHBJUU1kK2c3NjFiZHdqTHpweEJKZHpzakwySE5HMzhaeWxZTXc1LzBIT1FOTHFoCmt2VW9rMVJHQkxhOGFuOFA2eGdvNXptMHY3SW11MGs4RVQ0bjI5aDJJclVVSm1pWkZMM3dBMThDZ1lFQXkrbXEKUUR3NktJc1lDTU44NnBzSXAvQU5zTWErVlJ5cFJRMWYzQStSSjdnNkhYRGxqOVlLQWIxSEdobnMwKy80ZzRrRQpoTUczZmVNM2hoc281aVhCNjRPS3RkQkFodVA1UkF0RDU0eDY2MHhtYnZqY3YyVFJFS0pLMGs5SXBSU1ppa2pGCmVVTG1rbzZnUmU1aVI5NU5NQ0o5NFRBQW80SXNEZjJxQ0pqRU1FMENnWUJFRjZMeUxISFk2YTdKOEYxRjFaMlIKSkUrUFZhWjZSSitUSm5WTFpyZkZQRkRRWHIrbE00TWdPd01EekxFOGhpK0ZMVlc4akk4K01wdWs3eHVQaFMrcAovbDFyL3VsUDlkQ0Q0ZHI1Y2VNR25vdHNMeHd6K1YyMGQyYXlEUFZlaGlKWjRjVVZmT0RUMzBXM1EzeXVQNDZ6Ckc4SmxqUExpS0huV2lmTFVMQktvbHdLQmdBeU45Mmg3RE0yZ09ydVhaYUtBSnhsSDQxL2w3S1FLM3JVY3JMRTgKMkNBTTdLOFJXMkR1dWJEL3VWZjNEcWpCMXBncW9IZVlBYmNqZlRDcGpXd1dHUWxxRU9rK3lDcEY4UHZxZ2FUSQo5bXgwU0w0K1hrRCtjUXpJSVRrdm5uWmpmVXlSVEc4NTJqNWR2NnB1a2VpQTNGbkJWZVMrY3R1ZVVSNFBaeCtlCllEM2xBb0dBRWRId3VVa2ZBYWZJMFdkOXdlYXNDb1daTUlDTm1MK1ZDcXEzMEZyamVFMXg0cElTNjdFODJkU2UKKy9TbDNkWXpucHkxODJVdGJQNzlSYThoNXloOUVCTzVYMXlweGlCeCt3MmdwNWVJWG00aStOUWZxbjNOUm42awo2VW1RY1hCMXV4djNKWENuYWxVUk5tVWdudEFzaEQxaGd0Y1VvSzArS0NXT3VhUnFqSTQ9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/dubhe-server/common-k8s/src/main/resources/kubeconfig_test b/dubhe-server/common-k8s/src/main/resources/kubeconfig_test index bac8fb6..8e38fd5 100644 --- a/dubhe-server/common-k8s/src/main/resources/kubeconfig_test +++ b/dubhe-server/common-k8s/src/main/resources/kubeconfig_test @@ -1,8 +1,8 @@ apiVersion: v1 clusters: - cluster: - certificate-authority-data: - server: https://127.0.0.1:6443 + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1USXdPREE0TWpnd00xb1hEVE14TVRJd05qQTRNamd3TTFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWVKCkRjTThQUHFlV0tHazE3S2JtN3kwbzkzTGtxb2w5dG1QbVN0YTVyUXZSWU9rOFJYK2QvVEpVaXE2S2c4NXFyMGUKZFZ3SG5vTjg3Q0VFbXhOK2o4OUhlcnY1dDFaRFBHMVRHS3U5N1l1WnV4RCt2R1hGQkhDZ2ViT0s0Zmc0am1iSgpwUmlJeCtUZmlyUjM4TlVhZTBkR1g5c1o1TWF2NkhiSVVEK2hNTm9VU2pyWUZ2bTcxbGtYdlhVWW9wd1FSR3FvCkZiWG5KTFFVakd3UFFRclRCQ0wycFE2c1dGbkpkQ0VBSHhXU0grTUN4cXV0L25SUHhORlFOWGFMZmU3clB3a3MKd3NrVlFlTzgyL205aFZkWnNnNUFnRDJtTS9jS2VybE9acVBGcWFEaFBoYnpkRzF4QjRtcVZmTGJVUUppa3I1SQpYQ1FhTWR2c2FRTnhPcmNPOHQwQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZFek92OGhVUGtRYS90bGNhQTE0OERmcmF5TXRNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBMTFqcDR4V1hyZkRPNDJlblBpdm9HNXZxY3hqbysxaktZV1YxR1NIdHNVdlBNU1UzVAptdVFCQWU2VVRzM0JtSTgzMUQ4aFd4cDlka3BkMlhWbCt6OTZabFIvQ2Qrb3NSWWE1MUpZd2RvMUp4Z29hbUk2CmszOS9sNTRBQVpWMkFPWWNQU0pKNkNzdVcxZ2NTMDlaTlA0djlhK0NIKzY5OVpQMGlhUStEc0FPRFJzUnhRcVMKK08zNUU4K0RzRjBSOWZCYlhvbkxiK05VSHlTQ2pQSnVVT21lS0RYMFpQQm81blNscmh5cUIxbHVNZnhEaUltdgpIU3BscXB4M1duMzJnNEZYOCs3clZBNUZjWkIvNDg3bUZmdG1CTXc3aENidlFJeWY2Ymd6VTUycUdaeFZGTTVjCnUxeEZyRXlXaFA1eElJY09lY1M1RCtJa29ZOHUyT3ozQldEeQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + server: https://test-master:6443 name: kubernetes contexts: - context: @@ -15,5 +15,5 @@ preferences: {} users: - name: kubernetes-admin user: - client-certificate-data: - client-key-data: \ No newline at end of file + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lJQjUxN3hkT3V2cnd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TVRFeU1EZ3dPREk0TUROYUZ3MHlNakV5TURnd09ESTRNRFZhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXlFaURrYVJ1WmVUbVJ4WmsKRkxUbkRPMUxIM1k5dE80WXpXREt4QVA2bFphTUJwZXJWRWpiU1VMOGIzdG9WWFppVnVnaHhvMzRHdjF4dlJKLwpDN3NhWGVaU3EzcTVMcEFhQjBrelRDNFRtbGtrdE03YkJTZFNhMG9SeGJXTUdvVEM0WUVBckJCNS9FRmdUYjRCCnE4aG1USUlmdUcwa05sNmd5a2M4UnhSL3N5Q1lSVjZCTTZTYWFzODR1Tmx3ZU9vV2RaQkQ0UVZQbzJueUI4NGQKUnc0d0w2VEQ5MlUwMG5KcjY3bUtpTjhBNGtBampFbWNrNm1JbHdsUTB2MHR5Uk8zTFBXZFdNazZ5QVIyVEZ6RgpBZCs4dUd5L3VDQlB6UDlVcDFERkI5TDF0Y25sd1lTeTYycGdFWC9wMzl6ZjZ0RFpZRVJwK0Jpd21Eb3RmVmFaClNLam9md0lEQVFBQm8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0h3WURWUjBqQkJnd0ZvQVVUTTYveUZRK1JCcisyVnhvRFhqd04rdHJJeTB3RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFCQ01US2VEV1dHa0VFZWp6dlN3TTFKZit0QUJaYklrd0FoZEt1MjFnUkxyQktzbHQ4Y0NHMG1KCmc3eDhndFJYcW0xRGlEU01DMnlWMW9OWTFSdGlLZVhZT29RbVY2SFh2bEQybWZyY2VzS3VSNHlIbCtlWHcyL1AKb2FLZVFEM21yVm9ZZ1ZMMVNqOWpzcDRBa1hEM1ErUzdKYXN4Y3MvVkR3SmlRQzhGamkzaWZMVFFKQkxCdm1WVwpCZ3QvM0xPZ1lCdWxYRWg3cFlQUzduTVFqNFpxQ0QwWTNJOENicUM0VGlZQjB1aTZlbVRDd3cvRnBNSzNIRFZ4CnZOd3FPRHBaWW5DK0IxSzR5RUk4NGIvM1o3ZXA0MUE1NGk3QkpxTklORUVtRjdGcmpQQy91V1RuYlpzVUNFQjAKWTFUR1Y3czNBUTZMZXNCRFF6UVRzVnBENWFRRk9VYz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBeUVpRGthUnVaZVRtUnhaa0ZMVG5ETzFMSDNZOXRPNFl6V0RLeEFQNmxaYU1CcGVyClZFamJTVUw4YjN0b1ZYWmlWdWdoeG8zNEd2MXh2UkovQzdzYVhlWlNxM3E1THBBYUIwa3pUQzRUbWxra3RNN2IKQlNkU2Ewb1J4YldNR29UQzRZRUFyQkI1L0VGZ1RiNEJxOGhtVElJZnVHMGtObDZneWtjOFJ4Ui9zeUNZUlY2QgpNNlNhYXM4NHVObHdlT29XZFpCRDRRVlBvMm55Qjg0ZFJ3NHdMNlREOTJVMDBuSnI2N21LaU44QTRrQWpqRW1jCms2bUlsd2xRMHYwdHlSTzNMUFdkV01rNnlBUjJURnpGQWQrOHVHeS91Q0JQelA5VXAxREZCOUwxdGNubHdZU3kKNjJwZ0VYL3AzOXpmNnREWllFUnArQml3bURvdGZWYVpTS2pvZndJREFRQUJBb0lCQUgrbHY3ME9TSkpHZmdHbQpvcWlUMTRKa3BuRnA0ZEF6dzdqNXpLRjdTN1VWR3krRWNOeXFCcUM5d2JlbnRvcHBoaW1Qang5R0VtL1pRaWxYCjVZTHJmOVdDMndPUmx2NjNOdStYMXNyaHZ2cXJmL3FBc0JTcnlCcTdQWEo5ejhxQy9OWE9hMGcreEJCaTltYjIKQjRpZGs1MkZmWVFFZzRUbmNLRWJINjdKd00yL2dmZ3BXMGNEd0RiNVc2aHIxclUzNkNGc1AyVTlEYytMVkV5cQpvK3FZZ0RFZ0MxSWxucGVIdzROejJmK3RkbFpLVHN5dzV1ZG5xVm1kWGNkLyszR3NvQVM3Q25DTmQ3a1pKQ1BPClNpRktZSDZZUS93ZmJLZjFibC84eERXU1UvKytQUTZyODJGV21mMFBjd3ExbWV5OXIvUW1kaHc5YndiamkxVDQKdURjSzUva0NnWUVBOURIZi8xZnN4czlqVldoaE94UXFzUFN3UzdDbVZYcHV2S2Q3MjlMRGFIV1VQT3E5Y0l4QQpaV01Ibi9EbEN3U2dxaitPL0FNSDdaQVpPbDV5V2RiZFVEM2I0WWVKLzlubXdvN25ReG9ndTlUdTlNZEVnM0pvCm1wTzVqQTYzYTE2Z0sxRWlmTDJkdllRQkJGQzg2aDJzYlV0WHNGRDV6R3QzTU96bVF6SFFQM3NDZ1lFQTBmY3gKMzlUR0YrZFdVb3MzOXRLaE9MZVpFaWtUSmtqckdrTUdldHpYNEVCRE5oRUUvdWNkYUZCWmJoU3ltZHAwMmtKNgpKQkpvaEJRaEN4Zk9nUXhRdFp2dUt3WUMvTEYrMVF1TG1MaERGV2NEa05Ma1E0RWU5WVRFV1VuNE9NNkJJTmVOCldseklzcncyeHRvaUVoMXZFWW92ODdobEZCQ0E4N0xYSElpa1NjMENnWUFnZ1pNajFueDZhcGo2Z1k2UDRydGMKR3ZaczNQTUhaZWpmekJ1OHcyMm50aDhwak1YeStYaUpCb201VE1Qd2w0a2JvS0pVQWNOSWFHb3pUdWRCOWt3Mwpwa1JpM1R3Tmh1QWsvZ2ppOGROeHJZS3hxdVQvNm5icVBNZnlVUE14bzhNR2dTSnFJSU9pK3JOMzg1ZHlRc2oxCk01WlVyakxTZDJ2Q1k0YXpZeW9waVFLQmdBS0RBb1c2OWVPelVNVU1CVTllbGE4b0F4ekVnUVlrT1N2SFFYeVcKeDc1WEJuYlBIVkF2VTVxNzIxWUZ5VjB0ejlnTWs0bDY3dVVsbUgrWmVVN3g1c1ZGYUVQN2ZtMm5jZXo2aG1EOQpVMUFlTzF5d0tTcmxrSWsvWFZuMEdKUVZaRllRUGhDbXEzLzM0Ry9nakNmTFVsalRYbDk4QlRtSU9RS3hVUUYyCllRREJBb0dBRTB5Q0NzcFE3Y1lYaVFyQnF4VVkxU2N5SjViVm1iaFN5cmgyb3FiUmRMRmxzWHFjUnBIK2tORjYKdXdjUzhZalJodmdQazgyZm1QcXlieE5ZWGxTYWoydGdzSFdpcGNETXZSM1dtSlQzTGpjeXZObDh6cUlkZDNBdQpJT1pNeG0vWWVMemNFOUdteDRsQkhmMnU5MkhiRjRiSCtqeVp3d3RWcHpRdmphaFF6dGs9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/config/RecycleConfig.java b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/config/RecycleConfig.java index 56cb46a..c52f23d 100644 --- a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/config/RecycleConfig.java +++ b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/config/RecycleConfig.java @@ -72,5 +72,9 @@ public class RecycleConfig { * 用户删除tadl算法版本文件后，文件最大有效时长，以天为单位 */ private Integer tadlValid; + /** + * 用户删除点云数据集文件后，文件最大有效时长，以天为单位 + */ + private Integer pointCloudValid; } \ No newline at end of file diff --git a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleModuleEnum.java b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleModuleEnum.java index 5fbf13f..c3d86c7 100644 --- a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleModuleEnum.java +++ b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleModuleEnum.java @@ -40,7 +40,8 @@ public enum RecycleModuleEnum { BIZ_DATAMEDICINE(8, "医学影像",SERVER_DATA_DCM), BIZ_MEASURE(9, "度量管理",SERVER_MEASURE), BIZ_SERVING(10, "云端部署", SERVER_SERVING), - BIZ_TADL(11,"自动机器学习",SERVER_TADL); + BIZ_TADL(11,"自动机器学习",SERVER_TADL), + BIZ_POINT_CLOUD(12,"3D点云",SERVER_POINT_CLOUD); private Integer value; diff --git a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java index 6915e28..6d3056d 100644 --- a/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java +++ b/dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java @@ -86,8 +86,16 @@ public enum RecycleResourceEnum { /** * 模型优化文件回收 */ - MODEL_OPT_RECYCLE_FILE("modelOptRecycleFile","模型优化文件回收"), - ; + MODEL_OPT_RECYCLE_FILE("modelOptRecycleFile", "模型优化文件回收"), + /** + * 点云文件回收 + */ + PC_DATASET_RECYCLE_FILE("pcDatasetRecycleFile", "点云数据集文件回收"), + + /** + * 训练任务文件回收 + */ + TRAIN_JOB_RECYCLE_FILE("trainJobRecycleFile", "训练任务文件回收"); private String className; diff --git a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/dao/PtTrainAlgorithmMapper.java b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/dao/PtTrainAlgorithmMapper.java index 63b28dc..b0bf6fd 100644 --- a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/dao/PtTrainAlgorithmMapper.java +++ b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/dao/PtTrainAlgorithmMapper.java @@ -72,6 +72,6 @@ public interface PtTrainAlgorithmMapper extends BaseMapper { * 查询可推理预置算法 * @return List 返回可推理预置算法集合 */ - @Select("select * from pt_train_algorithm where deleted = 0 and inference=1 and algorithm_source=2 order by id desc") + @Select("select * from pt_train_algorithm where deleted = 0 and algorithm_source=2 order by id desc") List selectPreAlgorithm(); } diff --git a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/dto/PtTrainAlgorithmCreateDTO.java b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/dto/PtTrainAlgorithmCreateDTO.java index f1dc9dd..6809570 100644 --- a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/dto/PtTrainAlgorithmCreateDTO.java +++ b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/dto/PtTrainAlgorithmCreateDTO.java @@ -70,9 +70,6 @@ public class PtTrainAlgorithmCreateDTO implements Serializable { @Length(max = MagicNumConstant.ONE_HUNDRED_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") private String runCommand; - @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)，管理员使用") - private JSONObject runParams; - @ApiModelProperty("算法来源(1为我的算法，2为预置算法)，管理员使用") private Integer algorithmSource; @@ -80,6 +77,7 @@ public class PtTrainAlgorithmCreateDTO implements Serializable { @Length(max = MagicNumConstant.ONE_HUNDRED_TWENTY_EIGHT, message = "算法用途-输入长度不能超过128个字符") private String algorithmUsage; + @ApiModelProperty("是否输出训练结果，不填则默认为true") private Boolean isTrainModelOut; diff --git a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/dto/PtTrainAlgorithmQueryDTO.java b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/dto/PtTrainAlgorithmQueryDTO.java index 6f5e9d7..cbf145b 100644 --- a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/dto/PtTrainAlgorithmQueryDTO.java +++ b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/dto/PtTrainAlgorithmQueryDTO.java @@ -54,7 +54,4 @@ public class PtTrainAlgorithmQueryDTO extends PageQueryBase implements Serializa @ApiModelProperty(value = "算法用途") private String algorithmUsage; - - @ApiModelProperty(value = "上传算法是否支持推理(true:可推理，false：不可推理)") - private Boolean inference; } diff --git a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/entity/PtTrainAlgorithm.java b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/entity/PtTrainAlgorithm.java index cb55ccf..91e4528 100644 --- a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/entity/PtTrainAlgorithm.java +++ b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/entity/PtTrainAlgorithm.java @@ -85,18 +85,13 @@ public class PtTrainAlgorithm extends BaseEntity { @TableField(value = "run_command") private String runCommand; - /** - * 运行参数 - */ - @TableField(value = "run_params", typeHandler = FastjsonTypeHandler.class) - private JSONObject runParams; - /** * 算法用途 */ @TableField(value = "algorithm_usage") private String algorithmUsage; + /** * 算法精度 */ @@ -109,12 +104,6 @@ public class PtTrainAlgorithm extends BaseEntity { @TableField(value = "p4_inference_speed") private Integer p4InferenceSpeed; - /** - * 算法是否支持推理（1可推理，0不可推理） - */ - @TableField(value = "inference") - private Boolean inference; - /** * 训练结果输出（1是，0否） */ diff --git a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/vo/PtTrainAlgorithmQueryVO.java b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/vo/PtTrainAlgorithmQueryVO.java index 4593b62..6146753 100644 --- a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/vo/PtTrainAlgorithmQueryVO.java +++ b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/domain/vo/PtTrainAlgorithmQueryVO.java @@ -60,9 +60,6 @@ public class PtTrainAlgorithmQueryVO implements Serializable { @ApiModelProperty(value = "运行命令") private String runCommand; - @ApiModelProperty(value = "运行参数") - private JSONObject runParams; - @ApiModelProperty(value = "算法用途") private String algorithmUsage; @@ -81,9 +78,6 @@ public class PtTrainAlgorithmQueryVO implements Serializable { @ApiModelProperty(value = "可视化日志（1是，0否）") private Boolean isVisualizedLog; - @ApiModelProperty(value = "算法是否支持推理（1可推理，0不可推理）") - private Boolean inference; - @ApiModelProperty(value = "创建人") private Long createUserId; diff --git a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/rest/PtTrainAlgorithmController.java b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/rest/PtTrainAlgorithmController.java index 72ed003..9a68b33 100644 --- a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/rest/PtTrainAlgorithmController.java +++ b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/rest/PtTrainAlgorithmController.java @@ -23,15 +23,13 @@ import org.dubhe.algorithm.domain.dto.PtTrainAlgorithmCreateDTO; import org.dubhe.algorithm.domain.dto.PtTrainAlgorithmDeleteDTO; import org.dubhe.algorithm.domain.dto.PtTrainAlgorithmQueryDTO; import org.dubhe.algorithm.domain.dto.PtTrainAlgorithmUpdateDTO; +import org.dubhe.algorithm.domain.vo.PtTrainAlgorithmQueryVO; import org.dubhe.algorithm.service.PtTrainAlgorithmService; -import org.dubhe.biz.base.annotation.ApiVersion; import org.dubhe.biz.base.constant.Permissions; -import org.dubhe.biz.base.dto.ModelOptAlgorithmCreateDTO; -import org.dubhe.biz.base.dto.TrainAlgorithmSelectAllBatchIdDTO; -import org.dubhe.biz.base.dto.TrainAlgorithmSelectAllByIdDTO; -import org.dubhe.biz.base.dto.TrainAlgorithmSelectByIdDTO; +import org.dubhe.biz.base.dto.*; import org.dubhe.biz.base.vo.DataResponseBody; import org.dubhe.biz.base.vo.TrainAlgorithmQureyVO; +import org.dubhe.biz.dataresponse.factory.DataResponseFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.validation.annotation.Validated; @@ -121,4 +119,25 @@ public class PtTrainAlgorithmController { public DataResponseBody getInferenceAlgorithm() { return new DataResponseBody(ptTrainAlgorithmService.getInferenceAlgorithm()); } + + @GetMapping("/listIdByName") + @ApiOperation("根据名称查询算法") + @PreAuthorize(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody listIdByName(@Validated TrainAlgorithmSelectByNameDTO trainAlgorithmSelectByNameDTO) { + return new DataResponseBody(ptTrainAlgorithmService.listIdByName(trainAlgorithmSelectByNameDTO.getAlgorithmName())); + } + + @ApiOperation("根据名称获取算法信息") + @GetMapping("/findAlgorithmByName") + public DataResponseBody findAlgorithmByName(@RequestParam String name) { + return DataResponseFactory.success(ptTrainAlgorithmService.findAlgorithmByName(name)); + } + + @GetMapping("/getAll") + @ApiOperation("获取所有算法(我的算法和预置算法)") + @PreAuthorize(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody getAll(@RequestParam(required = false) String algorithmUsage) { + return DataResponseFactory.success(ptTrainAlgorithmService.getAll(algorithmUsage)); + } + } diff --git a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/service/PtTrainAlgorithmService.java b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/service/PtTrainAlgorithmService.java index 95bb9c7..d1eda1d 100644 --- a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/service/PtTrainAlgorithmService.java +++ b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/service/PtTrainAlgorithmService.java @@ -118,4 +118,29 @@ public interface PtTrainAlgorithmService { * @return List 返回可推理算法集合 */ List getInferenceAlgorithm(); + + /** + * 根据算法名称获取算法id + * + * @param algorithmName 算法名称 + * @return + */ + List listIdByName(String algorithmName); + + /** + * 根据算法名称获取算法信息 + * + * @param algorithmName 算法名称 + * @return {@code TrainAlgorithmQureyVO} 算法对象 + */ + TrainAlgorithmQureyVO findAlgorithmByName(String algorithmName); + + + /** + * 获取所有算法(预置算法和我的算法) + * + * @return + */ + List getAll(String algorithmUsage); + } diff --git a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/service/impl/PtTrainAlgorithmServiceImpl.java b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/service/impl/PtTrainAlgorithmServiceImpl.java index d324d87..0acaa1b 100644 --- a/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/service/impl/PtTrainAlgorithmServiceImpl.java +++ b/dubhe-server/dubhe-algorithm/src/main/java/org/dubhe/algorithm/service/impl/PtTrainAlgorithmServiceImpl.java @@ -1,12 +1,12 @@ /** * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,11 +17,15 @@ package org.dubhe.algorithm.service.impl; +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.util.RandomUtil; import cn.hutool.core.util.StrUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.google.common.collect.Lists; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.map.HashedMap; import org.dubhe.algorithm.async.TrainAlgorithmUploadAsync; @@ -39,9 +43,11 @@ import org.dubhe.algorithm.domain.vo.PtTrainAlgorithmQueryVO; import org.dubhe.algorithm.service.PtTrainAlgorithmService; import org.dubhe.biz.base.constant.MagicNumConstant; import org.dubhe.biz.base.constant.NumberConstant; +import org.dubhe.biz.base.constant.SymbolConstant; import org.dubhe.biz.base.context.UserContext; import org.dubhe.biz.base.dto.*; import org.dubhe.biz.base.enums.AlgorithmSourceEnum; +import org.dubhe.biz.base.enums.AlgorithmStatusEnum; import org.dubhe.biz.base.enums.DatasetTypeEnum; import org.dubhe.biz.base.enums.ImageTypeEnum; import org.dubhe.biz.base.exception.BusinessException; @@ -149,10 +155,6 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { if (ptTrainAlgorithmQueryDTO.getAlgorithmUsage() != null) { wrapper.like("algorithm_usage", ptTrainAlgorithmQueryDTO.getAlgorithmUsage()); } - //根据算法是否可推理筛选 - if (ptTrainAlgorithmQueryDTO.getInference() != null) { - wrapper.eq("inference", ptTrainAlgorithmQueryDTO.getInference()); - } if (!StringUtils.isEmpty(ptTrainAlgorithmQueryDTO.getAlgorithmName())) { wrapper.and(qw -> qw.eq("id", ptTrainAlgorithmQueryDTO.getAlgorithmName()).or().like("algorithm_name", ptTrainAlgorithmQueryDTO.getAlgorithmName())); @@ -197,11 +199,10 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { public List create(PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO) { //获取用户信息 UserContext user = userContext.getCurUser(); - //获取镜像url - BaseImageDTO baseImageDTO = new BaseImageDTO(); - BeanUtils.copyProperties(ptTrainAlgorithmCreateDTO, baseImageDTO); - if (StringUtils.isNotBlank(ptTrainAlgorithmCreateDTO.getImageName()) && StringUtils.isNotBlank(ptTrainAlgorithmCreateDTO.getImageTag())) { - ptTrainAlgorithmCreateDTO.setImageName(getImageUrl(baseImageDTO, user)); + String imageName = ptTrainAlgorithmCreateDTO.getImageName(); + String imageTag = ptTrainAlgorithmCreateDTO.getImageTag(); + if (StringUtils.isNotBlank(imageName) && StringUtils.isNotBlank(imageTag)) { + ptTrainAlgorithmCreateDTO.setImageName(imageName + SymbolConstant.COLON + imageTag); } //创建算法校验DTO并设置默认值 setAlgorithmDtoDefault(ptTrainAlgorithmCreateDTO); @@ -348,8 +349,6 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { } //算法路径 ptTrainAlgorithm.setCodeDir(targetPath); - //算法文件可推理 - ptTrainAlgorithm.setInference(true); } /** @@ -457,6 +456,7 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { /** * 根据Id查询所有数据(包含已被软删除的数据) + * * @param trainAlgorithmSelectAllByIdDTO 算法id * @return TrainAlgorithmQureyVO返回查询数据(包含已被软删除的数据) */ @@ -471,7 +471,8 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { /** * 根据Id查询 - * @param trainAlgorithmSelectByIdDTO 算法id + * + * @param trainAlgorithmSelectByIdDTO 算法id * @return TrainAlgorithmQureyVO 返回查询数据 */ @Override @@ -479,12 +480,15 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { public TrainAlgorithmQureyVO selectById(TrainAlgorithmSelectByIdDTO trainAlgorithmSelectByIdDTO) { PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(trainAlgorithmSelectByIdDTO.getId()); TrainAlgorithmQureyVO trainAlgorithmQureyVO = new TrainAlgorithmQureyVO(); - BeanUtils.copyProperties(ptTrainAlgorithm, trainAlgorithmQureyVO); + if (ptTrainAlgorithm != null) { + BeanUtils.copyProperties(ptTrainAlgorithm, trainAlgorithmQureyVO); + } return trainAlgorithmQureyVO; } /** * 根据Id批量查询 + * * @param trainAlgorithmSelectAllBatchIdDTO 算法ids * @return List 返回查询数据 */ @@ -507,9 +511,9 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { * @param ptTrainAlgorithmQueryVO 镜像名称与版本 */ private void getImageNameAndImageTag(PtTrainAlgorithm trainAlgorithm, PtTrainAlgorithmQueryVO ptTrainAlgorithmQueryVO) { + String image = trainAlgorithm.getImageName(); if (StringUtils.isNotBlank(trainAlgorithm.getImageName())) { - String imageNameSuffix = trainAlgorithm.getImageName().substring(trainAlgorithm.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); - String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); + String[] imageNameSuffixArray = image.split(StrUtil.COLON); ptTrainAlgorithmQueryVO.setImageName(imageNameSuffixArray[0]); ptTrainAlgorithmQueryVO.setImageTag(imageNameSuffixArray[1]); } @@ -556,7 +560,10 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { PtImageQueryUrlDTO ptImageQueryUrlDTO = new PtImageQueryUrlDTO(); ptImageQueryUrlDTO.setImageTag(baseImageDto.getImageTag()); ptImageQueryUrlDTO.setImageName(baseImageDto.getImageName()); - ptImageQueryUrlDTO.setProjectType(ImageTypeEnum.TRAIN.getType()); + List trainImageType = new ArrayList() {{ + add(ImageTypeEnum.TRAIN.getType()); + }}; + ptImageQueryUrlDTO.setImageTypes(trainImageType); DataResponseBody dataResponseBody = imageClient.getImageUrl(ptImageQueryUrlDTO); if (!dataResponseBody.succeed()) { LogUtil.error(LogEnum.BIZ_TRAIN, " User {} gets image ,the imageName is {}, the imageTag is {}, and the result of dubhe-image service call failed", user.getUsername(), baseImageDto.getImageName(), baseImageDto.getImageTag()); @@ -572,14 +579,15 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { } /** - * * @param modelOptAlgorithmCreateDTO 模型优化上传算法入参 * @return PtTrainAlgorithm 新增算法信息 */ @Override public ModelOptAlgorithmQureyVO modelOptimizationUploadAlgorithm(ModelOptAlgorithmCreateDTO modelOptAlgorithmCreateDTO) { PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO = new PtTrainAlgorithmCreateDTO(); - ptTrainAlgorithmCreateDTO.setAlgorithmName(modelOptAlgorithmCreateDTO.getName()).setCodeDir(modelOptAlgorithmCreateDTO.getPath()).setAlgorithmUsage("模型优化").setIsTrainModelOut(false).setIsTrainOut(false).setIsVisualizedLog(false); + ptTrainAlgorithmCreateDTO.setAlgorithmName(modelOptAlgorithmCreateDTO.getName()) + .setCodeDir(modelOptAlgorithmCreateDTO.getPath()).setAlgorithmUsage("5001") + .setIsTrainModelOut(false).setIsTrainOut(false).setIsVisualizedLog(false); List ids = create(ptTrainAlgorithmCreateDTO); PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(ids.get(NumberConstant.NUMBER_0)); ModelOptAlgorithmQureyVO modelOptAlgorithmQureyVO = new ModelOptAlgorithmQureyVO(); @@ -589,6 +597,7 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { /** * 算法删除文件还原 + * * @param dto 还原实体 */ @Override @@ -616,6 +625,7 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { /** * 查询可推理算法 + * * @return List 返回可推理算法集合 */ @Override @@ -623,7 +633,6 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { //获取用户信息 UserContext user = userContext.getCurUser(); QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.eq("inference", true).orderByDesc("id"); List ptTrainAlgorithms = ptTrainAlgorithmMapper.selectList(wrapper); List ptTrainAlgorithmQueryResult = new ArrayList<>(); if (CollectionUtils.isNotEmpty(ptTrainAlgorithms)) { @@ -653,4 +662,80 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { return ptTrainAlgorithmQueryResult; } + @Override + public List listIdByName(String algorithmName) { + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.lambda() + .like(PtTrainAlgorithm::getAlgorithmName, algorithmName); + List ptTrainAlgorithms = ptTrainAlgorithmMapper.selectList(wrapper); + List ids = Lists.newArrayList(); + if (CollectionUtils.isEmpty(ptTrainAlgorithms)) { + return ids; + } + ids = ptTrainAlgorithms.stream() + .map(PtTrainAlgorithm::getId) + .collect(Collectors.toList()); + return ids; + } + + @Override + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + public TrainAlgorithmQureyVO findAlgorithmByName(String algorithmName) { + + TrainAlgorithmQureyVO atlasTrainAlgorithmVO = new TrainAlgorithmQureyVO(); + List ptTrainAlgorithms = ptTrainAlgorithmMapper.selectList(new LambdaQueryWrapper() + .eq(PtTrainAlgorithm::getAlgorithmName, algorithmName)); + if (CollUtil.isNotEmpty(ptTrainAlgorithms)) { + BeanUtils.copyProperties(ptTrainAlgorithms.get(0), atlasTrainAlgorithmVO); + //获取镜像名称与版本 + if (StrUtil.isNotBlank(ptTrainAlgorithms.get(0).getImageName())) { + String imageNameSuffix = ptTrainAlgorithms.get(0).getImageName().substring(ptTrainAlgorithms.get(0).getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); + String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); + atlasTrainAlgorithmVO.setImageName(imageNameSuffixArray[0]); + atlasTrainAlgorithmVO.setImageTag(imageNameSuffixArray[1]); + } + } + return atlasTrainAlgorithmVO; + } + + @Override + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + public List getAll(String algorithmUsage) { + List all = new ArrayList<>(); + all.addAll(getAlgorithmListBySource(AlgorithmSourceEnum.MINE.getStatus(), algorithmUsage)); + all.addAll(getAlgorithmListBySource(AlgorithmSourceEnum.PRE.getStatus(), algorithmUsage)); + return all; + } + + public List getAlgorithmListBySource(Integer algorithmSource, String algorithmUsage) { + List result = new ArrayList<>(); + //获取用户信息 + UserContext user = userContext.getCurUser(); + // 获取我的算法 + QueryWrapper ptTrainAlgorithmQueryWrapper = new QueryWrapper<>(); + //判断算法来源 + if (AlgorithmSourceEnum.MINE.getStatus().equals(algorithmSource)) { + if (!BaseService.isAdmin(user)) { + ptTrainAlgorithmQueryWrapper.lambda().eq(PtTrainAlgorithm::getCreateUserId, userContext.getCurUserId()); + } + } + if (StringUtils.isNotEmpty(algorithmUsage)) { + ptTrainAlgorithmQueryWrapper.lambda().eq(PtTrainAlgorithm::getAlgorithmUsage, algorithmUsage); + } + ptTrainAlgorithmQueryWrapper.lambda().eq(PtTrainAlgorithm::getAlgorithmStatus, AlgorithmStatusEnum.SUCCESS.getCode()); + ptTrainAlgorithmQueryWrapper.lambda().eq(PtTrainAlgorithm::getAlgorithmSource, algorithmSource); + List ptTrainAlgorithmList = ptTrainAlgorithmMapper.selectList(ptTrainAlgorithmQueryWrapper); + if (CollectionUtil.isNotEmpty(ptTrainAlgorithmList)) { + result = ptTrainAlgorithmList.stream().map(x -> { + PtTrainAlgorithmQueryVO ptTrainAlgorithmQueryVO = new PtTrainAlgorithmQueryVO(); + BeanUtils.copyProperties(x, ptTrainAlgorithmQueryVO); + //获取镜像名称与版本 + getImageNameAndImageTag(x, ptTrainAlgorithmQueryVO); + return ptTrainAlgorithmQueryVO; + }).collect(Collectors.toList()); + } + return result; + } + } + diff --git a/dubhe-server/dubhe-algorithm/src/main/resources/bootstrap.yml b/dubhe-server/dubhe-algorithm/src/main/resources/bootstrap.yml index 94dd2df..d34de31 100644 --- a/dubhe-server/dubhe-algorithm/src/main/resources/bootstrap.yml +++ b/dubhe-server/dubhe-algorithm/src/main/resources/bootstrap.yml @@ -10,7 +10,7 @@ spring: nacos: config: enabled: true - namespace: dubhe-server-cloud-prod + namespace: dubhe-prod server-addr: 127.0.0.1:8848 shared-configs[0]: data-id: common-biz.yaml @@ -29,9 +29,13 @@ spring: data-id: dubhe-algorithm.yaml group: dubhe refresh: true + shared-configs[4]: + data-id: kubeconfig.yaml + group: dubhe + refresh: true discovery: enabled: true - namespace: dubhe-server-cloud-dev + namespace: dubhe-prod group: dubhe server-addr: 127.0.0.1:8848 diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/constant/DcmConstant.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/constant/DcmConstant.java index b11e872..d5fcabb 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/constant/DcmConstant.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/constant/DcmConstant.java @@ -78,6 +78,11 @@ public class DcmConstant { /** * dcm文件上传 */ - public static final String DCM_UPLOAD = "ssh %s@%s \"docker run --rm -v %s:/nfs dcm4che/dcm4che-tools:5.10.5 storescu -c DCM4CHEE@%s:%s %s\""; + public static final String DCM_UPLOAD = "docker run --rm -v %s:/nfs dcm4che/dcm4che-tools:5.10.5 storescu -c DCM4CHEE@%s:%s %s"; + + /** + * dcm数据存储路径 + */ + public static final String DCM_DATA_URL = "dataset/dcm/%s"; } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/dao/DataMedicineFileMapper.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/dao/DataMedicineFileMapper.java index 9668b6e..8944f4d 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/dao/DataMedicineFileMapper.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/dao/DataMedicineFileMapper.java @@ -120,4 +120,20 @@ public interface DataMedicineFileMapper extends BaseMapper { */ @Delete("DELETE FROM data_medicine_file WHERE medicine_id= #{id} ") void deleteByDatasetId(@Param("id") Long id); + + /** + * 根据医学数据集id删除医学数据集文件数据 + * + * @param datasetId 医学数据集ID + */ + List getFileStatusListByDataset(@Param("datasetId") Long datasetId); + + /** + * 清理所有已标注信息 + * + * @param datasetId + */ + @Update("update data_medicine_file set `status` = 101 where medicine_id = #{datasetId}") + void cleanAnnotation(@Param("datasetId") Long datasetId); + } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/dao/DataMedicineMapper.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/dao/DataMedicineMapper.java index bc15f85..a9b56fb 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/dao/DataMedicineMapper.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/dao/DataMedicineMapper.java @@ -23,6 +23,9 @@ import org.apache.ibatis.annotations.Select; import org.apache.ibatis.annotations.Update; import org.dubhe.biz.base.annotation.DataPermission; import org.dubhe.dcm.domain.entity.DataMedicine; +import org.dubhe.dcm.domain.vo.DataMedicineSmallVO; + +import java.util.List; /** * @description 医学数据集管理 Mapper 接口 @@ -86,4 +89,12 @@ public interface DataMedicineMapper extends BaseMapper { */ @Select("select * from data_medicine where id = #{id} and deleted = 1") DataMedicine findDataMedicineByIdAndDeleteIsFalse(@Param("id") Long datasetId); + + /** + * 获取医学数据集列表 + * + * @return + */ + List getList(); + } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/dto/MedicineAutoAnnotationDTO.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/dto/MedicineAutoAnnotationDTO.java index d4e87d2..33b220f 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/dto/MedicineAutoAnnotationDTO.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/dto/MedicineAutoAnnotationDTO.java @@ -21,7 +21,9 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; - +import org.dubhe.biz.base.annotation.EnumValue; +import org.dubhe.data.constant.Constant; +import org.dubhe.data.constant.FileTypeEnum; import javax.validation.constraints.NotNull; import java.io.Serializable; @@ -41,4 +43,16 @@ public class MedicineAutoAnnotationDTO implements Serializable { @NotNull(message = "自动标注的数据集不能为空") private Long medicalId; + @ApiModelProperty("模型服务ID") + @NotNull(message = "模型服务不能为空") + private Long modelServiceId; + + /** + * 文件状态 400-全部 304-无标注 303-有标注 + */ + @NotNull(message = "文件标注信息筛选不能空") + @EnumValue(enumClass = FileTypeEnum.class, enumMethod = "autoFileStatusIsValid", + message = Constant.ANNOTATE_FILE_TYPE_RULE) + private Integer fileStatus; + } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/entity/DataMedicine.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/entity/DataMedicine.java index 7b0da8c..ee75668 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/entity/DataMedicine.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/entity/DataMedicine.java @@ -16,10 +16,7 @@ */ package org.dubhe.dcm.domain.entity; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.annotation.*; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -77,6 +74,9 @@ public class DataMedicine extends BaseEntity implements Serializable { @ApiModelProperty(value = "标注类型: 1001.器官分割 2001.病灶检测之肺结节检测") private Integer annotateType; + @ApiModelProperty(value = "是否自动标注停止") + private Boolean stop; + public DataMedicine() { } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/vo/DataMedicineSmallVO.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/vo/DataMedicineSmallVO.java new file mode 100644 index 0000000..a8cfbf2 --- /dev/null +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/vo/DataMedicineSmallVO.java @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.dcm.domain.vo; + +import lombok.Data; + +/** + * @description 医学数据集完成标注文件VO + * @date 2022-05-18 + */ +@Data +public class DataMedicineSmallVO { + + private Long id; + private String name; + private String url; + +} diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/vo/DataMedicineVO.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/vo/DataMedicineVO.java index 68d26c1..2e5aafd 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/vo/DataMedicineVO.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/domain/vo/DataMedicineVO.java @@ -68,6 +68,11 @@ public class DataMedicineVO implements Serializable { @ApiModelProperty(value = "标注类型") private Integer annotateType; + @ApiModelProperty(value = "是否自动标注停止") + private Boolean stop; + + + /** * 医学数据集 转化 医学数据集详情VO 方法 * @@ -88,6 +93,7 @@ public class DataMedicineVO implements Serializable { dataMedicineVO.setName(dataMedicine.getName()); dataMedicineVO.setRemark(dataMedicine.getRemark()); dataMedicineVO.setAnnotateType(dataMedicine.getAnnotateType()); + dataMedicineVO.setStop(dataMedicine.getStop()); return dataMedicineVO; } } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/enums/DcmDataStateEnum.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/enums/DcmDataStateEnum.java index 669a075..d372beb 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/enums/DcmDataStateEnum.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/enums/DcmDataStateEnum.java @@ -18,6 +18,8 @@ package org.dubhe.dcm.machine.enums; import org.dubhe.dcm.machine.constant.DcmDataStateCodeConstant; +import java.util.Arrays; + /** * @description 数据集状态类 * @date 2020-08-28 @@ -121,4 +123,13 @@ public enum DcmDataStateEnum { return null; } + /** + * 检查当前状态是否可以自动标注 + * + * @return + */ + public static boolean checkCurrentStatusWhetherToAutoLabel(Integer status) { + return !Arrays.asList(AUTOMATIC_LABELING_STATE.getCode()).contains(status); + } + } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/enums/DcmFileStateEnum.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/enums/DcmFileStateEnum.java index e657faa..288d193 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/enums/DcmFileStateEnum.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/enums/DcmFileStateEnum.java @@ -16,11 +16,10 @@ */ package org.dubhe.dcm.machine.enums; - +import org.dubhe.data.constant.FileTypeEnum; import org.dubhe.dcm.machine.constant.DcmFileStateCodeConstant; -import java.util.HashSet; -import java.util.Set; +import java.util.*; /** @@ -135,4 +134,26 @@ public enum DcmFileStateEnum { return null; } + /** + * 自动标注中筛选条件对应文件状态集 + * + * @return + */ + public static List getFileStatusFromAutoLabelScreen(Integer status) { + List noAnnotation = Arrays.asList(NOT_ANNOTATION_FILE_STATE.getCode()); + List hasAnnotation = Arrays.asList(ANNOTATION_FILE_STATE.getCode(), + AUTO_ANNOTATION_COMPLETE_FILE_STATE.getCode(), ANNOTATION_COMPLETE_FILE_STATE.getCode()); + switch (status) { + case 304: + return noAnnotation; + case 303: + return hasAnnotation; + default: + List result = new ArrayList<>(); + result.addAll(noAnnotation); + result.addAll(hasAnnotation); + return result; + } + } + } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AnnotationCompleteDcmState.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AnnotationCompleteDcmState.java index e343f1f..c78b58e 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AnnotationCompleteDcmState.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AnnotationCompleteDcmState.java @@ -70,4 +70,18 @@ public class AnnotationCompleteDcmState extends AbstractDataMedicineState { LogUtil.debug(LogEnum.STATE_MACHINE, " 【标注完成】 执行事件后内存状态机的切换： {}", dcmDataMedicineStateMachine.getMemoryDataMedicineState()); } + /** + * 数据集 未标注-->自动标注-->自动标注中 + * + * @param medical 医学数据集对象 + */ + @Override + public void autoAnnotationSaveEvent(DataMedicine medical) { + LogUtil.debug(LogEnum.STATE_MACHINE, " 【未标注】 执行事件前内存中状态机的状态 :{} ", dcmDataMedicineStateMachine.getMemoryDataMedicineState()); + LogUtil.debug(LogEnum.STATE_MACHINE, " 接受参数： {} ", medical); + dataMedicineMapper.updateStatus(medical.getId(), DcmDataStateEnum.AUTOMATIC_LABELING_STATE.getCode()); + dcmDataMedicineStateMachine.setMemoryDataMedicineState(dcmDataMedicineStateMachine.getAutomaticLabelingDcmState()); + LogUtil.debug(LogEnum.STATE_MACHINE, " 【未标注】 执行事件后内存状态机的切换： {}", dcmDataMedicineStateMachine.getMemoryDataMedicineState()); + } + } \ No newline at end of file diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AnnotationDataState.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AnnotationDataState.java index 5eecf99..6b12645 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AnnotationDataState.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AnnotationDataState.java @@ -83,4 +83,18 @@ public class AnnotationDataState extends AbstractDataMedicineState { LogUtil.debug(LogEnum.STATE_MACHINE, " 【标注中】 执行事件后内存状态机的切换： {}", dcmDataMedicineStateMachine.getMemoryDataMedicineState()); } + /** + * 数据集 标注中-->自动标注-->自动标注中 + * + * @param medical 医学数据集对象 + */ + @Override + public void autoAnnotationSaveEvent(DataMedicine medical) { + LogUtil.debug(LogEnum.STATE_MACHINE, " 【未标注】 执行事件前内存中状态机的状态 :{} ", dcmDataMedicineStateMachine.getMemoryDataMedicineState()); + LogUtil.debug(LogEnum.STATE_MACHINE, " 接受参数： {} ", medical); + dataMedicineMapper.updateStatus(medical.getId(), DcmDataStateEnum.AUTOMATIC_LABELING_STATE.getCode()); + dcmDataMedicineStateMachine.setMemoryDataMedicineState(dcmDataMedicineStateMachine.getAutomaticLabelingDcmState()); + LogUtil.debug(LogEnum.STATE_MACHINE, " 【未标注】 执行事件后内存状态机的切换： {}", dcmDataMedicineStateMachine.getMemoryDataMedicineState()); + } + } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AutoAnnotationCompleteDcmState.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AutoAnnotationCompleteDcmState.java index dde76ca..93f8f15 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AutoAnnotationCompleteDcmState.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/state/specific/datamedicine/AutoAnnotationCompleteDcmState.java @@ -69,4 +69,18 @@ public class AutoAnnotationCompleteDcmState extends AbstractDataMedicineState { LogUtil.debug(LogEnum.STATE_MACHINE, " 【自动标注完成】 执行事件后内存状态机的切换： {}", dcmDataMedicineStateMachine.getMemoryDataMedicineState()); } + /** + * 数据集 未标注-->自动标注-->自动标注中 + * + * @param medical 医学数据集对象 + */ + @Override + public void autoAnnotationSaveEvent(DataMedicine medical) { + LogUtil.debug(LogEnum.STATE_MACHINE, " 【未标注】 执行事件前内存中状态机的状态 :{} ", dcmDataMedicineStateMachine.getMemoryDataMedicineState()); + LogUtil.debug(LogEnum.STATE_MACHINE, " 接受参数： {} ", medical); + dataMedicineMapper.updateStatus(medical.getId(), DcmDataStateEnum.AUTOMATIC_LABELING_STATE.getCode()); + dcmDataMedicineStateMachine.setMemoryDataMedicineState(dcmDataMedicineStateMachine.getAutomaticLabelingDcmState()); + LogUtil.debug(LogEnum.STATE_MACHINE, " 【未标注】 执行事件后内存状态机的切换： {}", dcmDataMedicineStateMachine.getMemoryDataMedicineState()); + } + } \ No newline at end of file diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/statemachine/DcmDataMedicineStateMachine.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/statemachine/DcmDataMedicineStateMachine.java index b1c8d8e..dcb487c 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/statemachine/DcmDataMedicineStateMachine.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/statemachine/DcmDataMedicineStateMachine.java @@ -139,10 +139,6 @@ public class DcmDataMedicineStateMachine extends AbstractDataMedicineState imple @Override public void autoAnnotationSaveEvent(DataMedicine medical) { initMemoryDataState(medical); - if (memoryDataMedicineState != notAnnotationDcmState) { - - throw new StateMachineException(ErrorMessageConstant.DATASET_CHANGE_ERR_MESSAGE); - } memoryDataMedicineState.autoAnnotationSaveEvent(medical); } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/DcmStateIdentifyUtil.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/DcmStateIdentifyUtil.java new file mode 100644 index 0000000..8a6bda6 --- /dev/null +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/DcmStateIdentifyUtil.java @@ -0,0 +1,129 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dcm.machine.utils; + +import org.dubhe.data.domain.entity.Dataset; +import org.dubhe.data.machine.enums.DataStateEnum; +import org.dubhe.dcm.machine.enums.DcmDataStateEnum; +import org.dubhe.dcm.machine.utils.identify.data.DcmDataHub; +import org.dubhe.dcm.machine.utils.identify.setting.DcmStateIdentifySetting; +import org.dubhe.dcm.machine.utils.identify.setting.DcmStateSelect; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.ReflectionUtils; + +import java.lang.reflect.Method; +import java.util.List; +import java.util.Set; + +/** + * @description 状态判断实现类 + * @date 2020-09-24 + */ +@Component +public class DcmStateIdentifyUtil { + + + /** + * 数据查询处理 + */ + @Autowired + private DcmDataHub dataHub; + + /** + * 状态判断类 + */ + @Autowired + private DcmStateSelect stateSelect; + + /** + * 状态判断中所有的自定义方法数组 + */ + private final Method[] method = ReflectionUtils.getDeclaredMethods(DcmStateSelect.class); + + /** + * 获取数据集状态(指定版本) + * + * @param datasetId 数据集id + * @param needFileStateDoIdentify 是否需要查询文件状态判断 + * @return DatasetStatusEnum 数据集状态(指定版本) + */ + public DcmDataStateEnum getStatus(Long datasetId, boolean needFileStateDoIdentify) { + return needFileStateDoIdentify ? + new IdentifyDatasetStateByFileState(datasetId, DcmStateIdentifySetting.NEED_FILE_STATE_DO_IDENTIFY).getStatus() + : dataHub.getDatasetStatus(datasetId); + } + + /** + * 获取数据集状态(未指定版本) + * + * @param dataset 数据集 + * @param needFileStateDoIdentify 是否需要查询文件状态判断 + * @return DatasetStatusEnum 数据集状态(指定版本) + */ + public DcmDataStateEnum getStatus(Dataset dataset, boolean needFileStateDoIdentify) { + return needFileStateDoIdentify ? + new IdentifyDatasetStateByFileState(dataset.getId(), DcmStateIdentifySetting.NEED_FILE_STATE_DO_IDENTIFY).getStatus() + : dataHub.getDatasetStatus(dataset.getId()); + } + + /** + * 获取数据集状态(自动标注/目标跟踪回滚使用) + * + * @param datasetId 数据集id + * @return DatasetStatusEnum 数据集状态(指定版本) + */ + public DcmDataStateEnum getStatusForRollback(Long datasetId) { + return new IdentifyDatasetStateByFileState(datasetId, DcmStateIdentifySetting.ROLL_BACK_FOR_STATE).getStatus(); + } + + + class IdentifyDatasetStateByFileState { + + /** + * 判断得到的数据集状态 + */ + public DcmDataStateEnum state; + + /** + * 会查询文件的状态去对数据集的状态做判断 + * + * @param datasetId 数据集ID + */ + public IdentifyDatasetStateByFileState(Long datasetId, Set dataStateEnums) { + state = dataHub.getDatasetStatus(datasetId); + if (dataStateEnums.contains(state)) { + List stateList = dataHub.getFileStatusListByDataset(datasetId); + if (stateList == null || stateList.isEmpty()) { + state = DcmDataStateEnum.NOT_ANNOTATION_STATE; + return; + } + for (Method stateSelectMethod : method) { + state = (DcmDataStateEnum) ReflectionUtils.invokeMethod(stateSelectMethod, stateSelect, new Object[]{stateList}); + if (state != null) { + return; + } + } + } + } + + DcmDataStateEnum getStatus() { + return this.state; + } + } +} \ No newline at end of file diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/identify/data/DcmDataHub.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/identify/data/DcmDataHub.java new file mode 100644 index 0000000..8c24860 --- /dev/null +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/identify/data/DcmDataHub.java @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.dcm.machine.utils.identify.data; + +import org.dubhe.data.machine.enums.DataStateEnum; +import org.dubhe.dcm.machine.enums.DcmDataStateEnum; +import org.dubhe.dcm.service.DataMedicineFileService; +import org.dubhe.dcm.service.DataMedicineService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * @description 数据查询/加工 + * @date 2020-09-24 + */ +@Component +public class DcmDataHub { + + /** + * 数据集版本文件关系服务类 + */ + @Autowired + private DataMedicineFileService medicineFileService; + + /** + * 数据集服务类 + */ + @Autowired + @Lazy + private DataMedicineService medicineService; + + /** + * 获取数据集的状态 + * @param datasetId 数据集ID + * @return 数据集状态枚举 + */ + public DcmDataStateEnum getDatasetStatus(Long datasetId){ + return DcmDataStateEnum.getState(medicineService.getOneById(datasetId).getStatus()); + } + + /** + * 获取数据集下文件的状态（数据经过去重处理） + * @param datasetId 数据集ID + * @return 数据集下文件状态的并集 + */ + public List getFileStatusListByDataset(Long datasetId) { + return medicineFileService.getFileStatusListByDataset(datasetId); + } + +} diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/identify/setting/DcmStateIdentifySetting.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/identify/setting/DcmStateIdentifySetting.java new file mode 100644 index 0000000..8b73245 --- /dev/null +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/identify/setting/DcmStateIdentifySetting.java @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dcm.machine.utils.identify.setting; + +import org.dubhe.dcm.machine.enums.DcmDataStateEnum; +import org.springframework.stereotype.Component; + +import java.util.HashSet; +import java.util.Set; + +/** + * @description 状态判断设置类 + * @date 2020-09-24 + */ +@Component +public class DcmStateIdentifySetting { + + /** + * 回退状态（自动标注失败/目标跟踪失败）使用 + */ + public static final Set ROLL_BACK_FOR_STATE = new HashSet() {{ + //自动标注中 + add(DcmDataStateEnum.AUTOMATIC_LABELING_STATE); + }}; + + + /** + * 数据集状态需要使用文件状态去判断的 + */ + public static final Set NEED_FILE_STATE_DO_IDENTIFY = new HashSet() {{ + //未标注 + add(DcmDataStateEnum.NOT_ANNOTATION_STATE); + //手动标注中 + add(DcmDataStateEnum.ANNOTATION_DATA_STATE); + //自动标注完成 + add(DcmDataStateEnum.AUTO_ANNOTATION_COMPLETE_STATE); + //标注完成 + add(DcmDataStateEnum.ANNOTATION_COMPLETE_STATE); + }}; + +} diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/identify/setting/DcmStateSelect.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/identify/setting/DcmStateSelect.java new file mode 100644 index 0000000..928fd79 --- /dev/null +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/machine/utils/identify/setting/DcmStateSelect.java @@ -0,0 +1,106 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dcm.machine.utils.identify.setting; + +import org.dubhe.biz.base.constant.NumberConstant; +import org.dubhe.data.machine.constant.FileStateCodeConstant; +import org.dubhe.data.machine.enums.DataStateEnum; +import org.dubhe.dcm.machine.constant.DcmFileStateCodeConstant; +import org.dubhe.dcm.machine.enums.DcmDataStateEnum; +import org.springframework.stereotype.Component; + +import java.util.HashSet; +import java.util.List; + +/** + * @description 状态判断类 + * @date 2020-09-24 + */ +@Component +public class DcmStateSelect { + + + /** + * 手动标注中 + * + * @param stateList 数据集下文件状态的并集 + * @return 数据集状态枚举 + */ + public DcmDataStateEnum isManualAnnotating(List stateList) { + if (stateList.size() > 1 && stateList.contains(DcmFileStateCodeConstant.NOT_ANNOTATION_FILE_STATE)) { + return DcmDataStateEnum.ANNOTATION_DATA_STATE; + } + return stateList.contains(DcmFileStateCodeConstant.ANNOTATION_FILE_STATE) ? DcmDataStateEnum.ANNOTATION_DATA_STATE : null; + } + + /** + * 自动标注完成 + * + * @param stateList 数据集下文件状态的并集 + * @return 数据集状态枚举 + */ + public DcmDataStateEnum isAutoFinished(List stateList) { + HashSet states = new HashSet() {{ + add(DcmFileStateCodeConstant.AUTO_ANNOTATION_COMPLETE_FILE_STATE); + add(DcmFileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE); + }}; + switch (stateList.size()) { + case NumberConstant.NUMBER_1: + if (stateList.contains(DcmFileStateCodeConstant.AUTO_ANNOTATION_COMPLETE_FILE_STATE)){ + return DcmDataStateEnum.AUTO_ANNOTATION_COMPLETE_STATE; + } + return null; + case NumberConstant.NUMBER_2: + case NumberConstant.NUMBER_3: + case NumberConstant.NUMBER_4: + for (Integer fileState : stateList) { + if (!states.contains(fileState)) { + return null; + }; + } + return DcmDataStateEnum.AUTO_ANNOTATION_COMPLETE_STATE; + default: + return null; + } + } + + /** + * 标注完成 + * + * @param stateList 数据集下文件状态的并集 + * @return 数据集状态枚举 + */ + public DcmDataStateEnum isFinished(List stateList) { + return stateList.contains(DcmFileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE)&&stateList.size()==NumberConstant.NUMBER_1 ? + DcmDataStateEnum.ANNOTATION_COMPLETE_STATE : null; + } + + /** + * 未标注 + * + * @param stateList 数据集下文件状态的并集 + * @return 数据集状态枚举 + */ + public DcmDataStateEnum isInit(List stateList) { + if (stateList.size() == NumberConstant.NUMBER_1 && stateList.contains(DcmFileStateCodeConstant.NOT_ANNOTATION_FILE_STATE)){ + return DcmDataStateEnum.NOT_ANNOTATION_STATE; + } + return null; + } + +} \ No newline at end of file diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/rest/DataMedicineController.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/rest/DataMedicineController.java index 93234ee..8aeb44b 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/rest/DataMedicineController.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/rest/DataMedicineController.java @@ -96,4 +96,20 @@ public class DataMedicineController { public DataResponseBody getAuto(@PathVariable(name = "medicalId") Long medicalId) { return new DataResponseBody(dataMedicineService.getAuto(medicalId)); } + + @ApiOperation(value = "获取数据集列表") + @GetMapping("/list") + @PreAuthorize(Permissions.DATA) + public DataResponseBody list() { + return new DataResponseBody(dataMedicineService.getList()); + } + + @ApiOperation(value = "任务停止") + @PutMapping(value = "/task/{medicalId}/stop") + @PreAuthorize(Permissions.DATA) + public DataResponseBody taskStop(@PathVariable(name = "medicalId") Long medicalId) { + dataMedicineService.taskStop(medicalId); + return new DataResponseBody(); + } + } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/DataMedicineFileService.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/DataMedicineFileService.java index 7cecf21..70be672 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/DataMedicineFileService.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/DataMedicineFileService.java @@ -84,4 +84,14 @@ public interface DataMedicineFileService { * @param datasetId 医学数据集Id */ void deleteByDatasetId(Long datasetId); + + List getFileStatusListByDataset(Long datasetId); + + /** + * 清理已有的标注数据 + * + * @param datasetId + */ + void deleteAnnotation(Long datasetId); + } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/DataMedicineService.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/DataMedicineService.java index 411abbd..60f8e7e 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/DataMedicineService.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/DataMedicineService.java @@ -22,9 +22,11 @@ import org.dubhe.biz.base.enums.OperationTypeEnum; import org.dubhe.dcm.domain.dto.*; import org.dubhe.dcm.domain.entity.DataMedicine; import org.dubhe.dcm.domain.vo.DataMedicineCompleteAnnotationVO; +import org.dubhe.dcm.domain.vo.DataMedicineSmallVO; import org.dubhe.dcm.domain.vo.DataMedicineVO; import org.dubhe.recycle.domain.dto.RecycleCreateDTO; +import java.util.List; import java.util.Map; /** @@ -146,4 +148,15 @@ public interface DataMedicineService { * @param datasetId 医学数据集Id */ void deleteByDatasetId(Long datasetId); + + /** + * 获取医学数据集列表 + * + * @return + */ + List getList(); + + void taskStop(Long medicalId); + + DataMedicine getOneById(Long datasetId); } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/MedicineAnnotationService.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/MedicineAnnotationService.java index 84c7a5e..f6850a5 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/MedicineAnnotationService.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/MedicineAnnotationService.java @@ -17,6 +17,7 @@ package org.dubhe.dcm.service; +import com.alibaba.fastjson.JSONObject; import org.dubhe.dcm.domain.dto.MedicineAnnotationDTO; import org.dubhe.dcm.domain.dto.MedicineAutoAnnotationDTO; import org.dubhe.dcm.domain.entity.DataMedicineFile; @@ -43,7 +44,7 @@ public interface MedicineAnnotationService { * * @return boolean 是否有任务 */ - boolean finishAuto(); + void finishAuto(JSONObject taskDetail); /** * 标注保存 diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/DataMedicineFileServiceImpl.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/DataMedicineFileServiceImpl.java index 26ab532..9bb0cf0 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/DataMedicineFileServiceImpl.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/DataMedicineFileServiceImpl.java @@ -17,6 +17,7 @@ package org.dubhe.dcm.service.impl; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import org.apache.commons.lang3.StringUtils; import org.dcm4che3.data.Attributes; @@ -33,6 +34,7 @@ import org.dubhe.dcm.dao.DataMedicineFileMapper; import org.dubhe.dcm.domain.dto.DataMedicineFileCreateDTO; import org.dubhe.dcm.domain.entity.DataMedicine; import org.dubhe.dcm.domain.entity.DataMedicineFile; +import org.dubhe.dcm.machine.enums.DcmFileStateEnum; import org.dubhe.dcm.service.DataMedicineFileService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -198,5 +200,18 @@ public class DataMedicineFileServiceImpl extends ServiceImpl getFileStatusListByDataset(Long datasetId) { + if (datasetId == null) { + LogUtil.error(LogEnum.BIZ_DATASET, "datasetId isEmpty"); + return null; + } + return baseMapper.getFileStatusListByDataset(datasetId); + } + + @Override + public void deleteAnnotation(Long datasetId) { + baseMapper.cleanAnnotation(datasetId); + } } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/DataMedicineServiceImpl.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/DataMedicineServiceImpl.java index 3a5ce1f..aee2315 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/DataMedicineServiceImpl.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/DataMedicineServiceImpl.java @@ -1,12 +1,12 @@ /** * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,9 +17,12 @@ package org.dubhe.dcm.service.impl; import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.ObjectUtil; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import org.dubhe.biz.permission.annotation.DataPermissionMethod; @@ -38,16 +41,28 @@ import org.dubhe.biz.file.utils.MinioUtil; import org.dubhe.biz.log.enums.LogEnum; import org.dubhe.biz.log.utils.LogUtil; import org.dubhe.cloud.authconfig.utils.JwtUtils; +import org.dubhe.data.constant.DataTaskTypeEnum; import org.dubhe.data.constant.ErrorEnum; +import org.dubhe.data.constant.TaskStatusEnum; +import org.dubhe.data.domain.entity.Task; import org.dubhe.data.machine.constant.DataStateCodeConstant; +import org.dubhe.data.machine.enums.DataStateEnum; +import org.dubhe.data.service.TaskService; import org.dubhe.dcm.constant.DcmConstant; import org.dubhe.dcm.dao.DataMedicineMapper; import org.dubhe.dcm.domain.dto.*; import org.dubhe.dcm.domain.entity.DataMedicine; +import org.dubhe.dcm.domain.entity.DataMedicineFile; import org.dubhe.dcm.domain.vo.DataMedicineCompleteAnnotationVO; +import org.dubhe.dcm.domain.vo.DataMedicineSmallVO; import org.dubhe.dcm.domain.vo.DataMedicineVO; +import org.dubhe.dcm.machine.constant.DcmDataStateCodeConstant; +import org.dubhe.dcm.machine.enums.DcmDataStateEnum; +import org.dubhe.dcm.machine.utils.DcmStateIdentifyUtil; import org.dubhe.dcm.service.DataMedicineFileService; import org.dubhe.dcm.service.DataMedicineService; +import org.dubhe.dcm.service.MedicineAnnotationService; +import org.dubhe.dcm.util.StringUtil; import org.dubhe.recycle.domain.dto.RecycleCreateDTO; import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO; import org.dubhe.recycle.enums.RecycleModuleEnum; @@ -64,10 +79,7 @@ import org.springframework.util.CollectionUtils; import java.io.File; import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; import static org.dubhe.data.constant.ErrorEnum.DATASET_PUBLIC_LIMIT_ERROR; @@ -95,6 +107,9 @@ public class DataMedicineServiceImpl extends ServiceImpl detailList = new ArrayList<>(); @@ -351,7 +373,7 @@ public class DataMedicineServiceImpl extends ServiceImpl getList() { + List list = baseMapper.getList(); + if (CollectionUtil.isNotEmpty(list)) { + list.stream().forEach(dataMedicineSmallVO -> { + dataMedicineSmallVO.setUrl(StringUtil.getDcmUrl(dataMedicineSmallVO.getId())); + }); + } + return list; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void taskStop(Long medicalId) { + DataMedicine medicine = baseMapper.selectById(medicalId); + checkDatasetForTask(medicine); + + //停止任务 + Task task = taskService.selectRunningDcmTask(medicalId); + task.setStop(true); + taskService.updateByTaskId(task); + + //更新状态 + DcmDataStateEnum status = stateIdentify.getStatusForRollback(medicalId); + medicine.setStatus(status.getCode()); + baseMapper.updateById(medicine); + //合并标注 + QueryWrapper objectQueryWrapper = new QueryWrapper<>(); + objectQueryWrapper.lambda().eq(DataMedicineFile::getMedicineId, medicalId); + List dataMedicineFiles = dataMedicineFileService.listFile(objectQueryWrapper); + List dataMedicineFilesOrderByAsc = dataMedicineFileService + .insertInstanceAndSort(dataMedicineFiles, medicalId); + medicineAnnotationService.mergeAnnotation(medicalId, dataMedicineFilesOrderByAsc); + medicine.setStop(true); + baseMapper.updateById(medicine); + } + + @Override + public DataMedicine getOneById(Long datasetId) { + return baseMapper.selectById(datasetId); + } + + public void checkDatasetForTask(DataMedicine medicine) { + if (ObjectUtil.isEmpty(medicine)) { + throw new BusinessException("数据集不存在"); + } + HashSet set = new HashSet() {{ + add(DcmDataStateCodeConstant.AUTOMATIC_LABELING_STATE); + }}; + if (!set.contains(medicine.getStatus())) { + throw new BusinessException("当前数据集状态不允许操作"); + } + } + } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/MedicineAnnotationServiceImpl.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/MedicineAnnotationServiceImpl.java index ddf494b..ae1ed88 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/MedicineAnnotationServiceImpl.java +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/service/impl/MedicineAnnotationServiceImpl.java @@ -38,11 +38,14 @@ import org.dubhe.biz.statemachine.dto.StateChangeDTO; import org.dubhe.cloud.authconfig.utils.JwtUtils; import org.dubhe.data.constant.DataTaskTypeEnum; import org.dubhe.data.constant.ErrorEnum; +import org.dubhe.data.constant.FileTypeEnum; import org.dubhe.data.constant.TaskStatusEnum; import org.dubhe.data.domain.entity.Task; import org.dubhe.data.machine.constant.DataStateCodeConstant; import org.dubhe.data.machine.enums.DataStateEnum; +import org.dubhe.data.service.AutoLabelModelServiceService; import org.dubhe.data.service.TaskService; +import org.dubhe.data.util.TaskUtils; import org.dubhe.dcm.constant.DcmConstant; import org.dubhe.dcm.dao.DataMedicineFileMapper; import org.dubhe.dcm.domain.dto.MedicineAnnotationDTO; @@ -54,6 +57,7 @@ import org.dubhe.dcm.machine.constant.DcmDataStateMachineConstant; import org.dubhe.dcm.machine.constant.DcmFileStateCodeConstant; import org.dubhe.dcm.machine.constant.DcmFileStateMachineConstant; import org.dubhe.dcm.machine.enums.DcmDataStateEnum; +import org.dubhe.dcm.machine.enums.DcmFileStateEnum; import org.dubhe.dcm.machine.utils.DcmStateMachineUtil; import org.dubhe.dcm.service.DataLesionSliceService; import org.dubhe.dcm.service.DataMedicineFileService; @@ -65,6 +69,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.*; @@ -108,12 +113,18 @@ public class MedicineAnnotationServiceImpl implements MedicineAnnotationService @Autowired private DataLesionSliceService dataLesionSliceService; + @Autowired + private TaskUtils taskUtils; + /** * bucketName */ @Value("${minio.bucketName}") private String bucketName; + @Autowired + private AutoLabelModelServiceService autoLabelModelServiceService; + /** * 医学标注算法处理中任务队列 @@ -136,22 +147,34 @@ public class MedicineAnnotationServiceImpl implements MedicineAnnotationService if (medicineAutoAnnotationDTO.getMedicalId() == null) { return; } + // 判断模型服务是否存在 + if (ObjectUtil.isNull(medicineAutoAnnotationDTO.getModelServiceId()) + || ObjectUtil.isNull(autoLabelModelServiceService.getOneById(medicineAutoAnnotationDTO.getModelServiceId()))) { + throw new BusinessException(ErrorEnum.MODEL_SERVER_NOT_EXIST); + } dataMedicineService.checkPublic(medicineAutoAnnotationDTO.getMedicalId(), OperationTypeEnum.UPDATE); Long medicineId = medicineAutoAnnotationDTO.getMedicalId(); DataMedicine dataMedicine = dataMedicineService.getDataMedicineById(medicineId); - if (!dataMedicine.getStatus().equals(DataStateCodeConstant.NOT_ANNOTATION_STATE)) { + // 只能指定状态下才能进行自动标注 + if (!DcmDataStateEnum.checkCurrentStatusWhetherToAutoLabel(dataMedicine.getStatus())) { throw new BusinessException(ErrorEnum.MEDICINE_AUTO_DATASET_ERROR); } QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.lambda().eq(DataMedicineFile::getMedicineId, medicineId).eq(DataMedicineFile::getStatus, - DataStateEnum.NOT_ANNOTATION_STATE.getCode()); + queryWrapper.lambda() + .eq(DataMedicineFile::getMedicineId, medicineId) + .in(DataMedicineFile::getStatus, DcmFileStateEnum.getFileStatusFromAutoLabelScreen(medicineAutoAnnotationDTO.getFileStatus())); Integer medicineFilesCount = dataMedicineFileService.getCountByMedicineId(queryWrapper); + if (medicineFilesCount == 0) { + throw new BusinessException(ErrorEnum.MEDICINE_AUTO_NO_LABEL_FILE); + } Task task = Task.builder() .status(TaskStatusEnum.INIT.getValue()) .datasetId(medicineId) .total(medicineFilesCount) + .modelServiceId(medicineAutoAnnotationDTO.getModelServiceId()) .type(DataTaskTypeEnum.MEDICINE_ANNOTATION.getValue()) .labels("") + .fileType(medicineAutoAnnotationDTO.getFileStatus()) .build(); taskService.createTask(task); DcmStateMachineUtil.stateChange(new StateChangeDTO() {{ @@ -169,19 +192,29 @@ public class MedicineAnnotationServiceImpl implements MedicineAnnotationService */ @Override @Transactional(rollbackFor = Exception.class) - public boolean finishAuto() { - Object object = redisUtils.lpop(MEDICINE_FINISHED_QUEUE); - if (ObjectUtil.isNotNull(object)) { - JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(redisUtils.get(object.toString()))); - String detailId = jsonObject.getString("reTaskId"); - JSONObject jsonDetail = JSON.parseObject(JSON.toJSONString(redisUtils.get(detailId))); - Long taskId = jsonDetail.getLong("taskId"); - JSONArray dcmsArray = jsonDetail.getJSONArray("dcms"); + public void finishAuto(JSONObject taskDetail) { + JSONObject jsonObject = JSON.parseObject(taskDetail.get("object").toString(),JSONObject.class); + if (ObjectUtil.isNotNull(taskDetail)) { + Long taskId = taskDetail.getLong("taskId"); + JSONArray dcmsArray = taskDetail.getJSONArray("dcms"); + JSONArray jsonArray = taskDetail.getJSONArray("annotations"); String[] dcms = dcmsArray.toArray(new String[dcmsArray.size()]); QueryWrapper taskQueryWrapper = new QueryWrapper<>(); taskQueryWrapper.lambda().eq(Task::getId, taskId); Task task = taskService.selectOne(taskQueryWrapper); - List medicineFileIds = JSON.parseObject(jsonDetail.getString("medicineFileIds"), ArrayList.class); + JSONArray annotationsArray = jsonObject.getJSONArray("annotations"); + for (int i =0;i medicineFileIds = JSON.parseObject(taskDetail.getString("medicineFileIds"), ArrayList.class); DcmStateMachineUtil.stateChange(new StateChangeDTO() {{ setObjectParam(new Object[]{medicineFileIds}); setStateMachineType(DcmFileStateMachineConstant.DCM_FILE_STATE_MACHINE); @@ -205,7 +238,6 @@ public class MedicineAnnotationServiceImpl implements MedicineAnnotationService task.setFinished(finished); taskService.updateByTaskId(task); } - return ObjectUtil.isNotNull(object); } /** @@ -260,6 +292,8 @@ public class MedicineAnnotationServiceImpl implements MedicineAnnotationService DcmFileStateMachineConstant.ANNOTATION_COMPLETE_EVENT : DcmFileStateMachineConstant.ANNOTATION_SAVE_EVENT); }}); modifyUpdataUserId(medicineAnnotationDTO.getMedicalId()); + medical.setStop(false); + dataMedicineService.updateByMedicineId(medical); return true; } @@ -345,7 +379,9 @@ public class MedicineAnnotationServiceImpl implements MedicineAnnotationService LogUtil.error(LogEnum.BIZ_DATASET, "get medicine annotation json failed, {}", e); } finally { try { - inputStream.close(); + if (inputStream != null) { + inputStream.close(); + } } catch (IOException e) { LogUtil.error(LogEnum.BIZ_DATASET, "close inputStream failed, {}", e); } diff --git a/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/util/StringUtil.java b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/util/StringUtil.java new file mode 100644 index 0000000..f517bc8 --- /dev/null +++ b/dubhe-server/dubhe-data-dcm/src/main/java/org/dubhe/dcm/util/StringUtil.java @@ -0,0 +1,38 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dcm.util; + +import org.dubhe.dcm.constant.DcmConstant; + +/** + * @description 字符串处理工具类 + * @date 2020-11-11 + */ +public class StringUtil { + + /** + * 获取医学数据集存储路径 + * + * @param dataMedicineId 医学数据集ID + * @return + */ + public static String getDcmUrl(Long dataMedicineId) { + return String.format(DcmConstant.DCM_DATA_URL, dataMedicineId); + } + +} diff --git a/dubhe-server/dubhe-data-dcm/src/main/resources/bootstrap.yml b/dubhe-server/dubhe-data-dcm/src/main/resources/bootstrap.yml index adc8619..eedfcd3 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/resources/bootstrap.yml +++ b/dubhe-server/dubhe-data-dcm/src/main/resources/bootstrap.yml @@ -14,7 +14,7 @@ spring: config: enabled: true server-addr: 127.0.0.1:8848 - namespace: dubhe-server-cloud-prod + namespace: dubhe-prod shared-configs[0]: data-id: common-biz.yaml group: dubhe @@ -39,10 +39,14 @@ spring: data-id: dubhe-data-dcm.yaml group: dubhe refresh: true + shared-configs[6]: + data-id: kubeconfig.yaml + group: dubhe + refresh: true discovery: enabled: true - namespace: dubhe-server-cloud-dev + namespace: dubhe-prod group: dubhe server-addr: 127.0.0.1:8848 # 配置允许后面的Bean覆盖前面名称重复的Bean diff --git a/dubhe-server/dubhe-data-dcm/src/main/resources/mapper/DataMedicineFileMapper.xml b/dubhe-server/dubhe-data-dcm/src/main/resources/mapper/DataMedicineFileMapper.xml index 28d8c61..a8b27e5 100644 --- a/dubhe-server/dubhe-data-dcm/src/main/resources/mapper/DataMedicineFileMapper.xml +++ b/dubhe-server/dubhe-data-dcm/src/main/resources/mapper/DataMedicineFileMapper.xml @@ -10,6 +10,9 @@ group by medicine_id,status + diff --git a/dubhe-server/dubhe-data-dcm/src/main/resources/mapper/DataMedicineMapper.xml b/dubhe-server/dubhe-data-dcm/src/main/resources/mapper/DataMedicineMapper.xml new file mode 100644 index 0000000..8ab2a69 --- /dev/null +++ b/dubhe-server/dubhe-data-dcm/src/main/resources/mapper/DataMedicineMapper.xml @@ -0,0 +1,9 @@ + + + + + + \ No newline at end of file diff --git a/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/constant/DataAlgorithmEnum.java b/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/constant/DataAlgorithmEnum.java new file mode 100644 index 0000000..e657bdd --- /dev/null +++ b/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/constant/DataAlgorithmEnum.java @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.task.constant; + +import lombok.Getter; +import org.dubhe.task.execute.impl.*; + +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +@Getter +public enum DataAlgorithmEnum { + + STANDARD_ANNOTATION(101, AnnotationQueueExecute.class, "标注标注"), + TEXT_CLASSIFICATION(102, TextClassificationQueueExecute.class,"文本分类"), + TRACK(103, TrackQueueExecute.class,"目标跟踪"), + ENHANCE(104, EnhanceQueueExecute.class,"数据增强"), + MEDICINE_ANNOTATION(105,MedicineAnnotataionQueueExecute.class,"医学标注"), + OF_RECORD(106,OfRecordQueueExecute.class,"ofrecord转换"), + VIDEO_SAMPLE(107,VideoSampleQueueExecute.class,"视频抽帧"); + + private int algorithmType; + + private Class className; + + private String desc; + + private static final Map map = new HashMap<>(); + + static{ + for (DataAlgorithmEnum dataAlgorithmEnum: EnumSet.allOf(DataAlgorithmEnum.class)){ + map.put(dataAlgorithmEnum.getAlgorithmType(),dataAlgorithmEnum); + } + } + + DataAlgorithmEnum(int algorithmType,Class clazz,String desc){ + this.algorithmType = algorithmType; + this.className = clazz; + this.desc = desc; + } + + public static DataAlgorithmEnum getType(int algorithmType){ + return map.get(algorithmType); + } +} diff --git a/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/constant/TaskQueueNameEnum.java b/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/constant/TaskQueueNameEnum.java new file mode 100644 index 0000000..b62e3d5 --- /dev/null +++ b/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/constant/TaskQueueNameEnum.java @@ -0,0 +1,220 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + + +package org.dubhe.task.constant; + +import lombok.Getter; +import org.springframework.util.ObjectUtils; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * + *└── dataset + * └── {算法名称} + * ├── {dataset_id} + * │ ├── annotation + * │ │ └── {taskId} + * │ │ ├── B.json + * │ │ └── C.json + * │ ├── detail + * │ │ └── {taskId} + * │ │ ├── B.json + * │ │ ├── C.json + * │ │ ├── E.json + * │ │ ├── F.json + * │ │ └── H.json + * │ ├── failed + * │ │ └── {taskId} + * │ │ └── task_E + * │ ├── finished + * │ │ └── {taskId} + * │ │ ├── task_B + * │ │ └── task_C + * │ ├── processing + * │ │ └── {taskId} + * │ │ └── task_F + * │ └── task + * │ └── {taskId} + * │ └── task_H + * ├── failed_task + * │ └── task_D + * ├── finished_task + * │ └── task_A + * └── pre_task + * └── task_G + * + */ +@Getter +public enum TaskQueueNameEnum { + + + /** + * 任务详情 + */ + DETAIL("dataset:%s:%s:%s:detail:%s","任务详情"), + + + /** + * 完成队列 + */ + FINISHED("dataset:%s:%s:%s:finished:%s","完成队列"), + + + /** + * 进行中队列 + */ + PROCESSING("dataset:%s:%s:%s:processing:%s","进行中队列"), + + + /** + * 标注内容 + */ + ANNOTATION("dataset:%s:%s:%s:annotation:%s","标注内容"), + + + /** + * 待处理队列 + */ + TASK("dataset:%s:%s:%s:task:001","待处理队列"), + + + /** + * 处理失败队列 + */ + FAILED("dataset:%s:%s:%s:failed:%s","处理失败队列"), + + /** + * failed flag + */ + FAILED_TASK("dataset:%s","flag"), + + /** + * finish flag + */ + FINISHED_TASK("dataset:%s","flag"), + + STOP_KEY("dataset:%s:%s:%s:*", "停止时队列前缀"), + + /** + * flag + */ + PRE_TASK("dataset:%s","flag"); + + TaskQueueNameEnum(String template, String desc) { + this.template = template; + this.desc = desc; + } + + /** + * redis 存储队列 + */ + private String template; + + /** + * 说明 + */ + private String desc; + + @Getter + public enum TaskQueueConfigEnum { + + /** + * 普通标算法 + */ + ANNOTATION("annotation", "普通标算法"), + + + /** + * 图片分类算法 + */ + IMAGENET("imagenet", "图片分类算法"), + + + /** + * 图片增强 + */ + IMGPROCESS("imgprocess", "图片增强"), + + + /** + * 肺部分割 + */ + LUNG_SEGMENTATION("lung_segmentation", "肺部分割"), + + + /** + * ofrecord转换 + */ + OFRECORD("ofrecord", "ofrecord转换"), + + + /** + * 文本分类 + */ + TEXT_CLASSIFICATION("text_classification", "文本分类"), + + + /** + * 目标跟踪 + */ + TRACK("track", "目标跟踪"), + + + /** + * 视频抽帧算法 + */ + VIDEOSAMPLE("videosample", "视频抽帧算法"), + + ALL("*","所有算法"); + + TaskQueueConfigEnum(String name, String desc) { + this.name = name; + this.desc = desc; + } + + private String name; + private String desc; + } + + public static String getTemplate(TaskQueueNameEnum queueNameEnum,TaskQueueConfigEnum configEnum,String ...args) { + if (ObjectUtils.isEmpty(args)){ + return String.format(queueNameEnum.template, configEnum.name); + } + Object[] array = new ArrayList() {{ + add(configEnum.name); + addAll(Arrays.asList(args)); + }}.toArray(); + return String.format(queueNameEnum.template,array); + } + + public static String getTaskNamespace(Integer type) { + switch (type) { + case 1: + return TaskQueueConfigEnum.OFRECORD.name; + case 3: + return TaskQueueConfigEnum.IMGPROCESS.name; + case 5: + return TaskQueueConfigEnum.VIDEOSAMPLE.name; + default: + return null; + } + } + +} diff --git a/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/AnnotationQueueExecuteThread.java b/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/AnnotationQueueExecuteThread.java deleted file mode 100644 index bcc3a90..0000000 --- a/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/AnnotationQueueExecuteThread.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.task.data; - -import cn.hutool.core.util.ObjectUtil; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; -import lombok.extern.slf4j.Slf4j; -import org.dubhe.biz.base.constant.MagicNumConstant; -import org.dubhe.biz.log.enums.LogEnum; -import org.dubhe.biz.log.utils.LogUtil; -import org.dubhe.biz.redis.utils.RedisUtils; -import org.dubhe.data.domain.bo.TaskSplitBO; -import org.dubhe.data.domain.dto.AnnotationInfoCreateDTO; -import org.dubhe.data.domain.dto.BatchAnnotationInfoCreateDTO; -import org.dubhe.data.service.AnnotationService; -import org.dubhe.data.util.TaskUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * @description 自动标注完成队列处理 - * @date 2020-08-28 - */ -@Slf4j -@Component -public class AnnotationQueueExecuteThread implements Runnable { - - @Autowired - private RedisUtils redisUtils; - @Autowired - private AnnotationService annotationService; - @Autowired - private TaskUtils taskUtils; - - /** - * 标注算法执行中任务队列 - */ - private static final String ANNOTATION_START_QUEUE = "annotation_processing_queue"; - /** - * 标注算法未执行任务队列 - */ - private static final String ANNOTATION_PENDING_QUEUE = "annotation_task_queue"; - /** - * 标注算法已完成任务队列 - */ - private static final String ANNOTATION_FINISHED_QUEUE = "annotation_finished_queue"; - - /** - * 启动标注任务处理线程 - */ - @PostConstruct - public void start() { - Thread thread = new Thread(this, "自动标注完成任务处理队列"); - thread.start(); - } - - /** - * 标注任务处理方法 - */ - @Override - public void run() { - while (true) { - try { - Object object = redisUtils.lpop(ANNOTATION_FINISHED_QUEUE); - if (ObjectUtil.isNotNull(object)) { - Long start = System.currentTimeMillis(); - JSONObject jsonObject = JSONObject.parseObject(JSON.toJSONString(redisUtils.get(object.toString()))); - String detailId = jsonObject.getString("reTaskId"); - JSONObject taskDetail = JSON.parseObject(JSON.toJSONString(redisUtils.get(detailId))); - // 得到一个任务的拆分 包括多个file - TaskSplitBO taskSplitBO = JSON.parseObject(JSON.toJSONString(taskDetail), TaskSplitBO.class); - JSONArray jsonArray = jsonObject.getJSONArray("annotations"); - - List list = new ArrayList<>(); - for (int i = 0; i < jsonArray.size(); i++) { - list.add(JSON.toJavaObject(jsonArray.getJSONObject(i), AnnotationInfoCreateDTO.class)); - } - BatchAnnotationInfoCreateDTO batchAnnotationInfoCreateDTO = new BatchAnnotationInfoCreateDTO(); - batchAnnotationInfoCreateDTO.setAnnotations(list); - annotationService.doFinishAuto(taskSplitBO, batchAnnotationInfoCreateDTO.toMap()); - redisUtils.del(detailId); - LogUtil.info(LogEnum.BIZ_DATASET, "the time it takes to perform a task is {} second", (System.currentTimeMillis() - start)); - TimeUnit.MILLISECONDS.sleep(MagicNumConstant.TEN); - } else { - TimeUnit.MILLISECONDS.sleep(MagicNumConstant.THREE_THOUSAND); - } - } catch (Exception exception) { - LogUtil.error(LogEnum.BIZ_DATASET, "annotation exception:{}", exception); - } - } - } - - /** - * annotation任务是否过期 - */ - @Scheduled(cron = "*/15 * * * * ?") - public void expireAnnotationTask() { - taskUtils.restartTask(ANNOTATION_START_QUEUE, ANNOTATION_PENDING_QUEUE); - } - -} diff --git a/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/DataTaskExecuteThread.java b/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/DataTaskExecuteThread.java index 324b8d7..faf96df 100644 --- a/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/DataTaskExecuteThread.java +++ b/dubhe-server/dubhe-data-task/src/main/java/org/dubhe/task/data/DataTaskExecuteThread.java @@ -22,21 +22,27 @@ import cn.hutool.core.util.ObjectUtil; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.support.spring.FastJsonRedisSerializer; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.bytedeco.javacv.FFmpegFrameGrabber; import org.dubhe.biz.base.constant.DataStateCodeConstant; import org.dubhe.biz.base.constant.MagicNumConstant; import org.dubhe.biz.base.constant.NumberConstant; import org.dubhe.biz.base.utils.StringUtils; +import org.dubhe.biz.file.dto.FileDTO; +import org.dubhe.biz.file.utils.MinioUtil; import org.dubhe.biz.log.enums.LogEnum; import org.dubhe.biz.log.utils.LogUtil; import org.dubhe.biz.redis.utils.RedisUtils; import org.dubhe.biz.statemachine.dto.StateChangeDTO; import org.dubhe.data.constant.Constant; import org.dubhe.data.constant.DatasetLabelEnum; +import org.dubhe.data.constant.FileTypeEnum; +import org.dubhe.data.domain.bo.FileBO; import org.dubhe.data.domain.bo.TaskSplitBO; import org.dubhe.data.domain.dto.DatasetEnhanceRequestDTO; import org.dubhe.data.domain.dto.FileCreateDTO; @@ -48,8 +54,13 @@ import org.dubhe.data.machine.utils.StateMachineUtil; import org.dubhe.data.pool.BasePool; import org.dubhe.data.service.*; import org.dubhe.data.util.TaskUtils; +import org.dubhe.dcm.domain.entity.DataMedicine; import org.dubhe.dcm.domain.entity.DataMedicineFile; +import org.dubhe.dcm.machine.enums.DcmFileStateEnum; import org.dubhe.dcm.service.DataMedicineFileService; +import org.dubhe.dcm.service.DataMedicineService; +import org.dubhe.task.constant.DataAlgorithmEnum; +import org.dubhe.task.constant.TaskQueueNameEnum; import org.dubhe.task.util.TableDataUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -59,6 +70,7 @@ import org.springframework.data.redis.core.RedisCallback; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.ObjectUtils; import javax.annotation.PostConstruct; import javax.annotation.Resource; @@ -98,6 +110,9 @@ public class DataTaskExecuteThread implements Runnable { @Autowired private DataMedicineFileService dataMedicineFileService; + @Autowired + private DataMedicineService medicineService; + @Resource private TaskUtils taskUtils; @@ -110,6 +125,9 @@ public class DataTaskExecuteThread implements Runnable { @Autowired private TableDataUtil tableDataUtil; + @Autowired + private MinioUtil minioUtil; + /** * 线程池 */ @@ -126,31 +144,6 @@ public class DataTaskExecuteThread implements Runnable { */ private static final Integer ANNOTATION_BATCH_SIZE = MagicNumConstant.SIXTEEN * MagicNumConstant.TEN_THOUSAND; - /** - * 文本分类算法待处理任务队列 - */ - private static final String TC_TASK_QUEUE = "text_classification_task_queue"; - /** - * 标注算法待处理任务队列 - */ - private static final String ANNOTATION_TASK_QUEUE = "annotation_task_queue"; - /** - * ImageNet算法待处理任务队列 - */ - private static final String IMAGENET_TASK_QUEUE = "imagenet_task_queue"; - /** - * ofRecord算法待处理任务队列 - */ - private static final String OFRECORD_TASK_QUEUE = "ofrecord_task_queue"; - /** - * 目标跟踪算法待处理任务队列 - */ - private static final String TRACK_TASK_QUEUE = "track_task_queue"; - /** - * 医学标注算法待处理任务队列 - */ - private static final String MEDICINE_PENDING_QUEUE = "dcm_task_queue"; - /** * 启动生成任务线程 */ @@ -198,7 +191,7 @@ public class DataTaskExecuteThread implements Runnable { if (count != 0) { switch (task.getType()) { case MagicNumConstant.ZERO: - annotationExecute(NumberConstant.NUMBER_0,task); + annotationExecute(task); break; case MagicNumConstant.ONE: ofRecordExecute(task); @@ -209,9 +202,6 @@ public class DataTaskExecuteThread implements Runnable { case MagicNumConstant.THREE: enhanceExecute(task); break; - case MagicNumConstant.FIVE: - videoSampleExecute(task); - break; case MagicNumConstant.SIX: medicineExecute(task); break; @@ -220,7 +210,7 @@ public class DataTaskExecuteThread implements Runnable { break; case MagicNumConstant.EIGHT: annotationService.deleteAnnotating(task.getDatasetId()); - annotationExecute(NumberConstant.NUMBER_0,task); + annotationExecute(task); break; case MagicNumConstant.TEN: csvImport(task); @@ -254,8 +244,31 @@ public class DataTaskExecuteThread implements Runnable { images.add(file.getUrl().substring(file.getUrl().lastIndexOf("/") + 1, file.getUrl().length())); }); jsonObject.put("images", images); - redisUtils.set(taskId, jsonObject); - redisUtils.zSet(TRACK_TASK_QUEUE, -1, taskId); + jsonObject.put("algorithm", DataAlgorithmEnum.TRACK.getAlgorithmType()); + List