Browse Source

!13 取消域名解析功能

Merge pull request !13 from 之江天枢/master-cyn
pull/14/MERGE
之江天枢 Gitee 2 years ago
parent
commit
987a22253c
No known key found for this signature in database GPG Key ID: 173E9B9CA92EEF8F
97 changed files with 791 additions and 1514 deletions
  1. +2
    -2
      dubhe-server/README.md
  2. +0
    -1
      dubhe-server/admin/src/main/java/org/dubhe/admin/client/fallback/AuthServiceFallback.java
  3. +4
    -0
      dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/UserServiceImpl.java
  4. +0
    -1
      dubhe-server/admin/src/test/java/org/dubhe/admin/AdminApplicationTests.java
  5. +0
    -1
      dubhe-server/auth/src/test/java/org/dubhe/auth/AuthApplicationTests.java
  6. +0
    -37
      dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/HarborProperties.java
  7. +1
    -1
      dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/Permissions.java
  8. +0
    -3
      dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java
  9. +2
    -2
      dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BaseErrorCodeEnum.java
  10. +0
    -53
      dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/ImageStateEnum.java
  11. +4
    -2
      dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml
  12. +0
    -10
      dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-prod.yml
  13. +7
    -0
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/NodeApi.java
  14. +8
    -10
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/JupyterResourceApiImpl.java
  15. +4
    -46
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/ModelServingApiImpl.java
  16. +20
    -0
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/NodeApiImpl.java
  17. +5
    -2
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/PodApiImpl.java
  18. +1
    -1
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TerminalApiImpl.java
  19. +1
    -0
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/constant/K8sParamConstants.java
  20. +7
    -1
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/dto/BaseK8sDeploymentCallbackCreateDTO.java
  21. +3
    -0
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/vo/PtJupyterDeployVO.java
  22. +0
    -4
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/properties/ClusterProperties.java
  23. +3
    -1
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/BizConvertUtils.java
  24. +0
    -2
      dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sUtils.java
  25. +0
    -5
      dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java
  26. +0
    -1
      dubhe-server/dubhe-data-task/src/test/java/org/dubhe/task/DubheDataTaskApplicationTests.java
  27. +0
    -36
      dubhe-server/dubhe-data/src/main/java/org/dubhe/data/config/DataHarborConfig.java
  28. +21
    -9
      dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/impl/AutoLabelModelServiceServiceImpl.java
  29. +0
    -142
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/async/HarborImagePushAsync.java
  30. +0
    -3
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/dto/PtImageQueryDTO.java
  31. +4
    -4
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/dto/PtImageSaveDTO.java
  32. +23
    -5
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/dto/PtImageUpdateDTO.java
  33. +0
    -7
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/entity/PtImage.java
  34. +3
    -3
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/vo/PtImageQueryVO.java
  35. +21
    -8
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/rest/PtImageController.java
  36. +10
    -12
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/service/PtImageService.java
  37. +98
    -151
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/service/impl/PtImageServiceImpl.java
  38. +0
    -77
      dubhe-server/dubhe-image/src/main/java/org/dubhe/image/service/task/ImageRecycleFile.java
  39. +4
    -4
      dubhe-server/dubhe-image/src/test/java/org/dubhe/image/PtImageTest.java
  40. +1
    -1
      dubhe-server/dubhe-k8s/src/main/java/org/dubhe/dubhek8s/event/callback/DeploymentCallback.java
  41. +13
    -2
      dubhe-server/dubhe-k8s/src/main/java/org/dubhe/dubhek8s/observer/TrainJobObserver.java
  42. +2
    -2
      dubhe-server/dubhe-k8s/src/main/java/org/dubhe/dubhek8s/service/impl/SystemNodeServiceImpl.java
  43. +1
    -1
      dubhe-server/dubhe-k8s/src/test/java/org/dubhe/dubhek8s/k8s/PodCallbackTest.java
  44. +1
    -1
      dubhe-server/dubhe-measure/src/main/java/org/dubhe/measure/service/impl/PtAtlasCameraServiceImpl.java
  45. +10
    -9
      dubhe-server/dubhe-notebook/src/main/java/org/dubhe/notebook/service/impl/NoteBookServiceImpl.java
  46. +0
    -39
      dubhe-server/dubhe-point-cloud/src/main/java/org/dubhe/pointcloud/config/PointCloudHarborConfig.java
  47. +1
    -5
      dubhe-server/dubhe-point-cloud/src/main/java/org/dubhe/pointcloud/service/impl/PcDatasetServiceImpl.java
  48. +36
    -16
      dubhe-server/dubhe-serving-gateway/src/main/java/org/dubhe/servinggateway/config/GatewayServiceHandler.java
  49. +1
    -1
      dubhe-server/dubhe-serving-gateway/src/main/java/org/dubhe/servinggateway/config/MetricsGatewayFilterFactory.java
  50. +1
    -1
      dubhe-server/dubhe-serving-gateway/src/main/resources/mapper/GatewayRouteMapper.xml
  51. +0
    -39
      dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/config/TrainHarborConfig.java
  52. +2
    -5
      dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/service/impl/BatchServingServiceImpl.java
  53. +38
    -9
      dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/service/impl/ServingServiceImpl.java
  54. +12
    -6
      dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/task/DeployServingAsyncTask.java
  55. +7
    -45
      dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/utils/GrpcClient.java
  56. +0
    -7
      dubhe-server/dubhe-terminal/src/main/java/org/dubhe/terminal/domain/entity/PtImage.java
  57. +1
    -1
      dubhe-server/dubhe-terminal/src/main/java/org/dubhe/terminal/domain/entity/Terminal.java
  58. +48
    -25
      dubhe-server/dubhe-terminal/src/main/java/org/dubhe/terminal/service/impl/TerminalServiceImpl.java
  59. +0
    -1
      dubhe-server/dubhe-train/src/main/java/org/dubhe/train/client/fallback/ImageClientFallback.java
  60. +0
    -39
      dubhe-server/dubhe-train/src/main/java/org/dubhe/train/config/TrainHarborConfig.java
  61. +18
    -30
      dubhe-server/dubhe-train/src/main/java/org/dubhe/train/service/impl/PtTrainJobServiceImpl.java
  62. +3
    -1
      dubhe-server/sql/11-Dubhe-Patch-3.0.sql
  63. +2
    -2
      dubhe-server/yaml/admin.yaml
  64. +9
    -16
      dubhe-server/yaml/common-biz.yaml
  65. +34
    -19
      dubhe-server/yaml/common-k8s.yaml
  66. +2
    -2
      dubhe-server/yaml/common-shardingjdbc.yaml
  67. +1
    -1
      dubhe-server/yaml/dubhe-data-dcm.yaml
  68. +5
    -5
      dubhe-server/yaml/dubhe-data-task.yaml
  69. +6
    -6
      dubhe-server/yaml/dubhe-data.yaml
  70. +1
    -1
      dubhe-server/yaml/dubhe-notebook.yaml
  71. +2
    -3
      dubhe-server/yaml/dubhe-serving-gateway.yaml
  72. +0
    -52
      dubhe-server/yaml/dubhe-task.yaml
  73. +1
    -1
      dubhe-server/yaml/dubhe-terminal.yaml
  74. +136
    -146
      dubhe-server/yaml/gateway.yaml
  75. +3
    -9
      dubhe-server/yaml/image.yaml
  76. +0
    -19
      dubhe-server/yaml/kubeconfig.yaml
  77. +5
    -5
      dubhe-server/yaml/measure.yaml
  78. +3
    -3
      webapp/.env.pre
  79. +2
    -1
      webapp/.eslintignore
  80. +4
    -7
      webapp/babel.config.js
  81. +12
    -0
      webapp/package.json
  82. +2
    -10
      webapp/src/api/trainingImage/index.js
  83. +4
    -0
      webapp/src/assets/styles/atomic.scss
  84. +3
    -0
      webapp/src/components/BaseTooltip/index.vue
  85. +3
    -2
      webapp/src/components/Training/jobForm.vue
  86. +1
    -1
      webapp/src/utils/VisualUtils/download.js
  87. +18
    -0
      webapp/src/utils/constant.js
  88. +1
    -1
      webapp/src/utils/download.js
  89. +2
    -2
      webapp/src/views/algorithm/components/algorithmForm.vue
  90. +2
    -3
      webapp/src/views/cloudServing/components/forms/batchServingForm.vue
  91. +3
    -4
      webapp/src/views/cloudServing/components/forms/servingModelConfig.vue
  92. +7
    -3
      webapp/src/views/dataset/modelService/CreateModelService.vue
  93. +7
    -3
      webapp/src/views/dataset/modelService/ModifyModelService.vue
  94. +3
    -3
      webapp/src/views/dataset/pointCloud/components/auto-annotate-modal.vue
  95. +3
    -4
      webapp/src/views/development/components/CreateDialog.vue
  96. +52
    -238
      webapp/src/views/trainingImage/index.vue
  97. +0
    -9
      webapp/src/views/trainingJob/utils.js

+ 2
- 2
dubhe-server/README.md View File

@@ -11,8 +11,8 @@ Nacos + Fegin + Gateway + (Spring Security + JWT + OAuth2)


初始化sql位置 /sql 初始化sql位置 /sql


**地址:** 127.0.0.1:3306
**用户名:** test **密码:** test
**地址:** 127.0.0.1
**用户名:** test **密码:** zj12345678


### Nacos ### Nacos




+ 0
- 1
dubhe-server/admin/src/main/java/org/dubhe/admin/client/fallback/AuthServiceFallback.java View File

@@ -27,7 +27,6 @@ import java.util.Map;
* @description Feign 熔断处理类 * @description Feign 熔断处理类
* @date 2020-11-04 * @date 2020-11-04
*/ */
@Component
public class AuthServiceFallback implements AuthServiceClient { public class AuthServiceFallback implements AuthServiceClient {






+ 4
- 0
dubhe-server/admin/src/main/java/org/dubhe/admin/service/impl/UserServiceImpl.java View File

@@ -16,6 +16,7 @@
*/ */
package org.dubhe.admin.service.impl; package org.dubhe.admin.service.impl;


import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
import cn.hutool.crypto.asymmetric.KeyType; import cn.hutool.crypto.asymmetric.KeyType;
@@ -704,6 +705,9 @@ public class UserServiceImpl extends ServiceImpl<UserMapper, User> implements Us
Oauth2TokenDTO userDto = restResult.getData(); Oauth2TokenDTO userDto = restResult.getData();
UserDTO user = findByName(authUserDTO.getUsername()); UserDTO user = findByName(authUserDTO.getUsername());
Set<String> permissions = this.queryPermissionByUserId(user.getId()); Set<String> permissions = this.queryPermissionByUserId(user.getId());
if (CollUtil.isEmpty(permissions)) {
throw new BusinessException(BaseErrorCodeEnum.SYSTEM_ROLE_NOT_EXISTS);
}
// 返回 token 与 用户信息 // 返回 token 与 用户信息
authInfo.put("token", userDto.getTokenHead() + userDto.getToken()); authInfo.put("token", userDto.getTokenHead() + userDto.getToken());
authInfo.put("user", user); authInfo.put("user", user);


+ 0
- 1
dubhe-server/admin/src/test/java/org/dubhe/admin/AdminApplicationTests.java View File

@@ -14,7 +14,6 @@
* limitations under the License. * limitations under the License.
* ============================================================= * =============================================================
*/ */

package org.dubhe.admin; package org.dubhe.admin;


import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;


+ 0
- 1
dubhe-server/auth/src/test/java/org/dubhe/auth/AuthApplicationTests.java View File

@@ -14,7 +14,6 @@
* limitations under the License. * limitations under the License.
* ============================================================= * =============================================================
*/ */

package org.dubhe.auth; package org.dubhe.auth;


import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;


+ 0
- 37
dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/HarborProperties.java View File

@@ -1,37 +0,0 @@
/**
* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/
package org.dubhe.biz.base.constant;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

/**
* @description harbor相关配置
* @date 2020-07-17
*/
@Data
@Component
@ConfigurationProperties(prefix = "harbor")
public class HarborProperties {

private String address;

private String username;

private String password;
}

+ 1
- 1
dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/Permissions.java View File

@@ -110,7 +110,7 @@ public final class Permissions {
* 镜像管理 * 镜像管理
*/ */
public static final String IMAGE = "hasAuthority('ROLE_training:image')"; public static final String IMAGE = "hasAuthority('ROLE_training:image')";
public static final String IMAGE_UPLOAD = "hasAuthority('ROLE_training:image:upload')";
public static final String IMAGE_SAVE = "hasAuthority('ROLE_training:image:save')";
public static final String IMAGE_EDIT = "hasAuthority('ROLE_training:image:edit')"; public static final String IMAGE_EDIT = "hasAuthority('ROLE_training:image:edit')";
public static final String IMAGE_DELETE = "hasAuthority('ROLE_training:image:delete')"; public static final String IMAGE_DELETE = "hasAuthority('ROLE_training:image:delete')";
public static final String IMAGE_EDIT_DEFAULT = "hasAuthority('ROLE_training:image:editDefault')"; public static final String IMAGE_EDIT_DEFAULT = "hasAuthority('ROLE_training:image:editDefault')";


+ 0
- 3
dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/constant/StringConstant.java View File

@@ -126,9 +126,6 @@ public final class StringConstant {
public static final String CACHE_TASK_ID ="task_id"; public static final String CACHE_TASK_ID ="task_id";
public static final String CACHE_TASK_NAME ="task_name"; public static final String CACHE_TASK_NAME ="task_name";



public static final String DEFAULT_IMAGE_PROJECT ="dubhe";

/** /**
* python命令行参数格式 * python命令行参数格式
*/ */


+ 2
- 2
dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/BaseErrorCodeEnum.java View File

@@ -54,8 +54,8 @@ public enum BaseErrorCodeEnum implements ErrorCode {
SYSTEM_USER_EMAIL_NOT_EXISTS(20013, "该邮箱未注册!"), SYSTEM_USER_EMAIL_NOT_EXISTS(20013, "该邮箱未注册!"),
SYSTEM_USER_CANNOT_DELETE(20014, "系统默认用户不可删除!"), SYSTEM_USER_CANNOT_DELETE(20014, "系统默认用户不可删除!"),
SYSTEM_ROLE_CANNOT_DELETE(20015, "系统默认角色不可删除!"), SYSTEM_ROLE_CANNOT_DELETE(20015, "系统默认角色不可删除!"),
DATASET_ADMIN_PERMISSION_ERROR(1310,"无此权限,请联系管理员"),
SYSTEM_ROLE_NOT_EXISTS(20016, "用户绑定角色异常,请联系管理员!"),
DATASET_ADMIN_PERMISSION_ERROR(1310, "无此权限,请联系管理员"),


; ;




+ 0
- 53
dubhe-server/common-biz/base/src/main/java/org/dubhe/biz/base/enums/ImageStateEnum.java View File

@@ -1,53 +0,0 @@
/**
* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/
package org.dubhe.biz.base.enums;

/**
* @description 镜像运行状态枚举
* @date 2020-07-15
**/
public enum ImageStateEnum {

MAKING(0, "制作中"),
SUCCESS(1, "制作成功"),
FAIL(2, "制作失败");


/**
* 编码
*/
private Integer code;

/**
* 描述
*/
private String description;

ImageStateEnum(int code, String description) {
this.code = code;
this.description = description;
}

public Integer getCode() {
return code;
}

public String getDescription() {
return description;
}
}

dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-clod-prod.yml → dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-cloud-dev.yml View File

@@ -1,10 +1,12 @@
spring: spring:
cloud: cloud:
nacos: nacos:
username: nacos
password: Tianshu
context-path: /nacos context-path: /nacos
config: config:
namespace: dubhe-prod namespace: dubhe-prod
server-addr: 127.0.0.1:8848
server-addr: 10.105.1.132:8848
discovery: discovery:
namespace: dubhe-prod namespace: dubhe-prod
server-addr: 127.0.0.1:8848
server-addr: 10.105.1.132:8848

+ 0
- 10
dubhe-server/common-cloud/configuration/src/main/resources/bootstrap-prod.yml View File

@@ -1,10 +0,0 @@
spring:
cloud:
nacos:
context-path: /nacos
config:
namespace: dubhe-prod
server-addr: 127.0.0.1:8848
discovery:
namespace: dubhe-prod
server-addr: 127.0.0.1:8848

+ 7
- 0
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/NodeApi.java View File

@@ -207,4 +207,11 @@ public interface NodeApi {
* @return * @return
*/ */
List<BizTaint> geBizTaintListByUserId(); List<BizTaint> geBizTaintListByUserId();

/**
* 获取一个可用node的ip
*
* @return String
*/
String getAvailableNodeIp();
} }

+ 8
- 10
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/JupyterResourceApiImpl.java View File

@@ -389,14 +389,11 @@ public class JupyterResourceApiImpl implements JupyterResourceApi {
StatefulSet statefulSet = deployStatefulSet(); StatefulSet statefulSet = deployStatefulSet();
//部署svc //部署svc
Service service = deployService(); Service service = deployService();
//部署ingress
Ingress ingress = deployIngress();


if (delayDelete != null && delayDelete > ZERO) { if (delayDelete != null && delayDelete > ZERO) {
taskYamlBO.append(secret); taskYamlBO.append(secret);
taskYamlBO.append(statefulSet); taskYamlBO.append(statefulSet);
taskYamlBO.append(service); taskYamlBO.append(service);
taskYamlBO.append(ingress);


long stopUnixTime = System.currentTimeMillis() / THOUSAND_LONG + delayDelete * SIXTY_LONG; long stopUnixTime = System.currentTimeMillis() / THOUSAND_LONG + delayDelete * SIXTY_LONG;
Timestamp stopDisplayTime = new Timestamp(stopUnixTime * THOUSAND_LONG); Timestamp stopDisplayTime = new Timestamp(stopUnixTime * THOUSAND_LONG);
@@ -412,7 +409,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi {
k8sTaskService.createOrUpdateTask(k8sTask); k8sTaskService.createOrUpdateTask(k8sTask);
} }


return new PtJupyterDeployVO(secret, statefulSet, service, ingress);
return new PtJupyterDeployVO(secret, statefulSet, service, null);
} }


/** /**
@@ -440,7 +437,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi {
.build(); .build();


LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}", secretName); LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}", secretName);
secret = client.secrets().create(secret);
secret = client.secrets().inNamespace(namespace).create(secret);
LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", secretName); LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", secretName);
} }


@@ -647,7 +644,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi {
.build(); .build();
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", statefulSetName, YamlUtils.dumpAsYaml(statefulSet)); LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", statefulSetName, YamlUtils.dumpAsYaml(statefulSet));
resourceIisolationApi.addIisolationInfo(statefulSet); resourceIisolationApi.addIisolationInfo(statefulSet);
statefulSet = client.apps().statefulSets().create(statefulSet);
statefulSet = client.apps().statefulSets().inNamespace(namespace).create(statefulSet);
LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", statefulSetName); LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", statefulSetName);
return statefulSet; return statefulSet;
} }
@@ -678,22 +675,23 @@ public class JupyterResourceApiImpl implements JupyterResourceApi {
.withTargetPort(new IntOrString(CONTAINER_PORT)) .withTargetPort(new IntOrString(CONTAINER_PORT))
.withName(CONTAINER_NAME) .withName(CONTAINER_NAME)
.endPort() .endPort()
.withClusterIP("None")
.withSelector(podLabels) .withSelector(podLabels)
.withType("NodePort")
.endSpec() .endSpec()
.build(); .build();


LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", svcName, YamlUtils.dumpAsYaml(svc)); LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", svcName, YamlUtils.dumpAsYaml(svc));
svc = client.services().create(svc);
svc = client.services().inNamespace(svc.getMetadata().getNamespace()).create(svc);
LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", svcName); LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", svcName);
return svc; return svc;
} }


/** /**
* 部署ingress
* 部署ingress,弃用泛域名,暴露方式由ingress改为 service nodePort
* *
* @return Ingress 类 * @return Ingress 类
*/ */
@Deprecated
private Ingress deployIngress() { private Ingress deployIngress() {
Ingress ingress = null; Ingress ingress = null;
IngressList list = client.extensions().ingresses().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(baseName)).list(); IngressList list = client.extensions().ingresses().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(baseName)).list();
@@ -727,7 +725,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi {
.endSpec() .endSpec()
.build(); .build();
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", ingressName, YamlUtils.dumpAsYaml(ingress)); LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", ingressName, YamlUtils.dumpAsYaml(ingress));
ingress = client.extensions().ingresses().create(ingress);
ingress = client.extensions().ingresses().inNamespace(namespace).create(ingress);
LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", ingressName); LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", ingressName);
return ingress; return ingress;
} }


+ 4
- 46
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/ModelServingApiImpl.java View File

@@ -65,6 +65,7 @@ import org.dubhe.k8s.enums.K8sKindEnum;
import org.dubhe.k8s.enums.K8sResponseEnum; import org.dubhe.k8s.enums.K8sResponseEnum;
import org.dubhe.k8s.enums.LimitsOfResourcesEnum; import org.dubhe.k8s.enums.LimitsOfResourcesEnum;
import org.dubhe.k8s.enums.RestartPolicyEnum; import org.dubhe.k8s.enums.RestartPolicyEnum;
import org.dubhe.k8s.enums.ServiceTypeENum;
import org.dubhe.k8s.enums.ShellCommandEnum; import org.dubhe.k8s.enums.ShellCommandEnum;
import org.dubhe.k8s.utils.BizConvertUtils; import org.dubhe.k8s.utils.BizConvertUtils;
import org.dubhe.k8s.utils.K8sUtils; import org.dubhe.k8s.utils.K8sUtils;
@@ -102,15 +103,6 @@ public class ModelServingApiImpl implements ModelServingApi {
@Autowired @Autowired
private ResourceIisolationApi resourceIisolationApi; private ResourceIisolationApi resourceIisolationApi;


@Value("${k8s.serving.host}")
String servingHost;

@Value("${k8s.serving.tls-crt}")
String servingTlsCrt;

@Value("${k8s.serving.tls-key}")
String servingTlsKey;

private static final String MODEL_SERVING_MAX_UPLOAD_SIZE = "100m"; private static final String MODEL_SERVING_MAX_UPLOAD_SIZE = "100m";


public ModelServingApiImpl(K8sUtils k8sUtils) { public ModelServingApiImpl(K8sUtils k8sUtils) {
@@ -144,7 +136,6 @@ public class ModelServingApiImpl implements ModelServingApi {
//名称生成 //名称生成
String deploymentName = StrUtil.format(K8sParamConstants.RESOURCE_NAME_TEMPLATE, bo.getResourceName(), RandomUtil.randomString(MagicNumConstant.EIGHT)); String deploymentName = StrUtil.format(K8sParamConstants.RESOURCE_NAME_TEMPLATE, bo.getResourceName(), RandomUtil.randomString(MagicNumConstant.EIGHT));
String svcName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.SVC_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE)); String svcName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.SVC_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE));
String ingressName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.INGRESS_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE));


//标签生成 //标签生成
Map<String, String> baseLabels = LabelUtils.getBaseLabels(bo.getResourceName(), bo.getBusinessLabel()); Map<String, String> baseLabels = LabelUtils.getBaseLabels(bo.getResourceName(), bo.getBusinessLabel());
@@ -164,45 +155,12 @@ public class ModelServingApiImpl implements ModelServingApi {
if (bo.getGrpcPort() != null) { if (bo.getGrpcPort() != null) {
buildServiceBO.addPort(ResourceBuildUtils.buildServicePort(bo.getGrpcPort(), bo.getGrpcPort(), SymbolConstant.GRPC)); buildServiceBO.addPort(ResourceBuildUtils.buildServicePort(bo.getGrpcPort(), bo.getGrpcPort(), SymbolConstant.GRPC));
} }
buildServiceBO.setType(ServiceTypeENum.NODE_PORT.getType());
Service service = ResourceBuildUtils.buildService(buildServiceBO); Service service = ResourceBuildUtils.buildService(buildServiceBO);
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service));
Service serviceResult = client.services().create(service);

//部署ingress
BuildIngressBO buildIngressBO = new BuildIngressBO(bo.getNamespace(), ingressName, baseLabels);
if (StringUtils.isNotEmpty(buildIngressBO.getMaxUploadSize())) {
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_PROXY_BODY_SIZE_KEY, buildIngressBO.getMaxUploadSize());
}
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_READ_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN));
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_NEXT_UPSTREAM_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN));
if (bo.getHttpPort() != null) {
String httpHost = RandomUtil.randomString(MagicNumConstant.SIX) + SymbolConstant.DOT + servingHost;
buildIngressBO.addIngressRule(ResourceBuildUtils.buildIngressRule(httpHost, svcName, SymbolConstant.HTTP));
}
Secret secretResult = null;
if (bo.getGrpcPort() != null) {
String secretName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), SymbolConstant.TOKEN, RandomUtil.randomString(MagicNumConstant.FIVE));
Map<String, String> data = new HashMap<String, String>(MagicNumConstant.FOUR) {
{
put(K8sParamConstants.SECRET_TLS_TLS_CRT, servingTlsCrt);
put(K8sParamConstants.SECRET_TLS_TLS_KEY, servingTlsKey);
}
};
Secret secret = ResourceBuildUtils.buildTlsSecret(bo.getNamespace(), secretName, baseLabels, data);
secretResult = client.secrets().create(secret);

String grpcHost = RandomUtil.randomString(MagicNumConstant.SIX) + SymbolConstant.DOT + servingHost;
buildIngressBO.addIngressRule(ResourceBuildUtils.buildIngressRule(grpcHost, svcName, SymbolConstant.GRPC));
buildIngressBO.addIngressTLS(ResourceBuildUtils.buildIngressTLS(secretName, grpcHost));
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_CLASS_KEY, StringConstant.NGINX_LOWERCASE);
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_SSL_REDIRECT_KEY, StringConstant.TRUE_LOWERCASE);
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_BACKEND_PROTOCOL_KEY, StringConstant.GRPC_CAPITALIZE);
}
Ingress ingress = ResourceBuildUtils.buildIngress(buildIngressBO);
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", ingressName, YamlUtils.dumpAsYaml(ingress));
Ingress ingressResult = client.extensions().ingresses().create(ingress);
Service serviceResult = client.services().inNamespace(bo.getNamespace()).create(service);


return new ModelServingVO(BizConvertUtils.toBizSecret(secretResult), BizConvertUtils.toBizService(serviceResult), BizConvertUtils.toBizDeployment(deploymentResult), BizConvertUtils.toBizIngress(ingressResult));
return new ModelServingVO(null, BizConvertUtils.toBizService(serviceResult), BizConvertUtils.toBizDeployment(deploymentResult), null);
} catch (KubernetesClientException e) { } catch (KubernetesClientException e) {
LogUtil.error(LogEnum.BIZ_K8S, "ModelOptJobApiImpl.create error, param:{} error:", bo, e); LogUtil.error(LogEnum.BIZ_K8S, "ModelOptJobApiImpl.create error, param:{} error:", bo, e);
return new ModelServingVO().error(String.valueOf(e.getCode()), e.getMessage()); return new ModelServingVO().error(String.valueOf(e.getCode()), e.getMessage());


+ 20
- 0
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/NodeApiImpl.java View File

@@ -41,6 +41,7 @@ import org.dubhe.k8s.constant.K8sLabelConstants;
import org.dubhe.k8s.constant.K8sParamConstants; import org.dubhe.k8s.constant.K8sParamConstants;
import org.dubhe.k8s.domain.PtBaseResult; import org.dubhe.k8s.domain.PtBaseResult;
import org.dubhe.k8s.domain.resource.BizNode; import org.dubhe.k8s.domain.resource.BizNode;
import org.dubhe.k8s.domain.resource.BizNodeAddress;
import org.dubhe.k8s.domain.resource.BizTaint; import org.dubhe.k8s.domain.resource.BizTaint;
import org.dubhe.k8s.domain.vo.PtNodeMetricsVO; import org.dubhe.k8s.domain.vo.PtNodeMetricsVO;
import org.dubhe.k8s.enums.K8sResponseEnum; import org.dubhe.k8s.enums.K8sResponseEnum;
@@ -560,6 +561,25 @@ public class NodeApiImpl implements NodeApi {
return geBizTaintListByUserId(userContextService.getCurUserId()); return geBizTaintListByUserId(userContextService.getCurUserId());
} }


/**
* 获取一个可用node的ip
*
* @return String
*/
@Override
public String getAvailableNodeIp(){
List<BizNode> nodes = listAll();
for (BizNode node : nodes){
if (node.getReady() && !node.isUnschedulable()){
for (BizNodeAddress address : node.getAddresses()){
if (K8sParamConstants.INTERNAL_IP.equals(address.getType())){
return address.getAddress();
}
}
}
}
return "";
}


/** /**
* 查询节点内存资源是否可分配 * 查询节点内存资源是否可分配


+ 5
- 2
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/PodApiImpl.java View File

@@ -400,13 +400,16 @@ public class PodApiImpl implements PodApi {
public String getUrlByResourceName(String namespace, String resourceName) { public String getUrlByResourceName(String namespace, String resourceName) {
LogUtil.info(LogEnum.BIZ_K8S,"Start GetUrlByResourceName {} {}",namespace,resourceName); LogUtil.info(LogEnum.BIZ_K8S,"Start GetUrlByResourceName {} {}",namespace,resourceName);
PtJupyterDeployVO info = jupyterResourceApi.get(namespace, resourceName); PtJupyterDeployVO info = jupyterResourceApi.get(namespace, resourceName);
if (info != null && info.getIngressInfo() != null && CollectionUtil.isNotEmpty(info.getIngressInfo().getRules())) {
if (info != null && info.getServiceInfo() != null && CollectionUtil.isNotEmpty(info.getServiceInfo().getPorts())) {
String token = getTokenByResourceName(namespace, resourceName); String token = getTokenByResourceName(namespace, resourceName);
if (StringUtils.isBlank(token)) { if (StringUtils.isBlank(token)) {
LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter Notebook token not generated,[namespace]={}, [resourceName]={}", namespace, resourceName); LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter Notebook token not generated,[namespace]={}, [resourceName]={}", namespace, resourceName);
return ""; return "";
} }
String url = StrUtil.format(POD_URL, info.getIngressInfo().getRules().get(0).getHost(), k8sUtils.getPort(), token);

BizPod pod = getWithResourceName(namespace,resourceName);

String url = StrUtil.format(POD_URL, pod.getHostIP(), info.getServiceInfo().getPorts().get(0).getNodePort(), token);
return validateJupyterUrl(url); return validateJupyterUrl(url);
} }
LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter statefulset not created,[namespace]={}, [resourceName]={}",namespace,resourceName); LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter statefulset not created,[namespace]={}, [resourceName]={}",namespace,resourceName);


+ 1
- 1
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/api/impl/TerminalApiImpl.java View File

@@ -148,7 +148,7 @@ public class TerminalApiImpl implements TerminalApi {
} }
Service service = ResourceBuildUtils.buildService(buildServiceBO); Service service = ResourceBuildUtils.buildService(buildServiceBO);
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service));
Service serviceResult = client.services().create(service);
Service serviceResult = client.services().inNamespace(service.getMetadata().getNamespace()).create(service);
return new TerminalResourceVO(BizConvertUtils.toBizDeployment(deploymentResult),BizConvertUtils.toBizService(serviceResult)); return new TerminalResourceVO(BizConvertUtils.toBizDeployment(deploymentResult),BizConvertUtils.toBizService(serviceResult));
}catch (KubernetesClientException e) { }catch (KubernetesClientException e) {
LogUtil.error(LogEnum.BIZ_K8S, "TerminalApiImpl.create error, param:{} error:", bo, e); LogUtil.error(LogEnum.BIZ_K8S, "TerminalApiImpl.create error, param:{} error:", bo, e);


+ 1
- 0
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/constant/K8sParamConstants.java View File

@@ -101,4 +101,5 @@ public class K8sParamConstants {


public static final String WAITING_REASON_CONTAINER_CREATING = "ContainerCreating"; public static final String WAITING_REASON_CONTAINER_CREATING = "ContainerCreating";


public final static String INTERNAL_IP = "InternalIP";
} }

+ 7
- 1
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/dto/BaseK8sDeploymentCallbackCreateDTO.java View File

@@ -57,16 +57,21 @@ public class BaseK8sDeploymentCallbackCreateDTO {
@NotNull(message = "replicas 不能为空!") @NotNull(message = "replicas 不能为空!")
private Integer replicas; private Integer replicas;


@ApiModelProperty(required = true, value = "action")
@NotBlank(message = "action 不能为空!")
private String action;

public BaseK8sDeploymentCallbackCreateDTO() { public BaseK8sDeploymentCallbackCreateDTO() {


} }


public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas) {
public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas,String action) {
this.namespace = namespace; this.namespace = namespace;
this.resourceName = resourceName; this.resourceName = resourceName;
this.deploymentName = deploymentName; this.deploymentName = deploymentName;
this.readyReplicas = readyReplicas; this.readyReplicas = readyReplicas;
this.replicas = replicas; this.replicas = replicas;
this.action = action;
} }


@Override @Override
@@ -77,6 +82,7 @@ public class BaseK8sDeploymentCallbackCreateDTO {
", deploymentName='" + deploymentName + '\'' + ", deploymentName='" + deploymentName + '\'' +
", readyReplicas='" + readyReplicas + '\'' + ", readyReplicas='" + readyReplicas + '\'' +
", replicas='" + replicas + '\'' + ", replicas='" + replicas + '\'' +
", action='" + action + '\'' +
'}'; '}';
} }
} }

+ 3
- 0
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/domain/vo/PtJupyterDeployVO.java View File

@@ -30,6 +30,7 @@ import org.dubhe.k8s.annotation.K8sField;
import org.dubhe.k8s.constant.K8sParamConstants; import org.dubhe.k8s.constant.K8sParamConstants;
import org.dubhe.k8s.domain.PtBaseResult; import org.dubhe.k8s.domain.PtBaseResult;
import org.dubhe.k8s.domain.resource.BizContainer; import org.dubhe.k8s.domain.resource.BizContainer;
import org.dubhe.k8s.domain.resource.BizServicePort;
import org.dubhe.k8s.utils.MappingUtils; import org.dubhe.k8s.utils.MappingUtils;


import java.util.List; import java.util.List;
@@ -105,6 +106,8 @@ public class PtJupyterDeployVO extends PtBaseResult<PtJupyterDeployVO> {
private String namespace; private String namespace;
@K8sField("metadata:uid") @K8sField("metadata:uid")
private String uid; private String uid;
@K8sField("spec:ports")
private List<BizServicePort> ports;
} }


@Data @Data


+ 0
- 4
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/properties/ClusterProperties.java View File

@@ -33,10 +33,6 @@ public class ClusterProperties {


private String nfs; private String nfs;


private String host;

private String port;

private String kubeconfig; private String kubeconfig;


/** /**


+ 3
- 1
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/BizConvertUtils.java View File

@@ -218,7 +218,9 @@ public class BizConvertUtils {
*/ */
public static BizIngress toBizIngress(Ingress ingress) { public static BizIngress toBizIngress(Ingress ingress) {
BizIngress bizIngress = MappingUtils.mappingTo(ingress, BizIngress.class); BizIngress bizIngress = MappingUtils.mappingTo(ingress, BizIngress.class);
bizIngress.getRules().forEach(BizIngressRule::takeServicePort);
if (bizIngress != null){
bizIngress.getRules().forEach(BizIngressRule::takeServicePort);
}
return bizIngress; return bizIngress;
} }




+ 0
- 2
dubhe-server/common-k8s/src/main/java/org/dubhe/k8s/utils/K8sUtils.java View File

@@ -98,8 +98,6 @@ public class K8sUtils implements ApplicationContextAware {




nfs = clusterProperties.getNfs(); nfs = clusterProperties.getNfs();
host = clusterProperties.getHost();
port = clusterProperties.getPort();


//打印集群信息 //打印集群信息
LogUtil.info(LogEnum.BIZ_K8S, "ApiVersion : {}", client.getApiVersion()); LogUtil.info(LogEnum.BIZ_K8S, "ApiVersion : {}", client.getApiVersion());


+ 0
- 5
dubhe-server/common-recycle/src/main/java/org/dubhe/recycle/enums/RecycleResourceEnum.java View File

@@ -69,11 +69,6 @@ public enum RecycleResourceEnum {
*/ */
MEASURE_RECYCLE_FILE("measureRecycleFile", "度量文件回收"), MEASURE_RECYCLE_FILE("measureRecycleFile", "度量文件回收"),


/**
* 镜像回收
*/
IMAGE_RECYCLE_FILE("imageRecycleFile", "镜像回收"),

/** /**
* 算法文件回收 * 算法文件回收
*/ */


+ 0
- 1
dubhe-server/dubhe-data-task/src/test/java/org/dubhe/task/DubheDataTaskApplicationTests.java View File

@@ -14,7 +14,6 @@
* limitations under the License. * limitations under the License.
* ============================================================= * =============================================================
*/ */

package org.dubhe.task; package org.dubhe.task;


import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;


+ 0
- 36
dubhe-server/dubhe-data/src/main/java/org/dubhe/data/config/DataHarborConfig.java View File

@@ -1,36 +0,0 @@
/**
* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.data.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

@Data
@Component
@ConfigurationProperties(prefix = "harbor")
public class DataHarborConfig {

private String address;

private String username;

private String password;

private String modelName;
}

+ 21
- 9
dubhe-server/dubhe-data/src/main/java/org/dubhe/data/service/impl/AutoLabelModelServiceServiceImpl.java View File

@@ -28,9 +28,15 @@ import com.baomidou.mybatisplus.extension.service.IService;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.apache.commons.compress.utils.Lists; import org.apache.commons.compress.utils.Lists;
import org.dubhe.biz.base.constant.MagicNumConstant; import org.dubhe.biz.base.constant.MagicNumConstant;
import org.dubhe.biz.base.constant.SymbolConstant;
import org.dubhe.biz.base.context.UserContext; import org.dubhe.biz.base.context.UserContext;
import org.dubhe.biz.base.dto.*;
import org.dubhe.biz.base.dto.PtImageIdDTO;
import org.dubhe.biz.base.dto.PtImageIdsDTO;
import org.dubhe.biz.base.dto.PtModelBranchQueryByIdDTO;
import org.dubhe.biz.base.dto.PtModelBranchQueryByIdsDTO;
import org.dubhe.biz.base.dto.TrainAlgorithmSelectAllBatchIdDTO;
import org.dubhe.biz.base.dto.TrainAlgorithmSelectByIdDTO;
import org.dubhe.biz.base.dto.UserDTO;
import org.dubhe.biz.base.dto.UserSmallDTO;
import org.dubhe.biz.base.enums.BizEnum; import org.dubhe.biz.base.enums.BizEnum;
import org.dubhe.biz.base.enums.ResourcesPoolTypeEnum; import org.dubhe.biz.base.enums.ResourcesPoolTypeEnum;
import org.dubhe.biz.base.exception.BusinessException; import org.dubhe.biz.base.exception.BusinessException;
@@ -49,13 +55,15 @@ import org.dubhe.cloud.authconfig.service.AdminClient;
import org.dubhe.data.client.AlgorithmClient; import org.dubhe.data.client.AlgorithmClient;
import org.dubhe.data.client.ImageClient; import org.dubhe.data.client.ImageClient;
import org.dubhe.data.client.ModelClient; import org.dubhe.data.client.ModelClient;
import org.dubhe.data.config.DataHarborConfig;
import org.dubhe.data.config.DataRedisConfig; import org.dubhe.data.config.DataRedisConfig;
import org.dubhe.data.constant.AutoLabelModelServiceStatusEnum; import org.dubhe.data.constant.AutoLabelModelServiceStatusEnum;
import org.dubhe.data.constant.Constant; import org.dubhe.data.constant.Constant;
import org.dubhe.data.constant.ErrorEnum; import org.dubhe.data.constant.ErrorEnum;
import org.dubhe.data.dao.AutoLabelModelServiceMapper; import org.dubhe.data.dao.AutoLabelModelServiceMapper;
import org.dubhe.data.domain.dto.*;
import org.dubhe.data.domain.dto.AutoLabelModelServiceCreateDTO;
import org.dubhe.data.domain.dto.AutoLabelModelServiceQueryDTO;
import org.dubhe.data.domain.dto.AutoLabelModelServiceUpdateDTO;
import org.dubhe.data.domain.dto.DataK8sDeploymentCallbackCreateDTO;
import org.dubhe.data.domain.entity.AutoLabelModelService; import org.dubhe.data.domain.entity.AutoLabelModelService;
import org.dubhe.data.domain.vo.AutoLabelModelServicePodVO; import org.dubhe.data.domain.vo.AutoLabelModelServicePodVO;
import org.dubhe.data.domain.vo.AutoLabelModelServiceVO; import org.dubhe.data.domain.vo.AutoLabelModelServiceVO;
@@ -75,9 +83,16 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.DuplicateKeyException; import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;

import javax.annotation.Resource; import javax.annotation.Resource;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -116,9 +131,6 @@ public class AutoLabelModelServiceServiceImpl extends ServiceImpl<AutoLabelModel
@Resource @Resource
private UserContextService userContextService; private UserContextService userContextService;


@Resource
private DataHarborConfig dataHarborConfig;

@Resource @Resource
private DataRedisConfig dataRedisConfig; private DataRedisConfig dataRedisConfig;


@@ -197,7 +209,7 @@ public class AutoLabelModelServiceServiceImpl extends ServiceImpl<AutoLabelModel
modelServiceBO.setResourceName(k8sNameTool.generateResourceName(BizEnum.DATA, autoLabelModelService.getId().toString())); modelServiceBO.setResourceName(k8sNameTool.generateResourceName(BizEnum.DATA, autoLabelModelService.getId().toString()));
modelServiceBO.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.DATA)); modelServiceBO.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.DATA));
modelServiceBO.setReplicas(autoLabelModelService.getInstanceNum()); modelServiceBO.setReplicas(autoLabelModelService.getInstanceNum());
modelServiceBO.setImage(dataHarborConfig.getAddress() + SymbolConstant.SLASH + ptImageVO.getImageUrl());
modelServiceBO.setImage(ptImageVO.getImageUrl());
modelServiceBO.setTaskIdentifyLabel(taskIdentify); modelServiceBO.setTaskIdentifyLabel(taskIdentify);


String command = String.format(Constant.MODEL_SERVICE_COMMAND, modelServiceId, String command = String.format(Constant.MODEL_SERVICE_COMMAND, modelServiceId,


+ 0
- 142
dubhe-server/dubhe-image/src/main/java/org/dubhe/image/async/HarborImagePushAsync.java View File

@@ -1,142 +0,0 @@
/**
* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/
package org.dubhe.image.async;

import cn.hutool.core.util.StrUtil;
import org.dubhe.biz.base.constant.HarborProperties;
import org.dubhe.biz.base.constant.StringConstant;
import org.dubhe.biz.base.enums.ImageStateEnum;
import org.dubhe.biz.base.exception.BusinessException;
import org.dubhe.biz.base.utils.StringUtils;
import org.dubhe.biz.file.utils.IOUtil;
import org.dubhe.biz.log.enums.LogEnum;
import org.dubhe.biz.log.utils.LogUtil;
import org.dubhe.harbor.api.HarborApi;
import org.dubhe.image.dao.PtImageMapper;
import org.dubhe.image.domain.entity.PtImage;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;

import java.io.BufferedReader;
import java.io.InputStreamReader;

/**
* @description 异步推送镜像到harbor
* @date 2020-07-10
*/
@Component
public class HarborImagePushAsync {

@Autowired
private HarborProperties harborProperties;

@Autowired
private HarborApi harborApi;

@Autowired
private PtImageMapper ptImageMapper;

/**
* 组装shell脚本执行命令
*
* @param imagePath
* @param imageNameandTag
**/
@Async
public void execShell(String imagePath, String imageNameandTag, PtImage ptImage) {
try {
String imageResource = harborProperties.getAddress() + StrUtil.SLASH + StringConstant.DEFAULT_IMAGE_PROJECT
+ StrUtil.SLASH + imageNameandTag;
String cmdStr = "docker login --username=" + harborProperties.getUsername() + " " + harborProperties.getAddress() + " --password=" + harborProperties.getPassword() + " ; docker " +
"load < " + imagePath + " |awk '{print $3}' |xargs -I str docker tag str " + imageResource + " ; docker push " + imageResource + "; docker rmi " + imageResource;
String[] cmd = {"/bin/bash", "-c", cmdStr};
LogUtil.info(LogEnum.IMAGE, "镜像上传执行脚本参数:{}", cmd);

Process process = Runtime.getRuntime().exec(cmd);
if (checkImagePushIsOk(ptImage, process)) {
updateImageStatus(ptImage, ImageStateEnum.SUCCESS.getCode());
} else {
updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode());
}
} catch (Exception e) {
LogUtil.error(LogEnum.IMAGE, "upload image failed,exception {}", e);
updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode());
throw new BusinessException("上传镜像异常!");

}
}

/**
* 更新镜像上传状态
*
* @param ptImage
* @param status
**/
public void updateImageStatus(PtImage ptImage, Integer status) {
ptImage.setImageStatus(status);
ptImageMapper.updateById(ptImage);
}


/**
* 校验镜像是否上传成功
*
* @param ptImage 镜像信息
* @param process process对象
* @return 是否上传成功
*/
public boolean checkImagePushIsOk(PtImage ptImage, Process process) {
//读取标准输出流
BufferedReader brOut = new BufferedReader(new InputStreamReader(process.getInputStream()));
//读取标准错误流
BufferedReader brErr = new BufferedReader(new InputStreamReader(process.getErrorStream()));
String line;
StringBuilder outMessage = new StringBuilder();
StringBuilder errMessage = new StringBuilder();
boolean isPushOk = true;
try {
while ((line = brOut.readLine()) != null) {
outMessage.append(line);
}
if (StringUtils.isNotEmpty(outMessage)) {
LogUtil.info(LogEnum.IMAGE, "upload image with shell standard output message {}", outMessage.toString());
}
while ((line = brErr.readLine()) != null) {
errMessage.append(line);
}
if (StringUtils.isNotEmpty(errMessage)) {
LogUtil.error(LogEnum.IMAGE, "upload image with shell standard output message {}", errMessage.toString());
}
Integer status = process.waitFor();
LogUtil.info(LogEnum.IMAGE, "上传镜像状态:{}", status);
if (status == null) {
if (!harborApi.isExistImage(ptImage.getImageUrl())) {
isPushOk = false;
}
} else if (status != 0) {
isPushOk = false;
}
} catch (Exception e) {
LogUtil.error(LogEnum.IMAGE, "upload image failed,exception {}", e);
return false;
} finally {
IOUtil.close(brErr, brOut);
}
return isPushOk;
}
}

+ 0
- 3
dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/dto/PtImageQueryDTO.java View File

@@ -41,9 +41,6 @@ public class PtImageQueryDTO extends PageQueryBase implements Serializable {
@ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") @ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)")
private Integer imageResource; private Integer imageResource;


@ApiModelProperty(value = "镜像状态,0为制作中,1位制作成功,2位制作失败")
private Integer imageStatus;

@ApiModelProperty(value = "镜像名称或id") @ApiModelProperty(value = "镜像名称或id")
private String imageNameOrId; private String imageNameOrId;




dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/dto/PtImageUploadDTO.java → dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/dto/PtImageSaveDTO.java View File

@@ -38,7 +38,7 @@ import java.util.List;
*/ */
@Data @Data
@Accessors(chain = true) @Accessors(chain = true)
public class PtImageUploadDTO implements Serializable {
public class PtImageSaveDTO implements Serializable {


private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;


@@ -50,9 +50,9 @@ public class PtImageUploadDTO implements Serializable {
@FlagValidator(value = {"0", "1"}, message = "无效镜像来源") @FlagValidator(value = {"0", "1"}, message = "无效镜像来源")
private Integer imageResource; private Integer imageResource;


@ApiModelProperty(value = "镜像文件路径", required = true)
@NotBlank(message = "镜像文件路径不能为空")
private String imagePath;
@ApiModelProperty(value = "镜像地址", required = true)
@NotBlank(message = "镜像地址不能为空")
private String imageUrl;


@ApiModelProperty(value = "镜像名称", required = true) @ApiModelProperty(value = "镜像名称", required = true)
@NotBlank(message = "源镜像名称不能为空") @NotBlank(message = "源镜像名称不能为空")

+ 23
- 5
dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/dto/PtImageUpdateDTO.java View File

@@ -20,10 +20,13 @@ import io.swagger.annotations.ApiModelProperty;
import lombok.Data; import lombok.Data;
import lombok.experimental.Accessors; import lombok.experimental.Accessors;
import org.dubhe.biz.base.constant.MagicNumConstant; import org.dubhe.biz.base.constant.MagicNumConstant;
import org.dubhe.biz.base.constant.StringConstant;
import org.hibernate.validator.constraints.Length; import org.hibernate.validator.constraints.Length;


import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull; import javax.validation.constraints.NotNull;
import javax.validation.constraints.Pattern;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.List;


@@ -39,14 +42,29 @@ public class PtImageUpdateDTO implements Serializable {


@ApiModelProperty(value = "id", required = true) @ApiModelProperty(value = "id", required = true)
@NotNull(message = "镜像id不能为空") @NotNull(message = "镜像id不能为空")
private List<Long> ids;

@ApiModelProperty("镜像描述")
@Length(max = MagicNumConstant.BINARY_TEN_EXP, message = "镜像描述-输入长度不能超过1024个字符")
private String remark;
private Long id;


@ApiModelProperty(value = "镜像用途", required = true) @ApiModelProperty(value = "镜像用途", required = true)
@NotEmpty(message = "镜像用途(0:notebook , 1:train , 2:serving, 3:terminal, 4:point-cloud)不能为空") @NotEmpty(message = "镜像用途(0:notebook , 1:train , 2:serving, 3:terminal, 4:point-cloud)不能为空")
private List<Integer> imageTypes; private List<Integer> imageTypes;


@ApiModelProperty(value = "镜像路径", required = true)
@NotBlank(message = "镜像路径不能为空")
private String imageUrl;

@ApiModelProperty(value = "镜像名称", required = true)
@NotBlank(message = "源镜像名称不能为空")
@Length(min = MagicNumConstant.ONE, max = MagicNumConstant.SIXTY_FOUR, message = "镜像名称长度在1-64个字符")
@Pattern(regexp = StringConstant.REGEXP_NAME, message = "镜像名称支持字母、数字、英文横杠和下划线")
private String imageName;

@ApiModelProperty(value = "镜像版本号", required = true)
@NotBlank(message = "镜像版本号不能为空")
@Length(max = MagicNumConstant.SIXTY_FOUR, message = "镜像版本号长度在1-64个字符")
@Pattern(regexp = StringConstant.REGEXP_TAG, message = "镜像版本号支持字母、数字、英文横杠、英文.号和下划线")
private String imageTag;

@ApiModelProperty("镜像描述")
@Length(max = MagicNumConstant.BINARY_TEN_EXP, message = "镜像描述-输入长度不能超过1024个字符")
private String remark;
} }

+ 0
- 7
dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/entity/PtImage.java View File

@@ -80,13 +80,6 @@ public class PtImage extends BaseEntity {
@TableField(value = "image_resource") @TableField(value = "image_resource")
private Integer imageResource; private Integer imageResource;



/**
* 镜像状态
*/
@TableField(value = "image_status")
private Integer imageStatus;

/** /**
* 资源拥有者ID * 资源拥有者ID
*/ */


+ 3
- 3
dubhe-server/dubhe-image/src/main/java/org/dubhe/image/domain/vo/PtImageQueryVO.java View File

@@ -41,9 +41,6 @@ public class PtImageQueryVO implements Serializable {
@ApiModelProperty("镜像版本") @ApiModelProperty("镜像版本")
private String imageTag; private String imageTag;


@ApiModelProperty("镜像状态(0:上传中,1:上传成功,2:上传失败)")
private Integer imageStatus;

@ApiModelProperty("备注") @ApiModelProperty("备注")
private String remark; private String remark;


@@ -58,4 +55,7 @@ public class PtImageQueryVO implements Serializable {


@ApiModelProperty("镜像用途") @ApiModelProperty("镜像用途")
private List<Integer> imageTypes; private List<Integer> imageTypes;

@ApiModelProperty("镜像地址")
private String imageUrl;
} }

+ 21
- 8
dubhe-server/dubhe-image/src/main/java/org/dubhe/image/rest/PtImageController.java View File

@@ -24,12 +24,25 @@ import org.dubhe.biz.base.dto.PtImageIdDTO;
import org.dubhe.biz.base.dto.PtImageIdsDTO; import org.dubhe.biz.base.dto.PtImageIdsDTO;
import org.dubhe.biz.base.vo.DataResponseBody; import org.dubhe.biz.base.vo.DataResponseBody;
import org.dubhe.biz.base.vo.PtImageVO; import org.dubhe.biz.base.vo.PtImageVO;
import org.dubhe.image.domain.dto.*;
import org.dubhe.image.domain.dto.PtImageDeleteDTO;
import org.dubhe.image.domain.dto.PtImageQueryDTO;
import org.dubhe.image.domain.dto.PtImageQueryImageDTO;
import org.dubhe.image.domain.dto.PtImageQueryNameDTO;
import org.dubhe.image.domain.dto.PtImageQueryUrlDTO;
import org.dubhe.image.domain.dto.PtImageSaveDTO;
import org.dubhe.image.domain.dto.PtImageUpdateDTO;
import org.dubhe.image.service.PtImageService; import org.dubhe.image.service.PtImageService;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated; import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;


import java.util.List; import java.util.List;


@@ -45,7 +58,7 @@ public class PtImageController {
@Autowired @Autowired
private PtImageService ptImageService; private PtImageService ptImageService;


@GetMapping("/info")
@GetMapping("/list")
@ApiOperation("查询镜像") @ApiOperation("查询镜像")
@PreAuthorize(Permissions.IMAGE) @PreAuthorize(Permissions.IMAGE)
public DataResponseBody getImage(PtImageQueryDTO ptImageQueryDTO) { public DataResponseBody getImage(PtImageQueryDTO ptImageQueryDTO) {
@@ -58,11 +71,11 @@ public class PtImageController {
return new DataResponseBody(ptImageService.searchImages(ptImageQueryImageDTO)); return new DataResponseBody(ptImageService.searchImages(ptImageQueryImageDTO));
} }


@PostMapping("uploadImage")
@ApiOperation("上传镜像包到harbor")
@PreAuthorize(Permissions.IMAGE_UPLOAD)
public DataResponseBody uploadImage(@Validated @RequestBody PtImageUploadDTO ptImageUploadDTO) {
ptImageService.uploadImage(ptImageUploadDTO);
@PostMapping
@ApiOperation("保存镜像信息")
@PreAuthorize(Permissions.IMAGE_SAVE)
public DataResponseBody saveImageInfo(@Validated @RequestBody PtImageSaveDTO ptImageSaveDTO) {
ptImageService.saveImageInfo(ptImageSaveDTO);
return new DataResponseBody(); return new DataResponseBody();
} }




+ 10
- 12
dubhe-server/dubhe-image/src/main/java/org/dubhe/image/service/PtImageService.java View File

@@ -18,9 +18,14 @@
package org.dubhe.image.service; package org.dubhe.image.service;


import org.dubhe.biz.base.vo.PtImageVO; import org.dubhe.biz.base.vo.PtImageVO;
import org.dubhe.image.domain.dto.*;
import org.dubhe.image.domain.dto.PtImageDeleteDTO;
import org.dubhe.image.domain.dto.PtImageQueryDTO;
import org.dubhe.image.domain.dto.PtImageQueryImageDTO;
import org.dubhe.image.domain.dto.PtImageQueryNameDTO;
import org.dubhe.image.domain.dto.PtImageQueryUrlDTO;
import org.dubhe.image.domain.dto.PtImageSaveDTO;
import org.dubhe.image.domain.dto.PtImageUpdateDTO;
import org.dubhe.image.domain.entity.PtImage; import org.dubhe.image.domain.entity.PtImage;
import org.dubhe.recycle.domain.dto.RecycleCreateDTO;


import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -42,11 +47,11 @@ public interface PtImageService {




/** /**
* 上传镜像到harbor
* 保存镜像信息
* *
* @param ptImageUploadDTO 上传条件
* @param ptImageSaveDTO 镜像信息DTO
*/ */
void uploadImage(PtImageUploadDTO ptImageUploadDTO);
void saveImageInfo(PtImageSaveDTO ptImageSaveDTO);




/** /**
@@ -98,13 +103,6 @@ public interface PtImageService {
*/ */
String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO); String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO);


/**
* 镜像回收还原
*
* @param dto 还原DTO对象
*/
void recycleRollback(RecycleCreateDTO dto);

/** /**
* 获取终端镜像列表 * 获取终端镜像列表
* *


+ 98
- 151
dubhe-server/dubhe-image/src/main/java/org/dubhe/image/service/impl/PtImageServiceImpl.java View File

@@ -27,12 +27,14 @@ import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.compress.utils.Lists; import org.apache.commons.compress.utils.Lists;
import org.dubhe.biz.base.constant.*;
import org.dubhe.biz.base.constant.MagicNumConstant;
import org.dubhe.biz.base.constant.NumberConstant;
import org.dubhe.biz.base.constant.ResponseCode;
import org.dubhe.biz.base.constant.StringConstant;
import org.dubhe.biz.base.context.DataContext; import org.dubhe.biz.base.context.DataContext;
import org.dubhe.biz.base.context.UserContext; import org.dubhe.biz.base.context.UserContext;
import org.dubhe.biz.base.enums.DatasetTypeEnum; import org.dubhe.biz.base.enums.DatasetTypeEnum;
import org.dubhe.biz.base.enums.ImageSourceEnum; import org.dubhe.biz.base.enums.ImageSourceEnum;
import org.dubhe.biz.base.enums.ImageStateEnum;
import org.dubhe.biz.base.enums.ImageTypeEnum; import org.dubhe.biz.base.enums.ImageTypeEnum;
import org.dubhe.biz.base.exception.BusinessException; import org.dubhe.biz.base.exception.BusinessException;
import org.dubhe.biz.base.service.UserContextService; import org.dubhe.biz.base.service.UserContextService;
@@ -40,31 +42,34 @@ import org.dubhe.biz.base.utils.ReflectionUtils;
import org.dubhe.biz.base.utils.StringUtils; import org.dubhe.biz.base.utils.StringUtils;
import org.dubhe.biz.base.vo.PtImageVO; import org.dubhe.biz.base.vo.PtImageVO;
import org.dubhe.biz.db.utils.PageUtil; import org.dubhe.biz.db.utils.PageUtil;
import org.dubhe.biz.file.config.NfsConfig;
import org.dubhe.biz.log.enums.LogEnum; import org.dubhe.biz.log.enums.LogEnum;
import org.dubhe.biz.log.utils.LogUtil; import org.dubhe.biz.log.utils.LogUtil;
import org.dubhe.biz.permission.annotation.DataPermissionMethod; import org.dubhe.biz.permission.annotation.DataPermissionMethod;
import org.dubhe.biz.permission.base.BaseService; import org.dubhe.biz.permission.base.BaseService;
import org.dubhe.image.async.HarborImagePushAsync;
import org.dubhe.image.dao.PtImageMapper; import org.dubhe.image.dao.PtImageMapper;
import org.dubhe.image.domain.dto.*;
import org.dubhe.image.domain.dto.PtImageDeleteDTO;
import org.dubhe.image.domain.dto.PtImageQueryDTO;
import org.dubhe.image.domain.dto.PtImageQueryImageDTO;
import org.dubhe.image.domain.dto.PtImageQueryNameDTO;
import org.dubhe.image.domain.dto.PtImageQueryUrlDTO;
import org.dubhe.image.domain.dto.PtImageSaveDTO;
import org.dubhe.image.domain.dto.PtImageUpdateDTO;
import org.dubhe.image.domain.entity.PtImage; import org.dubhe.image.domain.entity.PtImage;
import org.dubhe.image.domain.vo.PtImageQueryVO; import org.dubhe.image.domain.vo.PtImageQueryVO;
import org.dubhe.image.service.PtImageService; import org.dubhe.image.service.PtImageService;
import org.dubhe.recycle.config.RecycleConfig;
import org.dubhe.recycle.domain.dto.RecycleCreateDTO;
import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO;
import org.dubhe.recycle.enums.RecycleModuleEnum;
import org.dubhe.recycle.enums.RecycleResourceEnum;
import org.dubhe.recycle.enums.RecycleTypeEnum;
import org.dubhe.recycle.service.RecycleService;
import org.dubhe.recycle.utils.RecycleTool;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;


import java.util.*;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors; import java.util.stream.Collectors;


/** /**
@@ -77,25 +82,9 @@ public class PtImageServiceImpl implements PtImageService {
@Autowired @Autowired
private PtImageMapper ptImageMapper; private PtImageMapper ptImageMapper;



@Autowired
private NfsConfig nfsConfig;

@Autowired
private HarborImagePushAsync imagePushAsync;

@Autowired
private HarborProperties harborProperties;

@Autowired @Autowired
private UserContextService userContextService; private UserContextService userContextService;


@Autowired
private RecycleService recycleService;

@Autowired
private RecycleConfig recycleConfig;



public final static List<String> FIELD_NAMES; public final static List<String> FIELD_NAMES;


@@ -119,9 +108,7 @@ public class PtImageServiceImpl implements PtImageService {


QueryWrapper<PtImage> query = new QueryWrapper<>(); QueryWrapper<PtImage> query = new QueryWrapper<>();
query.eq("deleted", NumberConstant.NUMBER_0); query.eq("deleted", NumberConstant.NUMBER_0);
if (ptImageQueryDTO.getImageStatus() != null) {
query.eq("image_status", ptImageQueryDTO.getImageStatus());
}

if (ptImageQueryDTO.getImageResource() != null) { if (ptImageQueryDTO.getImageResource() != null) {
query.eq("image_resource", ptImageQueryDTO.getImageResource()); query.eq("image_resource", ptImageQueryDTO.getImageResource());
} }
@@ -166,67 +153,55 @@ public class PtImageServiceImpl implements PtImageService {
} }


/** /**
* 上传镜像到harbor
* 保存镜像信息
* *
* @param ptImageUploadDTO 上传条件
* @param ptImageSaveDTO 镜像信息DTO
*/ */
@Override @Override
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public void uploadImage(PtImageUploadDTO ptImageUploadDTO) {
public void saveImageInfo(PtImageSaveDTO ptImageSaveDTO) {
UserContext user = userContextService.getCurUser(); UserContext user = userContextService.getCurUser();


//普通用户不支持上传预置镜像
if (ImageSourceEnum.PRE.getCode().equals(ptImageUploadDTO.getImageResource()) &&
validImageInfo(ptImageSaveDTO.getImageUrl(), ptImageSaveDTO.getImageName(), ptImageSaveDTO.getImageTag());

//普通用户不支持预置镜像信息的保存
if (ImageSourceEnum.PRE.getCode().equals(ptImageSaveDTO.getImageResource()) &&
!BaseService.isAdmin(user)) { !BaseService.isAdmin(user)) {
throw new BusinessException(ResponseCode.UNAUTHORIZED, "普通用户不支持上传预置镜像!");
throw new BusinessException(ResponseCode.UNAUTHORIZED, "普通用户不支持保存预置镜像!");
} }


//校验用户自定义镜像不能和预置镜像重名 //校验用户自定义镜像不能和预置镜像重名
List<PtImage> resList = checkUploadImage(ptImageUploadDTO, null, ImageSourceEnum.PRE.getCode());
List<PtImage> resList = checkSaveImage(ptImageSaveDTO, null, ImageSourceEnum.PRE.getCode());
if (CollUtil.isNotEmpty(resList)) { if (CollUtil.isNotEmpty(resList)) {
throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复,请重新上传!");
throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复");
} }


//同一用户上传镜像的(userId+imageName+imageTag)存在的情况下是不能重复上传的 //同一用户上传镜像的(userId+imageName+imageTag)存在的情况下是不能重复上传的
List<PtImage> imageList = checkUploadImage(ptImageUploadDTO, user, ImageSourceEnum.MINE.getCode());
if (CollUtil.isNotEmpty(imageList) && ImageStateEnum.SUCCESS.getCode().equals(imageList.get(0).getImageStatus())) {
throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在,不允许重复上传!");
List<PtImage> imageList = checkSaveImage(ptImageSaveDTO, user, ImageSourceEnum.MINE.getCode());
if (CollUtil.isNotEmpty(imageList)) {
throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在!");
} }


String harborImagePath = StringConstant.DEFAULT_IMAGE_PROJECT + StrUtil.SLASH + ptImageUploadDTO.getImageName() + StrUtil.DASHED + user.getId() +
StrUtil.COLON + ptImageUploadDTO.getImageTag();

//存储镜像信息 //存储镜像信息
PtImage ptImage = new PtImage(); PtImage ptImage = new PtImage();
ptImage.setImageName(ptImageUploadDTO.getImageName())
.setImageUrl(harborImagePath)
.setImageResource(ptImageUploadDTO.getImageResource())
.setImageStatus(ImageStateEnum.MAKING.getCode())
.setRemark(ptImageUploadDTO.getRemark())
.setImageTag(ptImageUploadDTO.getImageTag())
ptImage.setImageName(ptImageSaveDTO.getImageName())
.setImageUrl(ptImageSaveDTO.getImageUrl())
.setImageResource(ptImageSaveDTO.getImageResource())
.setRemark(ptImageSaveDTO.getRemark())
.setImageTag(ptImageSaveDTO.getImageTag())
.setCreateUserId(user.getId()); .setCreateUserId(user.getId());
if (ImageSourceEnum.PRE.getCode().equals(ptImageUploadDTO.getImageResource())) {
if (ImageSourceEnum.PRE.getCode().equals(ptImageSaveDTO.getImageResource())) {
ptImage.setOriginUserId(MagicNumConstant.ZERO_LONG); ptImage.setOriginUserId(MagicNumConstant.ZERO_LONG);
} else { } else {
ptImage.setOriginUserId(user.getId()); ptImage.setOriginUserId(user.getId());
} }


int count = ptImageMapper.insert(ptImage);
if (count < 1) {
imagePushAsync.updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode());
throw new BusinessException("内部错误!");
}
for (Integer imageType : ptImageUploadDTO.getImageTypes()) {
ptImageMapper.insert(ptImage);

for (Integer imageType : ptImageSaveDTO.getImageTypes()) {
ptImageMapper.insertImageType(ptImage.getId(), imageType); ptImageMapper.insertImageType(ptImage.getId(), imageType);
} }
//shell脚本上传镜像
try {
String imagePath = nfsConfig.getRootDir() + nfsConfig.getBucket().substring(1) + ptImageUploadDTO.getImagePath();
String imageNameAndTag = ptImageUploadDTO.getImageName() + StrUtil.DASHED + user.getId() + StrUtil.COLON + ptImageUploadDTO.getImageTag();
imagePushAsync.execShell(imagePath, imageNameAndTag, ptImage);
} catch (Exception e) {
LogUtil.error(LogEnum.IMAGE, "Image upload exception {}", e);
throw new BusinessException("镜像上传失败!");
}
} }


/** /**
@@ -240,9 +215,8 @@ public class PtImageServiceImpl implements PtImageService {
public List<PtImage> searchImages(PtImageQueryImageDTO ptImageQueryImageDTO) { public List<PtImage> searchImages(PtImageQueryImageDTO ptImageQueryImageDTO) {
LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(PtImage::getImageName, ptImageQueryImageDTO.getImageName()) queryWrapper.eq(PtImage::getImageName, ptImageQueryImageDTO.getImageName())
.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode())
.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); .eq(PtImage::getDeleted, NumberConstant.NUMBER_0);
List<PtImage> ptImages = new ArrayList<>();
List<PtImage> ptImages;
if (ptImageQueryImageDTO.getImageResource() != null) { if (ptImageQueryImageDTO.getImageResource() != null) {
queryWrapper.eq(PtImage::getImageResource, ptImageQueryImageDTO.getImageResource()); queryWrapper.eq(PtImage::getImageResource, ptImageQueryImageDTO.getImageResource());
} }
@@ -252,18 +226,13 @@ public class PtImageServiceImpl implements PtImageService {
ptImages = ptImageMapper.getImagesByTypes(queryWrapper, ptImageQueryImageDTO.getImageTypes()); ptImages = ptImageMapper.getImagesByTypes(queryWrapper, ptImageQueryImageDTO.getImageTypes());
} }


List<PtImage> list = new ArrayList<>();
if (CollUtil.isEmpty(ptImages)) { if (CollUtil.isEmpty(ptImages)) {
throw new BusinessException(ResponseCode.BADREQUEST, "未查询到镜像信息!"); throw new BusinessException(ResponseCode.BADREQUEST, "未查询到镜像信息!");
} }


ptImages = ptImages.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> ptImages = ptImages.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() ->
new TreeSet<>(Comparator.comparing(PtImage::getImageTag))), ArrayList::new)); new TreeSet<>(Comparator.comparing(PtImage::getImageTag))), ArrayList::new));
ptImages.stream().forEach(ptImage -> {
ptImage.setImageUrl(harborProperties.getAddress() + StrUtil.SLASH + ptImage.getImageUrl());
list.add(ptImage);
});
return list;
return ptImages;
} }




@@ -282,37 +251,9 @@ public class PtImageServiceImpl implements PtImageService {
//删除本地镜像 //删除本地镜像
imageList.forEach(image -> { imageList.forEach(image -> {
ptImageMapper.deleteById(image.getId()); ptImageMapper.deleteById(image.getId());
//创建镜像回收任务
createRecycleTask(image);
}); });
} }


/**
* 文件定时清理
*
* @param ptImage 镜像实体对象
*/
private void createRecycleTask(PtImage ptImage) {

String imageUrl = ImageStateEnum.SUCCESS.getCode().equals(ptImage.getImageStatus()) ? ptImage.getImageUrl() : "";
RecycleCreateDTO recycleCreateDTO = RecycleCreateDTO.builder()
.recycleModule(RecycleModuleEnum.BIZ_IMAGE.getValue())
.recycleDelayDate(recycleConfig.getImageValid())
.recycleNote(RecycleTool.generateRecycleNote("删除镜像", ptImage.getImageName(), ptImage.getId()))
.recycleCustom(RecycleResourceEnum.IMAGE_RECYCLE_FILE.getClassName())
.restoreCustom(RecycleResourceEnum.IMAGE_RECYCLE_FILE.getClassName())
.remark(String.valueOf(ptImage.getId()))
.build();
recycleCreateDTO.addRecycleDetailCreateDTO(RecycleDetailCreateDTO.builder()
.recycleCondition(imageUrl)
.recycleType(RecycleTypeEnum.FILE.getCode())
.recycleNote(RecycleTool.generateRecycleNote("删除镜像", ptImage.getImageName(), ptImage.getId()))
.remark(String.valueOf(ptImage.getId()))
.build()
);
recycleService.createRecycleTask(recycleCreateDTO);
}

/** /**
* 修改镜像信息 * 修改镜像信息
* *
@@ -324,29 +265,33 @@ public class PtImageServiceImpl implements PtImageService {
public void updateTrainImage(PtImageUpdateDTO imageUpdateDTO) { public void updateTrainImage(PtImageUpdateDTO imageUpdateDTO) {


UserContext curUser = userContextService.getCurUser(); UserContext curUser = userContextService.getCurUser();
List<PtImage> imageList = ptImageMapper.selectList(new LambdaQueryWrapper<PtImage>()
.in(PtImage::getId, imageUpdateDTO.getIds()));
PtImage image = ptImageMapper.selectById(imageUpdateDTO.getId());


if (CollUtil.isEmpty(imageList)) {
throw new BusinessException("内部错误");
if (image == null) {
throw new BusinessException("镜像不存在!");
} }
for (PtImage image : imageList) {
//非管理员禁止修改预置镜像
if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource()) && !BaseService.isAdmin(curUser)) {
throw new BusinessException("非管理员无权限修改预置镜像信息");
}
image.setRemark(imageUpdateDTO.getRemark());
ptImageMapper.updateById(image);
List<Integer> imageTypes = ptImageMapper.selectImageType(image.getId());
for (Integer imageType : imageUpdateDTO.getImageTypes()) {
if (!CollectionUtil.contains(imageTypes, imageType)) {
ptImageMapper.insertImageType(image.getId(), imageType);
}

validImageInfo(imageUpdateDTO.getImageUrl(), imageUpdateDTO.getImageName(), imageUpdateDTO.getImageTag());

//非管理员禁止修改预置镜像
if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource()) && !BaseService.isAdmin(curUser)) {
throw new BusinessException("非管理员无权限修改预置镜像信息");
}
image.setImageTypes(imageUpdateDTO.getImageTypes())
.setImageName(imageUpdateDTO.getImageName())
.setImageUrl(imageUpdateDTO.getImageUrl())
.setImageTag(imageUpdateDTO.getImageTag())
.setRemark(imageUpdateDTO.getRemark());
ptImageMapper.updateById(image);
List<Integer> imageTypes = ptImageMapper.selectImageType(image.getId());
for (Integer imageType : imageUpdateDTO.getImageTypes()) {
if (!CollectionUtil.contains(imageTypes, imageType)) {
ptImageMapper.insertImageType(image.getId(), imageType);
} }
for (Integer imageType : imageTypes) {
if (!CollectionUtil.contains(imageUpdateDTO.getImageTypes(), imageType)) {
ptImageMapper.deleteImageType(image.getId(), imageType);
}
}
for (Integer imageType : imageTypes) {
if (!CollectionUtil.contains(imageUpdateDTO.getImageTypes(), imageType)) {
ptImageMapper.deleteImageType(image.getId(), imageType);
} }
} }
} }
@@ -362,10 +307,8 @@ public class PtImageServiceImpl implements PtImageService {
//从会话中获取用户信息 //从会话中获取用户信息
UserContext user = userContextService.getCurUser(); UserContext user = userContextService.getCurUser();
QueryWrapper<PtImage> queryWrapper = new QueryWrapper<>(); QueryWrapper<PtImage> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("image_status", ImageStateEnum.SUCCESS.getCode());
queryWrapper.eq("deleted", NumberConstant.NUMBER_0); queryWrapper.eq("deleted", NumberConstant.NUMBER_0);
if (!BaseService.isAdmin(user)) { if (!BaseService.isAdmin(user)) {

queryWrapper.in("origin_user_id", user.getId(), 0L); queryWrapper.in("origin_user_id", user.getId(), 0L);
} }
if (ptImageQueryNameDTO.getImageResource() != null) { if (ptImageQueryNameDTO.getImageResource() != null) {
@@ -403,12 +346,8 @@ public class PtImageServiceImpl implements PtImageService {
throw new BusinessException(ResponseCode.BADREQUEST, "该镜像不存在或镜像用途不支持!"); throw new BusinessException(ResponseCode.BADREQUEST, "该镜像不存在或镜像用途不支持!");
} }


//仅支持[制作成功]状态镜像设置为默认镜像
if (!ImageStateEnum.SUCCESS.getCode().equals(image.getImageStatus())) {
throw new BusinessException(ResponseCode.BADREQUEST, "仅支持[制作成功]状态镜像设置为默认镜像!");
}


if (image.getImageResource() != ImageSourceEnum.PRE.getCode()) {
if (!image.getImageResource().equals(ImageSourceEnum.PRE.getCode())) {
throw new BusinessException(ResponseCode.BADREQUEST, "非预制镜像不能设置为默认镜像!"); throw new BusinessException(ResponseCode.BADREQUEST, "非预制镜像不能设置为默认镜像!");
} }


@@ -434,7 +373,7 @@ public class PtImageServiceImpl implements PtImageService {
/** /**
* 获取镜像URL * 获取镜像URL
* *
* @param imageQueryUrlDTO 查询镜像路径DTO
* @param imageQueryUrlDTO 查询镜像地址DTO
* @return String 镜像url * @return String 镜像url
*/ */
@Override @Override
@@ -453,7 +392,6 @@ public class PtImageServiceImpl implements PtImageService {
if (imageQueryUrlDTO.getIsDefault() != null) { if (imageQueryUrlDTO.getIsDefault() != null) {
queryWrapper.eq(PtImage::getIsDefault, imageQueryUrlDTO.getIsDefault()); queryWrapper.eq(PtImage::getIsDefault, imageQueryUrlDTO.getIsDefault());
} }
queryWrapper.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode());
queryWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); queryWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0);
List<PtImage> imageList = new ArrayList<>(); List<PtImage> imageList = new ArrayList<>();
if (CollectionUtil.isNotEmpty(imageQueryUrlDTO.getImageTypes())) { if (CollectionUtil.isNotEmpty(imageQueryUrlDTO.getImageTypes())) {
@@ -470,23 +408,13 @@ public class PtImageServiceImpl implements PtImageService {
return imageUrl; return imageUrl;
} }


/**
* 自定义镜像回收的还原实现
*
* @param dto 资源回收DTO对象
*/
@Override
public void recycleRollback(RecycleCreateDTO dto) {
String imageId = dto.getRemark();
ptImageMapper.updateDeletedById(Long.valueOf(imageId), false);
}

@Override @Override
@DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC)
public List<PtImage> getTerminalImageList() { public List<PtImage> getTerminalImageList() {
UserContext user = userContextService.getCurUser(); UserContext user = userContextService.getCurUser();
LambdaQueryWrapper<PtImage> queryTerminalWrapper = new LambdaQueryWrapper<>(); LambdaQueryWrapper<PtImage> queryTerminalWrapper = new LambdaQueryWrapper<>();
queryTerminalWrapper.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()).eq(PtImage::getDeleted, NumberConstant.NUMBER_0);;
queryTerminalWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0);
;
if (user != null && !BaseService.isAdmin()) { if (user != null && !BaseService.isAdmin()) {
queryTerminalWrapper.and(wrapper -> wrapper.eq(PtImage::getCreateUserId, user.getId()).or().eq(PtImage::getImageResource, ImageSourceEnum.PRE.getCode())) queryTerminalWrapper.and(wrapper -> wrapper.eq(PtImage::getCreateUserId, user.getId()).or().eq(PtImage::getImageResource, ImageSourceEnum.PRE.getCode()))
.and(wrapper -> wrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0)); .and(wrapper -> wrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0));
@@ -509,17 +437,16 @@ public class PtImageServiceImpl implements PtImageService {




/** /**
* @param ptImageUploadDTO 镜像上传逻辑校验
* @param ptImageSaveDTO 镜像保存逻辑校验
* @param user 用户 * @param user 用户
* @return List<PtImage> 镜像列表 * @return List<PtImage> 镜像列表
**/ **/
@DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC)
private List<PtImage> checkUploadImage(PtImageUploadDTO ptImageUploadDTO, UserContext user, Integer source) {
private List<PtImage> checkSaveImage(PtImageSaveDTO ptImageSaveDTO, UserContext user, Integer source) {


LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>();


queryWrapper.eq(PtImage::getImageName, ptImageUploadDTO.getImageName())
.eq(PtImage::getImageTag, ptImageUploadDTO.getImageTag())
queryWrapper.eq(PtImage::getImageUrl, ptImageSaveDTO.getImageUrl())
.eq(PtImage::getImageResource, source); .eq(PtImage::getImageResource, source);


if (user != null) { if (user != null) {
@@ -549,7 +476,7 @@ public class PtImageServiceImpl implements PtImageService {


ptImages.stream().forEach( ptImages.stream().forEach(
ptImage -> { ptImage -> {
PtImageVO ptImageVO =new PtImageVO();
PtImageVO ptImageVO = new PtImageVO();
ptImageVO.setId(ptImage.getId()); ptImageVO.setId(ptImage.getId());
ptImageVO.setName(ptImage.getImageName()); ptImageVO.setName(ptImage.getImageName());
ptImageVO.setTag(ptImage.getImageTag()); ptImageVO.setTag(ptImage.getImageTag());
@@ -558,4 +485,24 @@ public class PtImageServiceImpl implements PtImageService {
); );
return ptImageVOS; return ptImageVOS;
} }
}

/**
* 校验镜像地址与镜像名称、tag是否匹配
*
* @param imageUrl 镜像地址
* @param imageName 镜像名称
* @param imageTag 镜像tag
*/
private void validImageInfo(String imageUrl, String imageName, String imageTag) {
// 标准镜像地址:镜像地址域名/命名空间/镜像名:镜像tag

// 分解镜像地址
String[] split = imageUrl.substring(imageUrl.lastIndexOf(StrUtil.SLASH) + 1).split(StrUtil.COLON);
if (!split[0].equals(imageName)) {
throw new BusinessException("镜像名称不匹配");
}
if (!split[1].equals(imageTag)) {
throw new BusinessException("镜像版本号不匹配");
}
}
}

+ 0
- 77
dubhe-server/dubhe-image/src/main/java/org/dubhe/image/service/task/ImageRecycleFile.java View File

@@ -1,77 +0,0 @@
/**
* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/
package org.dubhe.image.service.task;

import cn.hutool.core.util.StrUtil;
import org.dubhe.biz.base.constant.HarborProperties;
import org.dubhe.biz.log.enums.LogEnum;
import org.dubhe.biz.log.utils.LogUtil;
import org.dubhe.harbor.api.HarborApi;
import org.dubhe.image.service.PtImageService;
import org.dubhe.recycle.domain.dto.RecycleCreateDTO;
import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO;
import org.dubhe.recycle.global.AbstractGlobalRecycle;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.stereotype.Component;

/**
* @description 镜像资源回收/还原
* @date 2021-03-23
*/
@RefreshScope
@Component(value = "imageRecycleFile")
public class ImageRecycleFile extends AbstractGlobalRecycle {

@Autowired
private HarborApi harborApi;

@Autowired
private HarborProperties harborProperties;

@Autowired
private PtImageService ptImageService;

/**
* 自定义回收镜像实现
*
* @param detail 数据清理详情参数
* @param dto 资源回收创建对象
* @return true 继续执行,false 中断任务详情回收(本次无法执行完毕,创建新任务到下次执行)
*/
@Override
protected boolean clearDetail(RecycleDetailCreateDTO detail, RecycleCreateDTO dto) throws Exception {
LogUtil.info(LogEnum.IMAGE, "image custom recycle file,params:{}", detail);
if (StrUtil.isNotBlank(detail.getRecycleCondition())) {
String imageUrl = harborProperties.getAddress() + StrUtil.SLASH + detail.getRecycleCondition();
LogUtil.info(LogEnum.IMAGE, "delete harbor image url:{}", imageUrl);
//同步删除harbor镜像
harborApi.deleteImageByTag(imageUrl);
}
return true;
}

/**
* 数据还原
*
* @param dto 数据清理参数
*/
@Override
protected void rollback(RecycleCreateDTO dto) {
ptImageService.recycleRollback(dto);
}
}

+ 4
- 4
dubhe-server/dubhe-image/src/test/java/org/dubhe/image/PtImageTest.java View File

@@ -21,8 +21,8 @@ import com.alibaba.fastjson.JSON;
import org.dubhe.biz.base.constant.AuthConst; import org.dubhe.biz.base.constant.AuthConst;
import org.dubhe.cloud.unittest.base.BaseTest; import org.dubhe.cloud.unittest.base.BaseTest;
import org.dubhe.image.domain.dto.PtImageDeleteDTO; import org.dubhe.image.domain.dto.PtImageDeleteDTO;
import org.dubhe.image.domain.dto.PtImageSaveDTO;
import org.dubhe.image.domain.dto.PtImageUpdateDTO; import org.dubhe.image.domain.dto.PtImageUpdateDTO;
import org.dubhe.image.domain.dto.PtImageUploadDTO;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
@@ -78,9 +78,9 @@ public class PtImageTest extends BaseTest {
*/ */
@Test @Test
public void uploadImageTest() throws Exception { public void uploadImageTest() throws Exception {
PtImageUploadDTO ptImageUploadDTO = new PtImageUploadDTO();
PtImageSaveDTO ptImageUploadDTO = new PtImageSaveDTO();
ptImageUploadDTO.setImageName("mysql"); ptImageUploadDTO.setImageName("mysql");
ptImageUploadDTO.setImagePath("F:/mysql.tar");
ptImageUploadDTO.setImageUrl("F:/mysql.tar");
ptImageUploadDTO.setImageTag("5.7"); ptImageUploadDTO.setImageTag("5.7");
ptImageUploadDTO.setRemark("测试上传镜像"); ptImageUploadDTO.setRemark("测试上传镜像");


@@ -94,7 +94,7 @@ public class PtImageTest extends BaseTest {
@Test @Test
public void updateImageTest() throws Exception { public void updateImageTest() throws Exception {
PtImageUpdateDTO imageUpdateDTO = new PtImageUpdateDTO(); PtImageUpdateDTO imageUpdateDTO = new PtImageUpdateDTO();
imageUpdateDTO.setIds(Arrays.asList());
imageUpdateDTO.setId(1L);
imageUpdateDTO.setRemark(""); imageUpdateDTO.setRemark("");
mockMvcTest(MockMvcRequestBuilders.put("/ptImage"), JSON.toJSONString(imageUpdateDTO), mockMvcTest(MockMvcRequestBuilders.put("/ptImage"), JSON.toJSONString(imageUpdateDTO),
MockMvcResultMatchers.status().is2xxSuccessful(), 200); MockMvcResultMatchers.status().is2xxSuccessful(), 200);


+ 1
- 1
dubhe-server/dubhe-k8s/src/main/java/org/dubhe/dubhek8s/event/callback/DeploymentCallback.java View File

@@ -75,7 +75,7 @@ public class DeploymentCallback extends Observable {
setChanged(); setChanged();
notifyObservers(deployment); notifyObservers(deployment);
if (StringUtils.isNotEmpty(businessLabel)){ if (StringUtils.isNotEmpty(businessLabel)){
BaseK8sDeploymentCallbackCreateDTO baseK8sDeploymentCallbackCreateDTO = new BaseK8sDeploymentCallbackCreateDTO(deployment.getNamespace(), deployment.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),deployment.getName(), deployment.getReadyReplicas() == null?0:deployment.getReadyReplicas(), deployment.getReplicas() == null?0:deployment.getReplicas());
BaseK8sDeploymentCallbackCreateDTO baseK8sDeploymentCallbackCreateDTO = new BaseK8sDeploymentCallbackCreateDTO(deployment.getNamespace(), deployment.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),deployment.getName(), deployment.getReadyReplicas() == null?0:deployment.getReadyReplicas(), deployment.getReplicas() == null?0:deployment.getReplicas(),watcherActionEnum.getAction());
String url = k8sCallBackTool.getDeploymentCallbackUrl(businessLabel); String url = k8sCallBackTool.getDeploymentCallbackUrl(businessLabel);
String token = k8sCallBackTool.generateToken(); String token = k8sCallBackTool.generateToken();




+ 13
- 2
dubhe-server/dubhe-k8s/src/main/java/org/dubhe/dubhek8s/observer/TrainJobObserver.java View File

@@ -17,6 +17,7 @@


package org.dubhe.dubhek8s.observer; package org.dubhe.dubhek8s.observer;


import org.dubhe.biz.base.constant.MagicNumConstant;
import org.dubhe.biz.base.enums.BizEnum; import org.dubhe.biz.base.enums.BizEnum;
import org.dubhe.biz.base.utils.SpringContextHolder; import org.dubhe.biz.base.utils.SpringContextHolder;
import org.dubhe.biz.log.enums.LogEnum; import org.dubhe.biz.log.enums.LogEnum;
@@ -54,8 +55,18 @@ public class TrainJobObserver implements Observer {
BizPod pod = (BizPod)arg; BizPod pod = (BizPod)arg;
boolean trainJobFailed = PodPhaseEnum.FAILED.getPhase().equals(pod.getPhase()) && BizEnum.ALGORITHM.getBizCode().equals(pod.getBusinessLabel()) && SpringContextHolder.getActiveProfile().equals(pod.getLabel(K8sLabelConstants.PLATFORM_RUNTIME_ENV)); boolean trainJobFailed = PodPhaseEnum.FAILED.getPhase().equals(pod.getPhase()) && BizEnum.ALGORITHM.getBizCode().equals(pod.getBusinessLabel()) && SpringContextHolder.getActiveProfile().equals(pod.getLabel(K8sLabelConstants.PLATFORM_RUNTIME_ENV));
if (trainJobFailed){ if (trainJobFailed){
LogUtil.warn(LogEnum.BIZ_K8S,"delete failed train job resourceName {};phase {};podName {}",pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),pod.getPhase(),pod.getName());
trainJobApi.delete(pod.getNamespace(),pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE));
new Thread(new Runnable(){
@Override
public void run(){
try {
Thread.sleep(MagicNumConstant.ONE_MINUTE);
} catch (InterruptedException e) {
LogUtil.error(LogEnum.BIZ_K8S,"TrainJobObserver update error {}",e.getMessage(),e);
}
LogUtil.warn(LogEnum.BIZ_K8S,"delete failed train job resourceName {};phase {};podName {}",pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),pod.getPhase(),pod.getName());
trainJobApi.delete(pod.getNamespace(),pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE));
}
}).start();
} }
} }
} }


+ 2
- 2
dubhe-server/dubhe-k8s/src/main/java/org/dubhe/dubhek8s/service/impl/SystemNodeServiceImpl.java View File

@@ -79,7 +79,7 @@ public class SystemNodeServiceImpl implements SystemNodeService {
@Resource @Resource
private AdminClient adminClient; private AdminClient adminClient;


private final static String INTERNAL_IP = "InternalIP";




/** /**
@@ -231,7 +231,7 @@ public class SystemNodeServiceImpl implements SystemNodeService {
nodeDTO.setUid(node.getUid()); nodeDTO.setUid(node.getUid());
nodeDTO.setName(node.getName()); nodeDTO.setName(node.getName());
node.getAddresses().stream().forEach(bizNodeAddress -> { node.getAddresses().stream().forEach(bizNodeAddress -> {
if (INTERNAL_IP.equals(bizNodeAddress.getType())) {
if (K8sParamConstants.INTERNAL_IP.equals(bizNodeAddress.getType())) {
nodeDTO.setIp(bizNodeAddress.getAddress()); nodeDTO.setIp(bizNodeAddress.getAddress());
return; return;
} }


+ 1
- 1
dubhe-server/dubhe-k8s/src/test/java/org/dubhe/dubhek8s/k8s/PodCallbackTest.java
File diff suppressed because it is too large
View File


+ 1
- 1
dubhe-server/dubhe-measure/src/main/java/org/dubhe/measure/service/impl/PtAtlasCameraServiceImpl.java View File

@@ -141,7 +141,7 @@ public class PtAtlasCameraServiceImpl implements PtAtlasCameraService {
ResponseEntity<Map> exchange = restTemplate.exchange(atlasUrlProperties.getVms() + "/checkCameraUrl", HttpMethod.POST, requestEntity, Map.class, param); ResponseEntity<Map> exchange = restTemplate.exchange(atlasUrlProperties.getVms() + "/checkCameraUrl", HttpMethod.POST, requestEntity, Map.class, param);
Map<String, String> statusMap = (Map) exchange.getBody().get("data"); Map<String, String> statusMap = (Map) exchange.getBody().get("data");
if (statusMap.containsKey("cameraUrl")) { if (statusMap.containsKey("cameraUrl")) {
ptAtlasCamera.setHlsUrl(statusMap.get("cameraUrl").replace("127.0.0.1", "127.0.0.1"));
ptAtlasCamera.setHlsUrl(statusMap.get("cameraUrl").replace("172.18.26.2", "10.105.10.51"));
ptAtlasCameraMapper.updateById(ptAtlasCamera); ptAtlasCameraMapper.updateById(ptAtlasCamera);
} }
} catch (Exception e) { } catch (Exception e) {


+ 10
- 9
dubhe-server/dubhe-notebook/src/main/java/org/dubhe/notebook/service/impl/NoteBookServiceImpl.java View File

@@ -21,13 +21,10 @@ import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateBetween; import cn.hutool.core.date.DateBetween;
import cn.hutool.core.date.DateUnit; import cn.hutool.core.date.DateUnit;
import cn.hutool.core.util.RandomUtil; import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.dubhe.biz.base.utils.StringUtils;
import org.dubhe.biz.base.constant.HarborProperties;
import org.dubhe.biz.base.constant.MagicNumConstant; import org.dubhe.biz.base.constant.MagicNumConstant;
import org.dubhe.biz.base.constant.NumberConstant; import org.dubhe.biz.base.constant.NumberConstant;
import org.dubhe.biz.base.constant.StringConstant; import org.dubhe.biz.base.constant.StringConstant;
@@ -45,8 +42,10 @@ import org.dubhe.biz.base.service.UserContextService;
import org.dubhe.biz.base.utils.HttpUtils; import org.dubhe.biz.base.utils.HttpUtils;
import org.dubhe.biz.base.utils.NumberUtil; import org.dubhe.biz.base.utils.NumberUtil;
import org.dubhe.biz.base.utils.ResultUtil; import org.dubhe.biz.base.utils.ResultUtil;
import org.dubhe.biz.base.utils.StringUtils;
import org.dubhe.biz.base.vo.DataResponseBody; import org.dubhe.biz.base.vo.DataResponseBody;
import org.dubhe.biz.base.vo.DatasetVO; import org.dubhe.biz.base.vo.DatasetVO;
import org.dubhe.biz.base.vo.NoteBookVO;
import org.dubhe.biz.db.utils.PageUtil; import org.dubhe.biz.db.utils.PageUtil;
import org.dubhe.biz.db.utils.WrapperHelp; import org.dubhe.biz.db.utils.WrapperHelp;
import org.dubhe.biz.file.api.FileStoreApi; import org.dubhe.biz.file.api.FileStoreApi;
@@ -79,7 +78,6 @@ import org.dubhe.notebook.enums.NoteBookStatusEnum;
import org.dubhe.notebook.service.NoteBookService; import org.dubhe.notebook.service.NoteBookService;
import org.dubhe.notebook.service.ProcessNotebookCommand; import org.dubhe.notebook.service.ProcessNotebookCommand;
import org.dubhe.notebook.utils.NotebookUtil; import org.dubhe.notebook.utils.NotebookUtil;
import org.dubhe.biz.base.vo.NoteBookVO;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
@@ -89,7 +87,13 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;


import java.io.File; import java.io.File;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;


/** /**
@@ -127,9 +131,6 @@ public class NoteBookServiceImpl implements NoteBookService {
@Autowired @Autowired
private ImageClient imageClient; private ImageClient imageClient;


@Autowired
private HarborProperties harborProperties;

@Autowired @Autowired
@Qualifier("hostFileStoreApiImpl") @Qualifier("hostFileStoreApiImpl")
private FileStoreApi fileStoreApi; private FileStoreApi fileStoreApi;
@@ -223,7 +224,7 @@ public class NoteBookServiceImpl implements NoteBookService {
throw new BusinessException(ImageTypeEnum.NOTEBOOK.getCode() + "未配置默认镜像!"); throw new BusinessException(ImageTypeEnum.NOTEBOOK.getCode() + "未配置默认镜像!");
} }


return harborProperties.getAddress() + StrUtil.SLASH + imageUrl;
return imageUrl;
} }


/** /**


+ 0
- 39
dubhe-server/dubhe-point-cloud/src/main/java/org/dubhe/pointcloud/config/PointCloudHarborConfig.java View File

@@ -1,39 +0,0 @@
/**
* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/
package org.dubhe.pointcloud.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

/**
* @description harbor相关配置
* @date 2020-07-17
*/
@Data
@Component
@ConfigurationProperties(prefix = "harbor")
public class PointCloudHarborConfig {

private String address;

private String username;

private String password;

private String modelName;
}

+ 1
- 5
dubhe-server/dubhe-point-cloud/src/main/java/org/dubhe/pointcloud/service/impl/PcDatasetServiceImpl.java View File

@@ -66,7 +66,6 @@ import org.dubhe.pointcloud.client.DataClient;
import org.dubhe.pointcloud.client.ImageClient; import org.dubhe.pointcloud.client.ImageClient;
import org.dubhe.pointcloud.client.ModelClient; import org.dubhe.pointcloud.client.ModelClient;
import org.dubhe.pointcloud.common.Constant; import org.dubhe.pointcloud.common.Constant;
import org.dubhe.pointcloud.config.PointCloudHarborConfig;
import org.dubhe.pointcloud.dao.PcAnnotationDetailMapper; import org.dubhe.pointcloud.dao.PcAnnotationDetailMapper;
import org.dubhe.pointcloud.dao.PcDatasetFileMapper; import org.dubhe.pointcloud.dao.PcDatasetFileMapper;
import org.dubhe.pointcloud.dao.PcDatasetMapper; import org.dubhe.pointcloud.dao.PcDatasetMapper;
@@ -141,9 +140,6 @@ public class PcDatasetServiceImpl implements PcDatasetService {
@Resource @Resource
private DeployAsyncTask deployAsyncTask; private DeployAsyncTask deployAsyncTask;


@Resource
private PointCloudHarborConfig pointCloudHarborConfig;

@Resource @Resource
private ImageClient imageClient; private ImageClient imageClient;


@@ -652,7 +648,7 @@ public class PcDatasetServiceImpl implements PcDatasetService {
if (!dataResponseBody.succeed()) { if (!dataResponseBody.succeed()) {
throw new BusinessException(ErrorEnum.CALL_IMAGE_SERVER_FAIL); throw new BusinessException(ErrorEnum.CALL_IMAGE_SERVER_FAIL);
} }
return pointCloudHarborConfig.getAddress() + SymbolConstant.SLASH + dataResponseBody.getData();
return dataResponseBody.getData();
} }


/** /**


+ 36
- 16
dubhe-server/dubhe-serving-gateway/src/main/java/org/dubhe/servinggateway/config/GatewayServiceHandler.java View File

@@ -18,8 +18,10 @@
package org.dubhe.servinggateway.config; package org.dubhe.servinggateway.config;


import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.dubhe.biz.base.constant.MagicNumConstant;
import org.dubhe.biz.base.constant.NumberConstant; import org.dubhe.biz.base.constant.NumberConstant;
import org.dubhe.biz.base.constant.SymbolConstant; import org.dubhe.biz.base.constant.SymbolConstant;
import org.dubhe.biz.base.utils.StringUtils;
import org.dubhe.biz.log.enums.LogEnum; import org.dubhe.biz.log.enums.LogEnum;
import org.dubhe.biz.log.utils.LogUtil; import org.dubhe.biz.log.utils.LogUtil;
import org.dubhe.servinggateway.constant.GatewayConstant; import org.dubhe.servinggateway.constant.GatewayConstant;
@@ -38,8 +40,8 @@ import org.springframework.web.util.UriComponentsBuilder;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;


import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -55,21 +57,12 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co


private ApplicationEventPublisher publisher; private ApplicationEventPublisher publisher;


@Value("${serving.gateway.postfixUrl}")
private String postfixUrl;

@Resource @Resource
private RedisRouteDefinitionRepository routeDefinitionRepository; private RedisRouteDefinitionRepository routeDefinitionRepository;


@Resource @Resource
private GatewayRouteService gatewayRouteService; private GatewayRouteService gatewayRouteService;


/**
* 模型服务部署使用的端口
*/
@Value("${k8s.port}")
private String httpPort;

@Override @Override
public void run(String... args) { public void run(String... args) {
this.loadRouteConfig(); this.loadRouteConfig();
@@ -87,6 +80,9 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co
LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin load route config"); LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin load route config");
List<GatewayRouteQueryVO> activeRoutes = gatewayRouteService.findActiveRoutes(); List<GatewayRouteQueryVO> activeRoutes = gatewayRouteService.findActiveRoutes();
for (GatewayRouteQueryVO activeRoute : activeRoutes) { for (GatewayRouteQueryVO activeRoute : activeRoutes) {
if (StringUtils.isEmpty(activeRoute.getUri())){
continue;
}
RouteDefinition definition = this.convert2RouteDefinition(activeRoute); RouteDefinition definition = this.convert2RouteDefinition(activeRoute);
routeDefinitionRepository.save(Mono.just(definition)).subscribe(); routeDefinitionRepository.save(Mono.just(definition)).subscribe();
} }
@@ -103,7 +99,19 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co
public void saveRouteByRouteId(Long id) { public void saveRouteByRouteId(Long id) {
LogUtil.info(LogEnum.SERVING_GATEWAY, "Deal save route event"); LogUtil.info(LogEnum.SERVING_GATEWAY, "Deal save route event");
GatewayRouteQueryVO gatewayRouteQueryVO = gatewayRouteService.findActiveById(id); GatewayRouteQueryVO gatewayRouteQueryVO = gatewayRouteService.findActiveById(id);
if (Objects.nonNull(gatewayRouteQueryVO)) {
//查不到数据进行重试
int count = 0;
while (gatewayRouteQueryVO == null && count < MagicNumConstant.THREE){
LogUtil.info(LogEnum.SERVING_GATEWAY, "gatewayRouteQueryVO is null retry:"+count +" id="+id);
gatewayRouteQueryVO = gatewayRouteService.findActiveById(id);
count++;
try {
Thread.sleep(MagicNumConstant.FIVE_HUNDRED);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
if (Objects.nonNull(gatewayRouteQueryVO) && StringUtils.isNotEmpty(gatewayRouteQueryVO.getUri())) {
routeDefinitionRepository.save(Mono.just(this.convert2RouteDefinition(gatewayRouteQueryVO))).subscribe(); routeDefinitionRepository.save(Mono.just(this.convert2RouteDefinition(gatewayRouteQueryVO))).subscribe();
} }
// 发布刷新路由事件 // 发布刷新路由事件
@@ -131,17 +139,19 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co
* @return 网关所需的RouteDefinition * @return 网关所需的RouteDefinition
*/ */
private RouteDefinition convert2RouteDefinition(GatewayRouteQueryVO gatewayRouteQueryVO) { private RouteDefinition convert2RouteDefinition(GatewayRouteQueryVO gatewayRouteQueryVO) {
if(StringUtils.isEmpty(gatewayRouteQueryVO.getUri())){
return null;
}
RouteDefinition definition = new RouteDefinition(); RouteDefinition definition = new RouteDefinition();
// 设置路由基础信息 // 设置路由基础信息
definition.setId(GatewayConstant.ROUTE_PREFIX + gatewayRouteQueryVO.getId()); definition.setId(GatewayConstant.ROUTE_PREFIX + gatewayRouteQueryVO.getId());
definition.setUri(UriComponentsBuilder.fromHttpUrl(SymbolConstant.HTTP_SLASH + gatewayRouteQueryVO.getUri() +
SymbolConstant.COLON + httpPort).build().toUri());
definition.setUri(UriComponentsBuilder.fromHttpUrl(SymbolConstant.HTTP_SLASH + gatewayRouteQueryVO.getUri()).build().toUri());
// 定义url匹配规则的断言 // 定义url匹配规则的断言
// 举例 {abc}.dubhe.ai 匹配{abc}部分 // 举例 {abc}.dubhe.ai 匹配{abc}部分
PredicateDefinition pathPredicate = new PredicateDefinition(); PredicateDefinition pathPredicate = new PredicateDefinition();
pathPredicate.setName("Host");
pathPredicate.setName("Path");
Map<String, String> predicateParams = new HashMap<>(NumberConstant.NUMBER_8); Map<String, String> predicateParams = new HashMap<>(NumberConstant.NUMBER_8);
predicateParams.put("pattern", gatewayRouteQueryVO.getPatternPath() + postfixUrl);
predicateParams.put("pattern", "/"+gatewayRouteQueryVO.getPatternPath() + "/**");
pathPredicate.setArgs(predicateParams); pathPredicate.setArgs(predicateParams);
// 定义权重断言 // 定义权重断言
// 根据配置的权重信息进行分流 // 根据配置的权重信息进行分流
@@ -152,11 +162,21 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co
weightParams.put("weight.weight", gatewayRouteQueryVO.getWeight()); weightParams.put("weight.weight", gatewayRouteQueryVO.getWeight());
weightPredicate.setArgs(weightParams); weightPredicate.setArgs(weightParams);
definition.setPredicates(Arrays.asList(pathPredicate, weightPredicate)); definition.setPredicates(Arrays.asList(pathPredicate, weightPredicate));

List<FilterDefinition> filters =new ArrayList<>();
// 传入自定义的监控指标过滤器 // 传入自定义的监控指标过滤器
FilterDefinition filterDefinition = new FilterDefinition(); FilterDefinition filterDefinition = new FilterDefinition();
// name设置为定义的MetricsGatewayFilterFactory中GatewayFilterFactory前部分 // name设置为定义的MetricsGatewayFilterFactory中GatewayFilterFactory前部分
filterDefinition.setName("Metrics"); filterDefinition.setName("Metrics");
definition.setFilters(Collections.singletonList(filterDefinition));
filters.add(filterDefinition);

//去除一级url转发
FilterDefinition filterDefinitionUrl = new FilterDefinition();
filterDefinitionUrl.setName("StripPrefix");
filterDefinitionUrl.addArg("parts","1");
filters.add(filterDefinitionUrl);

definition.setFilters(filters);
// 塞入自定义数据,供之后filter使用 // 塞入自定义数据,供之后filter使用
HashMap<String, Object> metadata = new HashMap<>(NumberConstant.NUMBER_4); HashMap<String, Object> metadata = new HashMap<>(NumberConstant.NUMBER_4);
metadata.put("servingInfoId", gatewayRouteQueryVO.getServiceInfoId()); metadata.put("servingInfoId", gatewayRouteQueryVO.getServiceInfoId());


+ 1
- 1
dubhe-server/dubhe-serving-gateway/src/main/java/org/dubhe/servinggateway/config/MetricsGatewayFilterFactory.java View File

@@ -120,7 +120,7 @@ public class MetricsGatewayFilterFactory extends AbstractGatewayFilterFactory<Me
DataBufferUtils.join(body) DataBufferUtils.join(body)
.doOnNext(dataBuffer -> { .doOnNext(dataBuffer -> {
// 判断该请求是否为推理请求 // 判断该请求是否为推理请求
boolean isInference = exchange.getRequest().getPath().toString().startsWith(GatewayConstant.INFERENCE_INTERFACE_NAME);
boolean isInference = exchange.getRequest().getPath().toString().endsWith(GatewayConstant.INFERENCE_INTERFACE_NAME);
if (isInference) { if (isInference) {
LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin deal inference request filter"); LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin deal inference request filter");
// 调用请求是否成功 // 调用请求是否成功


+ 1
- 1
dubhe-server/dubhe-serving-gateway/src/main/resources/mapper/GatewayRouteMapper.xml View File

@@ -7,7 +7,7 @@
select smc.id id, si.id serviceInfoId, si.uuid patternPath, smc.release_rate weight, smc.url uri select smc.id id, si.id serviceInfoId, si.uuid patternPath, smc.release_rate weight, smc.url uri
from serving_model_config smc from serving_model_config smc
left join serving_info si on si.id = smc.serving_id left join serving_info si on si.id = smc.serving_id
where si.`status` = 2 and si.deleted = 0 and smc.ready_replicas > 0 and smc.deleted = 0
where si.deleted = 0 and smc.deleted = 0
and smc.id = #{id} and smc.id = #{id}
</select> </select>




+ 0
- 39
dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/config/TrainHarborConfig.java View File

@@ -1,39 +0,0 @@
/**
* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/
package org.dubhe.serving.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

/**
* @description harbor相关配置
* @date 2020-07-17
*/
@Data
@Component
@ConfigurationProperties(prefix = "harbor")
public class TrainHarborConfig {

private String address;

private String username;

private String password;

private String modelName;
}

+ 2
- 5
dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/service/impl/BatchServingServiceImpl.java View File

@@ -71,7 +71,6 @@ import org.dubhe.serving.client.AlgorithmClient;
import org.dubhe.serving.client.ImageClient; import org.dubhe.serving.client.ImageClient;
import org.dubhe.serving.client.ModelBranchClient; import org.dubhe.serving.client.ModelBranchClient;
import org.dubhe.serving.client.ModelInfoClient; import org.dubhe.serving.client.ModelInfoClient;
import org.dubhe.serving.config.TrainHarborConfig;
import org.dubhe.serving.constant.ServingConstant; import org.dubhe.serving.constant.ServingConstant;
import org.dubhe.serving.dao.BatchServingMapper; import org.dubhe.serving.dao.BatchServingMapper;
import org.dubhe.serving.domain.dto.*; import org.dubhe.serving.domain.dto.*;
@@ -135,8 +134,6 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat
@Resource @Resource
private ImageClient imageClient; private ImageClient imageClient;
@Resource @Resource
private TrainHarborConfig trainHarborConfig;
@Resource
private AlgorithmClient algorithmClient; private AlgorithmClient algorithmClient;
@Autowired @Autowired
private RedisUtils redisUtils; private RedisUtils redisUtils;
@@ -238,7 +235,7 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat
BeanUtils.copyProperties(batchServingCreateDTO, batchServing); BeanUtils.copyProperties(batchServingCreateDTO, batchServing);


String imageUrl = getImageUrl(batchServingCreateDTO.getImageName(), batchServingCreateDTO.getImageTag()); String imageUrl = getImageUrl(batchServingCreateDTO.getImageName(), batchServingCreateDTO.getImageTag());
batchServing.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + imageUrl);
batchServing.setImage(imageUrl);
PtModelInfoQueryVO ptModelInfoQueryVO = getPtModelInfo(batchServingCreateDTO.getModelId()); PtModelInfoQueryVO ptModelInfoQueryVO = getPtModelInfo(batchServingCreateDTO.getModelId());
batchServing.setModelAddress(ptModelInfoQueryVO.getModelAddress()); batchServing.setModelAddress(ptModelInfoQueryVO.getModelAddress());
if (ptModelInfoQueryVO.getFrameType() > NumberConstant.NUMBER_4) { if (ptModelInfoQueryVO.getFrameType() > NumberConstant.NUMBER_4) {
@@ -421,7 +418,7 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat
BeanUtils.copyProperties(batchServingUpdateDTO, batchServing); BeanUtils.copyProperties(batchServingUpdateDTO, batchServing);


String imageUrl = getImageUrl(batchServingUpdateDTO.getImageName(), batchServingUpdateDTO.getImageTag()); String imageUrl = getImageUrl(batchServingUpdateDTO.getImageName(), batchServingUpdateDTO.getImageTag());
batchServing.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + imageUrl);
batchServing.setImage(imageUrl);
checkScriptPath(batchServing); checkScriptPath(batchServing);
checkResourceType(batchServing.getFrameType(), batchServing.getResourcesPoolType()); checkResourceType(batchServing.getFrameType(), batchServing.getResourcesPoolType());
batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus());


+ 38
- 9
dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/service/impl/ServingServiceImpl.java View File

@@ -18,6 +18,7 @@
package org.dubhe.serving.service.impl; package org.dubhe.serving.service.impl;


import cn.hutool.core.bean.BeanUtil; import cn.hutool.core.bean.BeanUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@@ -56,11 +57,16 @@ import org.dubhe.biz.permission.annotation.DataPermissionMethod;
import org.dubhe.biz.permission.base.BaseService; import org.dubhe.biz.permission.base.BaseService;
import org.dubhe.biz.redis.utils.RedisUtils; import org.dubhe.biz.redis.utils.RedisUtils;
import org.dubhe.k8s.api.MetricsApi; import org.dubhe.k8s.api.MetricsApi;
import org.dubhe.k8s.api.ModelServingApi;
import org.dubhe.k8s.api.NodeApi;
import org.dubhe.k8s.cache.ResourceCache; import org.dubhe.k8s.cache.ResourceCache;
import org.dubhe.k8s.domain.dto.PodQueryDTO; import org.dubhe.k8s.domain.dto.PodQueryDTO;
import org.dubhe.k8s.domain.resource.BizServicePort;
import org.dubhe.k8s.domain.vo.ModelServingVO;
import org.dubhe.k8s.domain.vo.PodVO; import org.dubhe.k8s.domain.vo.PodVO;
import org.dubhe.k8s.domain.vo.PtPodsVO; import org.dubhe.k8s.domain.vo.PtPodsVO;
import org.dubhe.k8s.enums.PodPhaseEnum; import org.dubhe.k8s.enums.PodPhaseEnum;
import org.dubhe.k8s.enums.WatcherActionEnum;
import org.dubhe.k8s.service.PodService; import org.dubhe.k8s.service.PodService;
import org.dubhe.k8s.utils.K8sNameTool; import org.dubhe.k8s.utils.K8sNameTool;
import org.dubhe.recycle.config.RecycleConfig; import org.dubhe.recycle.config.RecycleConfig;
@@ -75,7 +81,6 @@ import org.dubhe.serving.client.AlgorithmClient;
import org.dubhe.serving.client.ImageClient; import org.dubhe.serving.client.ImageClient;
import org.dubhe.serving.client.ModelBranchClient; import org.dubhe.serving.client.ModelBranchClient;
import org.dubhe.serving.client.ModelInfoClient; import org.dubhe.serving.client.ModelInfoClient;
import org.dubhe.serving.config.TrainHarborConfig;
import org.dubhe.serving.constant.ServingConstant; import org.dubhe.serving.constant.ServingConstant;
import org.dubhe.serving.dao.ServingInfoMapper; import org.dubhe.serving.dao.ServingInfoMapper;
import org.dubhe.serving.dao.ServingModelConfigMapper; import org.dubhe.serving.dao.ServingModelConfigMapper;
@@ -178,8 +183,6 @@ public class ServingServiceImpl implements ServingService {
@Resource @Resource
private ImageClient imageClient; private ImageClient imageClient;
@Resource @Resource
private TrainHarborConfig trainHarborConfig;
@Resource
private AlgorithmClient algorithmClient; private AlgorithmClient algorithmClient;
@Resource @Resource
private RecycleConfig recycleConfig; private RecycleConfig recycleConfig;
@@ -191,6 +194,10 @@ public class ServingServiceImpl implements ServingService {
private ResourceCache resourceCache; private ResourceCache resourceCache;
@Value("Task:Serving:" + "${spring.profiles.active}_serving_id_") @Value("Task:Serving:" + "${spring.profiles.active}_serving_id_")
private String servingIdPrefix; private String servingIdPrefix;
@Resource
private ModelServingApi modelServingApi;
@Resource
private NodeApi nodeApi;


private final static List<String> FILE_NAMES; private final static List<String> FILE_NAMES;


@@ -265,7 +272,7 @@ public class ServingServiceImpl implements ServingService {
List<ServingInfoQueryVO> queryList = servingInfos.getRecords().stream().map(servingInfo -> { List<ServingInfoQueryVO> queryList = servingInfos.getRecords().stream().map(servingInfo -> {
ServingInfoQueryVO servingInfoQueryVO = new ServingInfoQueryVO(); ServingInfoQueryVO servingInfoQueryVO = new ServingInfoQueryVO();
BeanUtils.copyProperties(servingInfo, servingInfoQueryVO); BeanUtils.copyProperties(servingInfo, servingInfoQueryVO);
servingInfoQueryVO.setUrl(servingInfo.getUuid() + GATEWAY_URI_POSTFIX);
servingInfoQueryVO.setUrl(GATEWAY_URI_POSTFIX+"/"+servingInfo.getUuid());
Map<String, String> statistics = servingLuaScriptService.countCallsByServingInfoId(servingInfo.getId()); Map<String, String> statistics = servingLuaScriptService.countCallsByServingInfoId(servingInfo.getId());
servingInfoQueryVO.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO)); servingInfoQueryVO.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO));
servingInfoQueryVO.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO)); servingInfoQueryVO.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO));
@@ -419,7 +426,7 @@ public class ServingServiceImpl implements ServingService {
throw new BusinessException(ServingErrorEnum.CALL_IMAGE_SERVER_FAIL); throw new BusinessException(ServingErrorEnum.CALL_IMAGE_SERVER_FAIL);
} }
servingModelConfig servingModelConfig
.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + dataResponseBody.getData());
.setImage(dataResponseBody.getData());


// 校验模型文件是否存在 // 校验模型文件是否存在
String path = k8sNameTool.getAbsolutePath(servingModelConfig.getModelAddress()); String path = k8sNameTool.getAbsolutePath(servingModelConfig.getModelAddress());
@@ -933,8 +940,8 @@ public class ServingServiceImpl implements ServingService {
predictParamVO.setOutputs(outputs); predictParamVO.setOutputs(outputs);
predictParamVO.setOther(other); predictParamVO.setOther(other);
} else if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { } else if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) {
String url = "http://" + servingInfo.getUuid() + GATEWAY_URI_POSTFIX
+ ServingConstant.INFERENCE_INTERFACE_NAME;
String url = "http://" + GATEWAY_URI_POSTFIX
+"/"+ servingInfo.getUuid()+ ServingConstant.INFERENCE_INTERFACE_NAME;
predictParamVO.setUrl(url); predictParamVO.setUrl(url);
Map<String, String> inputs = new HashMap<>(); Map<String, String> inputs = new HashMap<>();
inputs.put("files", "File"); inputs.put("files", "File");
@@ -1079,6 +1086,7 @@ public class ServingServiceImpl implements ServingService {
*/ */
@Override @Override
public boolean servingDeploymentCallback(ServingK8sDeploymentCallbackCreateDTO req) { public boolean servingDeploymentCallback(ServingK8sDeploymentCallbackCreateDTO req) {
LogUtil.info(LogEnum.BIZ_K8S,"servingDeploymentCallback:{}", JSON.toJSONString(req));
// 根据namespace和podName找到模型配置 // 根据namespace和podName找到模型配置
String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.SERVING, req.getResourceName()); String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.SERVING, req.getResourceName());
if (StringUtils.isBlank(resourceInfo)) { if (StringUtils.isBlank(resourceInfo)) {
@@ -1105,13 +1113,13 @@ public class ServingServiceImpl implements ServingService {
return false; return false;
} }
// 增加发送路由信息 // 增加发送路由信息
if (req.getReadyReplicas() > NumberConstant.NUMBER_0
if (WatcherActionEnum.ADDED.getAction().equals(req.getAction())
&& ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) {
this.notifyUpdateServingRoute(Collections.singletonList(servingModelConfig.getId()), this.notifyUpdateServingRoute(Collections.singletonList(servingModelConfig.getId()),
Collections.emptyList()); Collections.emptyList());
} }
// 增加删除路由信息 // 增加删除路由信息
if (req.getReadyReplicas() == NumberConstant.NUMBER_0
if (WatcherActionEnum.DELETED.getAction().equals(req.getAction())
&& ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) {
this.notifyUpdateServingRoute(Collections.emptyList(), this.notifyUpdateServingRoute(Collections.emptyList(),
Collections.singletonList(servingModelConfig.getId())); Collections.singletonList(servingModelConfig.getId()));
@@ -1148,8 +1156,29 @@ public class ServingServiceImpl implements ServingService {
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public boolean updateByCallback(ServingK8sDeploymentCallbackCreateDTO req, ServingModelConfig servingModelConfig, public boolean updateByCallback(ServingK8sDeploymentCallbackCreateDTO req, ServingModelConfig servingModelConfig,
ServingInfo servingInfo) { ServingInfo servingInfo) {

//更新url
if (StringUtils.isEmpty(servingModelConfig.getUrl())){
//查询
ModelServingVO modelServingVO = modelServingApi.get(req.getNamespace(),req.getResourceName());
if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) {
// 获取pod对应的url,并修改模型部署状态
List<BizServicePort> ports = modelServingVO.getBizService().getPorts();

if (CollectionUtils.isNotEmpty(ports)) {
//取第一个url
String url = "";
if (ports.get(NumberConstant.NUMBER_0).getNodePort() != null) {
url = nodeApi.getAvailableNodeIp() + ":" + ports.get(NumberConstant.NUMBER_0).getNodePort();
}
servingModelConfig.setUrl(url);
}
}
}

// 更新当前模型配置有效节点数 // 更新当前模型配置有效节点数
servingModelConfig.setReadyReplicas(req.getReadyReplicas()); servingModelConfig.setReadyReplicas(req.getReadyReplicas());
servingModelConfig.setResourceInfo(null);
int result = servingModelConfigMapper.updateById(servingModelConfig); int result = servingModelConfigMapper.updateById(servingModelConfig);
if (result < NumberConstant.NUMBER_1) { if (result < NumberConstant.NUMBER_1) {
return true; return true;


+ 12
- 6
dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/task/DeployServingAsyncTask.java View File

@@ -31,6 +31,7 @@ import org.dubhe.biz.log.enums.LogEnum;
import org.dubhe.biz.log.utils.LogUtil; import org.dubhe.biz.log.utils.LogUtil;
import org.dubhe.k8s.api.DistributeTrainApi; import org.dubhe.k8s.api.DistributeTrainApi;
import org.dubhe.k8s.api.ModelServingApi; import org.dubhe.k8s.api.ModelServingApi;
import org.dubhe.k8s.api.NodeApi;
import org.dubhe.k8s.api.TrainJobApi; import org.dubhe.k8s.api.TrainJobApi;
import org.dubhe.k8s.domain.PtBaseResult; import org.dubhe.k8s.domain.PtBaseResult;
import org.dubhe.k8s.domain.bo.DistributeTrainBO; import org.dubhe.k8s.domain.bo.DistributeTrainBO;
@@ -38,7 +39,7 @@ import org.dubhe.k8s.domain.bo.ModelServingBO;
import org.dubhe.k8s.domain.bo.PtJupyterJobBO; import org.dubhe.k8s.domain.bo.PtJupyterJobBO;
import org.dubhe.k8s.domain.bo.PtMountDirBO; import org.dubhe.k8s.domain.bo.PtMountDirBO;
import org.dubhe.k8s.domain.resource.BizDistributeTrain; import org.dubhe.k8s.domain.resource.BizDistributeTrain;
import org.dubhe.k8s.domain.resource.BizIngressRule;
import org.dubhe.k8s.domain.resource.BizServicePort;
import org.dubhe.k8s.domain.vo.ModelServingVO; import org.dubhe.k8s.domain.vo.ModelServingVO;
import org.dubhe.k8s.domain.vo.PtJupyterJobVO; import org.dubhe.k8s.domain.vo.PtJupyterJobVO;
import org.dubhe.k8s.utils.K8sNameTool; import org.dubhe.k8s.utils.K8sNameTool;
@@ -88,6 +89,9 @@ public class DeployServingAsyncTask {
@Resource @Resource
private ModelServingApi modelServingApi; private ModelServingApi modelServingApi;


@Resource
private NodeApi nodeApi;

@Resource @Resource
private K8sNameTool k8sNameTool; private K8sNameTool k8sNameTool;


@@ -143,11 +147,14 @@ public class DeployServingAsyncTask {
String statusDetailKey = ServingStatusDetailDescUtil.getServingStatusDetailKey(ServingStatusDetailDescUtil.CONTAINER_DEPLOYMENT_EXCEPTION, uniqueName); String statusDetailKey = ServingStatusDetailDescUtil.getServingStatusDetailKey(ServingStatusDetailDescUtil.CONTAINER_DEPLOYMENT_EXCEPTION, uniqueName);
if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) {
// 获取pod对应的url,并修改模型部署状态 // 获取pod对应的url,并修改模型部署状态
List<BizIngressRule> rules = modelServingVO.getBizIngress().getRules();
List<BizServicePort> ports = modelServingVO.getBizService().getPorts();


if (CollectionUtils.isNotEmpty(rules)) {
if (CollectionUtils.isNotEmpty(ports)) {
//取第一个url //取第一个url
String url = rules.get(NumberConstant.NUMBER_0).getHost();
String url = "";
if (ports.get(NumberConstant.NUMBER_0).getNodePort() != null){
url = nodeApi.getAvailableNodeIp()+":"+ports.get(NumberConstant.NUMBER_0).getNodePort();
}
servingModelConfig.setUrl(url); servingModelConfig.setUrl(url);
flag = true; flag = true;
servingInfo.removeStatusDetail(statusDetailKey); servingInfo.removeStatusDetail(statusDetailKey);
@@ -176,7 +183,6 @@ public class DeployServingAsyncTask {
servingInfo.putStatusDetail(statusDetailKey, e.getMessage()); servingInfo.putStatusDetail(statusDetailKey, e.getMessage());
LogUtil.error(LogEnum.SERVING, "User {} create serving failed.The name of serving is {}", user.getUsername(), servingInfo.getName(), e); LogUtil.error(LogEnum.SERVING, "User {} create serving failed.The name of serving is {}", user.getUsername(), servingInfo.getName(), e);
} }

} }


//修改服务状态 //修改服务状态
@@ -276,7 +282,7 @@ public class DeployServingAsyncTask {
return null; return null;
} }
} else { } else {
LogUtil.info(LogEnum.SERVING, "User {} failed to copy the inference script, script {} not exist", user.getUsername(), pyPathList.get(0));
LogUtil.info(LogEnum.SERVING, "User {} failed to copy the inference script, script {} not exist", user.getUsername(), pyPathList);
return null; return null;
} }
servingPath = fileStoreApi.formatPath(targetPath + getSourceName(sourcePath)); servingPath = fileStoreApi.formatPath(targetPath + getSourceName(sourcePath));


+ 7
- 45
dubhe-server/dubhe-serving/src/main/java/org/dubhe/serving/utils/GrpcClient.java View File

@@ -17,13 +17,9 @@


package org.dubhe.serving.utils; package org.dubhe.serving.utils;


import cn.hutool.core.codec.Base64;
import cn.hutool.core.io.FileUtil;
import io.grpc.ManagedChannel; import io.grpc.ManagedChannel;
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
import io.grpc.netty.shaded.io.grpc.netty.NegotiationType; import io.grpc.netty.shaded.io.grpc.netty.NegotiationType;
import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
import org.dubhe.biz.base.constant.NumberConstant; import org.dubhe.biz.base.constant.NumberConstant;
import org.dubhe.biz.base.context.UserContext; import org.dubhe.biz.base.context.UserContext;
import org.dubhe.biz.log.enums.LogEnum; import org.dubhe.biz.log.enums.LogEnum;
@@ -31,12 +27,9 @@ import org.dubhe.biz.log.utils.LogUtil;
import org.dubhe.serving.domain.entity.DataInfo; import org.dubhe.serving.domain.entity.DataInfo;
import org.dubhe.serving.proto.Inference; import org.dubhe.serving.proto.Inference;
import org.dubhe.serving.proto.InferenceServiceGrpc; import org.dubhe.serving.proto.InferenceServiceGrpc;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;


import javax.net.ssl.SSLException; import javax.net.ssl.SSLException;
import java.io.File;
import java.io.InputStream;
import java.util.List; import java.util.List;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@@ -52,51 +45,20 @@ public class GrpcClient {
* <服务id, grpc客户端channel> * <服务id, grpc客户端channel>
*/ */
public static ConcurrentHashMap<Long, ManagedChannel> channelMap = new ConcurrentHashMap<>(); public static ConcurrentHashMap<Long, ManagedChannel> channelMap = new ConcurrentHashMap<>();
/**
* 项目根路径
*/
private final static String USER_DIR_SYSTEM_PROPERTY = "user.dir";
/**
* k8s crt证书文件名
*/
private final static String TLS_CRT = "server.crt";
/**
* k8s grpc端口
*/
@Value("${k8s.https-port:31365}")
private String grpcPort;
/**
* k8s 模型部署tls证书 crt
*/
@Value("${k8s.serving.tls-crt}")
private String tlsCrt;


/** /**
* 创建带tls认证的grpc通道
* 创建grpc通道
* *
* @param url tls证书地址
* @param url
* @return ManagedChannel 通道 * @return ManagedChannel 通道
* @throws SSLException * @throws SSLException
*/ */
public ManagedChannel createTlsChannel(String url) throws Exception {
String path = getTlsCrt();
SslContext sslContext = GrpcSslContexts.forClient().trustManager(new File(path)).build();
ManagedChannel channel = NettyChannelBuilder.forAddress(url, Integer.parseInt(grpcPort)).maxInboundMessageSize(NumberConstant.MAX_MESSAGE_LENGTH).negotiationType(NegotiationType.TLS).sslContext(sslContext).build();
public ManagedChannel createChannel(String url) {
String[] host = url.split(":");
ManagedChannel channel = NettyChannelBuilder.forAddress(host[0], Integer.parseInt(host[1])).maxInboundMessageSize(NumberConstant.MAX_MESSAGE_LENGTH).negotiationType(NegotiationType.PLAINTEXT).build();
return channel; return channel;
} }


/**
* 获取crt证书路径
*/
public String getTlsCrt() {
String path = System.getProperty(USER_DIR_SYSTEM_PROPERTY) + File.separator.concat(TLS_CRT);
if(!FileUtil.exist(path)) {
FileUtil.writeUtf8String(Base64.decodeStr(tlsCrt), path);
}
return path;
}


/** /**
* 关闭通道 * 关闭通道
* *
@@ -148,7 +110,7 @@ public class GrpcClient {
} }
ManagedChannel channel = null; ManagedChannel channel = null;
try { try {
channel = this.createTlsChannel(url);
channel = this.createChannel(url);
channelMap.put(servingId, channel); channelMap.put(servingId, channel);
} catch (Exception e) { } catch (Exception e) {
LogUtil.error(LogEnum.SERVING, "An Exception occurred when user {} creating the grpc channel, service id:{}", user.getUsername(), servingId, e); LogUtil.error(LogEnum.SERVING, "An Exception occurred when user {} creating the grpc channel, service id:{}", user.getUsername(), servingId, e);
@@ -169,7 +131,7 @@ public class GrpcClient {
} }
ManagedChannel channel = null; ManagedChannel channel = null;
try { try {
channel = this.createTlsChannel(url);
channel = this.createChannel(url);
channelMap.put(servingId, channel); channelMap.put(servingId, channel);
} catch (Exception e) { } catch (Exception e) {
LogUtil.error(LogEnum.SERVING, "An Exception occurred when getting grpc channel, service id:{}", servingId, e); LogUtil.error(LogEnum.SERVING, "An Exception occurred when getting grpc channel, service id:{}", servingId, e);


+ 0
- 7
dubhe-server/dubhe-terminal/src/main/java/org/dubhe/terminal/domain/entity/PtImage.java View File

@@ -71,13 +71,6 @@ public class PtImage extends BaseEntity {
@TableField(value = "image_resource") @TableField(value = "image_resource")
private Integer imageResource; private Integer imageResource;



/**
* 镜像状态
*/
@TableField(value = "image_status")
private Integer imageStatus;

/** /**
* 资源拥有者ID * 资源拥有者ID
*/ */


+ 1
- 1
dubhe-server/dubhe-terminal/src/main/java/org/dubhe/terminal/domain/entity/Terminal.java View File

@@ -182,6 +182,6 @@ public class Terminal extends BaseEntity {
return null; return null;
} }
String[] strings = imageUrl.split(SymbolConstant.SLASH); String[] strings = imageUrl.split(SymbolConstant.SLASH);
return strings.length > 0 ? strings[MagicNumConstant.ZERO] : null;
return strings.length > 0 ? strings[MagicNumConstant.ONE] : null;
} }
} }

+ 48
- 25
dubhe-server/dubhe-terminal/src/main/java/org/dubhe/terminal/service/impl/TerminalServiceImpl.java View File

@@ -16,7 +16,6 @@
*/ */


package org.dubhe.terminal.service.impl; package org.dubhe.terminal.service.impl;
import java.sql.Timestamp;


import cn.hutool.core.util.RandomUtil; import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
@@ -38,6 +37,7 @@ import org.dubhe.biz.log.utils.LogUtil;
import org.dubhe.biz.permission.base.BaseService; import org.dubhe.biz.permission.base.BaseService;
import org.dubhe.biz.redis.utils.RedisUtils; import org.dubhe.biz.redis.utils.RedisUtils;
import org.dubhe.docker.api.DockerApi; import org.dubhe.docker.api.DockerApi;
import org.dubhe.docker.callback.TerminalPushImageResultCallback;
import org.dubhe.docker.config.DockerClientFactory; import org.dubhe.docker.config.DockerClientFactory;
import org.dubhe.docker.enums.DockerOperationEnum; import org.dubhe.docker.enums.DockerOperationEnum;
import org.dubhe.docker.utils.DockerCallbackTool; import org.dubhe.docker.utils.DockerCallbackTool;
@@ -53,7 +53,6 @@ import org.dubhe.k8s.domain.vo.TerminalResourceVO;
import org.dubhe.k8s.enums.BusinessLabelServiceNameEnum; import org.dubhe.k8s.enums.BusinessLabelServiceNameEnum;
import org.dubhe.k8s.enums.PodPhaseEnum; import org.dubhe.k8s.enums.PodPhaseEnum;
import org.dubhe.k8s.utils.K8sNameTool; import org.dubhe.k8s.utils.K8sNameTool;
import org.dubhe.docker.callback.TerminalPushImageResultCallback;
import org.dubhe.terminal.config.TerminalConfig; import org.dubhe.terminal.config.TerminalConfig;
import org.dubhe.terminal.constant.TerminalConstant; import org.dubhe.terminal.constant.TerminalConstant;
import org.dubhe.terminal.dao.PtImageMapper; import org.dubhe.terminal.dao.PtImageMapper;
@@ -77,6 +76,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;


import java.sql.Timestamp;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
@@ -174,38 +174,43 @@ public class TerminalServiceImpl implements TerminalService {


@Override @Override
public boolean preserve(TerminalPreserveDTO terminalPreserveDTO) { public boolean preserve(TerminalPreserveDTO terminalPreserveDTO) {

Terminal terminal = terminalMapper.selectById(terminalPreserveDTO.getId()); Terminal terminal = terminalMapper.selectById(terminalPreserveDTO.getId());
if (terminal == null){
LogUtil.error(LogEnum.TERMINAL,"preserve terminal 数据不存在 terminalPreserveDTO: {}",terminalPreserveDTO);

if (checkImageExists(terminal.getImageProject(), terminalPreserveDTO.getImageName(), terminalPreserveDTO.getImageTag())) {
throw new BusinessException("请确认镜像版本号是否重复!");
}
if (terminal == null) {
LogUtil.error(LogEnum.TERMINAL, "preserve terminal 数据不存在 terminalPreserveDTO: {}", terminalPreserveDTO);
throw new BusinessException("数据不存在"); throw new BusinessException("数据不存在");
} }


LambdaQueryWrapper<TerminalInfo> wrapper = new LambdaQueryWrapper<>(); LambdaQueryWrapper<TerminalInfo> wrapper = new LambdaQueryWrapper<>();
wrapper.eq(TerminalInfo::getTerminalId, terminalPreserveDTO.getId()); wrapper.eq(TerminalInfo::getTerminalId, terminalPreserveDTO.getId());
List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectList(wrapper); List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectList(wrapper);
if (CollectionUtils.isEmpty(terminalInfoList)){
LogUtil.error(LogEnum.TERMINAL,"preserve terminalInfoList 数据不存在 terminalPreserveDTO: {}",terminalPreserveDTO);
if (CollectionUtils.isEmpty(terminalInfoList)) {
LogUtil.error(LogEnum.TERMINAL, "preserve terminalInfoList 数据不存在 terminalPreserveDTO: {}", terminalPreserveDTO);
throw new BusinessException("数据不存在"); throw new BusinessException("数据不存在");
} }


TerminalInfo masterTerminalInfo = null; TerminalInfo masterTerminalInfo = null;
for (TerminalInfo terminalInfo : terminalInfoList) { for (TerminalInfo terminalInfo : terminalInfoList) {
if (terminalInfo.isMasterFlag()){
if (terminalInfo.isMasterFlag()) {
masterTerminalInfo = terminalInfo; masterTerminalInfo = terminalInfo;
} }
} }
if (masterTerminalInfo == null){
LogUtil.error(LogEnum.TERMINAL,"master 节点不存在 terminalPreserveDTO:{}",terminalPreserveDTO);
if (masterTerminalInfo == null) {
LogUtil.error(LogEnum.TERMINAL, "master 节点不存在 terminalPreserveDTO:{}", terminalPreserveDTO);
throw new BusinessException("master 节点不存在"); throw new BusinessException("master 节点不存在");
} }


BizPod pod = podApi.getWithResourceName(k8sNameTool.getNamespace(terminal.getCreateUserId()),masterTerminalInfo.getK8sResourceName());
if (pod == null){
LogUtil.error(LogEnum.TERMINAL,"master 容器不存在 terminalPreserveDTO:{}",terminalPreserveDTO);
BizPod pod = podApi.getWithResourceName(k8sNameTool.getNamespace(terminal.getCreateUserId()), masterTerminalInfo.getK8sResourceName());
if (pod == null) {
LogUtil.error(LogEnum.TERMINAL, "master 容器不存在 terminalPreserveDTO:{}", terminalPreserveDTO);
throw new BusinessException("master 容器不存在"); throw new BusinessException("master 容器不存在");
} }
if (!PodPhaseEnum.RUNNING.getPhase().equals(pod.getPhase()) || pod.getPodIp() == null || StringUtils.isNotEmpty(pod.getContainerStateMessages())){
LogUtil.error(LogEnum.TERMINAL,"master 容器未运行 terminalPreserveDTO:{}",terminalPreserveDTO);
if (!PodPhaseEnum.RUNNING.getPhase().equals(pod.getPhase()) || pod.getPodIp() == null || StringUtils.isNotEmpty(pod.getContainerStateMessages())) {
LogUtil.error(LogEnum.TERMINAL, "master 容器未运行 terminalPreserveDTO:{}", terminalPreserveDTO);
throw new BusinessException("master 容器未运行"); throw new BusinessException("master 容器未运行");
} }
String containerID = pod.getContainerId(); String containerID = pod.getContainerId();
@@ -217,14 +222,13 @@ public class TerminalServiceImpl implements TerminalService {
terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"commit 镜像..."); terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"commit 镜像...");
terminalMapper.updateById(terminal); terminalMapper.updateById(terminal);
DockerClient dockerClient = dockerClientFactory.getDockerClient(pod.getHostIP()); DockerClient dockerClient = dockerClientFactory.getDockerClient(pod.getHostIP());
String newImagePath = terminal.getImageProject()+SymbolConstant.SLASH+terminal.getCreateUserId()+SymbolConstant.SLASH+terminalPreserveDTO.getImageName();
String newImageRepository = terminalConfig.getHarborAddress()+SymbolConstant.SLASH+newImagePath;
String newImagePath = terminalConfig.getHarborAddress() + StrUtil.SLASH + terminal.getImageProject() + SymbolConstant.SLASH + terminalPreserveDTO.getImageName();
try { try {
dockerApi.commit(dockerClient,containerID,newImageRepository,terminalPreserveDTO.getImageTag());
dockerApi.commit(dockerClient,containerID,newImagePath,terminalPreserveDTO.getImageTag());
terminal.setStatus(null); terminal.setStatus(null);
terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"push 镜像..."); terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"push 镜像...");
terminalMapper.updateById(terminal); terminalMapper.updateById(terminal);
boolean pushResult = dockerApi.push(dockerClient,newImageRepository+SymbolConstant.COLON+terminalPreserveDTO.getImageTag(),new TerminalPushImageResultCallback(dockerCallbackTool.getCallbackUrl(SymbolConstant.LOCAL_HOST,terminalConfig.getServerPort(), DockerOperationEnum.PUSH.getType()),terminal.getId(),dockerClient,terminal.getCreateUserId()));
boolean pushResult = dockerApi.push(dockerClient,newImagePath+SymbolConstant.COLON+terminalPreserveDTO.getImageTag(),new TerminalPushImageResultCallback(dockerCallbackTool.getCallbackUrl(SymbolConstant.LOCAL_HOST,terminalConfig.getServerPort(), DockerOperationEnum.PUSH.getType()),terminal.getId(),dockerClient,terminal.getCreateUserId()));
if (!pushResult){ if (!pushResult){
LogUtil.error(LogEnum.TERMINAL,"master 推送镜像错误 terminalPreserveDTO:{}",terminalPreserveDTO); LogUtil.error(LogEnum.TERMINAL,"master 推送镜像错误 terminalPreserveDTO:{}",terminalPreserveDTO);
throw new BusinessException("推送镜像错误:"); throw new BusinessException("推送镜像错误:");
@@ -234,7 +238,7 @@ public class TerminalServiceImpl implements TerminalService {
throw new BusinessException("保存容器错误:"+e.getMessage()); throw new BusinessException("保存容器错误:"+e.getMessage());
} }


terminal.setImageUrl(newImagePath+SymbolConstant.COLON+terminalPreserveDTO.getImageTag());
terminal.setImageUrl(newImagePath + SymbolConstant.COLON + terminalPreserveDTO.getImageTag());
terminal.setImageName(terminalPreserveDTO.getImageName()); terminal.setImageName(terminalPreserveDTO.getImageName());
terminal.setDescription(terminalPreserveDTO.getImageRemark()); terminal.setDescription(terminalPreserveDTO.getImageRemark());
terminal.setImageTag(terminalPreserveDTO.getImageTag()); terminal.setImageTag(terminalPreserveDTO.getImageTag());
@@ -244,13 +248,24 @@ public class TerminalServiceImpl implements TerminalService {
return terminalMapper.updateById(terminal) > 0; return terminalMapper.updateById(terminal) > 0;
} }


private boolean checkImageExists(String imageProject, String imageName, String imageTag) {
// 拼接镜像的完整路径
String imageUrl = terminalConfig.getHarborAddress() + StrUtil.SLASH + imageProject + SymbolConstant.SLASH + imageName + StrUtil.COLON + imageTag;

// 校验该待保存镜像是否存在
LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(PtImage::getImageUrl, imageUrl);
queryWrapper.ne(PtImage::getCreateUserId, userContextService.getCurUserId());
return ptImageMapper.selectCount(queryWrapper) > 0;
}

@Override @Override
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public boolean delete(TerminalDTO terminalDTO) { public boolean delete(TerminalDTO terminalDTO) {
try{
try {
Terminal terminal = terminalMapper.selectById(terminalDTO.getId()); Terminal terminal = terminalMapper.selectById(terminalDTO.getId());
if (terminal == null){
LogUtil.error(LogEnum.TERMINAL,"delete 数据不存在 terminalDTO: {}",terminalDTO);
if (terminal == null) {
LogUtil.error(LogEnum.TERMINAL, "delete 数据不存在 terminalDTO: {}", terminalDTO);
throw new BusinessException("数据不存在"); throw new BusinessException("数据不存在");
} }
terminal.setDeleted(true); terminal.setDeleted(true);
@@ -259,7 +274,7 @@ public class TerminalServiceImpl implements TerminalService {
terminalMapper.deleteById(terminal); terminalMapper.deleteById(terminal);
// 删除任务缓存 // 删除任务缓存
String taskIdentify = (String) redisUtils.get(terminalIdPrefix + String.valueOf(terminal.getId())); String taskIdentify = (String) redisUtils.get(terminalIdPrefix + String.valueOf(terminal.getId()));
if (StringUtils.isNotEmpty(taskIdentify)){
if (StringUtils.isNotEmpty(taskIdentify)) {
redisUtils.del(taskIdentify, terminalIdPrefix + String.valueOf(terminal.getId())); redisUtils.del(taskIdentify, terminalIdPrefix + String.valueOf(terminal.getId()));
} }
String namespace = k8sNameTool.getNamespace(terminal.getCreateUserId()); String namespace = k8sNameTool.getNamespace(terminal.getCreateUserId());
@@ -477,6 +492,13 @@ public class TerminalServiceImpl implements TerminalService {
return; return;
} }
Terminal terminal = terminalMapper.selectById(terminalId); Terminal terminal = terminalMapper.selectById(terminalId);
List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectByTerminalId(terminalId);
TerminalInfo masterTerminalInfo = null;
for (TerminalInfo info : terminalInfoList){
if (info.isMasterFlag()){
masterTerminalInfo = info;
}
}
if (terminal == null){ if (terminal == null){
LogUtil.error(LogEnum.TERMINAL,"pushImageComplete no terminal found id:{}",terminalId); LogUtil.error(LogEnum.TERMINAL,"pushImageComplete no terminal found id:{}",terminalId);
return; return;
@@ -499,10 +521,11 @@ public class TerminalServiceImpl implements TerminalService {
ptImage.setImageTag(terminal.getImageTag()); ptImage.setImageTag(terminal.getImageTag());
ptImage.setRemark(terminal.getDescription()); ptImage.setRemark(terminal.getDescription());
ptImage.setImageResource(MagicNumConstant.ZERO); ptImage.setImageResource(MagicNumConstant.ZERO);
ptImage.setImageStatus(MagicNumConstant.ONE);
ptImage.setDeleted(false); ptImage.setDeleted(false);
ptImage.setUpdateUserId(userId); ptImage.setUpdateUserId(userId);
ptImage.setUpdateTime(new Timestamp(new java.util.Date().getTime())); ptImage.setUpdateTime(new Timestamp(new java.util.Date().getTime()));
ptImage.setSshUser(masterTerminalInfo.getSshUser());
ptImage.setSshPwd(masterTerminalInfo.getSshPassword());


if (ptImage.getId() != null){ if (ptImage.getId() != null){
ptImageMapper.updateById(ptImage); ptImageMapper.updateById(ptImage);
@@ -563,7 +586,7 @@ public class TerminalServiceImpl implements TerminalService {
terminalBO.setGpuNum(terminalInfo.getGpuNum()); terminalBO.setGpuNum(terminalInfo.getGpuNum());
terminalBO.setMemNum(terminalInfo.getMemNum()); terminalBO.setMemNum(terminalInfo.getMemNum());
terminalBO.setCpuNum(terminalInfo.getCpuNum()); terminalBO.setCpuNum(terminalInfo.getCpuNum());
terminalBO.setImage(terminalConfig.getHarborAddress()+SymbolConstant.SLASH+terminalCreateDTO.getImageUrl());
terminalBO.setImage(terminalCreateDTO.getImageUrl());
terminalBO.setFsMounts(Maps.newHashMap()); terminalBO.setFsMounts(Maps.newHashMap());
terminalBO.setBusinessLabel(BusinessLabelServiceNameEnum.TERMINAL.getBusinessLabel()); terminalBO.setBusinessLabel(BusinessLabelServiceNameEnum.TERMINAL.getBusinessLabel());
terminalBO.setTaskIdentifyLabel(taskIdentifyLabel); terminalBO.setTaskIdentifyLabel(taskIdentifyLabel);


+ 0
- 1
dubhe-server/dubhe-train/src/main/java/org/dubhe/train/client/fallback/ImageClientFallback.java View File

@@ -26,7 +26,6 @@ import org.springframework.stereotype.Component;
* @description 镜像远程服务调用熔断类 * @description 镜像远程服务调用熔断类
* @date 2020-12-15 * @date 2020-12-15
*/ */
@Component
public class ImageClientFallback implements ImageClient { public class ImageClientFallback implements ImageClient {


@Override @Override


+ 0
- 39
dubhe-server/dubhe-train/src/main/java/org/dubhe/train/config/TrainHarborConfig.java View File

@@ -1,39 +0,0 @@
/**
* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/
package org.dubhe.train.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

/**
* @description harbor相关配置
* @date 2020-07-17
*/
@Data
@Component
@ConfigurationProperties(prefix = "harbor")
public class TrainHarborConfig {

private String address;

private String username;

private String password;

private String modelName;
}

+ 18
- 30
dubhe-server/dubhe-train/src/main/java/org/dubhe/train/service/impl/PtTrainJobServiceImpl.java View File

@@ -96,7 +96,6 @@ import org.dubhe.train.client.ModelBranchClient;
import org.dubhe.train.client.ModelInfoClient; import org.dubhe.train.client.ModelInfoClient;
import org.dubhe.train.client.NoteBookClient; import org.dubhe.train.client.NoteBookClient;
import org.dubhe.train.client.ResourceSpecsClient; import org.dubhe.train.client.ResourceSpecsClient;
import org.dubhe.train.config.TrainHarborConfig;
import org.dubhe.train.config.TrainJobConfig; import org.dubhe.train.config.TrainJobConfig;
import org.dubhe.train.constant.ATlasTrainConstant; import org.dubhe.train.constant.ATlasTrainConstant;
import org.dubhe.train.constant.TrainConstant; import org.dubhe.train.constant.TrainConstant;
@@ -202,9 +201,6 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
@Autowired @Autowired
private TrainJobConfig trainJobConfig; private TrainJobConfig trainJobConfig;


@Autowired
private TrainHarborConfig trainHarborConfig;

@Autowired @Autowired
private K8sNameTool k8sNameTool; private K8sNameTool k8sNameTool;


@@ -484,7 +480,12 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
public void buildImageAndTagInfo(PtJobParam ptJobParam, PtTrainJobDetailVO ptTrainJobDetailVO) { public void buildImageAndTagInfo(PtJobParam ptJobParam, PtTrainJobDetailVO ptTrainJobDetailVO) {
//image信息拼装 //image信息拼装
if (StringUtils.isNotBlank(ptJobParam.getImageName())) { if (StringUtils.isNotBlank(ptJobParam.getImageName())) {
String imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE);
String imageNameSuffix;
if (ptJobParam.getImageName().contains(StrUtil.SLASH)) {
imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE);
} else {
imageNameSuffix = ptJobParam.getImageName();
}
String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON);
ptTrainJobDetailVO.setImageName(imageNameSuffixArray[0]); ptTrainJobDetailVO.setImageName(imageNameSuffixArray[0]);
ptTrainJobDetailVO.setImageTag(imageNameSuffixArray[1]); ptTrainJobDetailVO.setImageTag(imageNameSuffixArray[1]);
@@ -563,11 +564,10 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
.setMemNum(ptTrainJobCreateDTO.getMemNum()) .setMemNum(ptTrainJobCreateDTO.getMemNum())
.setWorkspaceRequest(ptTrainJobCreateDTO.getWorkspaceRequest()) .setWorkspaceRequest(ptTrainJobCreateDTO.getWorkspaceRequest())
.setTaskIdentify(taskIdentify); .setTaskIdentify(taskIdentify);

//例如: 将harbor.dubhe.ai/notebook/notebook:v1 去掉 harbor地址
String userImageName = trimHarborAddress(ptImageAndAlgorithmVO.getImageName());
String imageNameAndTag = ptTrainJobCreateDTO.getImageName() + StrUtil.COLON + ptTrainJobCreateDTO.getImageTag();
//结果集处理 //结果集处理
PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, userContextService.getCurUser(), userImageName, trainKey, baseTrainJobDTO);
PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, userContextService.getCurUser(), imageNameAndTag, trainKey, baseTrainJobDTO);
//添加任务缓存 //添加任务缓存
resourceCache.addTaskCache(taskIdentify, ptTrainJob.getTrainId(), ptTrainJobCreateDTO.getTrainName(), trainIdPrefix); resourceCache.addTaskCache(taskIdentify, ptTrainJob.getTrainId(), ptTrainJobCreateDTO.getTrainName(), trainIdPrefix);
// 提交job // 提交job
@@ -575,16 +575,6 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
return Collections.singletonList(ptTrainJob.getTrainId()); return Collections.singletonList(ptTrainJob.getTrainId());
} }


/**
* 去掉harbor地址
*
* @param imageName
* @return
*/
private String trimHarborAddress(String imageName) {
return StringUtils.isBlank(imageName) ? StringUtils.EMPTY : imageName.replace(trainHarborConfig.getAddress() + StrUtil.SLASH, StringUtils.EMPTY);
}

/** /**
* 构建镜像和算法目录VO 考虑到无算法创建 * 构建镜像和算法目录VO 考虑到无算法创建
* *
@@ -611,9 +601,8 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
String imageUrl = imageUtil.getImageUrl(ptTrainJobBaseDTO, userContextService.getCurUser()); String imageUrl = imageUtil.getImageUrl(ptTrainJobBaseDTO, userContextService.getCurUser());
String userImageName = imageUrl.split(StrUtil.SLASH)[0] + StrUtil.SLASH + ptTrainJobBaseDTO.getImageName() + StrUtil.COLON + ptTrainJobBaseDTO.getImageTag(); String userImageName = imageUrl.split(StrUtil.SLASH)[0] + StrUtil.SLASH + ptTrainJobBaseDTO.getImageName() + StrUtil.COLON + ptTrainJobBaseDTO.getImageTag();
ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobBaseDTO.getAlgorithmId()); ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobBaseDTO.getAlgorithmId());
String imageName = trainHarborConfig.getAddress() + StrUtil.SLASH + userImageName;
ptImageAndAlgorithmVO.setImageName(imageName);
ptImageAndAlgorithmVO.setImageUrl(trainHarborConfig.getAddress() + StrUtil.SLASH + imageUrl);
ptImageAndAlgorithmVO.setImageName(ptTrainJobBaseDTO.getImageName());
ptImageAndAlgorithmVO.setImageUrl(imageUrl);


} }
ptImageAndAlgorithmVO.setRunCommand(ptTrainJobBaseDTO.getRunCommand()); ptImageAndAlgorithmVO.setRunCommand(ptTrainJobBaseDTO.getRunCommand());
@@ -676,13 +665,13 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
* *
* @param ptTrainJobCreateDTO 创建训练任务DTO * @param ptTrainJobCreateDTO 创建训练任务DTO
* @param currentUser 用户 * @param currentUser 用户
* @param imageName 镜像名称
* @param imageNameAndTag 镜像名称和tag
* @param trainKey 训练key * @param trainKey 训练key
* @param baseTrainJobDTO 基础训练参数 * @param baseTrainJobDTO 基础训练参数
* @return PtTrain 训练 * @return PtTrain 训练
*/ */
private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserContext currentUser, private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserContext currentUser,
String imageName, String trainKey, BaseTrainJobDTO baseTrainJobDTO) {
String imageNameAndTag, String trainKey, BaseTrainJobDTO baseTrainJobDTO) {
// 添加train表 // 添加train表
PtTrain ptTrain = new PtTrain(); PtTrain ptTrain = new PtTrain();
ptTrain.setTrainName(ptTrainJobCreateDTO.getTrainName()) ptTrain.setTrainName(ptTrainJobCreateDTO.getTrainName())
@@ -732,7 +721,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
ptJobParam.setTrainJobId(ptTrainJob.getId()) ptJobParam.setTrainJobId(ptTrainJob.getId())
.setAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId()) .setAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId())
.setRunCommand(ptTrainJobCreateDTO.getRunCommand()) .setRunCommand(ptTrainJobCreateDTO.getRunCommand())
.setImageName(imageName)
.setImageName(imageNameAndTag)
.setRunParams(ptTrainJobCreateDTO.getRunParams()) .setRunParams(ptTrainJobCreateDTO.getRunParams())
.setRunParamsNameMap(ptTrainJobCreateDTO.getRunParamsNameMap()) .setRunParamsNameMap(ptTrainJobCreateDTO.getRunParamsNameMap())
.setCreateUserId(currentUser.getId()); .setCreateUserId(currentUser.getId());
@@ -1010,9 +999,8 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
.setWorkspaceRequest(ptTrainJobUpdateDTO.getWorkspaceRequest()) .setWorkspaceRequest(ptTrainJobUpdateDTO.getWorkspaceRequest())
.setTaskIdentify(taskIdentify); .setTaskIdentify(taskIdentify);


String userImageName = trimHarborAddress(ptImageAndAlgorithmVO.getImageName());
//结果集处理 //结果集处理
PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, userContextService.getCurUser(), existPtTrainJob, userImageName, ptTrain, baseTrainJobDTO);
PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, userContextService.getCurUser(), existPtTrainJob, ptTrainJobUpdateDTO.getImageName() + StrUtil.COLON + ptTrainJobUpdateDTO.getImageTag(), ptTrain, baseTrainJobDTO);
//提交job //提交job
asyncManager.execute(baseTrainJobDTO, ptTrain.getCreateUserId(), ptImageAndAlgorithmVO, ptTrainJob); asyncManager.execute(baseTrainJobDTO, ptTrain.getCreateUserId(), ptImageAndAlgorithmVO, ptTrainJob);


@@ -1025,13 +1013,13 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
* @param ptTrainJobUpdateDTO 更新训练任务DTO * @param ptTrainJobUpdateDTO 更新训练任务DTO
* @param currentUser 当前用户 * @param currentUser 当前用户
* @param existPtTrainJob 存在的训练任务 * @param existPtTrainJob 存在的训练任务
* @param imageName 镜像名称
* @param imageNameAndTag 镜像名称和tag
* @param ptTrain 训练 * @param ptTrain 训练
* @param baseTrainJobDTO 基本训练信息 * @param baseTrainJobDTO 基本训练信息
* @return PtTrainJob 训练任务 * @return PtTrainJob 训练任务
*/ */
private PtTrainJob updateTrainJobTableData(PtTrainJobUpdateDTO ptTrainJobUpdateDTO, UserContext private PtTrainJob updateTrainJobTableData(PtTrainJobUpdateDTO ptTrainJobUpdateDTO, UserContext
currentUser, PtTrainJob existPtTrainJob, String imageName, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) {
currentUser, PtTrainJob existPtTrainJob, String imageNameAndTag, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) {


//检查模型是否合法,合法则保存其路径地址 //检查模型是否合法,合法则保存其路径地址
checkModelAndSavePath(currentUser, baseTrainJobDTO); checkModelAndSavePath(currentUser, baseTrainJobDTO);
@@ -1070,7 +1058,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService {
ptJobParam.setTrainJobId(ptTrainJob.getId()) ptJobParam.setTrainJobId(ptTrainJob.getId())
.setAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId()) .setAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId())
.setRunCommand(ptTrainJobUpdateDTO.getRunCommand()) .setRunCommand(ptTrainJobUpdateDTO.getRunCommand())
.setImageName(imageName)
.setImageName(imageNameAndTag)
.setRunParams(ptTrainJobUpdateDTO.getRunParams()) .setRunParams(ptTrainJobUpdateDTO.getRunParams())
.setRunParamsNameMap(ptTrainJobUpdateDTO.getRunParamsNameMap()) .setRunParamsNameMap(ptTrainJobUpdateDTO.getRunParamsNameMap())
.setCreateUserId(ptTrain.getCreateUserId()); .setCreateUserId(ptTrain.getCreateUserId());


+ 3
- 1
dubhe-server/sql/11-Dubhe-Patch-3.0.sql View File

@@ -252,10 +252,10 @@ ALTER TABLE `data_task` ADD COLUMN `file_type` smallint(3) NULL COMMENT '待处
ALTER TABLE `data_task` ADD COLUMN `stop` bit(1) NULL COMMENT '是否停止 0-没有 1-已停止' AFTER `file_type`; ALTER TABLE `data_task` ADD COLUMN `stop` bit(1) NULL COMMENT '是否停止 0-没有 1-已停止' AFTER `file_type`;
ALTER TABLE `data_task` ADD COLUMN `of_record_version` varchar(255) NULL COMMENT '生成ofRecord的版本号' AFTER `stop`; ALTER TABLE `data_task` ADD COLUMN `of_record_version` varchar(255) NULL COMMENT '生成ofRecord的版本号' AFTER `stop`;



-- 医学表 -- 医学表
ALTER TABLE `data_medicine` ADD COLUMN `stop` bit(1) NULL AFTER `annotate_type`; ALTER TABLE `data_medicine` ADD COLUMN `stop` bit(1) NULL AFTER `annotate_type`;



-- 点云数据集表 -- 点云数据集表
create table if not exists `pc_dataset` ( create table if not exists `pc_dataset` (
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键id', `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键id',
@@ -397,3 +397,5 @@ CREATE TABLE `pt_atlas_camera` (
PRIMARY KEY (`id`), PRIMARY KEY (`id`),
UNIQUE KEY `camera_index_uniq` (`camera_index_code`) COMMENT '设备标识唯一' UNIQUE KEY `camera_index_uniq` (`camera_index_code`) COMMENT '设备标识唯一'
) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb4 COMMENT='模型炼知视频流管理'; ) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb4 COMMENT='模型炼知视频流管理';

ALTER TABLE pt_image DROP image_status;

+ 2
- 2
dubhe-server/yaml/admin.yaml View File

@@ -33,11 +33,11 @@ ribbon:
dubhe-proxy: dubhe-proxy:
visual: visual:
keyword: visual keyword: visual
server: 127.0.0.1
server: 10.5.26.88
port: 9898 port: 9898
refine: refine:
keyword: refine keyword: refine
server: 127.0.0.1
server: localhost
port: 9797 port: 9797
## 测试标识 ## 测试标识
debug: debug:


+ 9
- 16
dubhe-server/yaml/common-biz.yaml View File

@@ -17,7 +17,7 @@ spring:
redis: redis:
#数据库索引 #数据库索引
database: 4 database: 4
host: 127.0.0.1
host: 10.5.26.88
port: 6379 port: 6379
password: password:
#连接超时时间 #连接超时时间
@@ -26,9 +26,9 @@ spring:
druid: druid:
db-type: com.alibaba.druid.pool.DruidDataSource db-type: com.alibaba.druid.pool.DruidDataSource
driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
username: test username: test
password: test
password: zj12345678


# 初始化配置 # 初始化配置
initial-size: 3 initial-size: 3
@@ -67,17 +67,17 @@ logging.config:
# 文件存储服务配置 # 文件存储服务配置
storage: storage:
# 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 # 文件存储服务暴露的IP地址 如需测试需修改为合适的地址
file-store: 127.0.0.1
file-store: 10.5.26.234
# 文件存储服务端 共享目录 # 文件存储服务端 共享目录
file-store-root-path: /nfs/ file-store-root-path: /nfs/
file-store-root-windows-path: "Z:" file-store-root-windows-path: "Z:"


# minio配置 # minio配置
minio: minio:
url: http://127.0.0.1:9000/
url: http://10.5.26.90:9000/
accessKey: admin accessKey: admin
secretKey: 123@abc.com secretKey: 123@abc.com
bucketName: dubhe-prod
bucketName: dubhe-cloud-test
presignedUrlExpiryTime: 300 presignedUrlExpiryTime: 300
annotation: /annotation/ annotation: /annotation/


@@ -121,13 +121,6 @@ feign:
okhttp: okhttp:
enabled: false enabled: false


#配置harbor
harbor:
address: harbor.dubhe.ai
username: admin
password: Harbor12345
model-name: train

#配置dev环境单元测试用户名与密码 #配置dev环境单元测试用户名与密码
unittest: unittest:
username: admin username: admin
@@ -136,10 +129,10 @@ unittest:
model: model:
measuring: measuring:
url: url:
package: http://127.0.0.1:32760/model_measure/package
json: http://127.0.0.1:32760/model_measure/measure
package: http://10.5.26.90:32760/model_measure/package
json: http://10.5.26.90:32760/model_measure/measure
converter: converter:
url: http://127.0.0.1:32230/model_convert
url: http://10.5.26.90:32230/model_convert


user: user:
config: config:


+ 34
- 19
dubhe-server/yaml/common-k8s.yaml View File

@@ -5,21 +5,17 @@ k8s:
# k8s集群配置文件 # k8s集群配置文件
kubeconfig: kubeconfig_test kubeconfig: kubeconfig_test
# nfs服务暴露的IP地址 如需测试需修改为合适的地址 # nfs服务暴露的IP地址 如需测试需修改为合适的地址
nfs: 127.0.0.1
nfs: 10.5.26.234
#nfs服务端 共享目录 #nfs服务端 共享目录
nfs-root-path: /nfs/ nfs-root-path: /nfs/
nfs-root-windows-path: "Z:" nfs-root-windows-path: "Z:"
# 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 # 文件存储服务暴露的IP地址 如需测试需修改为合适的地址
file-store: 127.0.0.1
file-store: 10.5.26.234
#文件存储服务端 共享目录 #文件存储服务端 共享目录
file-store-root-path: /nfs/ file-store-root-path: /nfs/
file-store-root-windows-path: "Z:" file-store-root-windows-path: "Z:"
# 命名空间关键字 # 命名空间关键字
namespace: namespace namespace: namespace
# k8s ingress域名 如需测试需修改为合适的域名,注意:需要为此域名配置泛域名解析到 k8s集群master节点的ip
host: notebooktest.dubhe.club
# k8s ingress-controller 对外port
port: 30865
# k8s ingress-controller 对外grpc port # k8s ingress-controller 对外grpc port
https-port: 31287 https-port: 31287
# k8s 模型部署配置 # k8s 模型部署配置
@@ -27,12 +23,12 @@ k8s:
# 在线服务模型部署后容器域名(k8s ingress域名),解析地址为k8s集群地址,如需测试需修改为合适的域名 # 在线服务模型部署后容器域名(k8s ingress域名),解析地址为k8s集群地址,如需测试需修改为合适的域名
host: servingtest.dubhe.club host: servingtest.dubhe.club
# tls 证书 crt # tls 证书 crt
tls-crt:
tls-crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURYekNDQWtlZ0F3SUJBZ0lKQUp3Z0VCYlhGdk9HTUEwR0NTcUdTSWIzRFFFQkN3VUFNRVl4SVRBZkJnTlYKQkFNTUdDb3VjMlZ5ZG1sdVozUmxjM1F1WkhWaWFHVXVZMngxWWpFaE1COEdBMVVFQ2d3WUtpNXpaWEoyYVc1bgpkR1Z6ZEM1a2RXSm9aUzVqYkhWaU1CNFhEVEl4TURVeE9UQTNNRGN5TTFvWERUTXhNRFV4TnpBM01EY3lNMW93ClJqRWhNQjhHQTFVRUF3d1lLaTV6WlhKMmFXNW5kR1Z6ZEM1a2RXSm9aUzVqYkhWaU1TRXdId1lEVlFRS0RCZ3EKTG5ObGNuWnBibWQwWlhOMExtUjFZbWhsTG1Oc2RXSXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBdwpnZ0VLQW9JQkFRREwxNDBMUXk2dXFHUi9WckVXeVNUdFppVGwvOEo1eGlHenB5WUJtTldUTzVQSTczTHRKbXdkCmFPMGNCNXhBcy96SXlpYjM3b0loR0FQejZNWlBBZjJEeUMxNXJLb3Via3h2Vkswa3hJY0k2L1dsWFUxMjdZZzQKdUllNzJEbFNIak9tbStVQ2JDWkxtc0VlQVI2S3RwNW9lVUFUQVgwOWEyL012ZkdtNU1URWZFSVplTENoL21rVAppZGFKcnl0NThwYVFLZmpJcVdKQnlnUGpJYlFMUlZnSG5mckIzdlB5b3RmMFBZZWVKVHlEa1lNQnNzblMxS044ClY2Qldvb2dzWUxQcCs2VE1wOXBHS3pBdksxdXJMWkFsa1N0SUw1TkppQWJOZ1lMOXN1UVE3OU9aano2c1ZrbWQKait6aHZHSGUrYk9zdlBrYmtqNS9lbzQyQUdQcW9zdTlBZ01CQUFHalVEQk9NQjBHQTFVZERnUVdCQlFzWDlCegpxVEhNN21xMUJrOFRNWXgzY1ZRaFB6QWZCZ05WSFNNRUdEQVdnQlFzWDlCenFUSE03bXExQms4VE1ZeDNjVlFoClB6QU1CZ05WSFJNRUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCc1BJSUpPMm1qUFNJV1lTaloKQmdjbFJoUTRudEkvdjFWczVjb0xOVzlrZUhqUGlCYlhvVzgzZExqSWh5ZmVYSk93Y3lpRnJkaVlZeVV5SEprTAprTmlxajRxVmZuaGkwcUoweDQyN3MzUG40TkZZRnNJYVM2OHA1WlZpUlJSZGNWY0dIMkNiRUxzYW41ZUhkNExYCnlTUUdDZmVkUVdtMEVyWWxDMGJWSER1d3RCa0NzWElPRmJEb0tSMW5wMDc3YlBrbEpIRS9rUi82QVpJR3hFM28KbDd1SDVRWWRsWldoanY5WFZRM1plVnZLUzVkM1pHYjcxZ0dwSEU2b2hmL1IweVhoS1RadUREYkhjK2lFc1dWbQowa24xenlvTUVHSW5nM0N6cVk4aE1tdHhRVStLekJrSEliVzh0UnkrS0MxZ2ZvNktFa01JVGpjK2pwQU4wZ2VDCm1LWWYKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
# tls 证书 key # tls 证书 key
tls-key:
tls-key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRREwxNDBMUXk2dXFHUi8KVnJFV3lTVHRaaVRsLzhKNXhpR3pweVlCbU5XVE81UEk3M0x0Sm13ZGFPMGNCNXhBcy96SXlpYjM3b0loR0FQego2TVpQQWYyRHlDMTVyS291Ymt4dlZLMGt4SWNJNi9XbFhVMTI3WWc0dUllNzJEbFNIak9tbStVQ2JDWkxtc0VlCkFSNkt0cDVvZVVBVEFYMDlhMi9NdmZHbTVNVEVmRUlaZUxDaC9ta1RpZGFKcnl0NThwYVFLZmpJcVdKQnlnUGoKSWJRTFJWZ0huZnJCM3ZQeW90ZjBQWWVlSlR5RGtZTUJzc25TMUtOOFY2Qldvb2dzWUxQcCs2VE1wOXBHS3pBdgpLMXVyTFpBbGtTdElMNU5KaUFiTmdZTDlzdVFRNzlPWmp6NnNWa21kait6aHZHSGUrYk9zdlBrYmtqNS9lbzQyCkFHUHFvc3U5QWdNQkFBRUNnZ0VCQUtJRG83WDA0dEN0UkpzMVV6RnlFK2lnUyttR201TGdHSDlLOWRsRkdWT1oKZ2IrdUdhRkY3WjUrZkI3UUthLy9ub1lVcmw1VzhwVE5HcTh0THhreTBzV0FRQ3k2UU5VQWs3M1ZCbnozdXdBcgpVeVhvLzUzcjgxNXB4SEJYems5bmM5UVRpalNmc3R4YWx1MTdKRVJLRzZPYjQ0SjNwNHcrclRDRk0rRmJhTTFsCkxuWUh2bGR4ZnRoMmhBWHdaU2dFYXdCNXRBbTY2YnIxbUc4ZW5saEIvbisrY09iYkZWYnk4OUcyUmVMMjJxdHIKa0R0RHVQOGY1NVdxNVB0a25TRE4zaGoweEYydG9aSC80RHpKOWVibVNMVHpUK1pQSjVjK1dlZ3BNdCtVd3YzbwpNNFdGQnFUUUZSR0MyQVJTaXVFTEdtRUJnVk50VDRmRVhRSmFveGJKeWYwQ2dZRUE2MDRJOXlDZUttY2hjSEJvCmRFeFJ2bnNhcGhwbkJqc3pHazZIWVhhTFI0NGpvalRCWm5ZWVhHYklreDdZaytla1RQYjMwRUpzQjlRRmxpVUUKQ25oZXhCS0pjTk5VcTdWdHFGTXBhT0JKWG9Md0cySmVwVmNhN1MzcFo3THZMWGRQR2wvOXFTTW5pMXROV1kxdwpmTGl1Zzlqb3JCeEJaL250NU43MDJFTWpaL2NDZ1lFQTNjVWV2U3U1VmlHaGNPSmFVSlBoUjdnclViMW9EWDF4CnROdDdXNS9pNkFyNDBsY1FZK2RCcll2VEljRllEL3piZnRWNGFnMDhsLzVHamxQK2grWW01QWNRb1c1Z3BaaHgKd1ZhTFFzRTg4ZUdPY2pLakVqd2krTEZRNE5HRmdXdU9pc1hZWk5GTlBvT1JDbHRUbkRMRnFjS0FNY2tBTEgwVwpRT2poTkJqeGhPc0NnWUFRaHRPVVgrTWNBVkJVOEdBMXd1Sis2WENPdUE0Q1h3Z1EwZkVxUkVRMkMyS3ZVdHMrCmtnN1Y5cFloMXluSkFaMEZsdGNDOXBkVjJXdG5CMFNJWTduc05ZMFhzcEFnMjBaUGF6L2VVTnAyVytYM1ZtcGwKWEgvVXBzUGM5N0ZhMVNWbUtkWE1HbDc5cDdVQUZESVJZSHRKWVdPK0t1SGhKcW14eUlNVDZXdEVNd0tCZ0R0ZAo4WVNpbDlLKzNnRGlGMXRLdXh2LzZWalFZM0o1Q2w5b0FmWGRMMWorMXUwMzhXTk5IUC9nVm56S3pWQTZXR2Z6CnJYQjJhcW9sbjYycVBwRVN3NFozZmJRNVlCWDBZVDlvYzQ5RE81VmsxRVV0MlFtZ241d0RtNnNUYTdIaG9SNzEKSjZDVmh5QWRDRTdGYy9SMGd3V1cwOHFBREZQY2lJQ0gxd0dqUzhSUkFvR0JBTklUNW5yTzNwZmNKYTROZWFZeApKMzd5RzFodUJGcWdRRWpVU3pmbHQ3UWwvNjI3LzNWT2lIRjVqSkhES0JqODBEdzh1UjhoQ2VhWDY2WkJqUmxKCk5vOUEvUm43Q2F4YUl2dkNjWG4rUW5oLy9BVlNHRzU4WWI2UUlQNFVqT1RKOW1GUlFFajN0cm9KNnVpL3FCZTQKUkhXSE44WC91Z242dzZUSGRWNktJbSs4Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
# elasticsearch暴露的服务地址 # elasticsearch暴露的服务地址
elasticsearch: elasticsearch:
hostlist: ${eshostlist:127.0.0.1:30498}
hostlist: ${eshostlist:10.5.26.90:30498}
# 日志采集配置信息 # 日志采集配置信息
log: log:
type: _doc type: _doc
@@ -50,9 +46,9 @@ k8s:
# 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址 # 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址
pod: pod:
metrics: metrics:
grafanaUrl: http://127.0.0.1:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod=
grafanaUrl: http://10.5.26.90:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod=
prometheus: prometheus:
url: http://127.0.0.1:30003/
url: http://10.5.26.90:30003/
query: api/v1/query query: api/v1/query
query-range: api/v1/query_range query-range: api/v1/query_range
gpu-query-param: sum(DCGM_FI_DEV_GPU_UTIL{pod="pod-name-placeholder"})by(pod,UUID) gpu-query-param: sum(DCGM_FI_DEV_GPU_UTIL{pod="pod-name-placeholder"})by(pod,UUID)
@@ -75,21 +71,40 @@ k8s:
cpu: 10 cpu: 10
memory: 32 memory: 32
gpu: 2 gpu: 2
#配置harbor
harbor:
address: harbor.dubhe.ai
username: admin
password: Harbor12345
model-name: train


# minio配置 # minio配置
minio: minio:
url: http://127.0.0.1:9000/
url: http://10.5.26.90:9000/
accessKey: admin accessKey: admin
secretKey: 123@abc.com secretKey: 123@abc.com
bucketName: dubhe-prod
bucketName: dubhe-cloud-test
presignedUrlExpiryTime: 300 presignedUrlExpiryTime: 300
annotation: /annotation/ annotation: /annotation/


docker: docker:
remote-api-port: 2375 remote-api-port: 2375

clusters:
- cluster:
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1USXdPREE0TWpnd00xb1hEVE14TVRJd05qQTRNamd3TTFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWVKCkRjTThQUHFlV0tHazE3S2JtN3kwbzkzTGtxb2w5dG1QbVN0YTVyUXZSWU9rOFJYK2QvVEpVaXE2S2c4NXFyMGUKZFZ3SG5vTjg3Q0VFbXhOK2o4OUhlcnY1dDFaRFBHMVRHS3U5N1l1WnV4RCt2R1hGQkhDZ2ViT0s0Zmc0am1iSgpwUmlJeCtUZmlyUjM4TlVhZTBkR1g5c1o1TWF2NkhiSVVEK2hNTm9VU2pyWUZ2bTcxbGtYdlhVWW9wd1FSR3FvCkZiWG5KTFFVakd3UFFRclRCQ0wycFE2c1dGbkpkQ0VBSHhXU0grTUN4cXV0L25SUHhORlFOWGFMZmU3clB3a3MKd3NrVlFlTzgyL205aFZkWnNnNUFnRDJtTS9jS2VybE9acVBGcWFEaFBoYnpkRzF4QjRtcVZmTGJVUUppa3I1SQpYQ1FhTWR2c2FRTnhPcmNPOHQwQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZFek92OGhVUGtRYS90bGNhQTE0OERmcmF5TXRNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBMTFqcDR4V1hyZkRPNDJlblBpdm9HNXZxY3hqbysxaktZV1YxR1NIdHNVdlBNU1UzVAptdVFCQWU2VVRzM0JtSTgzMUQ4aFd4cDlka3BkMlhWbCt6OTZabFIvQ2Qrb3NSWWE1MUpZd2RvMUp4Z29hbUk2CmszOS9sNTRBQVpWMkFPWWNQU0pKNkNzdVcxZ2NTMDlaTlA0djlhK0NIKzY5OVpQMGlhUStEc0FPRFJzUnhRcVMKK08zNUU4K0RzRjBSOWZCYlhvbkxiK05VSHlTQ2pQSnVVT21lS0RYMFpQQm81blNscmh5cUIxbHVNZnhEaUltdgpIU3BscXB4M1duMzJnNEZYOCs3clZBNUZjWkIvNDg3bUZmdG1CTXc3aENidlFJeWY2Ymd6VTUycUdaeFZGTTVjCnUxeEZyRXlXaFA1eElJY09lY1M1RCtJa29ZOHUyT3ozQldEeQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
server: https://10.105.0.6:6443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: kubernetes-admin
name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin
user:
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lJQjUxN3hkT3V2cnd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TVRFeU1EZ3dPREk0TUROYUZ3MHlNakV5TURnd09ESTRNRFZhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXlFaURrYVJ1WmVUbVJ4WmsKRkxUbkRPMUxIM1k5dE80WXpXREt4QVA2bFphTUJwZXJWRWpiU1VMOGIzdG9WWFppVnVnaHhvMzRHdjF4dlJKLwpDN3NhWGVaU3EzcTVMcEFhQjBrelRDNFRtbGtrdE03YkJTZFNhMG9SeGJXTUdvVEM0WUVBckJCNS9FRmdUYjRCCnE4aG1USUlmdUcwa05sNmd5a2M4UnhSL3N5Q1lSVjZCTTZTYWFzODR1Tmx3ZU9vV2RaQkQ0UVZQbzJueUI4NGQKUnc0d0w2VEQ5MlUwMG5KcjY3bUtpTjhBNGtBampFbWNrNm1JbHdsUTB2MHR5Uk8zTFBXZFdNazZ5QVIyVEZ6RgpBZCs4dUd5L3VDQlB6UDlVcDFERkI5TDF0Y25sd1lTeTYycGdFWC9wMzl6ZjZ0RFpZRVJwK0Jpd21Eb3RmVmFaClNLam9md0lEQVFBQm8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0h3WURWUjBqQkJnd0ZvQVVUTTYveUZRK1JCcisyVnhvRFhqd04rdHJJeTB3RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFCQ01US2VEV1dHa0VFZWp6dlN3TTFKZit0QUJaYklrd0FoZEt1MjFnUkxyQktzbHQ4Y0NHMG1KCmc3eDhndFJYcW0xRGlEU01DMnlWMW9OWTFSdGlLZVhZT29RbVY2SFh2bEQybWZyY2VzS3VSNHlIbCtlWHcyL1AKb2FLZVFEM21yVm9ZZ1ZMMVNqOWpzcDRBa1hEM1ErUzdKYXN4Y3MvVkR3SmlRQzhGamkzaWZMVFFKQkxCdm1WVwpCZ3QvM0xPZ1lCdWxYRWg3cFlQUzduTVFqNFpxQ0QwWTNJOENicUM0VGlZQjB1aTZlbVRDd3cvRnBNSzNIRFZ4CnZOd3FPRHBaWW5DK0IxSzR5RUk4NGIvM1o3ZXA0MUE1NGk3QkpxTklORUVtRjdGcmpQQy91V1RuYlpzVUNFQjAKWTFUR1Y3czNBUTZMZXNCRFF6UVRzVnBENWFRRk9VYz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBeUVpRGthUnVaZVRtUnhaa0ZMVG5ETzFMSDNZOXRPNFl6V0RLeEFQNmxaYU1CcGVyClZFamJTVUw4YjN0b1ZYWmlWdWdoeG8zNEd2MXh2UkovQzdzYVhlWlNxM3E1THBBYUIwa3pUQzRUbWxra3RNN2IKQlNkU2Ewb1J4YldNR29UQzRZRUFyQkI1L0VGZ1RiNEJxOGhtVElJZnVHMGtObDZneWtjOFJ4Ui9zeUNZUlY2QgpNNlNhYXM4NHVObHdlT29XZFpCRDRRVlBvMm55Qjg0ZFJ3NHdMNlREOTJVMDBuSnI2N21LaU44QTRrQWpqRW1jCms2bUlsd2xRMHYwdHlSTzNMUFdkV01rNnlBUjJURnpGQWQrOHVHeS91Q0JQelA5VXAxREZCOUwxdGNubHdZU3kKNjJwZ0VYL3AzOXpmNnREWllFUnArQml3bURvdGZWYVpTS2pvZndJREFRQUJBb0lCQUgrbHY3ME9TSkpHZmdHbQpvcWlUMTRKa3BuRnA0ZEF6dzdqNXpLRjdTN1VWR3krRWNOeXFCcUM5d2JlbnRvcHBoaW1Qang5R0VtL1pRaWxYCjVZTHJmOVdDMndPUmx2NjNOdStYMXNyaHZ2cXJmL3FBc0JTcnlCcTdQWEo5ejhxQy9OWE9hMGcreEJCaTltYjIKQjRpZGs1MkZmWVFFZzRUbmNLRWJINjdKd00yL2dmZ3BXMGNEd0RiNVc2aHIxclUzNkNGc1AyVTlEYytMVkV5cQpvK3FZZ0RFZ0MxSWxucGVIdzROejJmK3RkbFpLVHN5dzV1ZG5xVm1kWGNkLyszR3NvQVM3Q25DTmQ3a1pKQ1BPClNpRktZSDZZUS93ZmJLZjFibC84eERXU1UvKytQUTZyODJGV21mMFBjd3ExbWV5OXIvUW1kaHc5YndiamkxVDQKdURjSzUva0NnWUVBOURIZi8xZnN4czlqVldoaE94UXFzUFN3UzdDbVZYcHV2S2Q3MjlMRGFIV1VQT3E5Y0l4QQpaV01Ibi9EbEN3U2dxaitPL0FNSDdaQVpPbDV5V2RiZFVEM2I0WWVKLzlubXdvN25ReG9ndTlUdTlNZEVnM0pvCm1wTzVqQTYzYTE2Z0sxRWlmTDJkdllRQkJGQzg2aDJzYlV0WHNGRDV6R3QzTU96bVF6SFFQM3NDZ1lFQTBmY3gKMzlUR0YrZFdVb3MzOXRLaE9MZVpFaWtUSmtqckdrTUdldHpYNEVCRE5oRUUvdWNkYUZCWmJoU3ltZHAwMmtKNgpKQkpvaEJRaEN4Zk9nUXhRdFp2dUt3WUMvTEYrMVF1TG1MaERGV2NEa05Ma1E0RWU5WVRFV1VuNE9NNkJJTmVOCldseklzcncyeHRvaUVoMXZFWW92ODdobEZCQ0E4N0xYSElpa1NjMENnWUFnZ1pNajFueDZhcGo2Z1k2UDRydGMKR3ZaczNQTUhaZWpmekJ1OHcyMm50aDhwak1YeStYaUpCb201VE1Qd2w0a2JvS0pVQWNOSWFHb3pUdWRCOWt3Mwpwa1JpM1R3Tmh1QWsvZ2ppOGROeHJZS3hxdVQvNm5icVBNZnlVUE14bzhNR2dTSnFJSU9pK3JOMzg1ZHlRc2oxCk01WlVyakxTZDJ2Q1k0YXpZeW9waVFLQmdBS0RBb1c2OWVPelVNVU1CVTllbGE4b0F4ekVnUVlrT1N2SFFYeVcKeDc1WEJuYlBIVkF2VTVxNzIxWUZ5VjB0ejlnTWs0bDY3dVVsbUgrWmVVN3g1c1ZGYUVQN2ZtMm5jZXo2aG1EOQpVMUFlTzF5d0tTcmxrSWsvWFZuMEdKUVZaRllRUGhDbXEzLzM0Ry9nakNmTFVsalRYbDk4QlRtSU9RS3hVUUYyCllRREJBb0dBRTB5Q0NzcFE3Y1lYaVFyQnF4VVkxU2N5SjViVm1iaFN5cmgyb3FiUmRMRmxzWHFjUnBIK2tORjYKdXdjUzhZalJodmdQazgyZm1QcXlieE5ZWGxTYWoydGdzSFdpcGNETXZSM1dtSlQzTGpjeXZObDh6cUlkZDNBdQpJT1pNeG0vWWVMemNFOUdteDRsQkhmMnU5MkhiRjRiSCtqeVp3d3RWcHpRdmphaFF6dGs9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==

#配置harbor
harbor:
address: harbor.dubhe.ai
username: admin
password: Harbor12345

+ 2
- 2
dubhe-server/yaml/common-shardingjdbc.yaml View File

@@ -6,9 +6,9 @@ spring:
master: master:
type: com.alibaba.druid.pool.DruidDataSource type: com.alibaba.druid.pool.DruidDataSource
driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
username: test username: test
password: test
password: zj12345678
props: props:
sql: sql:
show: true show: true


+ 1
- 1
dubhe-server/yaml/dubhe-data-dcm.yaml View File

@@ -18,7 +18,7 @@ data:


# 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 # 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务
dcm: dcm:
host: 127.0.0.1
host: 10.5.29.100
port: 11112 port: 11112
# 资源回收 # 资源回收


+ 5
- 5
dubhe-server/yaml/dubhe-data-task.yaml View File

@@ -27,13 +27,13 @@ recycle:


# 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 # 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务
dcm: dcm:
host: 127.0.0.1
host: 10.5.29.100
port: 11112 port: 11112
# ES服务地址及端口 # ES服务地址及端口
es: es:
host: 127.0.0.1
serverPort: 31321
transportPort: 31322
clusterName: kubenets-logging
host: 10.5.29.100
serverPort: 9200
transportPort: 9300
clusterName: docker-cluster
index: dataset_text_test index: dataset_text_test

+ 6
- 6
dubhe-server/yaml/dubhe-data.yaml View File

@@ -32,11 +32,11 @@ spring:
precise-algorithm-class-name: org.dubhe.data.util.MyPreciseShardingAlgorithm precise-algorithm-class-name: org.dubhe.data.util.MyPreciseShardingAlgorithm
k8s: k8s:
# nfs服务暴露的IP地址 如需测试需修改为合适的地址 # nfs服务暴露的IP地址 如需测试需修改为合适的地址
nfs: 127.0.0.1
nfs: 10.5.26.234
#nfs服务端 共享目录 #nfs服务端 共享目录
nfs-root-path: /nfs/ nfs-root-path: /nfs/
nfs-root-windows-path: "Z:" nfs-root-windows-path: "Z:"
file-store: 127.0.0.1
file-store: 10.5.26.234
#文件存储服务端 共享目录 #文件存储服务端 共享目录
file-store-root-path: /nfs/ file-store-root-path: /nfs/
file-store-root-windows-path: "Z:" file-store-root-windows-path: "Z:"
@@ -80,8 +80,8 @@ recycle:
# ES服务地址及端口 # ES服务地址及端口
es: es:
host: 127.0.0.1
serverPort: 32321
transportPort: 32322
clusterName: kubenets-logging
host: 10.5.29.100
serverPort: 9200
transportPort: 9300
clusterName: docker-cluster
index: dataset_text_test index: dataset_text_test

+ 1
- 1
dubhe-server/yaml/dubhe-notebook.yaml View File

@@ -3,7 +3,7 @@ notebook-specs:
# cpu数量(核) # cpu数量(核)
cpu-num: 1 cpu-num: 1
# gpu数量(核) # gpu数量(核)
gpu-num: 1
gpu-num: 0
# 内存大小(M) # 内存大小(M)
mem-num: 1024 mem-num: 1024
# 工作空间配额(m) # 工作空间配额(m)


+ 2
- 3
dubhe-server/yaml/dubhe-serving-gateway.yaml View File

@@ -22,8 +22,8 @@ management:


serving: serving:
gateway: gateway:
# 在线服务http接口域名,解析地址为serving网关部署的服务器地址
postfixUrl: .openapi.dubhe.club
# 在线服务http接口 host:port,解析地址为serving网关部署的服务地址
postfixUrl: 10.101.12.41:8081
token: token:
secret-key: 1n3512x secret-key: 1n3512x
expire-seconds: 108000 expire-seconds: 108000
@@ -31,7 +31,6 @@ serving:
httpclient: httpclient:
connect-timeout: 1000 connect-timeout: 1000
response-timeout: 3000 response-timeout: 3000

#关闭熔断机制 #关闭熔断机制
feign: feign:
hystrix: hystrix:

+ 0
- 52
dubhe-server/yaml/dubhe-task.yaml View File

@@ -1,52 +0,0 @@
# serving预测上传文件大小限制
spring:
servlet:
multipart:
# 设置单个文件的大小
max-file-size: 10MB
# 设置单次请求的文件的总大小
max-request-size: 100MB

k8s:
# nfs服务暴露的IP地址 如需测试需修改为合适的地址
nfs: 127.0.0.1
#nfs服务端 共享目录
nfs-root-path: /nfs/
nfs-root-windows-path: "Z:"

# data模块配置
data:
annotation:
task:
# 自动标注任务分割的文件split size
splitSize: 16
# 单位ms
retryInterval: 5000
# 任务不更新置为失败的时间单位秒
failTime: 43200
server:
# 文件存储服务器用户名
userName: root
#数据集训练配置
ptversion: http://127.0.0.1:8000/

minioweb:
GetToken:
url: minio/webrpc
param:
id: 2
jsonrpc: 2.0
method: Web.Login
zip:
url: minio/zip?token=

#logback
logging.config:
classpath:logback-spring-dev.xml

# ES服务地址及端口
es:
host: 127.0.0.1
serverPort: 32321
transportPort: 32322
index: dataset_text_test

+ 1
- 1
dubhe-server/yaml/dubhe-terminal.yaml View File

@@ -5,7 +5,7 @@ terminal:
# 用户workspace目录 # 用户workspace目录
workspace-dir: "/workspace" workspace-dir: "/workspace"
# ssh主机 # ssh主机
ssh-host: 127.0.0.1
ssh-host: 10.5.26.91


# 可匿名访问路径 # 可匿名访问路径
security: security:


+ 136
- 146
dubhe-server/yaml/gateway.yaml View File

@@ -4,149 +4,139 @@ spring:
gateway: gateway:
enabled: true enabled: true
routes: routes:
# id唯一表示 路由至admin服务
- id: admin-route
# uri=> lb://服务名称
uri: lb://admin
# 路由规则
predicates:
- Path=/api/v1/admin/**
filters:
- StripPrefix=3
- id: visual-route
# uri=> lb://服务名称
uri: lb://admin
# 路由规则
predicates:
- Path=/visual/api/**
# 路由至auth服务
- id: auth-route
uri: lb://auth
predicates:
- Path=/api/v1/auth/**
filters:
- StripPrefix=3
# 路由至notebook服务
- id: notebook-route
uri: lb://dubhe-notebook
predicates:
- Path=/api/v1/notebook/**
filters:
- StripPrefix=3
# 路由至镜像管理服务
- id: image-route
uri: lb://dubhe-image
predicates:
- Path=/api/v1/image/**
filters:
- StripPrefix=3
# 路由至度量管理服务
- id: measure-route
uri: lb://dubhe-measure
predicates:
- Path=/api/v1/measure/**
filters:
- StripPrefix=3
# 路由至dubhe-k8s服务
- id: dubhe-k8s
uri: lb://dubhe-k8s
predicates:
- Path=/api/v1/k8s/**
filters:
- StripPrefix=3
# 路由至dubhe-model(模型管理)服务
- id: dubhe-model
uri: lb://dubhe-model
predicates:
- Path=/api/v1/model/**
filters:
- StripPrefix=3
# 路由至dubhe-algorithm(算法管理)服务
- id: dubhe-algorithm
uri: lb://dubhe-algorithm
predicates:
- Path=/api/v1/algorithm/**
filters:
- StripPrefix=3
# 路由至dubhe-train(训练管理)服务
- id: dubhe-train
uri: lb://dubhe-train
predicates:
- Path=/api/v1/train/**
filters:
- StripPrefix=3
# id唯一表示 路由至data(数据集)服务
- id: dubhe-data
# uri=> lb://服务名称
uri: lb://dubhe-data
# 路由规则
predicates:
- Path=/api/v1/data/**
filters:
- StripPrefix=3
# 路由至dubhe-optimize(模型优化)服务
- id: dubhe-optimize
uri: lb://dubhe-optimize
predicates:
- Path=/api/v1/optimize/**
filters:
- StripPrefix=3
# 路由至dubhe-serving(云端Serving)服务
- id: dubhe-serving
uri: lb://dubhe-serving
predicates:
- Path=/api/v1/serving/**,/api/v1/batchServing/**
filters:
- StripPrefix=3
# 负载均衡样例
- id: weight-route1
uri: lb://demo-client
# 路由规则
predicates:
# 接口匹配前缀
- Path=/api/v1/w/**
# 权重组weight-group1,权重比例 1/4
- Weight=weight-group1, 1
filters:
# 根据接口匹配前缀截取
- StripPrefix=3
- id: weight-route2
uri: lb://demo-provider
# 路由规则
predicates:
- Path=/api/v1/w/**
# 权重组weight-group1,权重比例 3/4
- Weight=weight-group1, 3
filters:
- StripPrefix=3
# 路由至dubhe-data-dcm(医学数据集)服务
- id: dubhe-data-dcm
uri: lb://dubhe-data-dcm
predicates:
- Path=/api/v1/dcm/**
filters:
- StripPrefix=3
# 路由至dubhe-terminal 天枢专业版终端
- id: dubhe-terminal
uri: lb://dubhe-terminal
predicates:
- Path=/api/v1/terminal/**
filters:
- StripPrefix=3
# 路由至dubhe-tadl服务
- id: dubhe-tadl
uri: lb://dubhe-tadl
predicates:
- Path=/api/v1/tadl/**
filters:
- StripPrefix=3
# 路由至dubhe-point-cloud服务
- id: dubhe-point-cloud
uri: lb://dubhe-point-cloud
predicates:
- Path=/api/v1/pointCloud/**
filters:
- StripPrefix=3
server:
maxPostSize: -1
maxHttpHeaderSize: 10240000
# id唯一表示 路由至admin服务
- id: admin-route
# uri=> lb://服务名称
uri: lb://admin
# 路由规则
predicates:
- Path=/api/v1/admin/**
filters:
- StripPrefix=3
- id: visual-route
# uri=> lb://服务名称
uri: lb://admin
# 路由规则
predicates:
- Path=/visual/api/**
# 路由至demo-provider服务
- id: nacos-server-route
uri: lb://demo-provider
predicates:
- Path=/api/v1/server/**,/api/v1/service/**
filters:
- StripPrefix=3
# 路由至auth服务
- id: auth-route
uri: lb://auth
predicates:
- Path=/api/v1/auth/**
filters:
- StripPrefix=3
# 路由至notebook服务
- id: notebook-route
uri: lb://dubhe-notebook
predicates:
- Path=/api/v1/notebook/**
filters:
- StripPrefix=3
# 路由至镜像管理服务
- id: image-route
uri: lb://dubhe-image
predicates:
- Path=/api/v1/image/**
filters:
- StripPrefix=3
# 路由至度量管理服务
- id: measure-route
uri: lb://dubhe-measure
predicates:
- Path=/api/v1/measure/**
filters:
- StripPrefix=3
# 路由至dubhe-k8s服务
- id: dubhe-k8s
uri: lb://dubhe-k8s
predicates:
- Path=/api/v1/k8s/**
filters:
- StripPrefix=3
# 路由至dubhe-model(模型管理)服务
- id: dubhe-model
uri: lb://dubhe-model
predicates:
- Path=/api/v1/model/**
filters:
- StripPrefix=3
# 路由至dubhe-algorithm(算法管理)服务
- id: dubhe-algorithm
uri: lb://dubhe-algorithm
predicates:
- Path=/api/v1/algorithm/**
filters:
- StripPrefix=3
# 路由至dubhe-train(训练管理)服务
- id: dubhe-train
uri: lb://dubhe-train
predicates:
- Path=/api/v1/train/**
filters:
- StripPrefix=3
# id唯一表示 路由至data(数据集)服务
- id: dubhe-data
# uri=> lb://服务名称
uri: lb://dubhe-data
# 路由规则
predicates:
- Path=/api/v1/data/**
filters:
- StripPrefix=3
# 路由至dubhe-optimize(模型优化)服务
- id: dubhe-optimize
uri: lb://dubhe-optimize
predicates:
- Path=/api/v1/optimize/**
filters:
- StripPrefix=3
# 路由至dubhe-serving(云端Serving)服务
- id: dubhe-serving
uri: lb://dubhe-serving
predicates:
- Path=/api/v1/serving/**,/api/v1/batchServing/**
filters:
- StripPrefix=3
# 负载均衡样例
- id: weight-route1
uri: lb://demo-client
# 路由规则
predicates:
# 接口匹配前缀
- Path=/api/v1/w/**
# 权重组weight-group1,权重比例 1/4
- Weight=weight-group1, 1
filters:
# 根据接口匹配前缀截取
- StripPrefix=3
- id: weight-route2
uri: lb://demo-provider
# 路由规则
predicates:
- Path=/api/v1/w/**
# 权重组weight-group1,权重比例 3/4
- Weight=weight-group1, 3
filters:
- StripPrefix=3
# 路由至dubhe-data-dcm(医学数据集)服务
- id: dubhe-data-dcm
uri: lb://dubhe-data-dcm
predicates:
- Path=/api/v1/dcm/**
filters:
- StripPrefix=3
# 路由至dubhe-tadl服务
- id: dubhe-tadl
uri: lb://dubhe-tadl
predicates:
- Path=/api/v1/tadl/**
filters:
- StripPrefix=3

+ 3
- 9
dubhe-server/yaml/image.yaml View File

@@ -3,7 +3,7 @@ spring:
redis: redis:
#数据库索引 #数据库索引
database: 1 database: 1
host: 127.0.0.1
host: 10.5.26.88
port: 6379 port: 6379
password: password:
#连接超时时间 #连接超时时间
@@ -12,9 +12,9 @@ spring:
druid: druid:
db-type: com.alibaba.druid.pool.DruidDataSource db-type: com.alibaba.druid.pool.DruidDataSource
driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true
url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true
username: test username: test
password: test
password: zj12345678


# 初始化配置 # 初始化配置
initial-size: 3 initial-size: 3
@@ -43,11 +43,5 @@ spring:
url-pattern: /* url-pattern: /*
exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*"


#配置harbor
harbor:
address: harbor.dubhe.ai
username: admin
password: Harbor12345






+ 0
- 19
dubhe-server/yaml/kubeconfig.yaml View File

@@ -1,19 +0,0 @@
apiVersion: v1
clusters:
- cluster:
certificate-authority-data:
server: https://127.0.0.1:6443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: kubernetes-admin
name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin
user:
client-certificate-data:
client-key-data:

+ 5
- 5
dubhe-server/yaml/measure.yaml View File

@@ -3,7 +3,7 @@ spring:
redis: redis:
#数据库索引 #数据库索引
database: 1 database: 1
host: 127.0.0.1
host: 10.5.26.88
port: 6379 port: 6379
password: password:
#连接超时时间 #连接超时时间
@@ -16,10 +16,10 @@ model:
atlas: atlas:
url: url:
# 流服务地址 # 流服务地址
rtmp: rtmp://127.0.0.1:1935/live/
rtmp: rtmp://10.5.30.42:1935/live/
# vms接口地址 # vms接口地址
vms: http://127.0.0.1:8000/api/vms
vms: http://10.105.10.51:8000/api/vms
# 推理服务地址 # 推理服务地址
video-serve: http://127.0.0.1:32761/video
video-serve: http://10.105.0.6:32761/video
# 拉流地址,需要拼接streamId # 拉流地址,需要拼接streamId
stream: http://127.0.0.1:8055/live?port=1935&app=live&stream=
stream: http://10.5.30.42:8055/live?port=1935&app=live&stream=

+ 3
- 3
webapp/.env.pre View File

@@ -14,16 +14,16 @@ VUE_APP_DATA_API = '/'
VUE_APP_VISUAL_API = '/' VUE_APP_VISUAL_API = '/'


# minio # minio
VUE_APP_MINIO_API = 'http://127.0.0.1:9000/minio'
VUE_APP_MINIO_API = 'http://10.105.1.132:9000/minio'


# atlas # atlas
VUE_APP_ATLAS_HOST = 'http://127.0.0.1'
VUE_APP_ATLAS_HOST = 'http://183.129.174.186'


# DCM4CHEE # DCM4CHEE
VUE_APP_DCM_API = 'http://pre.dubhe.club/dcm4chee/dcm4chee-arc/aets/DCM4CHEE_ADMIN' VUE_APP_DCM_API = 'http://pre.dubhe.club/dcm4chee/dcm4chee-arc/aets/DCM4CHEE_ADMIN'


# minIO 服务 IP # minIO 服务 IP
VUE_APP_MINIO_ENDPOINT = '127.0.0.1'
VUE_APP_MINIO_ENDPOINT = '10.105.1.132'
# minIO 服务 端口 # minIO 服务 端口
VUE_APP_MINIO_PORT = '9000' VUE_APP_MINIO_PORT = '9000'
# 是否开启 SSL # 是否开启 SSL


+ 2
- 1
webapp/.eslintignore View File

@@ -5,4 +5,5 @@ dist
src/components/Crud src/components/Crud
mock mock
src/views/visual src/views/visual
src/store/modules/Visual
src/store/modules/Visual
src/utils/VisualUtils

+ 4
- 7
webapp/babel.config.js View File

@@ -1,12 +1,9 @@
const plugins = [
"@vue/babel-plugin-transform-vue-jsx",
"@babel/plugin-proposal-optional-chaining",
];
const plugins = ['@vue/babel-plugin-transform-vue-jsx', '@babel/plugin-proposal-optional-chaining'];
// 生产环境移除console // 生产环境移除console
if (process.env.NODE_ENV === "production") {
plugins.push("transform-remove-console");
if (process.env.NODE_ENV === 'production') {
plugins.push('transform-remove-console');
} }
module.exports = { module.exports = {
plugins, plugins,
presets: [["@vue/app",{ useBuiltIns: "entry" }]],
presets: [['@vue/app', { useBuiltIns: 'entry' }]],
}; };

+ 12
- 0
webapp/package.json View File

@@ -24,6 +24,18 @@
"lint:style": "stylelint src/**/*.{html,vue,css,sass,scss}", "lint:style": "stylelint src/**/*.{html,vue,css,sass,scss}",
"fix:style": "stylelint src/**/*.{html,vue,css,sass,scss} --fix" "fix:style": "stylelint src/**/*.{html,vue,css,sass,scss} --fix"
}, },
"husky": {
"hooks": {
"pre-commit": "lint-staged",
"commit-msg": "commitlint -E HUSKY_GIT_PARAMS"
}
},
"lint-staged": {
"src/**/*.{js,vue}": [
"eslint --fix",
"git add"
]
},
"repository": { "repository": {
"type": "git", "type": "git",
"url": "git@codeup.teambition.com:zhejianglab/dubhe-web.git" "url": "git@codeup.teambition.com:zhejianglab/dubhe-web.git"


+ 2
- 10
webapp/src/api/trainingImage/index.js View File

@@ -19,7 +19,7 @@ import { API_MODULE_NAME } from '@/config';


export function list(params) { export function list(params) {
return request({ return request({
url: `/${API_MODULE_NAME.IMAGE}/ptImage/info`,
url: `/${API_MODULE_NAME.IMAGE}/ptImage/list`,
method: 'get', method: 'get',
params, params,
}); });
@@ -27,7 +27,7 @@ export function list(params) {


export function add(data) { export function add(data) {
return request({ return request({
url: `/${API_MODULE_NAME.IMAGE}/ptImage/uploadImage`,
url: `/${API_MODULE_NAME.IMAGE}/ptImage`,
method: 'post', method: 'post',
data, data,
}); });
@@ -65,14 +65,6 @@ export function getImageTagList(params) {
}); });
} }


export function setPrecast(params) {
return request({
url: `/${API_MODULE_NAME.IMAGE}/ptImage/imageResource`,
method: 'put',
params,
});
}

// 设置notebook默认镜像-获取默认镜像 // 设置notebook默认镜像-获取默认镜像
export function getDefaultImage(params) { export function getDefaultImage(params) {
return request({ return request({


+ 4
- 0
webapp/src/assets/styles/atomic.scss View File

@@ -332,6 +332,10 @@
width: 320px; width: 320px;
} }


.w-400 {
width: 400px;
}

.w-500 { .w-500 {
width: 500px; width: 500px;
} }


+ 3
- 0
webapp/src/components/BaseTooltip/index.vue View File

@@ -9,6 +9,9 @@ the License. * ============================================================= */
<template> <template>
<el-tooltip v-bind="mergedAttrs"> <el-tooltip v-bind="mergedAttrs">
<i class="primary f18 vm" :class="[icon]" /> <i class="primary f18 vm" :class="[icon]" />
<template #content>
<slot name="content" />
</template>
</el-tooltip> </el-tooltip>
</template> </template>




+ 3
- 2
webapp/src/components/Training/jobForm.vue View File

@@ -541,6 +541,7 @@ import {
RESOURCES_MODULE_ENUM, RESOURCES_MODULE_ENUM,
RESOURCES_POOL_TYPE_ENUM, RESOURCES_POOL_TYPE_ENUM,
RESOURCES_POOL_TYPE_MAP, RESOURCES_POOL_TYPE_MAP,
IMAGE_TYPE_ENUM,
} from '@/utils'; } from '@/utils';
import { list as getAlgorithmList, getAlgorithmInfo } from '@/api/algorithm/algorithm'; import { list as getAlgorithmList, getAlgorithmInfo } from '@/api/algorithm/algorithm';
import { getModelByResource } from '@/api/model/model'; import { getModelByResource } from '@/api/model/model';
@@ -551,7 +552,7 @@ import { list as getSpecsNames } from '@/api/system/resources';
import { list as getNotebooks } from '@/api/development/notebook'; import { list as getNotebooks } from '@/api/development/notebook';
import { trainConfig } from '@/config'; import { trainConfig } from '@/config';
import { NOTEBOOK_STATUS_ENUM } from '@/views/development/utils'; import { NOTEBOOK_STATUS_ENUM } from '@/views/development/utils';
import { IMAGE_TYPE, TRAINING_TYPE_ENUM } from '@/views/trainingJob/utils';
import { TRAINING_TYPE_ENUM } from '@/views/trainingJob/utils';
import BaseTooltip from '@/components/BaseTooltip'; import BaseTooltip from '@/components/BaseTooltip';
import DataSourceSelector from './dataSourceSelector'; import DataSourceSelector from './dataSourceSelector';


@@ -929,7 +930,7 @@ export default {
}, },
async getHarborProjects() { async getHarborProjects() {
this.harborProjectList = await getImageNameList({ this.harborProjectList = await getImageNameList({
imageTypes: [IMAGE_TYPE.TRAIN],
imageTypes: [IMAGE_TYPE_ENUM.TRAIN],
}); });
if ( if (
this.form.imageName && this.form.imageName &&


+ 1
- 1
webapp/src/utils/VisualUtils/download.js View File

@@ -39,7 +39,7 @@ const covertSVG2Image = (node, name, width, height, type = 'png') => {
const context = canvas.getContext('2d'); const context = canvas.getContext('2d');
context.fillStyle = '#fff'; context.fillStyle = '#fff';
context.fillRect(0, 0, 10000, 10000); context.fillRect(0, 0, 10000, 10000);
image.onload = function() {
image.onload = () => {
context.drawImage(image, 0, 0); context.drawImage(image, 0, 0);
const a = document.createElement('a'); const a = document.createElement('a');
a.download = `${name}.${type}`; a.download = `${name}.${type}`;


+ 18
- 0
webapp/src/utils/constant.js View File

@@ -133,6 +133,24 @@ export const K8S_BUSINESS_LABEL_MAP = {
[K8S_BUSINESS_LABEL_ENUM.POINT_CLOUD]: '点云数据集', [K8S_BUSINESS_LABEL_ENUM.POINT_CLOUD]: '点云数据集',
}; };


// 镜像分类枚举
export const IMAGE_TYPE_ENUM = {
NOTEBOOK: 0, // notebook镜像类型
TRAIN: 1, // 训练镜像和预置镜像类型
SERVING: 2, // serving镜像
TERMINAL: 3, // 终端镜像
POINTCLOUD: 4, // 点云镜像
DATASETMARKED: 5, // 数据标注镜像
};
export const IMAGE_TYPE_MAP = {
[IMAGE_TYPE_ENUM.NOTEBOOK]: 'Notebook镜像',
[IMAGE_TYPE_ENUM.TRAIN]: '训练镜像',
[IMAGE_TYPE_ENUM.SERVING]: 'Serving镜像',
[IMAGE_TYPE_ENUM.TERMINAL]: '终端镜像',
[IMAGE_TYPE_ENUM.POINTCLOUD]: '点云镜像',
[IMAGE_TYPE_ENUM.DATASETMARKED]: '数据标注镜像',
};

// 默认进度条颜色 // 默认进度条颜色
export const defaultProcessColors = [ export const defaultProcessColors = [
{ color: '#909399', percentage: 40 }, { color: '#909399', percentage: 40 },


+ 1
- 1
webapp/src/utils/download.js View File

@@ -20,7 +20,7 @@ import ZIP from './zip';


const pMap = require('p-map'); const pMap = require('p-map');


//streamSaver.mitm = 'https://static.zhejianglab.com/mitm.html';
streamSaver.mitm = 'https://static.zhejianglab.com/mitm.html';


// 默认名字解析 // 默认名字解析
const defaultName = (file) => file.name; const defaultName = (file) => file.name;


+ 2
- 2
webapp/src/views/algorithm/components/algorithmForm.vue View File

@@ -171,13 +171,13 @@ import {
invalidFileNameChar, invalidFileNameChar,
defaultProcessColors, defaultProcessColors,
ALGORITHM_RESOURCE_ENUM, ALGORITHM_RESOURCE_ENUM,
IMAGE_TYPE_ENUM,
} from '@/utils'; } from '@/utils';
import UploadInline from '@/components/UploadForm/inline'; import UploadInline from '@/components/UploadForm/inline';
import UploadProgress from '@/components/UploadProgress'; import UploadProgress from '@/components/UploadProgress';
import { useMapGetters } from '@/hooks'; import { useMapGetters } from '@/hooks';
import { algorithmConfig } from '@/config'; import { algorithmConfig } from '@/config';
import { getImageNameList, getImageTagList } from '@/api/trainingImage'; import { getImageNameList, getImageTagList } from '@/api/trainingImage';
import { IMAGE_TYPE } from '@/views/trainingJob/utils';


const defaultForm = { const defaultForm = {
id: null, id: null,
@@ -289,7 +289,7 @@ export default {
}; };
// 获取镜像名列表 // 获取镜像名列表
const getImageNames = async (keepValue = false) => { const getImageNames = async (keepValue = false) => {
state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.TRAIN] });
state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.TRAIN] });
if (!keepValue || !form.imageName) { if (!keepValue || !form.imageName) {
form.imageTag = null; form.imageTag = null;
} else if (!state.imageNameList.includes(form.imageName)) { } else if (!state.imageNameList.includes(form.imageName)) {


+ 2
- 3
webapp/src/views/cloudServing/components/forms/batchServingForm.vue View File

@@ -199,14 +199,13 @@
<script> <script>
import { mapGetters } from 'vuex'; import { mapGetters } from 'vuex';


import { getUniqueId, invalidFileNameChar, RESOURCES_MODULE_ENUM } from '@/utils';
import { getUniqueId, invalidFileNameChar, RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils';
import { getServingModel } from '@/api/model/model'; import { getServingModel } from '@/api/model/model';
import { list as getModelVersions } from '@/api/model/modelVersion'; import { list as getModelVersions } from '@/api/model/modelVersion';
import { getImageNameList, getImageTagList } from '@/api/trainingImage'; import { getImageNameList, getImageTagList } from '@/api/trainingImage';
import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm';
import { list as getSpecsNames } from '@/api/system/resources'; import { list as getSpecsNames } from '@/api/system/resources';
import { validateNameWithHyphen } from '@/utils/validate'; import { validateNameWithHyphen } from '@/utils/validate';
import { IMAGE_TYPE } from '@/views/trainingJob/utils';
import RunParamForm from '@/components/Training/runParamForm'; import RunParamForm from '@/components/Training/runParamForm';
import BaseModal from '@/components/BaseModal'; import BaseModal from '@/components/BaseModal';
import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; import AlgorithmForm from '@/views/algorithm/components/algorithmForm';
@@ -541,7 +540,7 @@ export default {


// 镜像选择 // 镜像选择
async getImageNames(keepValue = false) { async getImageNames(keepValue = false) {
this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.SERVING] });
this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.SERVING] });
if (!keepValue || !this.form.imageName) { if (!keepValue || !this.form.imageName) {
this.form.imageTag = null; this.form.imageTag = null;
} else if (!this.imageNameList.includes(this.form.imageName)) { } else if (!this.imageNameList.includes(this.form.imageName)) {


+ 3
- 4
webapp/src/views/cloudServing/components/forms/servingModelConfig.vue View File

@@ -204,8 +204,7 @@ import { getImageNameList, getImageTagList } from '@/api/trainingImage';
import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm';
import { list as getSpecsNames } from '@/api/system/resources'; import { list as getSpecsNames } from '@/api/system/resources';
import { servingConfig } from '@/config'; import { servingConfig } from '@/config';
import { RESOURCES_MODULE_ENUM } from '@/utils';
import { IMAGE_TYPE } from '@/views/trainingJob/utils';
import { RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils';
import RunParamForm from '@/components/Training/runParamForm'; import RunParamForm from '@/components/Training/runParamForm';
import BaseModal from '@/components/BaseModal'; import BaseModal from '@/components/BaseModal';
import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; import AlgorithmForm from '@/views/algorithm/components/algorithmForm';
@@ -540,7 +539,7 @@ export default {
// 镜像选择 // 镜像选择
// 获取镜像名称列表 // 获取镜像名称列表
async getImageNames(keepValue = false) { async getImageNames(keepValue = false) {
this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.SERVING] });
this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.SERVING] });
if (!keepValue || !this.form.imageName) { if (!keepValue || !this.form.imageName) {
this.form.imageTag = null; this.form.imageTag = null;
} else if (!this.imageNameList.includes(this.form.imageName)) { } else if (!this.imageNameList.includes(this.form.imageName)) {
@@ -554,7 +553,7 @@ export default {
async getImageTags(imageName, keepValue = false) { async getImageTags(imageName, keepValue = false) {
this.imageTagList = await getImageTagList({ this.imageTagList = await getImageTagList({
imageName, imageName,
imageTypes: [IMAGE_TYPE.SERVING],
imageTypes: [IMAGE_TYPE_ENUM.SERVING],
}); });
if (keepValue && this.form.imageTag) { if (keepValue && this.form.imageTag) {
if (!this.imageTagList.some((image) => image.imageTag === this.form.imageTag)) { if (!this.imageTagList.some((image) => image.imageTag === this.form.imageTag)) {


+ 7
- 3
webapp/src/views/dataset/modelService/CreateModelService.vue View File

@@ -134,8 +134,12 @@ import { list as listBranchModel } from '@/api/model/modelVersion';
import { getImageNameList, getImageTagList } from '@/api/trainingImage'; import { getImageNameList, getImageTagList } from '@/api/trainingImage';
import { list as listResourceSpec } from '@/api/system/resources'; import { list as listResourceSpec } from '@/api/system/resources';
import { types } from '@/utils/validate'; import { types } from '@/utils/validate';
import { RESOURCES_POOL_TYPE_ENUM, RESOURCES_MODULE_ENUM, modelTypeSymbol } from '@/utils/constant';
import { IMAGE_TYPE } from '@/views/trainingJob/utils';
import {
RESOURCES_POOL_TYPE_ENUM,
RESOURCES_MODULE_ENUM,
modelTypeSymbol,
IMAGE_TYPE_ENUM,
} from '@/utils/constant';


const initialForm = { const initialForm = {
name: undefined, name: undefined,
@@ -246,7 +250,7 @@ export default {
}; };


const queryImages = () => { const queryImages = () => {
return getImageNameList({ imageTypes: [IMAGE_TYPE.DATASETMARKED] });
return getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.DATASETMARKED] });
}; };


const handleModelParentChange = () => { const handleModelParentChange = () => {


+ 7
- 3
webapp/src/views/dataset/modelService/ModifyModelService.vue View File

@@ -143,8 +143,12 @@ import { list as listBranchModel } from '@/api/model/modelVersion';
import { getImageNameList, getImageTagList } from '@/api/trainingImage'; import { getImageNameList, getImageTagList } from '@/api/trainingImage';
import { list as listResourceSpec } from '@/api/system/resources'; import { list as listResourceSpec } from '@/api/system/resources';
import { types } from '@/utils/validate'; import { types } from '@/utils/validate';
import { RESOURCES_POOL_TYPE_ENUM, RESOURCES_MODULE_ENUM, modelTypeSymbol } from '@/utils/constant';
import { IMAGE_TYPE } from '@/views/trainingJob/utils';
import {
RESOURCES_POOL_TYPE_ENUM,
RESOURCES_MODULE_ENUM,
modelTypeSymbol,
IMAGE_TYPE_ENUM,
} from '@/utils/constant';


const initialForm = { const initialForm = {
name: undefined, name: undefined,
@@ -283,7 +287,7 @@ export default {


// 查询镜像列表 // 查询镜像列表
const queryImages = () => { const queryImages = () => {
return getImageNameList({ imageTypes: [IMAGE_TYPE.DATASETMARKED] });
return getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.DATASETMARKED] });
}; };


// 查询镜像版本列表 // 查询镜像版本列表


+ 3
- 3
webapp/src/views/dataset/pointCloud/components/auto-annotate-modal.vue View File

@@ -163,8 +163,8 @@ import {
ALGORITHM_RESOURCE_ENUM, ALGORITHM_RESOURCE_ENUM,
MODEL_RESOURCE_ENUM, MODEL_RESOURCE_ENUM,
RESOURCES_POOL_TYPE_ENUM, RESOURCES_POOL_TYPE_ENUM,
IMAGE_TYPE_ENUM,
} from '@/utils'; } from '@/utils';
import { IMAGE_TYPE } from '@/views/trainingJob/utils';


const initFormState = { const initFormState = {
modelId: null, modelId: null,
@@ -285,7 +285,7 @@ export default {
return Promise.reject(); return Promise.reject();
} }
return getImageTagList({ return getImageTagList({
imageTypes: IMAGE_TYPE.POINTCLOUD,
imageTypes: IMAGE_TYPE_ENUM.POINTCLOUD,
imageName: modelState.imageName, imageName: modelState.imageName,
}).then((res) => { }).then((res) => {
state.imageTagList = res; state.imageTagList = res;
@@ -294,7 +294,7 @@ export default {


// 镜像选择 // 镜像选择
const getImageNames = async () => { const getImageNames = async () => {
state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.POINTCLOUD] });
state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.POINTCLOUD] });
if ( if (
modelState.imageName && modelState.imageName &&
!state.imageNameList.some((image) => image === modelState.imageName) !state.imageNameList.some((image) => image === modelState.imageName)


+ 3
- 4
webapp/src/views/development/components/CreateDialog.vue View File

@@ -122,14 +122,13 @@
</template> </template>


<script> <script>
import { validateNameWithHyphen, RESOURCES_MODULE_ENUM } from '@/utils';
import { validateNameWithHyphen, RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils';
import BaseModal from '@/components/BaseModal'; import BaseModal from '@/components/BaseModal';
import InfoSelect from '@/components/InfoSelect'; import InfoSelect from '@/components/InfoSelect';
import { getPublishedDatasets, getDatasetVersions } from '@/api/preparation/dataset'; import { getPublishedDatasets, getDatasetVersions } from '@/api/preparation/dataset';
import { add as addNotebook } from '@/api/development/notebook'; import { add as addNotebook } from '@/api/development/notebook';
import { list as getSpecsNames } from '@/api/system/resources'; import { list as getSpecsNames } from '@/api/system/resources';
import { getImageNameList, getImageTagList } from '@/api/trainingImage/index'; import { getImageNameList, getImageTagList } from '@/api/trainingImage/index';
import { IMAGE_TYPE } from '@/views/trainingJob/utils';


const defaultForm = { const defaultForm = {
noteBookName: null, noteBookName: null,
@@ -208,7 +207,7 @@ export default {
} }
}, },
async getHarborProjects() { async getHarborProjects() {
this.harborProjectList = await getImageNameList({ imageTypes: [IMAGE_TYPE.NOTEBOOK] });
this.harborProjectList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.NOTEBOOK] });
if ( if (
this.form.imageName && this.form.imageName &&
!this.harborProjectList.some((project) => project === this.form.imageName) !this.harborProjectList.some((project) => project === this.form.imageName)
@@ -237,7 +236,7 @@ export default {
} }
return getImageTagList({ return getImageTagList({
imageName: this.form.imageName, imageName: this.form.imageName,
imageTypes: [IMAGE_TYPE.NOTEBOOK],
imageTypes: [IMAGE_TYPE_ENUM.NOTEBOOK],
}).then((res) => { }).then((res) => {
this.harborImageList = res; this.harborImageList = res;
}); });


+ 52
- 238
webapp/src/views/trainingImage/index.vue View File

@@ -48,7 +48,7 @@
<el-table <el-table
v-if="prefabricate" v-if="prefabricate"
ref="table" ref="table"
v-loading="crud.loading || disableEdit"
v-loading="crud.loading"
:data="crud.data" :data="crud.data"
highlight-current-row highlight-current-row
@selection-change="crud.selectionChangeHandler" @selection-change="crud.selectionChangeHandler"
@@ -57,21 +57,6 @@
<el-table-column v-if="!isPreset" prop="id" label="ID" sortable="custom" width="80px" /> <el-table-column v-if="!isPreset" prop="id" label="ID" sortable="custom" width="80px" />
<el-table-column prop="imageName" label="镜像名称" sortable="custom" /> <el-table-column prop="imageName" label="镜像名称" sortable="custom" />
<el-table-column prop="imageTag" label="镜像版本号" sortable="custom" /> <el-table-column prop="imageTag" label="镜像版本号" sortable="custom" />
<el-table-column prop="imageStatus" label="状态" width="130px">
<template #header>
<dropdown-header
title="状态"
:list="imageStatusList"
:filtered="Boolean(localQuery.imageStatus)"
@command="filterStatus"
/>
</template>
<template slot-scope="scope">
<el-tag effect="plain" :type="map[scope.row.imageStatus]">
{{ statusMap[scope.row.imageStatus] }}
</el-tag>
</template>
</el-table-column>
<el-table-column prop="imageTypes" label="镜像用途" width="180px"> <el-table-column prop="imageTypes" label="镜像用途" width="180px">
<template #header> <template #header>
<dropdown-header <dropdown-header
@@ -91,31 +76,8 @@
<span>{{ parseTime(scope.row.createTime) }}</span> <span>{{ parseTime(scope.row.createTime) }}</span>
</template> </template>
</el-table-column> </el-table-column>
<el-table-column v-if="isNotebook" prop="imageResource" label="是否为默认" align="center">
<template slot-scope="scope">
<i
:class="resourceObj(scope.row.imageResource).icon"
:style="{ color: resourceObj(scope.row.imageResource).color, fontSize: '20px' }"
></i>
</template>
</el-table-column>
<el-table-column v-if="isAdmin || isCustom" label="操作" width="200px" fixed="right"> <el-table-column v-if="isAdmin || isCustom" label="操作" width="200px" fixed="right">
<template slot-scope="scope"> <template slot-scope="scope">
<el-tooltip
v-if="isAdmin && isNotebook"
effect="dark"
content="设为在线编辑算法时创建nootbook的默认镜像"
placement="top"
>
<el-button
:id="`doPrecast_` + scope.$index"
:disabled="Boolean(scope.row.imageResource)"
type="text"
@click.stop="doPrecast(scope.row.id)"
>
{{ resourceObj(scope.row.imageResource).butText }}
</el-button>
</el-tooltip>
<el-button <el-button
v-if="(hasPermission('training:image:edit') && isCustom) || (isPreset && isAdmin)" v-if="(hasPermission('training:image:edit') && isCustom) || (isPreset && isAdmin)"
:id="`doEdit_` + scope.$index" :id="`doEdit_` + scope.$index"
@@ -143,15 +105,13 @@
:visible="crud.status.cu > 0" :visible="crud.status.cu > 0"
:title="crud.status.title" :title="crud.status.title"
:loading="crud.status.cu === 2" :loading="crud.status.cu === 2"
:disabled="loading"
width="600px" width="600px"
@open="onDialogOpen" @open="onDialogOpen"
@close="onDialogClose"
@cancel="crud.cancelCU" @cancel="crud.cancelCU"
@ok="crud.submitCU" @ok="crud.submitCU"
> >
<el-form ref="form" :model="form" :rules="rules" label-width="120px"> <el-form ref="form" :model="form" :rules="rules" label-width="120px">
<el-form-item v-if="isEdit && isAdmin" label="镜像类别" prop="imageResource">
<el-form-item v-if="isFormAdd && isAdmin" label="镜像类别" prop="imageResource">
<el-radio-group v-model="form.imageResource" @change="imageResourceChange"> <el-radio-group v-model="form.imageResource" @change="imageResourceChange">
<el-radio :label="Number(IMAGE_RESOURCE_ENUM.CUSTOM)" border class="mr-0" <el-radio :label="Number(IMAGE_RESOURCE_ENUM.CUSTOM)" border class="mr-0"
>我的镜像</el-radio >我的镜像</el-radio
@@ -159,21 +119,20 @@
<el-radio :label="Number(IMAGE_RESOURCE_ENUM.PRESET)" border>预置镜像</el-radio> <el-radio :label="Number(IMAGE_RESOURCE_ENUM.PRESET)" border>预置镜像</el-radio>
</el-radio-group> </el-radio-group>
</el-form-item> </el-form-item>
<el-form-item v-if="isEdit" label="镜像名称" prop="imageName">
<el-form-item label="镜像名称" prop="imageName">
<el-autocomplete <el-autocomplete
ref="imageName" ref="imageName"
v-model="form.imageName" v-model="form.imageName"
class="inline-input"
class="inline-input w-400"
:fetch-suggestions="querySearchAsync" :fetch-suggestions="querySearchAsync"
placeholder="请选择或输入镜像名称" placeholder="请选择或输入镜像名称"
style="width: 400px;"
></el-autocomplete> ></el-autocomplete>
</el-form-item> </el-form-item>
<el-form-item label="镜像用途"> <el-form-item label="镜像用途">
<el-select <el-select
v-model="form.imageTypes" v-model="form.imageTypes"
placeholder="请选择或输入镜像用途" placeholder="请选择或输入镜像用途"
style="width: 400px;"
class="w-400"
multiple multiple
filterable filterable
allow-create allow-create
@@ -187,36 +146,18 @@
/> />
</el-select> </el-select>
</el-form-item> </el-form-item>
<el-form-item v-if="isEdit" ref="imagePath" label="镜像文件路径" prop="imagePath">
<upload-inline
v-if="crud.status.cu > 0"
ref="upload"
action="fakeApi"
accept=".zip,.tar,.rar,.gz"
list-type="text"
:acceptSize="imageConfig.uploadFileAcceptSize"
:acceptSizeFormat="uploadSizeFomatter"
:params="uploadParams"
:show-file-count="false"
:auto-upload="true"
:filters="uploadFilters"
:limit="1"
:on-remove="onFileRemove"
@uploadStart="uploadStart"
@uploadSuccess="uploadSuccess"
@uploadError="uploadError"
/>
<upload-progress
v-if="loading"
:progress="progress"
:color="customColors"
:status="status"
:size="size"
@onSetProgress="onSetProgress"
/>
<el-form-item ref="imageUrl" label="镜像地址" prop="imageUrl">
<el-input v-model="form.imageUrl" placeholder="请输入镜像地址" class="w-400" />
<BaseTooltip icon="el-icon-warning" class="c-info">
<template #content>
<div>
镜像地址标准格式:镜像仓库域名/命名空间/镜像名:镜像版本号<br />示例:registry.cn-hangzhou.aliyuncs.com/enlin/notebook:v1
</div>
</template>
</BaseTooltip>
</el-form-item> </el-form-item>
<el-form-item v-if="isEdit" label="镜像版本号" prop="imageTag">
<el-input id="imageTag" v-model="form.imageTag" style="width: 400px;" />
<el-form-item label="镜像版本号" prop="imageTag">
<el-input id="imageTag" v-model="form.imageTag" class="w-400" />
</el-form-item> </el-form-item>
<el-form-item label="描述" prop="remark"> <el-form-item label="描述" prop="remark">
<el-input <el-input
@@ -227,19 +168,19 @@
maxlength="1024" maxlength="1024"
show-word-limit show-word-limit
placeholder placeholder
style="width: 400px;"
class="w-400"
/> />
</el-form-item> </el-form-item>
</el-form> </el-form>
</BaseModal> </BaseModal>
<!--Notebook默认镜像设置表单--> <!--Notebook默认镜像设置表单-->
<BaseModal <BaseModal
:visible.sync="formVisible"
:visible.sync="notebookFormVisible"
title="Notebook默认镜像设置" title="Notebook默认镜像设置"
:loading="formSubmitting"
:loading="notebookFormSubmitting"
width="800px" width="800px"
@cancel="formVisible = false"
@ok="onSubmitForm"
@cancel="notebookFormVisible = false"
@ok="onSubmitNotebookForm"
> >
<el-form <el-form
ref="noteBookFormRef" ref="noteBookFormRef"
@@ -253,7 +194,7 @@
id="defaultImage" id="defaultImage"
v-model="noteBookForm.defaultImage" v-model="noteBookForm.defaultImage"
placeholder="请选择镜像" placeholder="请选择镜像"
style="width: 400px;"
class="w-400"
filterable filterable
allow-create allow-create
default-first-option default-first-option
@@ -283,8 +224,6 @@


<script> <script>
import { mapGetters } from 'vuex'; import { mapGetters } from 'vuex';
// eslint-disable-next-line import/no-extraneous-dependencies
import { debounce } from 'throttle-debounce';


import cdOperation from '@crud/CD.operation'; import cdOperation from '@crud/CD.operation';
import rrOperation from '@crud/RR.operation'; import rrOperation from '@crud/RR.operation';
@@ -292,40 +231,36 @@ import pagination from '@crud/Pagination';
import CRUD, { presenter, header, form, crud } from '@crud/crud'; import CRUD, { presenter, header, form, crud } from '@crud/crud';
import trainingImageApi, { import trainingImageApi, {
del, del,
setPrecast,
setDefaultImage, setDefaultImage,
getDefaultImage, getDefaultImage,
getImageNameList, getImageNameList,
getImageTagList, getImageTagList,
} from '@/api/trainingImage/index'; } from '@/api/trainingImage/index';
import { import {
getUniqueId,
uploadSizeFomatter,
invalidFileNameChar,
ADMIN_ROLE_ID, ADMIN_ROLE_ID,
hasPermission, hasPermission,
validateImageName, validateImageName,
validateImageTag, validateImageTag,
IMAGE_TYPE_ENUM,
IMAGE_TYPE_MAP,
} from '@/utils'; } from '@/utils';
import BaseModal from '@/components/BaseModal'; import BaseModal from '@/components/BaseModal';
import UploadInline from '@/components/UploadForm/inline';
import BaseTooltip from '@/components/BaseTooltip';
import DropdownHeader from '@/components/DropdownHeader'; import DropdownHeader from '@/components/DropdownHeader';
import UploadProgress from '@/components/UploadProgress';
import { imageConfig } from '@/config'; import { imageConfig } from '@/config';


import { IMAGE_RESOURCE_ENUM, IMAGE_TYPE } from '../trainingJob/utils';
import { IMAGE_RESOURCE_ENUM } from '../trainingJob/utils';


const defaultForm = { const defaultForm = {
imagePath: null,
imageUrl: null,
imageTag: null, imageTag: null,
remark: null, remark: null,
imageTypes: Number(IMAGE_TYPE),
imageTypes: Number(IMAGE_TYPE_ENUM),
imageResource: Number(IMAGE_RESOURCE_ENUM.CUSTOM), imageResource: Number(IMAGE_RESOURCE_ENUM.CUSTOM),
imageName: null, imageName: null,
}; };


const defaultQuery = { const defaultQuery = {
imageStatus: null,
imageNameOrId: null, imageNameOrId: null,
imageTypes: null, imageTypes: null,
}; };
@@ -334,28 +269,27 @@ export default {
name: 'TrainingImage', name: 'TrainingImage',
components: { components: {
BaseModal, BaseModal,
BaseTooltip,
pagination, pagination,
cdOperation, cdOperation,
rrOperation, rrOperation,
UploadInline,
DropdownHeader, DropdownHeader,
UploadProgress,
}, },
cruds() { cruds() {
return CRUD({ return CRUD({
title: '镜像', title: '镜像',
crudMethod: { ...trainingImageApi }, crudMethod: { ...trainingImageApi },
optShow: { optShow: {
add: imageConfig.allowUploadImage && hasPermission('training:image:upload'),
add: imageConfig.allowUploadImage && hasPermission('training:image:save'),
del: false, del: false,
}, },
queryOnPresenterCreated: false, queryOnPresenterCreated: false,
props: { props: {
optText: { optText: {
add: '上传镜像',
add: '创建镜像',
}, },
optTitle: { optTitle: {
add: '上传',
add: '创建',
}, },
}, },
}); });
@@ -365,24 +299,6 @@ export default {
return { return {
active: IMAGE_RESOURCE_ENUM.CUSTOM, active: IMAGE_RESOURCE_ENUM.CUSTOM,
localQuery: { ...defaultQuery }, localQuery: { ...defaultQuery },
map: {
0: 'info',
1: 'success',
2: 'danger',
},
statusMap: {
0: '制作中',
1: '制作成功',
2: '制作失败',
},
imageTypesMap: {
0: 'Notebook镜像',
1: '训练镜像',
2: 'Serving镜像',
3: '终端镜像',
4: '点云镜像',
5: '数据标注镜像',
},
rules: { rules: {
imageTypes: [{ required: true, message: '请选择镜像类型', trigger: 'change' }], imageTypes: [{ required: true, message: '请选择镜像类型', trigger: 'change' }],
imageResource: [{ required: true, message: '请选择镜像来源', trigger: 'change' }], imageResource: [{ required: true, message: '请选择镜像来源', trigger: 'change' }],
@@ -390,7 +306,7 @@ export default {
{ required: true, message: '请选择项目名称', trigger: 'change' }, { required: true, message: '请选择项目名称', trigger: 'change' },
{ validator: validateImageName, trigger: ['blur', 'change'] }, { validator: validateImageName, trigger: ['blur', 'change'] },
], ],
imagePath: [{ required: true, message: '请输入镜像路径', trigger: ['blur', 'manual'] }],
imageUrl: [{ required: true, message: '请输入镜像地址', trigger: ['blur', 'manual'] }],
imageTag: [ imageTag: [
{ required: true, message: '请输入镜像版本号', trigger: 'blur' }, { required: true, message: '请输入镜像版本号', trigger: 'blur' },
{ validator: validateImageTag, trigger: ['blur', 'change'] }, { validator: validateImageTag, trigger: ['blur', 'change'] },
@@ -401,32 +317,17 @@ export default {
id: [{ required: true, message: '请选择默认镜像版本', trigger: 'blur' }], id: [{ required: true, message: '请选择默认镜像版本', trigger: 'blur' }],
}, },
harborProjectList: [], harborProjectList: [],
drawer: false,
uploadParams: {
objectPath: null, // 对象存储路径
},
progress: 0,
size: 0,
customColors: [
{ color: '#909399', percentage: 40 },
{ color: '#e6a23c', percentage: 80 },
{ color: '#67c23a', percentage: 100 },
],
disableEdit: false,
loading: false,
isEdit: false,
prefabricate: true, prefabricate: true,
// 以下为配置参数及常量参数 // 以下为配置参数及常量参数
imageConfig,
IMAGE_RESOURCE_ENUM, IMAGE_RESOURCE_ENUM,
IMAGE_TYPE,
uploadFilters: [invalidFileNameChar],
// 设置notebook默认镜像相关参数 // 设置notebook默认镜像相关参数
noteBookImages: [], noteBookImages: [],
noteBookTags: [], noteBookTags: [],
noteBookForm: { defaultImage: '', defaultTag: '', id: '' }, noteBookForm: { defaultImage: '', defaultTag: '', id: '' },
formVisible: false,
formSubmitting: false,

formType: 'add',
notebookFormVisible: false,
notebookFormSubmitting: false,
}; };
}, },
computed: { computed: {
@@ -441,14 +342,8 @@ export default {
isPreset() { isPreset() {
return this.active === IMAGE_RESOURCE_ENUM.PRESET; return this.active === IMAGE_RESOURCE_ENUM.PRESET;
}, },
isNotebook() {
return this.active === IMAGE_RESOURCE_ENUM.NOTEBOOK;
},
isTerminal() {
return this.active === IMAGE_RESOURCE_ENUM.TERMINAL;
},
disableAdd() { disableAdd() {
if (this.isAdmin) return this.isTerminal; // 管理员只有在终端镜像处无法点击上传
if (this.isAdmin) return false; // 管理员可以创建我的镜像和预置镜像
return !this.isCustom; // 其他角色只有在我的镜像处可以点击上传 return !this.isCustom; // 其他角色只有在我的镜像处可以点击上传
}, },
operationProps() { operationProps() {
@@ -456,37 +351,24 @@ export default {
disabled: this.disableAdd, disabled: this.disableAdd,
}; };
}, },
imageStatusList() {
const arr = [{ label: '全部', value: null }];
for (const key in this.statusMap) {
arr.push({ label: this.statusMap[key], value: key });
}
return arr;
},
imageTypesList() { imageTypesList() {
const arr = [{ label: '全部', value: null }]; const arr = [{ label: '全部', value: null }];
for (const key in this.imageTypesMap) {
arr.push({ label: this.imageTypesMap[key], value: +key });
for (const key in IMAGE_TYPE_MAP) {
arr.push({ label: IMAGE_TYPE_MAP[key], value: +key });
} }
return arr; return arr;
}, },
status() {
return this.progress === 100 ? 'success' : null;
isFormAdd() {
return this.formType === 'add';
}, },
}, },
mounted() { mounted() {
this.crud.refresh(); this.crud.refresh();
this.refetch = debounce(3000, this.crud.refresh);
this.updateImagePath();
}, },
methods: { methods: {
hasPermission, hasPermission,
getImageTypes(imageTypes) { getImageTypes(imageTypes) {
const usageStr = [];
imageTypes.forEach((item) => {
usageStr.push(this.imageTypesMap[item]);
});
return usageStr.join(',');
return imageTypes.map((type) => IMAGE_TYPE_MAP[type]).join(',');
}, },
getNoteBookTags() { getNoteBookTags() {
this.noteBookForm.defaultTag = null; this.noteBookForm.defaultTag = null;
@@ -497,7 +379,7 @@ export default {
} }
return getImageTagList({ return getImageTagList({
imageName: this.noteBookForm.defaultImage, imageName: this.noteBookForm.defaultImage,
imageTypes: IMAGE_TYPE.NOTEBOOK,
imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK,
imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET),
}).then((res) => { }).then((res) => {
this.noteBookTags = res; this.noteBookTags = res;
@@ -513,46 +395,11 @@ export default {
this.prefabricate = true; this.prefabricate = true;
}); });
}, },
onFileRemove() {
this.form.imagePath = null;
this.loading = false;
this.$refs.imagePath.validate('manual');
},
uploadStart(files) {
this.updateImagePath();
[this.loading, this.size, this.progress] = [true, files.size, 0];
},
onSetProgress(val) {
this.progress += val;
},
uploadSuccess(res) {
this.progress = 100;
setTimeout(() => {
this.loading = false;
}, 1000);
if (this.loading) {
this.form.imagePath = res[0].data.objectName;
this.$refs.imagePath.validate('manual');
}
},
uploadError() {
this.$message({
message: '上传文件失败',
type: 'error',
});
this.loading = false;
},
// hook // hook
[CRUD.HOOK.afterRefresh]() {
this.checkStatus();
},
[CRUD.HOOK.beforeToAdd]() { [CRUD.HOOK.beforeToAdd]() {
this.isEdit = true;
this.formType = 'add'; this.formType = 'add';
if (this.isPreset) { if (this.isPreset) {
this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.PRESET); this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.PRESET);
} else if (this.isNotebook) {
this.form.imageTypes = IMAGE_TYPE.NOTEBOOK;
} }
}, },
[CRUD.HOOK.beforeRefresh]() { [CRUD.HOOK.beforeRefresh]() {
@@ -567,7 +414,7 @@ export default {
} }
}, },
[CRUD.HOOK.beforeToEdit]() { [CRUD.HOOK.beforeToEdit]() {
this.isEdit = false;
this.formType = 'edit';
}, },
async querySearchAsync(queryString, cb) { async querySearchAsync(queryString, cb) {
let { harborProjectList } = this; let { harborProjectList } = this;
@@ -597,24 +444,9 @@ export default {
this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.CUSTOM); this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.CUSTOM);
this.getImageNameList(); this.getImageNameList();
}, },
onDialogClose() {
if (this.isEdit) {
this.$refs.upload.formRef.reset();
}
this.loading = false;
},
async onDialogOpen() { async onDialogOpen() {
this.getImageNameList(); this.getImageNameList();
}, },
checkStatus() {
if (this.crud.data.some((item) => [0].includes(item.imageStatus))) {
this.refetch();
}
},
filterStatus(status) {
this.localQuery.imageStatus = status;
this.crud.toQuery();
},
filterImageTypes(imageTypes) { filterImageTypes(imageTypes) {
this.localQuery.imageTypes = imageTypes; this.localQuery.imageTypes = imageTypes;
this.crud.toQuery(); this.crud.toQuery();
@@ -622,9 +454,6 @@ export default {
resetQuery() { resetQuery() {
this.localQuery = { ...defaultQuery }; this.localQuery = { ...defaultQuery };
}, },
updateImagePath() {
this.uploadParams.objectPath = `upload-temp/${this.user.id}/${getUniqueId()}`;
},
async doEdit(imageObj) { async doEdit(imageObj) {
const dataObj = { const dataObj = {
ids: [imageObj.id], ids: [imageObj.id],
@@ -642,21 +471,6 @@ export default {
this.crud.refresh(); this.crud.refresh();
}); });
}, },
doPrecast(i) {
setPrecast({ id: i }).then(() => {
this.$message({
message: '设置成功',
type: 'success',
});
this.crud.refresh();
});
},
resourceObj(resource) {
return resource
? { icon: 'el-icon-circle-check', color: '#67C23A', butText: '当前为默认' }
: { icon: 'el-icon-circle-close', color: '#F56C6C', butText: '设为默认' };
},
uploadSizeFomatter,


async doSetDefaultImage() { async doSetDefaultImage() {
// 获取默认镜像 // 获取默认镜像
@@ -666,20 +480,20 @@ export default {
this.noteBookForm.id = defaultImage.length ? defaultImage[0].id : ''; this.noteBookForm.id = defaultImage.length ? defaultImage[0].id : '';
// 获取镜像列表 // 获取镜像列表
this.noteBookImages = await getImageNameList({ this.noteBookImages = await getImageNameList({
imageTypes: IMAGE_TYPE.NOTEBOOK,
imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK,
imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET),
}); });
if (this.noteBookForm.defaultImage) { if (this.noteBookForm.defaultImage) {
this.noteBookTags = await getImageTagList({ this.noteBookTags = await getImageTagList({
imageName: this.noteBookForm.defaultImage, imageName: this.noteBookForm.defaultImage,
imageTypes: IMAGE_TYPE.NOTEBOOK,
imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK,
imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET),
}); });
} else { } else {
this.noteBookTags = []; this.noteBookTags = [];
} }


this.formVisible = true;
this.notebookFormVisible = true;
this.$nextTick(() => { this.$nextTick(() => {
this.clearValidate(); this.clearValidate();
}); });
@@ -690,17 +504,17 @@ export default {
validateField(field) { validateField(field) {
this.$refs.noteBookFormRef.validateField(field); this.$refs.noteBookFormRef.validateField(field);
}, },
onSubmitForm() {
onSubmitNotebookForm() {
this.$refs.noteBookFormRef.validate((valid) => { this.$refs.noteBookFormRef.validate((valid) => {
if (valid) { if (valid) {
this.formSubmitting = true;
this.notebookFormSubmitting = true;
setDefaultImage({ id: this.noteBookForm.id }) setDefaultImage({ id: this.noteBookForm.id })
.then(() => { .then(() => {
this.formVisible = false;
this.notebookFormVisible = false;
this.crud.toQuery(); this.crud.toQuery();
}) })
.finally(() => { .finally(() => {
this.formSubmitting = false;
this.notebookFormSubmitting = false;
}); });
} }
}); });


+ 0
- 9
webapp/src/views/trainingJob/utils.js View File

@@ -96,15 +96,6 @@ export const ATLAS_ALGORITHM_TYPE_ENUM = {
DDRL: 4, // ddrl DDRL: 4, // ddrl
}; };


export const IMAGE_TYPE = {
NOTEBOOK: 0, // notebook镜像类型
TRAIN: 1, // 训练镜像和预置镜像类型
SERVING: 2, // serving镜像
TERMINAL: 3, // 终端镜像
POINTCLOUD: 4, // 点云镜像
DATASETMARKED: 5, // 数据标注镜像
};

// 训练可视化列表页查询项 // 训练可视化列表页查询项
export const trainVisualQueryFormItems = [ export const trainVisualQueryFormItems = [
{ {


Loading…
Cancel
Save