| @@ -11,8 +11,8 @@ Nacos + Fegin + Gateway + (Spring Security + JWT + OAuth2) | |||||
| 初始化sql位置 /sql | 初始化sql位置 /sql | ||||
| **地址:** 127.0.0.1:3306 | |||||
| **用户名:** test **密码:** test | |||||
| **地址:** 127.0.0.1 | |||||
| **用户名:** test **密码:** zj12345678 | |||||
| ### Nacos | ### Nacos | ||||
| @@ -27,7 +27,6 @@ import java.util.Map; | |||||
| * @description Feign 熔断处理类 | * @description Feign 熔断处理类 | ||||
| * @date 2020-11-04 | * @date 2020-11-04 | ||||
| */ | */ | ||||
| @Component | |||||
| public class AuthServiceFallback implements AuthServiceClient { | public class AuthServiceFallback implements AuthServiceClient { | ||||
| @@ -16,6 +16,7 @@ | |||||
| */ | */ | ||||
| package org.dubhe.admin.service.impl; | package org.dubhe.admin.service.impl; | ||||
| import cn.hutool.core.collection.CollUtil; | |||||
| import cn.hutool.core.util.ObjectUtil; | import cn.hutool.core.util.ObjectUtil; | ||||
| import cn.hutool.core.util.StrUtil; | import cn.hutool.core.util.StrUtil; | ||||
| import cn.hutool.crypto.asymmetric.KeyType; | import cn.hutool.crypto.asymmetric.KeyType; | ||||
| @@ -704,6 +705,9 @@ public class UserServiceImpl extends ServiceImpl<UserMapper, User> implements Us | |||||
| Oauth2TokenDTO userDto = restResult.getData(); | Oauth2TokenDTO userDto = restResult.getData(); | ||||
| UserDTO user = findByName(authUserDTO.getUsername()); | UserDTO user = findByName(authUserDTO.getUsername()); | ||||
| Set<String> permissions = this.queryPermissionByUserId(user.getId()); | Set<String> permissions = this.queryPermissionByUserId(user.getId()); | ||||
| if (CollUtil.isEmpty(permissions)) { | |||||
| throw new BusinessException(BaseErrorCodeEnum.SYSTEM_ROLE_NOT_EXISTS); | |||||
| } | |||||
| // 返回 token 与 用户信息 | // 返回 token 与 用户信息 | ||||
| authInfo.put("token", userDto.getTokenHead() + userDto.getToken()); | authInfo.put("token", userDto.getTokenHead() + userDto.getToken()); | ||||
| authInfo.put("user", user); | authInfo.put("user", user); | ||||
| @@ -14,7 +14,6 @@ | |||||
| * limitations under the License. | * limitations under the License. | ||||
| * ============================================================= | * ============================================================= | ||||
| */ | */ | ||||
| package org.dubhe.admin; | package org.dubhe.admin; | ||||
| import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | ||||
| @@ -14,7 +14,6 @@ | |||||
| * limitations under the License. | * limitations under the License. | ||||
| * ============================================================= | * ============================================================= | ||||
| */ | */ | ||||
| package org.dubhe.auth; | package org.dubhe.auth; | ||||
| import org.junit.jupiter.api.Test; | import org.junit.jupiter.api.Test; | ||||
| @@ -1,37 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| * ============================================================= | |||||
| */ | |||||
| package org.dubhe.biz.base.constant; | |||||
| import lombok.Data; | |||||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||||
| import org.springframework.stereotype.Component; | |||||
| /** | |||||
| * @description harbor相关配置 | |||||
| * @date 2020-07-17 | |||||
| */ | |||||
| @Data | |||||
| @Component | |||||
| @ConfigurationProperties(prefix = "harbor") | |||||
| public class HarborProperties { | |||||
| private String address; | |||||
| private String username; | |||||
| private String password; | |||||
| } | |||||
| @@ -110,7 +110,7 @@ public final class Permissions { | |||||
| * 镜像管理 | * 镜像管理 | ||||
| */ | */ | ||||
| public static final String IMAGE = "hasAuthority('ROLE_training:image')"; | public static final String IMAGE = "hasAuthority('ROLE_training:image')"; | ||||
| public static final String IMAGE_UPLOAD = "hasAuthority('ROLE_training:image:upload')"; | |||||
| public static final String IMAGE_SAVE = "hasAuthority('ROLE_training:image:save')"; | |||||
| public static final String IMAGE_EDIT = "hasAuthority('ROLE_training:image:edit')"; | public static final String IMAGE_EDIT = "hasAuthority('ROLE_training:image:edit')"; | ||||
| public static final String IMAGE_DELETE = "hasAuthority('ROLE_training:image:delete')"; | public static final String IMAGE_DELETE = "hasAuthority('ROLE_training:image:delete')"; | ||||
| public static final String IMAGE_EDIT_DEFAULT = "hasAuthority('ROLE_training:image:editDefault')"; | public static final String IMAGE_EDIT_DEFAULT = "hasAuthority('ROLE_training:image:editDefault')"; | ||||
| @@ -126,9 +126,6 @@ public final class StringConstant { | |||||
| public static final String CACHE_TASK_ID ="task_id"; | public static final String CACHE_TASK_ID ="task_id"; | ||||
| public static final String CACHE_TASK_NAME ="task_name"; | public static final String CACHE_TASK_NAME ="task_name"; | ||||
| public static final String DEFAULT_IMAGE_PROJECT ="dubhe"; | |||||
| /** | /** | ||||
| * python命令行参数格式 | * python命令行参数格式 | ||||
| */ | */ | ||||
| @@ -54,8 +54,8 @@ public enum BaseErrorCodeEnum implements ErrorCode { | |||||
| SYSTEM_USER_EMAIL_NOT_EXISTS(20013, "该邮箱未注册!"), | SYSTEM_USER_EMAIL_NOT_EXISTS(20013, "该邮箱未注册!"), | ||||
| SYSTEM_USER_CANNOT_DELETE(20014, "系统默认用户不可删除!"), | SYSTEM_USER_CANNOT_DELETE(20014, "系统默认用户不可删除!"), | ||||
| SYSTEM_ROLE_CANNOT_DELETE(20015, "系统默认角色不可删除!"), | SYSTEM_ROLE_CANNOT_DELETE(20015, "系统默认角色不可删除!"), | ||||
| DATASET_ADMIN_PERMISSION_ERROR(1310,"无此权限,请联系管理员"), | |||||
| SYSTEM_ROLE_NOT_EXISTS(20016, "用户绑定角色异常,请联系管理员!"), | |||||
| DATASET_ADMIN_PERMISSION_ERROR(1310, "无此权限,请联系管理员"), | |||||
| ; | ; | ||||
| @@ -1,53 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| * ============================================================= | |||||
| */ | |||||
| package org.dubhe.biz.base.enums; | |||||
| /** | |||||
| * @description 镜像运行状态枚举 | |||||
| * @date 2020-07-15 | |||||
| **/ | |||||
| public enum ImageStateEnum { | |||||
| MAKING(0, "制作中"), | |||||
| SUCCESS(1, "制作成功"), | |||||
| FAIL(2, "制作失败"); | |||||
| /** | |||||
| * 编码 | |||||
| */ | |||||
| private Integer code; | |||||
| /** | |||||
| * 描述 | |||||
| */ | |||||
| private String description; | |||||
| ImageStateEnum(int code, String description) { | |||||
| this.code = code; | |||||
| this.description = description; | |||||
| } | |||||
| public Integer getCode() { | |||||
| return code; | |||||
| } | |||||
| public String getDescription() { | |||||
| return description; | |||||
| } | |||||
| } | |||||
| @@ -1,10 +1,12 @@ | |||||
| spring: | spring: | ||||
| cloud: | cloud: | ||||
| nacos: | nacos: | ||||
| username: nacos | |||||
| password: Tianshu | |||||
| context-path: /nacos | context-path: /nacos | ||||
| config: | config: | ||||
| namespace: dubhe-prod | namespace: dubhe-prod | ||||
| server-addr: 127.0.0.1:8848 | |||||
| server-addr: 10.105.1.132:8848 | |||||
| discovery: | discovery: | ||||
| namespace: dubhe-prod | namespace: dubhe-prod | ||||
| server-addr: 127.0.0.1:8848 | |||||
| server-addr: 10.105.1.132:8848 | |||||
| @@ -1,10 +0,0 @@ | |||||
| spring: | |||||
| cloud: | |||||
| nacos: | |||||
| context-path: /nacos | |||||
| config: | |||||
| namespace: dubhe-prod | |||||
| server-addr: 127.0.0.1:8848 | |||||
| discovery: | |||||
| namespace: dubhe-prod | |||||
| server-addr: 127.0.0.1:8848 | |||||
| @@ -207,4 +207,11 @@ public interface NodeApi { | |||||
| * @return | * @return | ||||
| */ | */ | ||||
| List<BizTaint> geBizTaintListByUserId(); | List<BizTaint> geBizTaintListByUserId(); | ||||
| /** | |||||
| * 获取一个可用node的ip | |||||
| * | |||||
| * @return String | |||||
| */ | |||||
| String getAvailableNodeIp(); | |||||
| } | } | ||||
| @@ -389,14 +389,11 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||||
| StatefulSet statefulSet = deployStatefulSet(); | StatefulSet statefulSet = deployStatefulSet(); | ||||
| //部署svc | //部署svc | ||||
| Service service = deployService(); | Service service = deployService(); | ||||
| //部署ingress | |||||
| Ingress ingress = deployIngress(); | |||||
| if (delayDelete != null && delayDelete > ZERO) { | if (delayDelete != null && delayDelete > ZERO) { | ||||
| taskYamlBO.append(secret); | taskYamlBO.append(secret); | ||||
| taskYamlBO.append(statefulSet); | taskYamlBO.append(statefulSet); | ||||
| taskYamlBO.append(service); | taskYamlBO.append(service); | ||||
| taskYamlBO.append(ingress); | |||||
| long stopUnixTime = System.currentTimeMillis() / THOUSAND_LONG + delayDelete * SIXTY_LONG; | long stopUnixTime = System.currentTimeMillis() / THOUSAND_LONG + delayDelete * SIXTY_LONG; | ||||
| Timestamp stopDisplayTime = new Timestamp(stopUnixTime * THOUSAND_LONG); | Timestamp stopDisplayTime = new Timestamp(stopUnixTime * THOUSAND_LONG); | ||||
| @@ -412,7 +409,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||||
| k8sTaskService.createOrUpdateTask(k8sTask); | k8sTaskService.createOrUpdateTask(k8sTask); | ||||
| } | } | ||||
| return new PtJupyterDeployVO(secret, statefulSet, service, ingress); | |||||
| return new PtJupyterDeployVO(secret, statefulSet, service, null); | |||||
| } | } | ||||
| /** | /** | ||||
| @@ -440,7 +437,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||||
| .build(); | .build(); | ||||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}", secretName); | LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}", secretName); | ||||
| secret = client.secrets().create(secret); | |||||
| secret = client.secrets().inNamespace(namespace).create(secret); | |||||
| LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", secretName); | LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", secretName); | ||||
| } | } | ||||
| @@ -647,7 +644,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||||
| .build(); | .build(); | ||||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", statefulSetName, YamlUtils.dumpAsYaml(statefulSet)); | LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", statefulSetName, YamlUtils.dumpAsYaml(statefulSet)); | ||||
| resourceIisolationApi.addIisolationInfo(statefulSet); | resourceIisolationApi.addIisolationInfo(statefulSet); | ||||
| statefulSet = client.apps().statefulSets().create(statefulSet); | |||||
| statefulSet = client.apps().statefulSets().inNamespace(namespace).create(statefulSet); | |||||
| LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", statefulSetName); | LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", statefulSetName); | ||||
| return statefulSet; | return statefulSet; | ||||
| } | } | ||||
| @@ -678,22 +675,23 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||||
| .withTargetPort(new IntOrString(CONTAINER_PORT)) | .withTargetPort(new IntOrString(CONTAINER_PORT)) | ||||
| .withName(CONTAINER_NAME) | .withName(CONTAINER_NAME) | ||||
| .endPort() | .endPort() | ||||
| .withClusterIP("None") | |||||
| .withSelector(podLabels) | .withSelector(podLabels) | ||||
| .withType("NodePort") | |||||
| .endSpec() | .endSpec() | ||||
| .build(); | .build(); | ||||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", svcName, YamlUtils.dumpAsYaml(svc)); | LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", svcName, YamlUtils.dumpAsYaml(svc)); | ||||
| svc = client.services().create(svc); | |||||
| svc = client.services().inNamespace(svc.getMetadata().getNamespace()).create(svc); | |||||
| LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", svcName); | LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", svcName); | ||||
| return svc; | return svc; | ||||
| } | } | ||||
| /** | /** | ||||
| * 部署ingress | |||||
| * 部署ingress,弃用泛域名,暴露方式由ingress改为 service nodePort | |||||
| * | * | ||||
| * @return Ingress 类 | * @return Ingress 类 | ||||
| */ | */ | ||||
| @Deprecated | |||||
| private Ingress deployIngress() { | private Ingress deployIngress() { | ||||
| Ingress ingress = null; | Ingress ingress = null; | ||||
| IngressList list = client.extensions().ingresses().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(baseName)).list(); | IngressList list = client.extensions().ingresses().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(baseName)).list(); | ||||
| @@ -727,7 +725,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||||
| .endSpec() | .endSpec() | ||||
| .build(); | .build(); | ||||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", ingressName, YamlUtils.dumpAsYaml(ingress)); | LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", ingressName, YamlUtils.dumpAsYaml(ingress)); | ||||
| ingress = client.extensions().ingresses().create(ingress); | |||||
| ingress = client.extensions().ingresses().inNamespace(namespace).create(ingress); | |||||
| LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", ingressName); | LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", ingressName); | ||||
| return ingress; | return ingress; | ||||
| } | } | ||||
| @@ -65,6 +65,7 @@ import org.dubhe.k8s.enums.K8sKindEnum; | |||||
| import org.dubhe.k8s.enums.K8sResponseEnum; | import org.dubhe.k8s.enums.K8sResponseEnum; | ||||
| import org.dubhe.k8s.enums.LimitsOfResourcesEnum; | import org.dubhe.k8s.enums.LimitsOfResourcesEnum; | ||||
| import org.dubhe.k8s.enums.RestartPolicyEnum; | import org.dubhe.k8s.enums.RestartPolicyEnum; | ||||
| import org.dubhe.k8s.enums.ServiceTypeENum; | |||||
| import org.dubhe.k8s.enums.ShellCommandEnum; | import org.dubhe.k8s.enums.ShellCommandEnum; | ||||
| import org.dubhe.k8s.utils.BizConvertUtils; | import org.dubhe.k8s.utils.BizConvertUtils; | ||||
| import org.dubhe.k8s.utils.K8sUtils; | import org.dubhe.k8s.utils.K8sUtils; | ||||
| @@ -102,15 +103,6 @@ public class ModelServingApiImpl implements ModelServingApi { | |||||
| @Autowired | @Autowired | ||||
| private ResourceIisolationApi resourceIisolationApi; | private ResourceIisolationApi resourceIisolationApi; | ||||
| @Value("${k8s.serving.host}") | |||||
| String servingHost; | |||||
| @Value("${k8s.serving.tls-crt}") | |||||
| String servingTlsCrt; | |||||
| @Value("${k8s.serving.tls-key}") | |||||
| String servingTlsKey; | |||||
| private static final String MODEL_SERVING_MAX_UPLOAD_SIZE = "100m"; | private static final String MODEL_SERVING_MAX_UPLOAD_SIZE = "100m"; | ||||
| public ModelServingApiImpl(K8sUtils k8sUtils) { | public ModelServingApiImpl(K8sUtils k8sUtils) { | ||||
| @@ -144,7 +136,6 @@ public class ModelServingApiImpl implements ModelServingApi { | |||||
| //名称生成 | //名称生成 | ||||
| String deploymentName = StrUtil.format(K8sParamConstants.RESOURCE_NAME_TEMPLATE, bo.getResourceName(), RandomUtil.randomString(MagicNumConstant.EIGHT)); | String deploymentName = StrUtil.format(K8sParamConstants.RESOURCE_NAME_TEMPLATE, bo.getResourceName(), RandomUtil.randomString(MagicNumConstant.EIGHT)); | ||||
| String svcName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.SVC_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE)); | String svcName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.SVC_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE)); | ||||
| String ingressName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.INGRESS_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE)); | |||||
| //标签生成 | //标签生成 | ||||
| Map<String, String> baseLabels = LabelUtils.getBaseLabels(bo.getResourceName(), bo.getBusinessLabel()); | Map<String, String> baseLabels = LabelUtils.getBaseLabels(bo.getResourceName(), bo.getBusinessLabel()); | ||||
| @@ -164,45 +155,12 @@ public class ModelServingApiImpl implements ModelServingApi { | |||||
| if (bo.getGrpcPort() != null) { | if (bo.getGrpcPort() != null) { | ||||
| buildServiceBO.addPort(ResourceBuildUtils.buildServicePort(bo.getGrpcPort(), bo.getGrpcPort(), SymbolConstant.GRPC)); | buildServiceBO.addPort(ResourceBuildUtils.buildServicePort(bo.getGrpcPort(), bo.getGrpcPort(), SymbolConstant.GRPC)); | ||||
| } | } | ||||
| buildServiceBO.setType(ServiceTypeENum.NODE_PORT.getType()); | |||||
| Service service = ResourceBuildUtils.buildService(buildServiceBO); | Service service = ResourceBuildUtils.buildService(buildServiceBO); | ||||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); | LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); | ||||
| Service serviceResult = client.services().create(service); | |||||
| //部署ingress | |||||
| BuildIngressBO buildIngressBO = new BuildIngressBO(bo.getNamespace(), ingressName, baseLabels); | |||||
| if (StringUtils.isNotEmpty(buildIngressBO.getMaxUploadSize())) { | |||||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_PROXY_BODY_SIZE_KEY, buildIngressBO.getMaxUploadSize()); | |||||
| } | |||||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_READ_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN)); | |||||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_NEXT_UPSTREAM_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN)); | |||||
| if (bo.getHttpPort() != null) { | |||||
| String httpHost = RandomUtil.randomString(MagicNumConstant.SIX) + SymbolConstant.DOT + servingHost; | |||||
| buildIngressBO.addIngressRule(ResourceBuildUtils.buildIngressRule(httpHost, svcName, SymbolConstant.HTTP)); | |||||
| } | |||||
| Secret secretResult = null; | |||||
| if (bo.getGrpcPort() != null) { | |||||
| String secretName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), SymbolConstant.TOKEN, RandomUtil.randomString(MagicNumConstant.FIVE)); | |||||
| Map<String, String> data = new HashMap<String, String>(MagicNumConstant.FOUR) { | |||||
| { | |||||
| put(K8sParamConstants.SECRET_TLS_TLS_CRT, servingTlsCrt); | |||||
| put(K8sParamConstants.SECRET_TLS_TLS_KEY, servingTlsKey); | |||||
| } | |||||
| }; | |||||
| Secret secret = ResourceBuildUtils.buildTlsSecret(bo.getNamespace(), secretName, baseLabels, data); | |||||
| secretResult = client.secrets().create(secret); | |||||
| String grpcHost = RandomUtil.randomString(MagicNumConstant.SIX) + SymbolConstant.DOT + servingHost; | |||||
| buildIngressBO.addIngressRule(ResourceBuildUtils.buildIngressRule(grpcHost, svcName, SymbolConstant.GRPC)); | |||||
| buildIngressBO.addIngressTLS(ResourceBuildUtils.buildIngressTLS(secretName, grpcHost)); | |||||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_CLASS_KEY, StringConstant.NGINX_LOWERCASE); | |||||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_SSL_REDIRECT_KEY, StringConstant.TRUE_LOWERCASE); | |||||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_BACKEND_PROTOCOL_KEY, StringConstant.GRPC_CAPITALIZE); | |||||
| } | |||||
| Ingress ingress = ResourceBuildUtils.buildIngress(buildIngressBO); | |||||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", ingressName, YamlUtils.dumpAsYaml(ingress)); | |||||
| Ingress ingressResult = client.extensions().ingresses().create(ingress); | |||||
| Service serviceResult = client.services().inNamespace(bo.getNamespace()).create(service); | |||||
| return new ModelServingVO(BizConvertUtils.toBizSecret(secretResult), BizConvertUtils.toBizService(serviceResult), BizConvertUtils.toBizDeployment(deploymentResult), BizConvertUtils.toBizIngress(ingressResult)); | |||||
| return new ModelServingVO(null, BizConvertUtils.toBizService(serviceResult), BizConvertUtils.toBizDeployment(deploymentResult), null); | |||||
| } catch (KubernetesClientException e) { | } catch (KubernetesClientException e) { | ||||
| LogUtil.error(LogEnum.BIZ_K8S, "ModelOptJobApiImpl.create error, param:{} error:", bo, e); | LogUtil.error(LogEnum.BIZ_K8S, "ModelOptJobApiImpl.create error, param:{} error:", bo, e); | ||||
| return new ModelServingVO().error(String.valueOf(e.getCode()), e.getMessage()); | return new ModelServingVO().error(String.valueOf(e.getCode()), e.getMessage()); | ||||
| @@ -41,6 +41,7 @@ import org.dubhe.k8s.constant.K8sLabelConstants; | |||||
| import org.dubhe.k8s.constant.K8sParamConstants; | import org.dubhe.k8s.constant.K8sParamConstants; | ||||
| import org.dubhe.k8s.domain.PtBaseResult; | import org.dubhe.k8s.domain.PtBaseResult; | ||||
| import org.dubhe.k8s.domain.resource.BizNode; | import org.dubhe.k8s.domain.resource.BizNode; | ||||
| import org.dubhe.k8s.domain.resource.BizNodeAddress; | |||||
| import org.dubhe.k8s.domain.resource.BizTaint; | import org.dubhe.k8s.domain.resource.BizTaint; | ||||
| import org.dubhe.k8s.domain.vo.PtNodeMetricsVO; | import org.dubhe.k8s.domain.vo.PtNodeMetricsVO; | ||||
| import org.dubhe.k8s.enums.K8sResponseEnum; | import org.dubhe.k8s.enums.K8sResponseEnum; | ||||
| @@ -560,6 +561,25 @@ public class NodeApiImpl implements NodeApi { | |||||
| return geBizTaintListByUserId(userContextService.getCurUserId()); | return geBizTaintListByUserId(userContextService.getCurUserId()); | ||||
| } | } | ||||
| /** | |||||
| * 获取一个可用node的ip | |||||
| * | |||||
| * @return String | |||||
| */ | |||||
| @Override | |||||
| public String getAvailableNodeIp(){ | |||||
| List<BizNode> nodes = listAll(); | |||||
| for (BizNode node : nodes){ | |||||
| if (node.getReady() && !node.isUnschedulable()){ | |||||
| for (BizNodeAddress address : node.getAddresses()){ | |||||
| if (K8sParamConstants.INTERNAL_IP.equals(address.getType())){ | |||||
| return address.getAddress(); | |||||
| } | |||||
| } | |||||
| } | |||||
| } | |||||
| return ""; | |||||
| } | |||||
| /** | /** | ||||
| * 查询节点内存资源是否可分配 | * 查询节点内存资源是否可分配 | ||||
| @@ -400,13 +400,16 @@ public class PodApiImpl implements PodApi { | |||||
| public String getUrlByResourceName(String namespace, String resourceName) { | public String getUrlByResourceName(String namespace, String resourceName) { | ||||
| LogUtil.info(LogEnum.BIZ_K8S,"Start GetUrlByResourceName {} {}",namespace,resourceName); | LogUtil.info(LogEnum.BIZ_K8S,"Start GetUrlByResourceName {} {}",namespace,resourceName); | ||||
| PtJupyterDeployVO info = jupyterResourceApi.get(namespace, resourceName); | PtJupyterDeployVO info = jupyterResourceApi.get(namespace, resourceName); | ||||
| if (info != null && info.getIngressInfo() != null && CollectionUtil.isNotEmpty(info.getIngressInfo().getRules())) { | |||||
| if (info != null && info.getServiceInfo() != null && CollectionUtil.isNotEmpty(info.getServiceInfo().getPorts())) { | |||||
| String token = getTokenByResourceName(namespace, resourceName); | String token = getTokenByResourceName(namespace, resourceName); | ||||
| if (StringUtils.isBlank(token)) { | if (StringUtils.isBlank(token)) { | ||||
| LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter Notebook token not generated,[namespace]={}, [resourceName]={}", namespace, resourceName); | LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter Notebook token not generated,[namespace]={}, [resourceName]={}", namespace, resourceName); | ||||
| return ""; | return ""; | ||||
| } | } | ||||
| String url = StrUtil.format(POD_URL, info.getIngressInfo().getRules().get(0).getHost(), k8sUtils.getPort(), token); | |||||
| BizPod pod = getWithResourceName(namespace,resourceName); | |||||
| String url = StrUtil.format(POD_URL, pod.getHostIP(), info.getServiceInfo().getPorts().get(0).getNodePort(), token); | |||||
| return validateJupyterUrl(url); | return validateJupyterUrl(url); | ||||
| } | } | ||||
| LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter statefulset not created,[namespace]={}, [resourceName]={}",namespace,resourceName); | LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter statefulset not created,[namespace]={}, [resourceName]={}",namespace,resourceName); | ||||
| @@ -148,7 +148,7 @@ public class TerminalApiImpl implements TerminalApi { | |||||
| } | } | ||||
| Service service = ResourceBuildUtils.buildService(buildServiceBO); | Service service = ResourceBuildUtils.buildService(buildServiceBO); | ||||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); | LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); | ||||
| Service serviceResult = client.services().create(service); | |||||
| Service serviceResult = client.services().inNamespace(service.getMetadata().getNamespace()).create(service); | |||||
| return new TerminalResourceVO(BizConvertUtils.toBizDeployment(deploymentResult),BizConvertUtils.toBizService(serviceResult)); | return new TerminalResourceVO(BizConvertUtils.toBizDeployment(deploymentResult),BizConvertUtils.toBizService(serviceResult)); | ||||
| }catch (KubernetesClientException e) { | }catch (KubernetesClientException e) { | ||||
| LogUtil.error(LogEnum.BIZ_K8S, "TerminalApiImpl.create error, param:{} error:", bo, e); | LogUtil.error(LogEnum.BIZ_K8S, "TerminalApiImpl.create error, param:{} error:", bo, e); | ||||
| @@ -101,4 +101,5 @@ public class K8sParamConstants { | |||||
| public static final String WAITING_REASON_CONTAINER_CREATING = "ContainerCreating"; | public static final String WAITING_REASON_CONTAINER_CREATING = "ContainerCreating"; | ||||
| public final static String INTERNAL_IP = "InternalIP"; | |||||
| } | } | ||||
| @@ -57,16 +57,21 @@ public class BaseK8sDeploymentCallbackCreateDTO { | |||||
| @NotNull(message = "replicas 不能为空!") | @NotNull(message = "replicas 不能为空!") | ||||
| private Integer replicas; | private Integer replicas; | ||||
| @ApiModelProperty(required = true, value = "action") | |||||
| @NotBlank(message = "action 不能为空!") | |||||
| private String action; | |||||
| public BaseK8sDeploymentCallbackCreateDTO() { | public BaseK8sDeploymentCallbackCreateDTO() { | ||||
| } | } | ||||
| public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas) { | |||||
| public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas,String action) { | |||||
| this.namespace = namespace; | this.namespace = namespace; | ||||
| this.resourceName = resourceName; | this.resourceName = resourceName; | ||||
| this.deploymentName = deploymentName; | this.deploymentName = deploymentName; | ||||
| this.readyReplicas = readyReplicas; | this.readyReplicas = readyReplicas; | ||||
| this.replicas = replicas; | this.replicas = replicas; | ||||
| this.action = action; | |||||
| } | } | ||||
| @Override | @Override | ||||
| @@ -77,6 +82,7 @@ public class BaseK8sDeploymentCallbackCreateDTO { | |||||
| ", deploymentName='" + deploymentName + '\'' + | ", deploymentName='" + deploymentName + '\'' + | ||||
| ", readyReplicas='" + readyReplicas + '\'' + | ", readyReplicas='" + readyReplicas + '\'' + | ||||
| ", replicas='" + replicas + '\'' + | ", replicas='" + replicas + '\'' + | ||||
| ", action='" + action + '\'' + | |||||
| '}'; | '}'; | ||||
| } | } | ||||
| } | } | ||||
| @@ -30,6 +30,7 @@ import org.dubhe.k8s.annotation.K8sField; | |||||
| import org.dubhe.k8s.constant.K8sParamConstants; | import org.dubhe.k8s.constant.K8sParamConstants; | ||||
| import org.dubhe.k8s.domain.PtBaseResult; | import org.dubhe.k8s.domain.PtBaseResult; | ||||
| import org.dubhe.k8s.domain.resource.BizContainer; | import org.dubhe.k8s.domain.resource.BizContainer; | ||||
| import org.dubhe.k8s.domain.resource.BizServicePort; | |||||
| import org.dubhe.k8s.utils.MappingUtils; | import org.dubhe.k8s.utils.MappingUtils; | ||||
| import java.util.List; | import java.util.List; | ||||
| @@ -105,6 +106,8 @@ public class PtJupyterDeployVO extends PtBaseResult<PtJupyterDeployVO> { | |||||
| private String namespace; | private String namespace; | ||||
| @K8sField("metadata:uid") | @K8sField("metadata:uid") | ||||
| private String uid; | private String uid; | ||||
| @K8sField("spec:ports") | |||||
| private List<BizServicePort> ports; | |||||
| } | } | ||||
| @Data | @Data | ||||
| @@ -33,10 +33,6 @@ public class ClusterProperties { | |||||
| private String nfs; | private String nfs; | ||||
| private String host; | |||||
| private String port; | |||||
| private String kubeconfig; | private String kubeconfig; | ||||
| /** | /** | ||||
| @@ -218,7 +218,9 @@ public class BizConvertUtils { | |||||
| */ | */ | ||||
| public static BizIngress toBizIngress(Ingress ingress) { | public static BizIngress toBizIngress(Ingress ingress) { | ||||
| BizIngress bizIngress = MappingUtils.mappingTo(ingress, BizIngress.class); | BizIngress bizIngress = MappingUtils.mappingTo(ingress, BizIngress.class); | ||||
| bizIngress.getRules().forEach(BizIngressRule::takeServicePort); | |||||
| if (bizIngress != null){ | |||||
| bizIngress.getRules().forEach(BizIngressRule::takeServicePort); | |||||
| } | |||||
| return bizIngress; | return bizIngress; | ||||
| } | } | ||||
| @@ -98,8 +98,6 @@ public class K8sUtils implements ApplicationContextAware { | |||||
| nfs = clusterProperties.getNfs(); | nfs = clusterProperties.getNfs(); | ||||
| host = clusterProperties.getHost(); | |||||
| port = clusterProperties.getPort(); | |||||
| //打印集群信息 | //打印集群信息 | ||||
| LogUtil.info(LogEnum.BIZ_K8S, "ApiVersion : {}", client.getApiVersion()); | LogUtil.info(LogEnum.BIZ_K8S, "ApiVersion : {}", client.getApiVersion()); | ||||
| @@ -69,11 +69,6 @@ public enum RecycleResourceEnum { | |||||
| */ | */ | ||||
| MEASURE_RECYCLE_FILE("measureRecycleFile", "度量文件回收"), | MEASURE_RECYCLE_FILE("measureRecycleFile", "度量文件回收"), | ||||
| /** | |||||
| * 镜像回收 | |||||
| */ | |||||
| IMAGE_RECYCLE_FILE("imageRecycleFile", "镜像回收"), | |||||
| /** | /** | ||||
| * 算法文件回收 | * 算法文件回收 | ||||
| */ | */ | ||||
| @@ -14,7 +14,6 @@ | |||||
| * limitations under the License. | * limitations under the License. | ||||
| * ============================================================= | * ============================================================= | ||||
| */ | */ | ||||
| package org.dubhe.task; | package org.dubhe.task; | ||||
| import org.junit.jupiter.api.Test; | import org.junit.jupiter.api.Test; | ||||
| @@ -1,36 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| * ============================================================= | |||||
| */ | |||||
| package org.dubhe.data.config; | |||||
| import lombok.Data; | |||||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||||
| import org.springframework.stereotype.Component; | |||||
| @Data | |||||
| @Component | |||||
| @ConfigurationProperties(prefix = "harbor") | |||||
| public class DataHarborConfig { | |||||
| private String address; | |||||
| private String username; | |||||
| private String password; | |||||
| private String modelName; | |||||
| } | |||||
| @@ -28,9 +28,15 @@ import com.baomidou.mybatisplus.extension.service.IService; | |||||
| import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; | import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; | ||||
| import org.apache.commons.compress.utils.Lists; | import org.apache.commons.compress.utils.Lists; | ||||
| import org.dubhe.biz.base.constant.MagicNumConstant; | import org.dubhe.biz.base.constant.MagicNumConstant; | ||||
| import org.dubhe.biz.base.constant.SymbolConstant; | |||||
| import org.dubhe.biz.base.context.UserContext; | import org.dubhe.biz.base.context.UserContext; | ||||
| import org.dubhe.biz.base.dto.*; | |||||
| import org.dubhe.biz.base.dto.PtImageIdDTO; | |||||
| import org.dubhe.biz.base.dto.PtImageIdsDTO; | |||||
| import org.dubhe.biz.base.dto.PtModelBranchQueryByIdDTO; | |||||
| import org.dubhe.biz.base.dto.PtModelBranchQueryByIdsDTO; | |||||
| import org.dubhe.biz.base.dto.TrainAlgorithmSelectAllBatchIdDTO; | |||||
| import org.dubhe.biz.base.dto.TrainAlgorithmSelectByIdDTO; | |||||
| import org.dubhe.biz.base.dto.UserDTO; | |||||
| import org.dubhe.biz.base.dto.UserSmallDTO; | |||||
| import org.dubhe.biz.base.enums.BizEnum; | import org.dubhe.biz.base.enums.BizEnum; | ||||
| import org.dubhe.biz.base.enums.ResourcesPoolTypeEnum; | import org.dubhe.biz.base.enums.ResourcesPoolTypeEnum; | ||||
| import org.dubhe.biz.base.exception.BusinessException; | import org.dubhe.biz.base.exception.BusinessException; | ||||
| @@ -49,13 +55,15 @@ import org.dubhe.cloud.authconfig.service.AdminClient; | |||||
| import org.dubhe.data.client.AlgorithmClient; | import org.dubhe.data.client.AlgorithmClient; | ||||
| import org.dubhe.data.client.ImageClient; | import org.dubhe.data.client.ImageClient; | ||||
| import org.dubhe.data.client.ModelClient; | import org.dubhe.data.client.ModelClient; | ||||
| import org.dubhe.data.config.DataHarborConfig; | |||||
| import org.dubhe.data.config.DataRedisConfig; | import org.dubhe.data.config.DataRedisConfig; | ||||
| import org.dubhe.data.constant.AutoLabelModelServiceStatusEnum; | import org.dubhe.data.constant.AutoLabelModelServiceStatusEnum; | ||||
| import org.dubhe.data.constant.Constant; | import org.dubhe.data.constant.Constant; | ||||
| import org.dubhe.data.constant.ErrorEnum; | import org.dubhe.data.constant.ErrorEnum; | ||||
| import org.dubhe.data.dao.AutoLabelModelServiceMapper; | import org.dubhe.data.dao.AutoLabelModelServiceMapper; | ||||
| import org.dubhe.data.domain.dto.*; | |||||
| import org.dubhe.data.domain.dto.AutoLabelModelServiceCreateDTO; | |||||
| import org.dubhe.data.domain.dto.AutoLabelModelServiceQueryDTO; | |||||
| import org.dubhe.data.domain.dto.AutoLabelModelServiceUpdateDTO; | |||||
| import org.dubhe.data.domain.dto.DataK8sDeploymentCallbackCreateDTO; | |||||
| import org.dubhe.data.domain.entity.AutoLabelModelService; | import org.dubhe.data.domain.entity.AutoLabelModelService; | ||||
| import org.dubhe.data.domain.vo.AutoLabelModelServicePodVO; | import org.dubhe.data.domain.vo.AutoLabelModelServicePodVO; | ||||
| import org.dubhe.data.domain.vo.AutoLabelModelServiceVO; | import org.dubhe.data.domain.vo.AutoLabelModelServiceVO; | ||||
| @@ -75,9 +83,16 @@ import org.springframework.beans.factory.annotation.Autowired; | |||||
| import org.springframework.beans.factory.annotation.Value; | import org.springframework.beans.factory.annotation.Value; | ||||
| import org.springframework.dao.DuplicateKeyException; | import org.springframework.dao.DuplicateKeyException; | ||||
| import org.springframework.stereotype.Service; | import org.springframework.stereotype.Service; | ||||
| import javax.annotation.Resource; | import javax.annotation.Resource; | ||||
| import java.sql.Timestamp; | import java.sql.Timestamp; | ||||
| import java.util.*; | |||||
| import java.util.ArrayList; | |||||
| import java.util.Arrays; | |||||
| import java.util.HashMap; | |||||
| import java.util.List; | |||||
| import java.util.Map; | |||||
| import java.util.Objects; | |||||
| import java.util.Set; | |||||
| import java.util.concurrent.TimeUnit; | import java.util.concurrent.TimeUnit; | ||||
| import java.util.function.Function; | import java.util.function.Function; | ||||
| import java.util.stream.Collectors; | import java.util.stream.Collectors; | ||||
| @@ -116,9 +131,6 @@ public class AutoLabelModelServiceServiceImpl extends ServiceImpl<AutoLabelModel | |||||
| @Resource | @Resource | ||||
| private UserContextService userContextService; | private UserContextService userContextService; | ||||
| @Resource | |||||
| private DataHarborConfig dataHarborConfig; | |||||
| @Resource | @Resource | ||||
| private DataRedisConfig dataRedisConfig; | private DataRedisConfig dataRedisConfig; | ||||
| @@ -197,7 +209,7 @@ public class AutoLabelModelServiceServiceImpl extends ServiceImpl<AutoLabelModel | |||||
| modelServiceBO.setResourceName(k8sNameTool.generateResourceName(BizEnum.DATA, autoLabelModelService.getId().toString())); | modelServiceBO.setResourceName(k8sNameTool.generateResourceName(BizEnum.DATA, autoLabelModelService.getId().toString())); | ||||
| modelServiceBO.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.DATA)); | modelServiceBO.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.DATA)); | ||||
| modelServiceBO.setReplicas(autoLabelModelService.getInstanceNum()); | modelServiceBO.setReplicas(autoLabelModelService.getInstanceNum()); | ||||
| modelServiceBO.setImage(dataHarborConfig.getAddress() + SymbolConstant.SLASH + ptImageVO.getImageUrl()); | |||||
| modelServiceBO.setImage(ptImageVO.getImageUrl()); | |||||
| modelServiceBO.setTaskIdentifyLabel(taskIdentify); | modelServiceBO.setTaskIdentifyLabel(taskIdentify); | ||||
| String command = String.format(Constant.MODEL_SERVICE_COMMAND, modelServiceId, | String command = String.format(Constant.MODEL_SERVICE_COMMAND, modelServiceId, | ||||
| @@ -1,142 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| * ============================================================= | |||||
| */ | |||||
| package org.dubhe.image.async; | |||||
| import cn.hutool.core.util.StrUtil; | |||||
| import org.dubhe.biz.base.constant.HarborProperties; | |||||
| import org.dubhe.biz.base.constant.StringConstant; | |||||
| import org.dubhe.biz.base.enums.ImageStateEnum; | |||||
| import org.dubhe.biz.base.exception.BusinessException; | |||||
| import org.dubhe.biz.base.utils.StringUtils; | |||||
| import org.dubhe.biz.file.utils.IOUtil; | |||||
| import org.dubhe.biz.log.enums.LogEnum; | |||||
| import org.dubhe.biz.log.utils.LogUtil; | |||||
| import org.dubhe.harbor.api.HarborApi; | |||||
| import org.dubhe.image.dao.PtImageMapper; | |||||
| import org.dubhe.image.domain.entity.PtImage; | |||||
| import org.springframework.beans.factory.annotation.Autowired; | |||||
| import org.springframework.scheduling.annotation.Async; | |||||
| import org.springframework.stereotype.Component; | |||||
| import java.io.BufferedReader; | |||||
| import java.io.InputStreamReader; | |||||
| /** | |||||
| * @description 异步推送镜像到harbor | |||||
| * @date 2020-07-10 | |||||
| */ | |||||
| @Component | |||||
| public class HarborImagePushAsync { | |||||
| @Autowired | |||||
| private HarborProperties harborProperties; | |||||
| @Autowired | |||||
| private HarborApi harborApi; | |||||
| @Autowired | |||||
| private PtImageMapper ptImageMapper; | |||||
| /** | |||||
| * 组装shell脚本执行命令 | |||||
| * | |||||
| * @param imagePath | |||||
| * @param imageNameandTag | |||||
| **/ | |||||
| @Async | |||||
| public void execShell(String imagePath, String imageNameandTag, PtImage ptImage) { | |||||
| try { | |||||
| String imageResource = harborProperties.getAddress() + StrUtil.SLASH + StringConstant.DEFAULT_IMAGE_PROJECT | |||||
| + StrUtil.SLASH + imageNameandTag; | |||||
| String cmdStr = "docker login --username=" + harborProperties.getUsername() + " " + harborProperties.getAddress() + " --password=" + harborProperties.getPassword() + " ; docker " + | |||||
| "load < " + imagePath + " |awk '{print $3}' |xargs -I str docker tag str " + imageResource + " ; docker push " + imageResource + "; docker rmi " + imageResource; | |||||
| String[] cmd = {"/bin/bash", "-c", cmdStr}; | |||||
| LogUtil.info(LogEnum.IMAGE, "镜像上传执行脚本参数:{}", cmd); | |||||
| Process process = Runtime.getRuntime().exec(cmd); | |||||
| if (checkImagePushIsOk(ptImage, process)) { | |||||
| updateImageStatus(ptImage, ImageStateEnum.SUCCESS.getCode()); | |||||
| } else { | |||||
| updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); | |||||
| } | |||||
| } catch (Exception e) { | |||||
| LogUtil.error(LogEnum.IMAGE, "upload image failed,exception {}", e); | |||||
| updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); | |||||
| throw new BusinessException("上传镜像异常!"); | |||||
| } | |||||
| } | |||||
| /** | |||||
| * 更新镜像上传状态 | |||||
| * | |||||
| * @param ptImage | |||||
| * @param status | |||||
| **/ | |||||
| public void updateImageStatus(PtImage ptImage, Integer status) { | |||||
| ptImage.setImageStatus(status); | |||||
| ptImageMapper.updateById(ptImage); | |||||
| } | |||||
| /** | |||||
| * 校验镜像是否上传成功 | |||||
| * | |||||
| * @param ptImage 镜像信息 | |||||
| * @param process process对象 | |||||
| * @return 是否上传成功 | |||||
| */ | |||||
| public boolean checkImagePushIsOk(PtImage ptImage, Process process) { | |||||
| //读取标准输出流 | |||||
| BufferedReader brOut = new BufferedReader(new InputStreamReader(process.getInputStream())); | |||||
| //读取标准错误流 | |||||
| BufferedReader brErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); | |||||
| String line; | |||||
| StringBuilder outMessage = new StringBuilder(); | |||||
| StringBuilder errMessage = new StringBuilder(); | |||||
| boolean isPushOk = true; | |||||
| try { | |||||
| while ((line = brOut.readLine()) != null) { | |||||
| outMessage.append(line); | |||||
| } | |||||
| if (StringUtils.isNotEmpty(outMessage)) { | |||||
| LogUtil.info(LogEnum.IMAGE, "upload image with shell standard output message {}", outMessage.toString()); | |||||
| } | |||||
| while ((line = brErr.readLine()) != null) { | |||||
| errMessage.append(line); | |||||
| } | |||||
| if (StringUtils.isNotEmpty(errMessage)) { | |||||
| LogUtil.error(LogEnum.IMAGE, "upload image with shell standard output message {}", errMessage.toString()); | |||||
| } | |||||
| Integer status = process.waitFor(); | |||||
| LogUtil.info(LogEnum.IMAGE, "上传镜像状态:{}", status); | |||||
| if (status == null) { | |||||
| if (!harborApi.isExistImage(ptImage.getImageUrl())) { | |||||
| isPushOk = false; | |||||
| } | |||||
| } else if (status != 0) { | |||||
| isPushOk = false; | |||||
| } | |||||
| } catch (Exception e) { | |||||
| LogUtil.error(LogEnum.IMAGE, "upload image failed,exception {}", e); | |||||
| return false; | |||||
| } finally { | |||||
| IOUtil.close(brErr, brOut); | |||||
| } | |||||
| return isPushOk; | |||||
| } | |||||
| } | |||||
| @@ -41,9 +41,6 @@ public class PtImageQueryDTO extends PageQueryBase implements Serializable { | |||||
| @ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") | @ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") | ||||
| private Integer imageResource; | private Integer imageResource; | ||||
| @ApiModelProperty(value = "镜像状态,0为制作中,1位制作成功,2位制作失败") | |||||
| private Integer imageStatus; | |||||
| @ApiModelProperty(value = "镜像名称或id") | @ApiModelProperty(value = "镜像名称或id") | ||||
| private String imageNameOrId; | private String imageNameOrId; | ||||
| @@ -38,7 +38,7 @@ import java.util.List; | |||||
| */ | */ | ||||
| @Data | @Data | ||||
| @Accessors(chain = true) | @Accessors(chain = true) | ||||
| public class PtImageUploadDTO implements Serializable { | |||||
| public class PtImageSaveDTO implements Serializable { | |||||
| private static final long serialVersionUID = 1L; | private static final long serialVersionUID = 1L; | ||||
| @@ -50,9 +50,9 @@ public class PtImageUploadDTO implements Serializable { | |||||
| @FlagValidator(value = {"0", "1"}, message = "无效镜像来源") | @FlagValidator(value = {"0", "1"}, message = "无效镜像来源") | ||||
| private Integer imageResource; | private Integer imageResource; | ||||
| @ApiModelProperty(value = "镜像文件路径", required = true) | |||||
| @NotBlank(message = "镜像文件路径不能为空") | |||||
| private String imagePath; | |||||
| @ApiModelProperty(value = "镜像地址", required = true) | |||||
| @NotBlank(message = "镜像地址不能为空") | |||||
| private String imageUrl; | |||||
| @ApiModelProperty(value = "镜像名称", required = true) | @ApiModelProperty(value = "镜像名称", required = true) | ||||
| @NotBlank(message = "源镜像名称不能为空") | @NotBlank(message = "源镜像名称不能为空") | ||||
| @@ -20,10 +20,13 @@ import io.swagger.annotations.ApiModelProperty; | |||||
| import lombok.Data; | import lombok.Data; | ||||
| import lombok.experimental.Accessors; | import lombok.experimental.Accessors; | ||||
| import org.dubhe.biz.base.constant.MagicNumConstant; | import org.dubhe.biz.base.constant.MagicNumConstant; | ||||
| import org.dubhe.biz.base.constant.StringConstant; | |||||
| import org.hibernate.validator.constraints.Length; | import org.hibernate.validator.constraints.Length; | ||||
| import javax.validation.constraints.NotBlank; | |||||
| import javax.validation.constraints.NotEmpty; | import javax.validation.constraints.NotEmpty; | ||||
| import javax.validation.constraints.NotNull; | import javax.validation.constraints.NotNull; | ||||
| import javax.validation.constraints.Pattern; | |||||
| import java.io.Serializable; | import java.io.Serializable; | ||||
| import java.util.List; | import java.util.List; | ||||
| @@ -39,14 +42,29 @@ public class PtImageUpdateDTO implements Serializable { | |||||
| @ApiModelProperty(value = "id", required = true) | @ApiModelProperty(value = "id", required = true) | ||||
| @NotNull(message = "镜像id不能为空") | @NotNull(message = "镜像id不能为空") | ||||
| private List<Long> ids; | |||||
| @ApiModelProperty("镜像描述") | |||||
| @Length(max = MagicNumConstant.BINARY_TEN_EXP, message = "镜像描述-输入长度不能超过1024个字符") | |||||
| private String remark; | |||||
| private Long id; | |||||
| @ApiModelProperty(value = "镜像用途", required = true) | @ApiModelProperty(value = "镜像用途", required = true) | ||||
| @NotEmpty(message = "镜像用途(0:notebook , 1:train , 2:serving, 3:terminal, 4:point-cloud)不能为空") | @NotEmpty(message = "镜像用途(0:notebook , 1:train , 2:serving, 3:terminal, 4:point-cloud)不能为空") | ||||
| private List<Integer> imageTypes; | private List<Integer> imageTypes; | ||||
| @ApiModelProperty(value = "镜像路径", required = true) | |||||
| @NotBlank(message = "镜像路径不能为空") | |||||
| private String imageUrl; | |||||
| @ApiModelProperty(value = "镜像名称", required = true) | |||||
| @NotBlank(message = "源镜像名称不能为空") | |||||
| @Length(min = MagicNumConstant.ONE, max = MagicNumConstant.SIXTY_FOUR, message = "镜像名称长度在1-64个字符") | |||||
| @Pattern(regexp = StringConstant.REGEXP_NAME, message = "镜像名称支持字母、数字、英文横杠和下划线") | |||||
| private String imageName; | |||||
| @ApiModelProperty(value = "镜像版本号", required = true) | |||||
| @NotBlank(message = "镜像版本号不能为空") | |||||
| @Length(max = MagicNumConstant.SIXTY_FOUR, message = "镜像版本号长度在1-64个字符") | |||||
| @Pattern(regexp = StringConstant.REGEXP_TAG, message = "镜像版本号支持字母、数字、英文横杠、英文.号和下划线") | |||||
| private String imageTag; | |||||
| @ApiModelProperty("镜像描述") | |||||
| @Length(max = MagicNumConstant.BINARY_TEN_EXP, message = "镜像描述-输入长度不能超过1024个字符") | |||||
| private String remark; | |||||
| } | } | ||||
| @@ -80,13 +80,6 @@ public class PtImage extends BaseEntity { | |||||
| @TableField(value = "image_resource") | @TableField(value = "image_resource") | ||||
| private Integer imageResource; | private Integer imageResource; | ||||
| /** | |||||
| * 镜像状态 | |||||
| */ | |||||
| @TableField(value = "image_status") | |||||
| private Integer imageStatus; | |||||
| /** | /** | ||||
| * 资源拥有者ID | * 资源拥有者ID | ||||
| */ | */ | ||||
| @@ -41,9 +41,6 @@ public class PtImageQueryVO implements Serializable { | |||||
| @ApiModelProperty("镜像版本") | @ApiModelProperty("镜像版本") | ||||
| private String imageTag; | private String imageTag; | ||||
| @ApiModelProperty("镜像状态(0:上传中,1:上传成功,2:上传失败)") | |||||
| private Integer imageStatus; | |||||
| @ApiModelProperty("备注") | @ApiModelProperty("备注") | ||||
| private String remark; | private String remark; | ||||
| @@ -58,4 +55,7 @@ public class PtImageQueryVO implements Serializable { | |||||
| @ApiModelProperty("镜像用途") | @ApiModelProperty("镜像用途") | ||||
| private List<Integer> imageTypes; | private List<Integer> imageTypes; | ||||
| @ApiModelProperty("镜像地址") | |||||
| private String imageUrl; | |||||
| } | } | ||||
| @@ -24,12 +24,25 @@ import org.dubhe.biz.base.dto.PtImageIdDTO; | |||||
| import org.dubhe.biz.base.dto.PtImageIdsDTO; | import org.dubhe.biz.base.dto.PtImageIdsDTO; | ||||
| import org.dubhe.biz.base.vo.DataResponseBody; | import org.dubhe.biz.base.vo.DataResponseBody; | ||||
| import org.dubhe.biz.base.vo.PtImageVO; | import org.dubhe.biz.base.vo.PtImageVO; | ||||
| import org.dubhe.image.domain.dto.*; | |||||
| import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryImageDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryNameDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryUrlDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||||
| import org.dubhe.image.service.PtImageService; | import org.dubhe.image.service.PtImageService; | ||||
| import org.springframework.beans.factory.annotation.Autowired; | import org.springframework.beans.factory.annotation.Autowired; | ||||
| import org.springframework.security.access.prepost.PreAuthorize; | import org.springframework.security.access.prepost.PreAuthorize; | ||||
| import org.springframework.validation.annotation.Validated; | import org.springframework.validation.annotation.Validated; | ||||
| import org.springframework.web.bind.annotation.*; | |||||
| import org.springframework.web.bind.annotation.DeleteMapping; | |||||
| import org.springframework.web.bind.annotation.GetMapping; | |||||
| import org.springframework.web.bind.annotation.PostMapping; | |||||
| import org.springframework.web.bind.annotation.PutMapping; | |||||
| import org.springframework.web.bind.annotation.RequestBody; | |||||
| import org.springframework.web.bind.annotation.RequestMapping; | |||||
| import org.springframework.web.bind.annotation.RequestParam; | |||||
| import org.springframework.web.bind.annotation.RestController; | |||||
| import java.util.List; | import java.util.List; | ||||
| @@ -45,7 +58,7 @@ public class PtImageController { | |||||
| @Autowired | @Autowired | ||||
| private PtImageService ptImageService; | private PtImageService ptImageService; | ||||
| @GetMapping("/info") | |||||
| @GetMapping("/list") | |||||
| @ApiOperation("查询镜像") | @ApiOperation("查询镜像") | ||||
| @PreAuthorize(Permissions.IMAGE) | @PreAuthorize(Permissions.IMAGE) | ||||
| public DataResponseBody getImage(PtImageQueryDTO ptImageQueryDTO) { | public DataResponseBody getImage(PtImageQueryDTO ptImageQueryDTO) { | ||||
| @@ -58,11 +71,11 @@ public class PtImageController { | |||||
| return new DataResponseBody(ptImageService.searchImages(ptImageQueryImageDTO)); | return new DataResponseBody(ptImageService.searchImages(ptImageQueryImageDTO)); | ||||
| } | } | ||||
| @PostMapping("uploadImage") | |||||
| @ApiOperation("上传镜像包到harbor") | |||||
| @PreAuthorize(Permissions.IMAGE_UPLOAD) | |||||
| public DataResponseBody uploadImage(@Validated @RequestBody PtImageUploadDTO ptImageUploadDTO) { | |||||
| ptImageService.uploadImage(ptImageUploadDTO); | |||||
| @PostMapping | |||||
| @ApiOperation("保存镜像信息") | |||||
| @PreAuthorize(Permissions.IMAGE_SAVE) | |||||
| public DataResponseBody saveImageInfo(@Validated @RequestBody PtImageSaveDTO ptImageSaveDTO) { | |||||
| ptImageService.saveImageInfo(ptImageSaveDTO); | |||||
| return new DataResponseBody(); | return new DataResponseBody(); | ||||
| } | } | ||||
| @@ -18,9 +18,14 @@ | |||||
| package org.dubhe.image.service; | package org.dubhe.image.service; | ||||
| import org.dubhe.biz.base.vo.PtImageVO; | import org.dubhe.biz.base.vo.PtImageVO; | ||||
| import org.dubhe.image.domain.dto.*; | |||||
| import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryImageDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryNameDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryUrlDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||||
| import org.dubhe.image.domain.entity.PtImage; | import org.dubhe.image.domain.entity.PtImage; | ||||
| import org.dubhe.recycle.domain.dto.RecycleCreateDTO; | |||||
| import java.util.List; | import java.util.List; | ||||
| import java.util.Map; | import java.util.Map; | ||||
| @@ -42,11 +47,11 @@ public interface PtImageService { | |||||
| /** | /** | ||||
| * 上传镜像到harbor | |||||
| * 保存镜像信息 | |||||
| * | * | ||||
| * @param ptImageUploadDTO 上传条件 | |||||
| * @param ptImageSaveDTO 镜像信息DTO | |||||
| */ | */ | ||||
| void uploadImage(PtImageUploadDTO ptImageUploadDTO); | |||||
| void saveImageInfo(PtImageSaveDTO ptImageSaveDTO); | |||||
| /** | /** | ||||
| @@ -98,13 +103,6 @@ public interface PtImageService { | |||||
| */ | */ | ||||
| String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO); | String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO); | ||||
| /** | |||||
| * 镜像回收还原 | |||||
| * | |||||
| * @param dto 还原DTO对象 | |||||
| */ | |||||
| void recycleRollback(RecycleCreateDTO dto); | |||||
| /** | /** | ||||
| * 获取终端镜像列表 | * 获取终端镜像列表 | ||||
| * | * | ||||
| @@ -27,12 +27,14 @@ import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; | |||||
| import com.baomidou.mybatisplus.core.metadata.IPage; | import com.baomidou.mybatisplus.core.metadata.IPage; | ||||
| import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | ||||
| import org.apache.commons.compress.utils.Lists; | import org.apache.commons.compress.utils.Lists; | ||||
| import org.dubhe.biz.base.constant.*; | |||||
| import org.dubhe.biz.base.constant.MagicNumConstant; | |||||
| import org.dubhe.biz.base.constant.NumberConstant; | |||||
| import org.dubhe.biz.base.constant.ResponseCode; | |||||
| import org.dubhe.biz.base.constant.StringConstant; | |||||
| import org.dubhe.biz.base.context.DataContext; | import org.dubhe.biz.base.context.DataContext; | ||||
| import org.dubhe.biz.base.context.UserContext; | import org.dubhe.biz.base.context.UserContext; | ||||
| import org.dubhe.biz.base.enums.DatasetTypeEnum; | import org.dubhe.biz.base.enums.DatasetTypeEnum; | ||||
| import org.dubhe.biz.base.enums.ImageSourceEnum; | import org.dubhe.biz.base.enums.ImageSourceEnum; | ||||
| import org.dubhe.biz.base.enums.ImageStateEnum; | |||||
| import org.dubhe.biz.base.enums.ImageTypeEnum; | import org.dubhe.biz.base.enums.ImageTypeEnum; | ||||
| import org.dubhe.biz.base.exception.BusinessException; | import org.dubhe.biz.base.exception.BusinessException; | ||||
| import org.dubhe.biz.base.service.UserContextService; | import org.dubhe.biz.base.service.UserContextService; | ||||
| @@ -40,31 +42,34 @@ import org.dubhe.biz.base.utils.ReflectionUtils; | |||||
| import org.dubhe.biz.base.utils.StringUtils; | import org.dubhe.biz.base.utils.StringUtils; | ||||
| import org.dubhe.biz.base.vo.PtImageVO; | import org.dubhe.biz.base.vo.PtImageVO; | ||||
| import org.dubhe.biz.db.utils.PageUtil; | import org.dubhe.biz.db.utils.PageUtil; | ||||
| import org.dubhe.biz.file.config.NfsConfig; | |||||
| import org.dubhe.biz.log.enums.LogEnum; | import org.dubhe.biz.log.enums.LogEnum; | ||||
| import org.dubhe.biz.log.utils.LogUtil; | import org.dubhe.biz.log.utils.LogUtil; | ||||
| import org.dubhe.biz.permission.annotation.DataPermissionMethod; | import org.dubhe.biz.permission.annotation.DataPermissionMethod; | ||||
| import org.dubhe.biz.permission.base.BaseService; | import org.dubhe.biz.permission.base.BaseService; | ||||
| import org.dubhe.image.async.HarborImagePushAsync; | |||||
| import org.dubhe.image.dao.PtImageMapper; | import org.dubhe.image.dao.PtImageMapper; | ||||
| import org.dubhe.image.domain.dto.*; | |||||
| import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryImageDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryNameDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageQueryUrlDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||||
| import org.dubhe.image.domain.entity.PtImage; | import org.dubhe.image.domain.entity.PtImage; | ||||
| import org.dubhe.image.domain.vo.PtImageQueryVO; | import org.dubhe.image.domain.vo.PtImageQueryVO; | ||||
| import org.dubhe.image.service.PtImageService; | import org.dubhe.image.service.PtImageService; | ||||
| import org.dubhe.recycle.config.RecycleConfig; | |||||
| import org.dubhe.recycle.domain.dto.RecycleCreateDTO; | |||||
| import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO; | |||||
| import org.dubhe.recycle.enums.RecycleModuleEnum; | |||||
| import org.dubhe.recycle.enums.RecycleResourceEnum; | |||||
| import org.dubhe.recycle.enums.RecycleTypeEnum; | |||||
| import org.dubhe.recycle.service.RecycleService; | |||||
| import org.dubhe.recycle.utils.RecycleTool; | |||||
| import org.springframework.beans.BeanUtils; | import org.springframework.beans.BeanUtils; | ||||
| import org.springframework.beans.factory.annotation.Autowired; | import org.springframework.beans.factory.annotation.Autowired; | ||||
| import org.springframework.stereotype.Service; | import org.springframework.stereotype.Service; | ||||
| import org.springframework.transaction.annotation.Transactional; | import org.springframework.transaction.annotation.Transactional; | ||||
| import java.util.*; | |||||
| import java.util.ArrayList; | |||||
| import java.util.Comparator; | |||||
| import java.util.HashSet; | |||||
| import java.util.List; | |||||
| import java.util.Map; | |||||
| import java.util.Objects; | |||||
| import java.util.Set; | |||||
| import java.util.TreeSet; | |||||
| import java.util.stream.Collectors; | import java.util.stream.Collectors; | ||||
| /** | /** | ||||
| @@ -77,25 +82,9 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| @Autowired | @Autowired | ||||
| private PtImageMapper ptImageMapper; | private PtImageMapper ptImageMapper; | ||||
| @Autowired | |||||
| private NfsConfig nfsConfig; | |||||
| @Autowired | |||||
| private HarborImagePushAsync imagePushAsync; | |||||
| @Autowired | |||||
| private HarborProperties harborProperties; | |||||
| @Autowired | @Autowired | ||||
| private UserContextService userContextService; | private UserContextService userContextService; | ||||
| @Autowired | |||||
| private RecycleService recycleService; | |||||
| @Autowired | |||||
| private RecycleConfig recycleConfig; | |||||
| public final static List<String> FIELD_NAMES; | public final static List<String> FIELD_NAMES; | ||||
| @@ -119,9 +108,7 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| QueryWrapper<PtImage> query = new QueryWrapper<>(); | QueryWrapper<PtImage> query = new QueryWrapper<>(); | ||||
| query.eq("deleted", NumberConstant.NUMBER_0); | query.eq("deleted", NumberConstant.NUMBER_0); | ||||
| if (ptImageQueryDTO.getImageStatus() != null) { | |||||
| query.eq("image_status", ptImageQueryDTO.getImageStatus()); | |||||
| } | |||||
| if (ptImageQueryDTO.getImageResource() != null) { | if (ptImageQueryDTO.getImageResource() != null) { | ||||
| query.eq("image_resource", ptImageQueryDTO.getImageResource()); | query.eq("image_resource", ptImageQueryDTO.getImageResource()); | ||||
| } | } | ||||
| @@ -166,67 +153,55 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| } | } | ||||
| /** | /** | ||||
| * 上传镜像到harbor | |||||
| * 保存镜像信息 | |||||
| * | * | ||||
| * @param ptImageUploadDTO 上传条件 | |||||
| * @param ptImageSaveDTO 镜像信息DTO | |||||
| */ | */ | ||||
| @Override | @Override | ||||
| @Transactional(rollbackFor = Exception.class) | @Transactional(rollbackFor = Exception.class) | ||||
| public void uploadImage(PtImageUploadDTO ptImageUploadDTO) { | |||||
| public void saveImageInfo(PtImageSaveDTO ptImageSaveDTO) { | |||||
| UserContext user = userContextService.getCurUser(); | UserContext user = userContextService.getCurUser(); | ||||
| //普通用户不支持上传预置镜像 | |||||
| if (ImageSourceEnum.PRE.getCode().equals(ptImageUploadDTO.getImageResource()) && | |||||
| validImageInfo(ptImageSaveDTO.getImageUrl(), ptImageSaveDTO.getImageName(), ptImageSaveDTO.getImageTag()); | |||||
| //普通用户不支持预置镜像信息的保存 | |||||
| if (ImageSourceEnum.PRE.getCode().equals(ptImageSaveDTO.getImageResource()) && | |||||
| !BaseService.isAdmin(user)) { | !BaseService.isAdmin(user)) { | ||||
| throw new BusinessException(ResponseCode.UNAUTHORIZED, "普通用户不支持上传预置镜像!"); | |||||
| throw new BusinessException(ResponseCode.UNAUTHORIZED, "普通用户不支持保存预置镜像!"); | |||||
| } | } | ||||
| //校验用户自定义镜像不能和预置镜像重名 | //校验用户自定义镜像不能和预置镜像重名 | ||||
| List<PtImage> resList = checkUploadImage(ptImageUploadDTO, null, ImageSourceEnum.PRE.getCode()); | |||||
| List<PtImage> resList = checkSaveImage(ptImageSaveDTO, null, ImageSourceEnum.PRE.getCode()); | |||||
| if (CollUtil.isNotEmpty(resList)) { | if (CollUtil.isNotEmpty(resList)) { | ||||
| throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复,请重新上传!"); | |||||
| throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复!"); | |||||
| } | } | ||||
| //同一用户上传镜像的(userId+imageName+imageTag)存在的情况下是不能重复上传的 | //同一用户上传镜像的(userId+imageName+imageTag)存在的情况下是不能重复上传的 | ||||
| List<PtImage> imageList = checkUploadImage(ptImageUploadDTO, user, ImageSourceEnum.MINE.getCode()); | |||||
| if (CollUtil.isNotEmpty(imageList) && ImageStateEnum.SUCCESS.getCode().equals(imageList.get(0).getImageStatus())) { | |||||
| throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在,不允许重复上传!"); | |||||
| List<PtImage> imageList = checkSaveImage(ptImageSaveDTO, user, ImageSourceEnum.MINE.getCode()); | |||||
| if (CollUtil.isNotEmpty(imageList)) { | |||||
| throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在!"); | |||||
| } | } | ||||
| String harborImagePath = StringConstant.DEFAULT_IMAGE_PROJECT + StrUtil.SLASH + ptImageUploadDTO.getImageName() + StrUtil.DASHED + user.getId() + | |||||
| StrUtil.COLON + ptImageUploadDTO.getImageTag(); | |||||
| //存储镜像信息 | //存储镜像信息 | ||||
| PtImage ptImage = new PtImage(); | PtImage ptImage = new PtImage(); | ||||
| ptImage.setImageName(ptImageUploadDTO.getImageName()) | |||||
| .setImageUrl(harborImagePath) | |||||
| .setImageResource(ptImageUploadDTO.getImageResource()) | |||||
| .setImageStatus(ImageStateEnum.MAKING.getCode()) | |||||
| .setRemark(ptImageUploadDTO.getRemark()) | |||||
| .setImageTag(ptImageUploadDTO.getImageTag()) | |||||
| ptImage.setImageName(ptImageSaveDTO.getImageName()) | |||||
| .setImageUrl(ptImageSaveDTO.getImageUrl()) | |||||
| .setImageResource(ptImageSaveDTO.getImageResource()) | |||||
| .setRemark(ptImageSaveDTO.getRemark()) | |||||
| .setImageTag(ptImageSaveDTO.getImageTag()) | |||||
| .setCreateUserId(user.getId()); | .setCreateUserId(user.getId()); | ||||
| if (ImageSourceEnum.PRE.getCode().equals(ptImageUploadDTO.getImageResource())) { | |||||
| if (ImageSourceEnum.PRE.getCode().equals(ptImageSaveDTO.getImageResource())) { | |||||
| ptImage.setOriginUserId(MagicNumConstant.ZERO_LONG); | ptImage.setOriginUserId(MagicNumConstant.ZERO_LONG); | ||||
| } else { | } else { | ||||
| ptImage.setOriginUserId(user.getId()); | ptImage.setOriginUserId(user.getId()); | ||||
| } | } | ||||
| int count = ptImageMapper.insert(ptImage); | |||||
| if (count < 1) { | |||||
| imagePushAsync.updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); | |||||
| throw new BusinessException("内部错误!"); | |||||
| } | |||||
| for (Integer imageType : ptImageUploadDTO.getImageTypes()) { | |||||
| ptImageMapper.insert(ptImage); | |||||
| for (Integer imageType : ptImageSaveDTO.getImageTypes()) { | |||||
| ptImageMapper.insertImageType(ptImage.getId(), imageType); | ptImageMapper.insertImageType(ptImage.getId(), imageType); | ||||
| } | } | ||||
| //shell脚本上传镜像 | |||||
| try { | |||||
| String imagePath = nfsConfig.getRootDir() + nfsConfig.getBucket().substring(1) + ptImageUploadDTO.getImagePath(); | |||||
| String imageNameAndTag = ptImageUploadDTO.getImageName() + StrUtil.DASHED + user.getId() + StrUtil.COLON + ptImageUploadDTO.getImageTag(); | |||||
| imagePushAsync.execShell(imagePath, imageNameAndTag, ptImage); | |||||
| } catch (Exception e) { | |||||
| LogUtil.error(LogEnum.IMAGE, "Image upload exception {}", e); | |||||
| throw new BusinessException("镜像上传失败!"); | |||||
| } | |||||
| } | } | ||||
| /** | /** | ||||
| @@ -240,9 +215,8 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| public List<PtImage> searchImages(PtImageQueryImageDTO ptImageQueryImageDTO) { | public List<PtImage> searchImages(PtImageQueryImageDTO ptImageQueryImageDTO) { | ||||
| LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | ||||
| queryWrapper.eq(PtImage::getImageName, ptImageQueryImageDTO.getImageName()) | queryWrapper.eq(PtImage::getImageName, ptImageQueryImageDTO.getImageName()) | ||||
| .eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()) | |||||
| .eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | .eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | ||||
| List<PtImage> ptImages = new ArrayList<>(); | |||||
| List<PtImage> ptImages; | |||||
| if (ptImageQueryImageDTO.getImageResource() != null) { | if (ptImageQueryImageDTO.getImageResource() != null) { | ||||
| queryWrapper.eq(PtImage::getImageResource, ptImageQueryImageDTO.getImageResource()); | queryWrapper.eq(PtImage::getImageResource, ptImageQueryImageDTO.getImageResource()); | ||||
| } | } | ||||
| @@ -252,18 +226,13 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| ptImages = ptImageMapper.getImagesByTypes(queryWrapper, ptImageQueryImageDTO.getImageTypes()); | ptImages = ptImageMapper.getImagesByTypes(queryWrapper, ptImageQueryImageDTO.getImageTypes()); | ||||
| } | } | ||||
| List<PtImage> list = new ArrayList<>(); | |||||
| if (CollUtil.isEmpty(ptImages)) { | if (CollUtil.isEmpty(ptImages)) { | ||||
| throw new BusinessException(ResponseCode.BADREQUEST, "未查询到镜像信息!"); | throw new BusinessException(ResponseCode.BADREQUEST, "未查询到镜像信息!"); | ||||
| } | } | ||||
| ptImages = ptImages.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> | ptImages = ptImages.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> | ||||
| new TreeSet<>(Comparator.comparing(PtImage::getImageTag))), ArrayList::new)); | new TreeSet<>(Comparator.comparing(PtImage::getImageTag))), ArrayList::new)); | ||||
| ptImages.stream().forEach(ptImage -> { | |||||
| ptImage.setImageUrl(harborProperties.getAddress() + StrUtil.SLASH + ptImage.getImageUrl()); | |||||
| list.add(ptImage); | |||||
| }); | |||||
| return list; | |||||
| return ptImages; | |||||
| } | } | ||||
| @@ -282,37 +251,9 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| //删除本地镜像 | //删除本地镜像 | ||||
| imageList.forEach(image -> { | imageList.forEach(image -> { | ||||
| ptImageMapper.deleteById(image.getId()); | ptImageMapper.deleteById(image.getId()); | ||||
| //创建镜像回收任务 | |||||
| createRecycleTask(image); | |||||
| }); | }); | ||||
| } | } | ||||
| /** | |||||
| * 文件定时清理 | |||||
| * | |||||
| * @param ptImage 镜像实体对象 | |||||
| */ | |||||
| private void createRecycleTask(PtImage ptImage) { | |||||
| String imageUrl = ImageStateEnum.SUCCESS.getCode().equals(ptImage.getImageStatus()) ? ptImage.getImageUrl() : ""; | |||||
| RecycleCreateDTO recycleCreateDTO = RecycleCreateDTO.builder() | |||||
| .recycleModule(RecycleModuleEnum.BIZ_IMAGE.getValue()) | |||||
| .recycleDelayDate(recycleConfig.getImageValid()) | |||||
| .recycleNote(RecycleTool.generateRecycleNote("删除镜像", ptImage.getImageName(), ptImage.getId())) | |||||
| .recycleCustom(RecycleResourceEnum.IMAGE_RECYCLE_FILE.getClassName()) | |||||
| .restoreCustom(RecycleResourceEnum.IMAGE_RECYCLE_FILE.getClassName()) | |||||
| .remark(String.valueOf(ptImage.getId())) | |||||
| .build(); | |||||
| recycleCreateDTO.addRecycleDetailCreateDTO(RecycleDetailCreateDTO.builder() | |||||
| .recycleCondition(imageUrl) | |||||
| .recycleType(RecycleTypeEnum.FILE.getCode()) | |||||
| .recycleNote(RecycleTool.generateRecycleNote("删除镜像", ptImage.getImageName(), ptImage.getId())) | |||||
| .remark(String.valueOf(ptImage.getId())) | |||||
| .build() | |||||
| ); | |||||
| recycleService.createRecycleTask(recycleCreateDTO); | |||||
| } | |||||
| /** | /** | ||||
| * 修改镜像信息 | * 修改镜像信息 | ||||
| * | * | ||||
| @@ -324,29 +265,33 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| public void updateTrainImage(PtImageUpdateDTO imageUpdateDTO) { | public void updateTrainImage(PtImageUpdateDTO imageUpdateDTO) { | ||||
| UserContext curUser = userContextService.getCurUser(); | UserContext curUser = userContextService.getCurUser(); | ||||
| List<PtImage> imageList = ptImageMapper.selectList(new LambdaQueryWrapper<PtImage>() | |||||
| .in(PtImage::getId, imageUpdateDTO.getIds())); | |||||
| PtImage image = ptImageMapper.selectById(imageUpdateDTO.getId()); | |||||
| if (CollUtil.isEmpty(imageList)) { | |||||
| throw new BusinessException("内部错误"); | |||||
| if (image == null) { | |||||
| throw new BusinessException("镜像不存在!"); | |||||
| } | } | ||||
| for (PtImage image : imageList) { | |||||
| //非管理员禁止修改预置镜像 | |||||
| if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource()) && !BaseService.isAdmin(curUser)) { | |||||
| throw new BusinessException("非管理员无权限修改预置镜像信息"); | |||||
| } | |||||
| image.setRemark(imageUpdateDTO.getRemark()); | |||||
| ptImageMapper.updateById(image); | |||||
| List<Integer> imageTypes = ptImageMapper.selectImageType(image.getId()); | |||||
| for (Integer imageType : imageUpdateDTO.getImageTypes()) { | |||||
| if (!CollectionUtil.contains(imageTypes, imageType)) { | |||||
| ptImageMapper.insertImageType(image.getId(), imageType); | |||||
| } | |||||
| validImageInfo(imageUpdateDTO.getImageUrl(), imageUpdateDTO.getImageName(), imageUpdateDTO.getImageTag()); | |||||
| //非管理员禁止修改预置镜像 | |||||
| if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource()) && !BaseService.isAdmin(curUser)) { | |||||
| throw new BusinessException("非管理员无权限修改预置镜像信息"); | |||||
| } | |||||
| image.setImageTypes(imageUpdateDTO.getImageTypes()) | |||||
| .setImageName(imageUpdateDTO.getImageName()) | |||||
| .setImageUrl(imageUpdateDTO.getImageUrl()) | |||||
| .setImageTag(imageUpdateDTO.getImageTag()) | |||||
| .setRemark(imageUpdateDTO.getRemark()); | |||||
| ptImageMapper.updateById(image); | |||||
| List<Integer> imageTypes = ptImageMapper.selectImageType(image.getId()); | |||||
| for (Integer imageType : imageUpdateDTO.getImageTypes()) { | |||||
| if (!CollectionUtil.contains(imageTypes, imageType)) { | |||||
| ptImageMapper.insertImageType(image.getId(), imageType); | |||||
| } | } | ||||
| for (Integer imageType : imageTypes) { | |||||
| if (!CollectionUtil.contains(imageUpdateDTO.getImageTypes(), imageType)) { | |||||
| ptImageMapper.deleteImageType(image.getId(), imageType); | |||||
| } | |||||
| } | |||||
| for (Integer imageType : imageTypes) { | |||||
| if (!CollectionUtil.contains(imageUpdateDTO.getImageTypes(), imageType)) { | |||||
| ptImageMapper.deleteImageType(image.getId(), imageType); | |||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| @@ -362,10 +307,8 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| //从会话中获取用户信息 | //从会话中获取用户信息 | ||||
| UserContext user = userContextService.getCurUser(); | UserContext user = userContextService.getCurUser(); | ||||
| QueryWrapper<PtImage> queryWrapper = new QueryWrapper<>(); | QueryWrapper<PtImage> queryWrapper = new QueryWrapper<>(); | ||||
| queryWrapper.eq("image_status", ImageStateEnum.SUCCESS.getCode()); | |||||
| queryWrapper.eq("deleted", NumberConstant.NUMBER_0); | queryWrapper.eq("deleted", NumberConstant.NUMBER_0); | ||||
| if (!BaseService.isAdmin(user)) { | if (!BaseService.isAdmin(user)) { | ||||
| queryWrapper.in("origin_user_id", user.getId(), 0L); | queryWrapper.in("origin_user_id", user.getId(), 0L); | ||||
| } | } | ||||
| if (ptImageQueryNameDTO.getImageResource() != null) { | if (ptImageQueryNameDTO.getImageResource() != null) { | ||||
| @@ -403,12 +346,8 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| throw new BusinessException(ResponseCode.BADREQUEST, "该镜像不存在或镜像用途不支持!"); | throw new BusinessException(ResponseCode.BADREQUEST, "该镜像不存在或镜像用途不支持!"); | ||||
| } | } | ||||
| //仅支持[制作成功]状态镜像设置为默认镜像 | |||||
| if (!ImageStateEnum.SUCCESS.getCode().equals(image.getImageStatus())) { | |||||
| throw new BusinessException(ResponseCode.BADREQUEST, "仅支持[制作成功]状态镜像设置为默认镜像!"); | |||||
| } | |||||
| if (image.getImageResource() != ImageSourceEnum.PRE.getCode()) { | |||||
| if (!image.getImageResource().equals(ImageSourceEnum.PRE.getCode())) { | |||||
| throw new BusinessException(ResponseCode.BADREQUEST, "非预制镜像不能设置为默认镜像!"); | throw new BusinessException(ResponseCode.BADREQUEST, "非预制镜像不能设置为默认镜像!"); | ||||
| } | } | ||||
| @@ -434,7 +373,7 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| /** | /** | ||||
| * 获取镜像URL | * 获取镜像URL | ||||
| * | * | ||||
| * @param imageQueryUrlDTO 查询镜像路径DTO | |||||
| * @param imageQueryUrlDTO 查询镜像地址DTO | |||||
| * @return String 镜像url | * @return String 镜像url | ||||
| */ | */ | ||||
| @Override | @Override | ||||
| @@ -453,7 +392,6 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| if (imageQueryUrlDTO.getIsDefault() != null) { | if (imageQueryUrlDTO.getIsDefault() != null) { | ||||
| queryWrapper.eq(PtImage::getIsDefault, imageQueryUrlDTO.getIsDefault()); | queryWrapper.eq(PtImage::getIsDefault, imageQueryUrlDTO.getIsDefault()); | ||||
| } | } | ||||
| queryWrapper.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()); | |||||
| queryWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | queryWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | ||||
| List<PtImage> imageList = new ArrayList<>(); | List<PtImage> imageList = new ArrayList<>(); | ||||
| if (CollectionUtil.isNotEmpty(imageQueryUrlDTO.getImageTypes())) { | if (CollectionUtil.isNotEmpty(imageQueryUrlDTO.getImageTypes())) { | ||||
| @@ -470,23 +408,13 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| return imageUrl; | return imageUrl; | ||||
| } | } | ||||
| /** | |||||
| * 自定义镜像回收的还原实现 | |||||
| * | |||||
| * @param dto 资源回收DTO对象 | |||||
| */ | |||||
| @Override | |||||
| public void recycleRollback(RecycleCreateDTO dto) { | |||||
| String imageId = dto.getRemark(); | |||||
| ptImageMapper.updateDeletedById(Long.valueOf(imageId), false); | |||||
| } | |||||
| @Override | @Override | ||||
| @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) | @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) | ||||
| public List<PtImage> getTerminalImageList() { | public List<PtImage> getTerminalImageList() { | ||||
| UserContext user = userContextService.getCurUser(); | UserContext user = userContextService.getCurUser(); | ||||
| LambdaQueryWrapper<PtImage> queryTerminalWrapper = new LambdaQueryWrapper<>(); | LambdaQueryWrapper<PtImage> queryTerminalWrapper = new LambdaQueryWrapper<>(); | ||||
| queryTerminalWrapper.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()).eq(PtImage::getDeleted, NumberConstant.NUMBER_0);; | |||||
| queryTerminalWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | |||||
| ; | |||||
| if (user != null && !BaseService.isAdmin()) { | if (user != null && !BaseService.isAdmin()) { | ||||
| queryTerminalWrapper.and(wrapper -> wrapper.eq(PtImage::getCreateUserId, user.getId()).or().eq(PtImage::getImageResource, ImageSourceEnum.PRE.getCode())) | queryTerminalWrapper.and(wrapper -> wrapper.eq(PtImage::getCreateUserId, user.getId()).or().eq(PtImage::getImageResource, ImageSourceEnum.PRE.getCode())) | ||||
| .and(wrapper -> wrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0)); | .and(wrapper -> wrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0)); | ||||
| @@ -509,17 +437,16 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| /** | /** | ||||
| * @param ptImageUploadDTO 镜像上传逻辑校验 | |||||
| * @param ptImageSaveDTO 镜像保存逻辑校验 | |||||
| * @param user 用户 | * @param user 用户 | ||||
| * @return List<PtImage> 镜像列表 | * @return List<PtImage> 镜像列表 | ||||
| **/ | **/ | ||||
| @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) | @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) | ||||
| private List<PtImage> checkUploadImage(PtImageUploadDTO ptImageUploadDTO, UserContext user, Integer source) { | |||||
| private List<PtImage> checkSaveImage(PtImageSaveDTO ptImageSaveDTO, UserContext user, Integer source) { | |||||
| LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | ||||
| queryWrapper.eq(PtImage::getImageName, ptImageUploadDTO.getImageName()) | |||||
| .eq(PtImage::getImageTag, ptImageUploadDTO.getImageTag()) | |||||
| queryWrapper.eq(PtImage::getImageUrl, ptImageSaveDTO.getImageUrl()) | |||||
| .eq(PtImage::getImageResource, source); | .eq(PtImage::getImageResource, source); | ||||
| if (user != null) { | if (user != null) { | ||||
| @@ -549,7 +476,7 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| ptImages.stream().forEach( | ptImages.stream().forEach( | ||||
| ptImage -> { | ptImage -> { | ||||
| PtImageVO ptImageVO =new PtImageVO(); | |||||
| PtImageVO ptImageVO = new PtImageVO(); | |||||
| ptImageVO.setId(ptImage.getId()); | ptImageVO.setId(ptImage.getId()); | ||||
| ptImageVO.setName(ptImage.getImageName()); | ptImageVO.setName(ptImage.getImageName()); | ||||
| ptImageVO.setTag(ptImage.getImageTag()); | ptImageVO.setTag(ptImage.getImageTag()); | ||||
| @@ -558,4 +485,24 @@ public class PtImageServiceImpl implements PtImageService { | |||||
| ); | ); | ||||
| return ptImageVOS; | return ptImageVOS; | ||||
| } | } | ||||
| } | |||||
| /** | |||||
| * 校验镜像地址与镜像名称、tag是否匹配 | |||||
| * | |||||
| * @param imageUrl 镜像地址 | |||||
| * @param imageName 镜像名称 | |||||
| * @param imageTag 镜像tag | |||||
| */ | |||||
| private void validImageInfo(String imageUrl, String imageName, String imageTag) { | |||||
| // 标准镜像地址:镜像地址域名/命名空间/镜像名:镜像tag | |||||
| // 分解镜像地址 | |||||
| String[] split = imageUrl.substring(imageUrl.lastIndexOf(StrUtil.SLASH) + 1).split(StrUtil.COLON); | |||||
| if (!split[0].equals(imageName)) { | |||||
| throw new BusinessException("镜像名称不匹配"); | |||||
| } | |||||
| if (!split[1].equals(imageTag)) { | |||||
| throw new BusinessException("镜像版本号不匹配"); | |||||
| } | |||||
| } | |||||
| } | |||||
| @@ -1,77 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| * ============================================================= | |||||
| */ | |||||
| package org.dubhe.image.service.task; | |||||
| import cn.hutool.core.util.StrUtil; | |||||
| import org.dubhe.biz.base.constant.HarborProperties; | |||||
| import org.dubhe.biz.log.enums.LogEnum; | |||||
| import org.dubhe.biz.log.utils.LogUtil; | |||||
| import org.dubhe.harbor.api.HarborApi; | |||||
| import org.dubhe.image.service.PtImageService; | |||||
| import org.dubhe.recycle.domain.dto.RecycleCreateDTO; | |||||
| import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO; | |||||
| import org.dubhe.recycle.global.AbstractGlobalRecycle; | |||||
| import org.springframework.beans.factory.annotation.Autowired; | |||||
| import org.springframework.cloud.context.config.annotation.RefreshScope; | |||||
| import org.springframework.stereotype.Component; | |||||
| /** | |||||
| * @description 镜像资源回收/还原 | |||||
| * @date 2021-03-23 | |||||
| */ | |||||
| @RefreshScope | |||||
| @Component(value = "imageRecycleFile") | |||||
| public class ImageRecycleFile extends AbstractGlobalRecycle { | |||||
| @Autowired | |||||
| private HarborApi harborApi; | |||||
| @Autowired | |||||
| private HarborProperties harborProperties; | |||||
| @Autowired | |||||
| private PtImageService ptImageService; | |||||
| /** | |||||
| * 自定义回收镜像实现 | |||||
| * | |||||
| * @param detail 数据清理详情参数 | |||||
| * @param dto 资源回收创建对象 | |||||
| * @return true 继续执行,false 中断任务详情回收(本次无法执行完毕,创建新任务到下次执行) | |||||
| */ | |||||
| @Override | |||||
| protected boolean clearDetail(RecycleDetailCreateDTO detail, RecycleCreateDTO dto) throws Exception { | |||||
| LogUtil.info(LogEnum.IMAGE, "image custom recycle file,params:{}", detail); | |||||
| if (StrUtil.isNotBlank(detail.getRecycleCondition())) { | |||||
| String imageUrl = harborProperties.getAddress() + StrUtil.SLASH + detail.getRecycleCondition(); | |||||
| LogUtil.info(LogEnum.IMAGE, "delete harbor image url:{}", imageUrl); | |||||
| //同步删除harbor镜像 | |||||
| harborApi.deleteImageByTag(imageUrl); | |||||
| } | |||||
| return true; | |||||
| } | |||||
| /** | |||||
| * 数据还原 | |||||
| * | |||||
| * @param dto 数据清理参数 | |||||
| */ | |||||
| @Override | |||||
| protected void rollback(RecycleCreateDTO dto) { | |||||
| ptImageService.recycleRollback(dto); | |||||
| } | |||||
| } | |||||
| @@ -21,8 +21,8 @@ import com.alibaba.fastjson.JSON; | |||||
| import org.dubhe.biz.base.constant.AuthConst; | import org.dubhe.biz.base.constant.AuthConst; | ||||
| import org.dubhe.cloud.unittest.base.BaseTest; | import org.dubhe.cloud.unittest.base.BaseTest; | ||||
| import org.dubhe.image.domain.dto.PtImageDeleteDTO; | import org.dubhe.image.domain.dto.PtImageDeleteDTO; | ||||
| import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||||
| import org.dubhe.image.domain.dto.PtImageUpdateDTO; | import org.dubhe.image.domain.dto.PtImageUpdateDTO; | ||||
| import org.dubhe.image.domain.dto.PtImageUploadDTO; | |||||
| import org.junit.Test; | import org.junit.Test; | ||||
| import org.junit.runner.RunWith; | import org.junit.runner.RunWith; | ||||
| import org.springframework.boot.test.context.SpringBootTest; | import org.springframework.boot.test.context.SpringBootTest; | ||||
| @@ -78,9 +78,9 @@ public class PtImageTest extends BaseTest { | |||||
| */ | */ | ||||
| @Test | @Test | ||||
| public void uploadImageTest() throws Exception { | public void uploadImageTest() throws Exception { | ||||
| PtImageUploadDTO ptImageUploadDTO = new PtImageUploadDTO(); | |||||
| PtImageSaveDTO ptImageUploadDTO = new PtImageSaveDTO(); | |||||
| ptImageUploadDTO.setImageName("mysql"); | ptImageUploadDTO.setImageName("mysql"); | ||||
| ptImageUploadDTO.setImagePath("F:/mysql.tar"); | |||||
| ptImageUploadDTO.setImageUrl("F:/mysql.tar"); | |||||
| ptImageUploadDTO.setImageTag("5.7"); | ptImageUploadDTO.setImageTag("5.7"); | ||||
| ptImageUploadDTO.setRemark("测试上传镜像"); | ptImageUploadDTO.setRemark("测试上传镜像"); | ||||
| @@ -94,7 +94,7 @@ public class PtImageTest extends BaseTest { | |||||
| @Test | @Test | ||||
| public void updateImageTest() throws Exception { | public void updateImageTest() throws Exception { | ||||
| PtImageUpdateDTO imageUpdateDTO = new PtImageUpdateDTO(); | PtImageUpdateDTO imageUpdateDTO = new PtImageUpdateDTO(); | ||||
| imageUpdateDTO.setIds(Arrays.asList()); | |||||
| imageUpdateDTO.setId(1L); | |||||
| imageUpdateDTO.setRemark(""); | imageUpdateDTO.setRemark(""); | ||||
| mockMvcTest(MockMvcRequestBuilders.put("/ptImage"), JSON.toJSONString(imageUpdateDTO), | mockMvcTest(MockMvcRequestBuilders.put("/ptImage"), JSON.toJSONString(imageUpdateDTO), | ||||
| MockMvcResultMatchers.status().is2xxSuccessful(), 200); | MockMvcResultMatchers.status().is2xxSuccessful(), 200); | ||||
| @@ -75,7 +75,7 @@ public class DeploymentCallback extends Observable { | |||||
| setChanged(); | setChanged(); | ||||
| notifyObservers(deployment); | notifyObservers(deployment); | ||||
| if (StringUtils.isNotEmpty(businessLabel)){ | if (StringUtils.isNotEmpty(businessLabel)){ | ||||
| BaseK8sDeploymentCallbackCreateDTO baseK8sDeploymentCallbackCreateDTO = new BaseK8sDeploymentCallbackCreateDTO(deployment.getNamespace(), deployment.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),deployment.getName(), deployment.getReadyReplicas() == null?0:deployment.getReadyReplicas(), deployment.getReplicas() == null?0:deployment.getReplicas()); | |||||
| BaseK8sDeploymentCallbackCreateDTO baseK8sDeploymentCallbackCreateDTO = new BaseK8sDeploymentCallbackCreateDTO(deployment.getNamespace(), deployment.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),deployment.getName(), deployment.getReadyReplicas() == null?0:deployment.getReadyReplicas(), deployment.getReplicas() == null?0:deployment.getReplicas(),watcherActionEnum.getAction()); | |||||
| String url = k8sCallBackTool.getDeploymentCallbackUrl(businessLabel); | String url = k8sCallBackTool.getDeploymentCallbackUrl(businessLabel); | ||||
| String token = k8sCallBackTool.generateToken(); | String token = k8sCallBackTool.generateToken(); | ||||
| @@ -17,6 +17,7 @@ | |||||
| package org.dubhe.dubhek8s.observer; | package org.dubhe.dubhek8s.observer; | ||||
| import org.dubhe.biz.base.constant.MagicNumConstant; | |||||
| import org.dubhe.biz.base.enums.BizEnum; | import org.dubhe.biz.base.enums.BizEnum; | ||||
| import org.dubhe.biz.base.utils.SpringContextHolder; | import org.dubhe.biz.base.utils.SpringContextHolder; | ||||
| import org.dubhe.biz.log.enums.LogEnum; | import org.dubhe.biz.log.enums.LogEnum; | ||||
| @@ -54,8 +55,18 @@ public class TrainJobObserver implements Observer { | |||||
| BizPod pod = (BizPod)arg; | BizPod pod = (BizPod)arg; | ||||
| boolean trainJobFailed = PodPhaseEnum.FAILED.getPhase().equals(pod.getPhase()) && BizEnum.ALGORITHM.getBizCode().equals(pod.getBusinessLabel()) && SpringContextHolder.getActiveProfile().equals(pod.getLabel(K8sLabelConstants.PLATFORM_RUNTIME_ENV)); | boolean trainJobFailed = PodPhaseEnum.FAILED.getPhase().equals(pod.getPhase()) && BizEnum.ALGORITHM.getBizCode().equals(pod.getBusinessLabel()) && SpringContextHolder.getActiveProfile().equals(pod.getLabel(K8sLabelConstants.PLATFORM_RUNTIME_ENV)); | ||||
| if (trainJobFailed){ | if (trainJobFailed){ | ||||
| LogUtil.warn(LogEnum.BIZ_K8S,"delete failed train job resourceName {};phase {};podName {}",pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),pod.getPhase(),pod.getName()); | |||||
| trainJobApi.delete(pod.getNamespace(),pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE)); | |||||
| new Thread(new Runnable(){ | |||||
| @Override | |||||
| public void run(){ | |||||
| try { | |||||
| Thread.sleep(MagicNumConstant.ONE_MINUTE); | |||||
| } catch (InterruptedException e) { | |||||
| LogUtil.error(LogEnum.BIZ_K8S,"TrainJobObserver update error {}",e.getMessage(),e); | |||||
| } | |||||
| LogUtil.warn(LogEnum.BIZ_K8S,"delete failed train job resourceName {};phase {};podName {}",pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),pod.getPhase(),pod.getName()); | |||||
| trainJobApi.delete(pod.getNamespace(),pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE)); | |||||
| } | |||||
| }).start(); | |||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| @@ -79,7 +79,7 @@ public class SystemNodeServiceImpl implements SystemNodeService { | |||||
| @Resource | @Resource | ||||
| private AdminClient adminClient; | private AdminClient adminClient; | ||||
| private final static String INTERNAL_IP = "InternalIP"; | |||||
| /** | /** | ||||
| @@ -231,7 +231,7 @@ public class SystemNodeServiceImpl implements SystemNodeService { | |||||
| nodeDTO.setUid(node.getUid()); | nodeDTO.setUid(node.getUid()); | ||||
| nodeDTO.setName(node.getName()); | nodeDTO.setName(node.getName()); | ||||
| node.getAddresses().stream().forEach(bizNodeAddress -> { | node.getAddresses().stream().forEach(bizNodeAddress -> { | ||||
| if (INTERNAL_IP.equals(bizNodeAddress.getType())) { | |||||
| if (K8sParamConstants.INTERNAL_IP.equals(bizNodeAddress.getType())) { | |||||
| nodeDTO.setIp(bizNodeAddress.getAddress()); | nodeDTO.setIp(bizNodeAddress.getAddress()); | ||||
| return; | return; | ||||
| } | } | ||||
| @@ -141,7 +141,7 @@ public class PtAtlasCameraServiceImpl implements PtAtlasCameraService { | |||||
| ResponseEntity<Map> exchange = restTemplate.exchange(atlasUrlProperties.getVms() + "/checkCameraUrl", HttpMethod.POST, requestEntity, Map.class, param); | ResponseEntity<Map> exchange = restTemplate.exchange(atlasUrlProperties.getVms() + "/checkCameraUrl", HttpMethod.POST, requestEntity, Map.class, param); | ||||
| Map<String, String> statusMap = (Map) exchange.getBody().get("data"); | Map<String, String> statusMap = (Map) exchange.getBody().get("data"); | ||||
| if (statusMap.containsKey("cameraUrl")) { | if (statusMap.containsKey("cameraUrl")) { | ||||
| ptAtlasCamera.setHlsUrl(statusMap.get("cameraUrl").replace("127.0.0.1", "127.0.0.1")); | |||||
| ptAtlasCamera.setHlsUrl(statusMap.get("cameraUrl").replace("172.18.26.2", "10.105.10.51")); | |||||
| ptAtlasCameraMapper.updateById(ptAtlasCamera); | ptAtlasCameraMapper.updateById(ptAtlasCamera); | ||||
| } | } | ||||
| } catch (Exception e) { | } catch (Exception e) { | ||||
| @@ -21,13 +21,10 @@ import cn.hutool.core.collection.CollUtil; | |||||
| import cn.hutool.core.date.DateBetween; | import cn.hutool.core.date.DateBetween; | ||||
| import cn.hutool.core.date.DateUnit; | import cn.hutool.core.date.DateUnit; | ||||
| import cn.hutool.core.util.RandomUtil; | import cn.hutool.core.util.RandomUtil; | ||||
| import cn.hutool.core.util.StrUtil; | |||||
| import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; | import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; | ||||
| import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; | import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; | ||||
| import com.baomidou.mybatisplus.core.metadata.IPage; | import com.baomidou.mybatisplus.core.metadata.IPage; | ||||
| import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | ||||
| import org.dubhe.biz.base.utils.StringUtils; | |||||
| import org.dubhe.biz.base.constant.HarborProperties; | |||||
| import org.dubhe.biz.base.constant.MagicNumConstant; | import org.dubhe.biz.base.constant.MagicNumConstant; | ||||
| import org.dubhe.biz.base.constant.NumberConstant; | import org.dubhe.biz.base.constant.NumberConstant; | ||||
| import org.dubhe.biz.base.constant.StringConstant; | import org.dubhe.biz.base.constant.StringConstant; | ||||
| @@ -45,8 +42,10 @@ import org.dubhe.biz.base.service.UserContextService; | |||||
| import org.dubhe.biz.base.utils.HttpUtils; | import org.dubhe.biz.base.utils.HttpUtils; | ||||
| import org.dubhe.biz.base.utils.NumberUtil; | import org.dubhe.biz.base.utils.NumberUtil; | ||||
| import org.dubhe.biz.base.utils.ResultUtil; | import org.dubhe.biz.base.utils.ResultUtil; | ||||
| import org.dubhe.biz.base.utils.StringUtils; | |||||
| import org.dubhe.biz.base.vo.DataResponseBody; | import org.dubhe.biz.base.vo.DataResponseBody; | ||||
| import org.dubhe.biz.base.vo.DatasetVO; | import org.dubhe.biz.base.vo.DatasetVO; | ||||
| import org.dubhe.biz.base.vo.NoteBookVO; | |||||
| import org.dubhe.biz.db.utils.PageUtil; | import org.dubhe.biz.db.utils.PageUtil; | ||||
| import org.dubhe.biz.db.utils.WrapperHelp; | import org.dubhe.biz.db.utils.WrapperHelp; | ||||
| import org.dubhe.biz.file.api.FileStoreApi; | import org.dubhe.biz.file.api.FileStoreApi; | ||||
| @@ -79,7 +78,6 @@ import org.dubhe.notebook.enums.NoteBookStatusEnum; | |||||
| import org.dubhe.notebook.service.NoteBookService; | import org.dubhe.notebook.service.NoteBookService; | ||||
| import org.dubhe.notebook.service.ProcessNotebookCommand; | import org.dubhe.notebook.service.ProcessNotebookCommand; | ||||
| import org.dubhe.notebook.utils.NotebookUtil; | import org.dubhe.notebook.utils.NotebookUtil; | ||||
| import org.dubhe.biz.base.vo.NoteBookVO; | |||||
| import org.springframework.beans.BeanUtils; | import org.springframework.beans.BeanUtils; | ||||
| import org.springframework.beans.factory.annotation.Autowired; | import org.springframework.beans.factory.annotation.Autowired; | ||||
| import org.springframework.beans.factory.annotation.Qualifier; | import org.springframework.beans.factory.annotation.Qualifier; | ||||
| @@ -89,7 +87,13 @@ import org.springframework.transaction.annotation.Transactional; | |||||
| import org.springframework.util.CollectionUtils; | import org.springframework.util.CollectionUtils; | ||||
| import java.io.File; | import java.io.File; | ||||
| import java.util.*; | |||||
| import java.util.ArrayList; | |||||
| import java.util.Arrays; | |||||
| import java.util.Collections; | |||||
| import java.util.Date; | |||||
| import java.util.List; | |||||
| import java.util.Map; | |||||
| import java.util.Set; | |||||
| import java.util.stream.Collectors; | import java.util.stream.Collectors; | ||||
| /** | /** | ||||
| @@ -127,9 +131,6 @@ public class NoteBookServiceImpl implements NoteBookService { | |||||
| @Autowired | @Autowired | ||||
| private ImageClient imageClient; | private ImageClient imageClient; | ||||
| @Autowired | |||||
| private HarborProperties harborProperties; | |||||
| @Autowired | @Autowired | ||||
| @Qualifier("hostFileStoreApiImpl") | @Qualifier("hostFileStoreApiImpl") | ||||
| private FileStoreApi fileStoreApi; | private FileStoreApi fileStoreApi; | ||||
| @@ -223,7 +224,7 @@ public class NoteBookServiceImpl implements NoteBookService { | |||||
| throw new BusinessException(ImageTypeEnum.NOTEBOOK.getCode() + "未配置默认镜像!"); | throw new BusinessException(ImageTypeEnum.NOTEBOOK.getCode() + "未配置默认镜像!"); | ||||
| } | } | ||||
| return harborProperties.getAddress() + StrUtil.SLASH + imageUrl; | |||||
| return imageUrl; | |||||
| } | } | ||||
| /** | /** | ||||
| @@ -1,39 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| * ============================================================= | |||||
| */ | |||||
| package org.dubhe.pointcloud.config; | |||||
| import lombok.Data; | |||||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||||
| import org.springframework.stereotype.Component; | |||||
| /** | |||||
| * @description harbor相关配置 | |||||
| * @date 2020-07-17 | |||||
| */ | |||||
| @Data | |||||
| @Component | |||||
| @ConfigurationProperties(prefix = "harbor") | |||||
| public class PointCloudHarborConfig { | |||||
| private String address; | |||||
| private String username; | |||||
| private String password; | |||||
| private String modelName; | |||||
| } | |||||
| @@ -66,7 +66,6 @@ import org.dubhe.pointcloud.client.DataClient; | |||||
| import org.dubhe.pointcloud.client.ImageClient; | import org.dubhe.pointcloud.client.ImageClient; | ||||
| import org.dubhe.pointcloud.client.ModelClient; | import org.dubhe.pointcloud.client.ModelClient; | ||||
| import org.dubhe.pointcloud.common.Constant; | import org.dubhe.pointcloud.common.Constant; | ||||
| import org.dubhe.pointcloud.config.PointCloudHarborConfig; | |||||
| import org.dubhe.pointcloud.dao.PcAnnotationDetailMapper; | import org.dubhe.pointcloud.dao.PcAnnotationDetailMapper; | ||||
| import org.dubhe.pointcloud.dao.PcDatasetFileMapper; | import org.dubhe.pointcloud.dao.PcDatasetFileMapper; | ||||
| import org.dubhe.pointcloud.dao.PcDatasetMapper; | import org.dubhe.pointcloud.dao.PcDatasetMapper; | ||||
| @@ -141,9 +140,6 @@ public class PcDatasetServiceImpl implements PcDatasetService { | |||||
| @Resource | @Resource | ||||
| private DeployAsyncTask deployAsyncTask; | private DeployAsyncTask deployAsyncTask; | ||||
| @Resource | |||||
| private PointCloudHarborConfig pointCloudHarborConfig; | |||||
| @Resource | @Resource | ||||
| private ImageClient imageClient; | private ImageClient imageClient; | ||||
| @@ -652,7 +648,7 @@ public class PcDatasetServiceImpl implements PcDatasetService { | |||||
| if (!dataResponseBody.succeed()) { | if (!dataResponseBody.succeed()) { | ||||
| throw new BusinessException(ErrorEnum.CALL_IMAGE_SERVER_FAIL); | throw new BusinessException(ErrorEnum.CALL_IMAGE_SERVER_FAIL); | ||||
| } | } | ||||
| return pointCloudHarborConfig.getAddress() + SymbolConstant.SLASH + dataResponseBody.getData(); | |||||
| return dataResponseBody.getData(); | |||||
| } | } | ||||
| /** | /** | ||||
| @@ -18,8 +18,10 @@ | |||||
| package org.dubhe.servinggateway.config; | package org.dubhe.servinggateway.config; | ||||
| import lombok.extern.slf4j.Slf4j; | import lombok.extern.slf4j.Slf4j; | ||||
| import org.dubhe.biz.base.constant.MagicNumConstant; | |||||
| import org.dubhe.biz.base.constant.NumberConstant; | import org.dubhe.biz.base.constant.NumberConstant; | ||||
| import org.dubhe.biz.base.constant.SymbolConstant; | import org.dubhe.biz.base.constant.SymbolConstant; | ||||
| import org.dubhe.biz.base.utils.StringUtils; | |||||
| import org.dubhe.biz.log.enums.LogEnum; | import org.dubhe.biz.log.enums.LogEnum; | ||||
| import org.dubhe.biz.log.utils.LogUtil; | import org.dubhe.biz.log.utils.LogUtil; | ||||
| import org.dubhe.servinggateway.constant.GatewayConstant; | import org.dubhe.servinggateway.constant.GatewayConstant; | ||||
| @@ -38,8 +40,8 @@ import org.springframework.web.util.UriComponentsBuilder; | |||||
| import reactor.core.publisher.Mono; | import reactor.core.publisher.Mono; | ||||
| import javax.annotation.Resource; | import javax.annotation.Resource; | ||||
| import java.util.ArrayList; | |||||
| import java.util.Arrays; | import java.util.Arrays; | ||||
| import java.util.Collections; | |||||
| import java.util.HashMap; | import java.util.HashMap; | ||||
| import java.util.List; | import java.util.List; | ||||
| import java.util.Map; | import java.util.Map; | ||||
| @@ -55,21 +57,12 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||||
| private ApplicationEventPublisher publisher; | private ApplicationEventPublisher publisher; | ||||
| @Value("${serving.gateway.postfixUrl}") | |||||
| private String postfixUrl; | |||||
| @Resource | @Resource | ||||
| private RedisRouteDefinitionRepository routeDefinitionRepository; | private RedisRouteDefinitionRepository routeDefinitionRepository; | ||||
| @Resource | @Resource | ||||
| private GatewayRouteService gatewayRouteService; | private GatewayRouteService gatewayRouteService; | ||||
| /** | |||||
| * 模型服务部署使用的端口 | |||||
| */ | |||||
| @Value("${k8s.port}") | |||||
| private String httpPort; | |||||
| @Override | @Override | ||||
| public void run(String... args) { | public void run(String... args) { | ||||
| this.loadRouteConfig(); | this.loadRouteConfig(); | ||||
| @@ -87,6 +80,9 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||||
| LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin load route config"); | LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin load route config"); | ||||
| List<GatewayRouteQueryVO> activeRoutes = gatewayRouteService.findActiveRoutes(); | List<GatewayRouteQueryVO> activeRoutes = gatewayRouteService.findActiveRoutes(); | ||||
| for (GatewayRouteQueryVO activeRoute : activeRoutes) { | for (GatewayRouteQueryVO activeRoute : activeRoutes) { | ||||
| if (StringUtils.isEmpty(activeRoute.getUri())){ | |||||
| continue; | |||||
| } | |||||
| RouteDefinition definition = this.convert2RouteDefinition(activeRoute); | RouteDefinition definition = this.convert2RouteDefinition(activeRoute); | ||||
| routeDefinitionRepository.save(Mono.just(definition)).subscribe(); | routeDefinitionRepository.save(Mono.just(definition)).subscribe(); | ||||
| } | } | ||||
| @@ -103,7 +99,19 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||||
| public void saveRouteByRouteId(Long id) { | public void saveRouteByRouteId(Long id) { | ||||
| LogUtil.info(LogEnum.SERVING_GATEWAY, "Deal save route event"); | LogUtil.info(LogEnum.SERVING_GATEWAY, "Deal save route event"); | ||||
| GatewayRouteQueryVO gatewayRouteQueryVO = gatewayRouteService.findActiveById(id); | GatewayRouteQueryVO gatewayRouteQueryVO = gatewayRouteService.findActiveById(id); | ||||
| if (Objects.nonNull(gatewayRouteQueryVO)) { | |||||
| //查不到数据进行重试 | |||||
| int count = 0; | |||||
| while (gatewayRouteQueryVO == null && count < MagicNumConstant.THREE){ | |||||
| LogUtil.info(LogEnum.SERVING_GATEWAY, "gatewayRouteQueryVO is null retry:"+count +" id="+id); | |||||
| gatewayRouteQueryVO = gatewayRouteService.findActiveById(id); | |||||
| count++; | |||||
| try { | |||||
| Thread.sleep(MagicNumConstant.FIVE_HUNDRED); | |||||
| } catch (InterruptedException e) { | |||||
| e.printStackTrace(); | |||||
| } | |||||
| } | |||||
| if (Objects.nonNull(gatewayRouteQueryVO) && StringUtils.isNotEmpty(gatewayRouteQueryVO.getUri())) { | |||||
| routeDefinitionRepository.save(Mono.just(this.convert2RouteDefinition(gatewayRouteQueryVO))).subscribe(); | routeDefinitionRepository.save(Mono.just(this.convert2RouteDefinition(gatewayRouteQueryVO))).subscribe(); | ||||
| } | } | ||||
| // 发布刷新路由事件 | // 发布刷新路由事件 | ||||
| @@ -131,17 +139,19 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||||
| * @return 网关所需的RouteDefinition | * @return 网关所需的RouteDefinition | ||||
| */ | */ | ||||
| private RouteDefinition convert2RouteDefinition(GatewayRouteQueryVO gatewayRouteQueryVO) { | private RouteDefinition convert2RouteDefinition(GatewayRouteQueryVO gatewayRouteQueryVO) { | ||||
| if(StringUtils.isEmpty(gatewayRouteQueryVO.getUri())){ | |||||
| return null; | |||||
| } | |||||
| RouteDefinition definition = new RouteDefinition(); | RouteDefinition definition = new RouteDefinition(); | ||||
| // 设置路由基础信息 | // 设置路由基础信息 | ||||
| definition.setId(GatewayConstant.ROUTE_PREFIX + gatewayRouteQueryVO.getId()); | definition.setId(GatewayConstant.ROUTE_PREFIX + gatewayRouteQueryVO.getId()); | ||||
| definition.setUri(UriComponentsBuilder.fromHttpUrl(SymbolConstant.HTTP_SLASH + gatewayRouteQueryVO.getUri() + | |||||
| SymbolConstant.COLON + httpPort).build().toUri()); | |||||
| definition.setUri(UriComponentsBuilder.fromHttpUrl(SymbolConstant.HTTP_SLASH + gatewayRouteQueryVO.getUri()).build().toUri()); | |||||
| // 定义url匹配规则的断言 | // 定义url匹配规则的断言 | ||||
| // 举例 {abc}.dubhe.ai 匹配{abc}部分 | // 举例 {abc}.dubhe.ai 匹配{abc}部分 | ||||
| PredicateDefinition pathPredicate = new PredicateDefinition(); | PredicateDefinition pathPredicate = new PredicateDefinition(); | ||||
| pathPredicate.setName("Host"); | |||||
| pathPredicate.setName("Path"); | |||||
| Map<String, String> predicateParams = new HashMap<>(NumberConstant.NUMBER_8); | Map<String, String> predicateParams = new HashMap<>(NumberConstant.NUMBER_8); | ||||
| predicateParams.put("pattern", gatewayRouteQueryVO.getPatternPath() + postfixUrl); | |||||
| predicateParams.put("pattern", "/"+gatewayRouteQueryVO.getPatternPath() + "/**"); | |||||
| pathPredicate.setArgs(predicateParams); | pathPredicate.setArgs(predicateParams); | ||||
| // 定义权重断言 | // 定义权重断言 | ||||
| // 根据配置的权重信息进行分流 | // 根据配置的权重信息进行分流 | ||||
| @@ -152,11 +162,21 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||||
| weightParams.put("weight.weight", gatewayRouteQueryVO.getWeight()); | weightParams.put("weight.weight", gatewayRouteQueryVO.getWeight()); | ||||
| weightPredicate.setArgs(weightParams); | weightPredicate.setArgs(weightParams); | ||||
| definition.setPredicates(Arrays.asList(pathPredicate, weightPredicate)); | definition.setPredicates(Arrays.asList(pathPredicate, weightPredicate)); | ||||
| List<FilterDefinition> filters =new ArrayList<>(); | |||||
| // 传入自定义的监控指标过滤器 | // 传入自定义的监控指标过滤器 | ||||
| FilterDefinition filterDefinition = new FilterDefinition(); | FilterDefinition filterDefinition = new FilterDefinition(); | ||||
| // name设置为定义的MetricsGatewayFilterFactory中GatewayFilterFactory前部分 | // name设置为定义的MetricsGatewayFilterFactory中GatewayFilterFactory前部分 | ||||
| filterDefinition.setName("Metrics"); | filterDefinition.setName("Metrics"); | ||||
| definition.setFilters(Collections.singletonList(filterDefinition)); | |||||
| filters.add(filterDefinition); | |||||
| //去除一级url转发 | |||||
| FilterDefinition filterDefinitionUrl = new FilterDefinition(); | |||||
| filterDefinitionUrl.setName("StripPrefix"); | |||||
| filterDefinitionUrl.addArg("parts","1"); | |||||
| filters.add(filterDefinitionUrl); | |||||
| definition.setFilters(filters); | |||||
| // 塞入自定义数据,供之后filter使用 | // 塞入自定义数据,供之后filter使用 | ||||
| HashMap<String, Object> metadata = new HashMap<>(NumberConstant.NUMBER_4); | HashMap<String, Object> metadata = new HashMap<>(NumberConstant.NUMBER_4); | ||||
| metadata.put("servingInfoId", gatewayRouteQueryVO.getServiceInfoId()); | metadata.put("servingInfoId", gatewayRouteQueryVO.getServiceInfoId()); | ||||
| @@ -120,7 +120,7 @@ public class MetricsGatewayFilterFactory extends AbstractGatewayFilterFactory<Me | |||||
| DataBufferUtils.join(body) | DataBufferUtils.join(body) | ||||
| .doOnNext(dataBuffer -> { | .doOnNext(dataBuffer -> { | ||||
| // 判断该请求是否为推理请求 | // 判断该请求是否为推理请求 | ||||
| boolean isInference = exchange.getRequest().getPath().toString().startsWith(GatewayConstant.INFERENCE_INTERFACE_NAME); | |||||
| boolean isInference = exchange.getRequest().getPath().toString().endsWith(GatewayConstant.INFERENCE_INTERFACE_NAME); | |||||
| if (isInference) { | if (isInference) { | ||||
| LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin deal inference request filter"); | LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin deal inference request filter"); | ||||
| // 调用请求是否成功 | // 调用请求是否成功 | ||||
| @@ -7,7 +7,7 @@ | |||||
| select smc.id id, si.id serviceInfoId, si.uuid patternPath, smc.release_rate weight, smc.url uri | select smc.id id, si.id serviceInfoId, si.uuid patternPath, smc.release_rate weight, smc.url uri | ||||
| from serving_model_config smc | from serving_model_config smc | ||||
| left join serving_info si on si.id = smc.serving_id | left join serving_info si on si.id = smc.serving_id | ||||
| where si.`status` = 2 and si.deleted = 0 and smc.ready_replicas > 0 and smc.deleted = 0 | |||||
| where si.deleted = 0 and smc.deleted = 0 | |||||
| and smc.id = #{id} | and smc.id = #{id} | ||||
| </select> | </select> | ||||
| @@ -1,39 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| * ============================================================= | |||||
| */ | |||||
| package org.dubhe.serving.config; | |||||
| import lombok.Data; | |||||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||||
| import org.springframework.stereotype.Component; | |||||
| /** | |||||
| * @description harbor相关配置 | |||||
| * @date 2020-07-17 | |||||
| */ | |||||
| @Data | |||||
| @Component | |||||
| @ConfigurationProperties(prefix = "harbor") | |||||
| public class TrainHarborConfig { | |||||
| private String address; | |||||
| private String username; | |||||
| private String password; | |||||
| private String modelName; | |||||
| } | |||||
| @@ -71,7 +71,6 @@ import org.dubhe.serving.client.AlgorithmClient; | |||||
| import org.dubhe.serving.client.ImageClient; | import org.dubhe.serving.client.ImageClient; | ||||
| import org.dubhe.serving.client.ModelBranchClient; | import org.dubhe.serving.client.ModelBranchClient; | ||||
| import org.dubhe.serving.client.ModelInfoClient; | import org.dubhe.serving.client.ModelInfoClient; | ||||
| import org.dubhe.serving.config.TrainHarborConfig; | |||||
| import org.dubhe.serving.constant.ServingConstant; | import org.dubhe.serving.constant.ServingConstant; | ||||
| import org.dubhe.serving.dao.BatchServingMapper; | import org.dubhe.serving.dao.BatchServingMapper; | ||||
| import org.dubhe.serving.domain.dto.*; | import org.dubhe.serving.domain.dto.*; | ||||
| @@ -135,8 +134,6 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat | |||||
| @Resource | @Resource | ||||
| private ImageClient imageClient; | private ImageClient imageClient; | ||||
| @Resource | @Resource | ||||
| private TrainHarborConfig trainHarborConfig; | |||||
| @Resource | |||||
| private AlgorithmClient algorithmClient; | private AlgorithmClient algorithmClient; | ||||
| @Autowired | @Autowired | ||||
| private RedisUtils redisUtils; | private RedisUtils redisUtils; | ||||
| @@ -238,7 +235,7 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat | |||||
| BeanUtils.copyProperties(batchServingCreateDTO, batchServing); | BeanUtils.copyProperties(batchServingCreateDTO, batchServing); | ||||
| String imageUrl = getImageUrl(batchServingCreateDTO.getImageName(), batchServingCreateDTO.getImageTag()); | String imageUrl = getImageUrl(batchServingCreateDTO.getImageName(), batchServingCreateDTO.getImageTag()); | ||||
| batchServing.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + imageUrl); | |||||
| batchServing.setImage(imageUrl); | |||||
| PtModelInfoQueryVO ptModelInfoQueryVO = getPtModelInfo(batchServingCreateDTO.getModelId()); | PtModelInfoQueryVO ptModelInfoQueryVO = getPtModelInfo(batchServingCreateDTO.getModelId()); | ||||
| batchServing.setModelAddress(ptModelInfoQueryVO.getModelAddress()); | batchServing.setModelAddress(ptModelInfoQueryVO.getModelAddress()); | ||||
| if (ptModelInfoQueryVO.getFrameType() > NumberConstant.NUMBER_4) { | if (ptModelInfoQueryVO.getFrameType() > NumberConstant.NUMBER_4) { | ||||
| @@ -421,7 +418,7 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat | |||||
| BeanUtils.copyProperties(batchServingUpdateDTO, batchServing); | BeanUtils.copyProperties(batchServingUpdateDTO, batchServing); | ||||
| String imageUrl = getImageUrl(batchServingUpdateDTO.getImageName(), batchServingUpdateDTO.getImageTag()); | String imageUrl = getImageUrl(batchServingUpdateDTO.getImageName(), batchServingUpdateDTO.getImageTag()); | ||||
| batchServing.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + imageUrl); | |||||
| batchServing.setImage(imageUrl); | |||||
| checkScriptPath(batchServing); | checkScriptPath(batchServing); | ||||
| checkResourceType(batchServing.getFrameType(), batchServing.getResourcesPoolType()); | checkResourceType(batchServing.getFrameType(), batchServing.getResourcesPoolType()); | ||||
| batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); | batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); | ||||
| @@ -18,6 +18,7 @@ | |||||
| package org.dubhe.serving.service.impl; | package org.dubhe.serving.service.impl; | ||||
| import cn.hutool.core.bean.BeanUtil; | import cn.hutool.core.bean.BeanUtil; | ||||
| import com.alibaba.fastjson.JSON; | |||||
| import com.alibaba.fastjson.JSONObject; | import com.alibaba.fastjson.JSONObject; | ||||
| import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; | import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; | ||||
| import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; | import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; | ||||
| @@ -56,11 +57,16 @@ import org.dubhe.biz.permission.annotation.DataPermissionMethod; | |||||
| import org.dubhe.biz.permission.base.BaseService; | import org.dubhe.biz.permission.base.BaseService; | ||||
| import org.dubhe.biz.redis.utils.RedisUtils; | import org.dubhe.biz.redis.utils.RedisUtils; | ||||
| import org.dubhe.k8s.api.MetricsApi; | import org.dubhe.k8s.api.MetricsApi; | ||||
| import org.dubhe.k8s.api.ModelServingApi; | |||||
| import org.dubhe.k8s.api.NodeApi; | |||||
| import org.dubhe.k8s.cache.ResourceCache; | import org.dubhe.k8s.cache.ResourceCache; | ||||
| import org.dubhe.k8s.domain.dto.PodQueryDTO; | import org.dubhe.k8s.domain.dto.PodQueryDTO; | ||||
| import org.dubhe.k8s.domain.resource.BizServicePort; | |||||
| import org.dubhe.k8s.domain.vo.ModelServingVO; | |||||
| import org.dubhe.k8s.domain.vo.PodVO; | import org.dubhe.k8s.domain.vo.PodVO; | ||||
| import org.dubhe.k8s.domain.vo.PtPodsVO; | import org.dubhe.k8s.domain.vo.PtPodsVO; | ||||
| import org.dubhe.k8s.enums.PodPhaseEnum; | import org.dubhe.k8s.enums.PodPhaseEnum; | ||||
| import org.dubhe.k8s.enums.WatcherActionEnum; | |||||
| import org.dubhe.k8s.service.PodService; | import org.dubhe.k8s.service.PodService; | ||||
| import org.dubhe.k8s.utils.K8sNameTool; | import org.dubhe.k8s.utils.K8sNameTool; | ||||
| import org.dubhe.recycle.config.RecycleConfig; | import org.dubhe.recycle.config.RecycleConfig; | ||||
| @@ -75,7 +81,6 @@ import org.dubhe.serving.client.AlgorithmClient; | |||||
| import org.dubhe.serving.client.ImageClient; | import org.dubhe.serving.client.ImageClient; | ||||
| import org.dubhe.serving.client.ModelBranchClient; | import org.dubhe.serving.client.ModelBranchClient; | ||||
| import org.dubhe.serving.client.ModelInfoClient; | import org.dubhe.serving.client.ModelInfoClient; | ||||
| import org.dubhe.serving.config.TrainHarborConfig; | |||||
| import org.dubhe.serving.constant.ServingConstant; | import org.dubhe.serving.constant.ServingConstant; | ||||
| import org.dubhe.serving.dao.ServingInfoMapper; | import org.dubhe.serving.dao.ServingInfoMapper; | ||||
| import org.dubhe.serving.dao.ServingModelConfigMapper; | import org.dubhe.serving.dao.ServingModelConfigMapper; | ||||
| @@ -178,8 +183,6 @@ public class ServingServiceImpl implements ServingService { | |||||
| @Resource | @Resource | ||||
| private ImageClient imageClient; | private ImageClient imageClient; | ||||
| @Resource | @Resource | ||||
| private TrainHarborConfig trainHarborConfig; | |||||
| @Resource | |||||
| private AlgorithmClient algorithmClient; | private AlgorithmClient algorithmClient; | ||||
| @Resource | @Resource | ||||
| private RecycleConfig recycleConfig; | private RecycleConfig recycleConfig; | ||||
| @@ -191,6 +194,10 @@ public class ServingServiceImpl implements ServingService { | |||||
| private ResourceCache resourceCache; | private ResourceCache resourceCache; | ||||
| @Value("Task:Serving:" + "${spring.profiles.active}_serving_id_") | @Value("Task:Serving:" + "${spring.profiles.active}_serving_id_") | ||||
| private String servingIdPrefix; | private String servingIdPrefix; | ||||
| @Resource | |||||
| private ModelServingApi modelServingApi; | |||||
| @Resource | |||||
| private NodeApi nodeApi; | |||||
| private final static List<String> FILE_NAMES; | private final static List<String> FILE_NAMES; | ||||
| @@ -265,7 +272,7 @@ public class ServingServiceImpl implements ServingService { | |||||
| List<ServingInfoQueryVO> queryList = servingInfos.getRecords().stream().map(servingInfo -> { | List<ServingInfoQueryVO> queryList = servingInfos.getRecords().stream().map(servingInfo -> { | ||||
| ServingInfoQueryVO servingInfoQueryVO = new ServingInfoQueryVO(); | ServingInfoQueryVO servingInfoQueryVO = new ServingInfoQueryVO(); | ||||
| BeanUtils.copyProperties(servingInfo, servingInfoQueryVO); | BeanUtils.copyProperties(servingInfo, servingInfoQueryVO); | ||||
| servingInfoQueryVO.setUrl(servingInfo.getUuid() + GATEWAY_URI_POSTFIX); | |||||
| servingInfoQueryVO.setUrl(GATEWAY_URI_POSTFIX+"/"+servingInfo.getUuid()); | |||||
| Map<String, String> statistics = servingLuaScriptService.countCallsByServingInfoId(servingInfo.getId()); | Map<String, String> statistics = servingLuaScriptService.countCallsByServingInfoId(servingInfo.getId()); | ||||
| servingInfoQueryVO.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO)); | servingInfoQueryVO.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO)); | ||||
| servingInfoQueryVO.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO)); | servingInfoQueryVO.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO)); | ||||
| @@ -419,7 +426,7 @@ public class ServingServiceImpl implements ServingService { | |||||
| throw new BusinessException(ServingErrorEnum.CALL_IMAGE_SERVER_FAIL); | throw new BusinessException(ServingErrorEnum.CALL_IMAGE_SERVER_FAIL); | ||||
| } | } | ||||
| servingModelConfig | servingModelConfig | ||||
| .setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + dataResponseBody.getData()); | |||||
| .setImage(dataResponseBody.getData()); | |||||
| // 校验模型文件是否存在 | // 校验模型文件是否存在 | ||||
| String path = k8sNameTool.getAbsolutePath(servingModelConfig.getModelAddress()); | String path = k8sNameTool.getAbsolutePath(servingModelConfig.getModelAddress()); | ||||
| @@ -933,8 +940,8 @@ public class ServingServiceImpl implements ServingService { | |||||
| predictParamVO.setOutputs(outputs); | predictParamVO.setOutputs(outputs); | ||||
| predictParamVO.setOther(other); | predictParamVO.setOther(other); | ||||
| } else if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | } else if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | ||||
| String url = "http://" + servingInfo.getUuid() + GATEWAY_URI_POSTFIX | |||||
| + ServingConstant.INFERENCE_INTERFACE_NAME; | |||||
| String url = "http://" + GATEWAY_URI_POSTFIX | |||||
| +"/"+ servingInfo.getUuid()+ ServingConstant.INFERENCE_INTERFACE_NAME; | |||||
| predictParamVO.setUrl(url); | predictParamVO.setUrl(url); | ||||
| Map<String, String> inputs = new HashMap<>(); | Map<String, String> inputs = new HashMap<>(); | ||||
| inputs.put("files", "File"); | inputs.put("files", "File"); | ||||
| @@ -1079,6 +1086,7 @@ public class ServingServiceImpl implements ServingService { | |||||
| */ | */ | ||||
| @Override | @Override | ||||
| public boolean servingDeploymentCallback(ServingK8sDeploymentCallbackCreateDTO req) { | public boolean servingDeploymentCallback(ServingK8sDeploymentCallbackCreateDTO req) { | ||||
| LogUtil.info(LogEnum.BIZ_K8S,"servingDeploymentCallback:{}", JSON.toJSONString(req)); | |||||
| // 根据namespace和podName找到模型配置 | // 根据namespace和podName找到模型配置 | ||||
| String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.SERVING, req.getResourceName()); | String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.SERVING, req.getResourceName()); | ||||
| if (StringUtils.isBlank(resourceInfo)) { | if (StringUtils.isBlank(resourceInfo)) { | ||||
| @@ -1105,13 +1113,13 @@ public class ServingServiceImpl implements ServingService { | |||||
| return false; | return false; | ||||
| } | } | ||||
| // 增加发送路由信息 | // 增加发送路由信息 | ||||
| if (req.getReadyReplicas() > NumberConstant.NUMBER_0 | |||||
| if (WatcherActionEnum.ADDED.getAction().equals(req.getAction()) | |||||
| && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | ||||
| this.notifyUpdateServingRoute(Collections.singletonList(servingModelConfig.getId()), | this.notifyUpdateServingRoute(Collections.singletonList(servingModelConfig.getId()), | ||||
| Collections.emptyList()); | Collections.emptyList()); | ||||
| } | } | ||||
| // 增加删除路由信息 | // 增加删除路由信息 | ||||
| if (req.getReadyReplicas() == NumberConstant.NUMBER_0 | |||||
| if (WatcherActionEnum.DELETED.getAction().equals(req.getAction()) | |||||
| && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | ||||
| this.notifyUpdateServingRoute(Collections.emptyList(), | this.notifyUpdateServingRoute(Collections.emptyList(), | ||||
| Collections.singletonList(servingModelConfig.getId())); | Collections.singletonList(servingModelConfig.getId())); | ||||
| @@ -1148,8 +1156,29 @@ public class ServingServiceImpl implements ServingService { | |||||
| @Transactional(rollbackFor = Exception.class) | @Transactional(rollbackFor = Exception.class) | ||||
| public boolean updateByCallback(ServingK8sDeploymentCallbackCreateDTO req, ServingModelConfig servingModelConfig, | public boolean updateByCallback(ServingK8sDeploymentCallbackCreateDTO req, ServingModelConfig servingModelConfig, | ||||
| ServingInfo servingInfo) { | ServingInfo servingInfo) { | ||||
| //更新url | |||||
| if (StringUtils.isEmpty(servingModelConfig.getUrl())){ | |||||
| //查询 | |||||
| ModelServingVO modelServingVO = modelServingApi.get(req.getNamespace(),req.getResourceName()); | |||||
| if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { | |||||
| // 获取pod对应的url,并修改模型部署状态 | |||||
| List<BizServicePort> ports = modelServingVO.getBizService().getPorts(); | |||||
| if (CollectionUtils.isNotEmpty(ports)) { | |||||
| //取第一个url | |||||
| String url = ""; | |||||
| if (ports.get(NumberConstant.NUMBER_0).getNodePort() != null) { | |||||
| url = nodeApi.getAvailableNodeIp() + ":" + ports.get(NumberConstant.NUMBER_0).getNodePort(); | |||||
| } | |||||
| servingModelConfig.setUrl(url); | |||||
| } | |||||
| } | |||||
| } | |||||
| // 更新当前模型配置有效节点数 | // 更新当前模型配置有效节点数 | ||||
| servingModelConfig.setReadyReplicas(req.getReadyReplicas()); | servingModelConfig.setReadyReplicas(req.getReadyReplicas()); | ||||
| servingModelConfig.setResourceInfo(null); | |||||
| int result = servingModelConfigMapper.updateById(servingModelConfig); | int result = servingModelConfigMapper.updateById(servingModelConfig); | ||||
| if (result < NumberConstant.NUMBER_1) { | if (result < NumberConstant.NUMBER_1) { | ||||
| return true; | return true; | ||||
| @@ -31,6 +31,7 @@ import org.dubhe.biz.log.enums.LogEnum; | |||||
| import org.dubhe.biz.log.utils.LogUtil; | import org.dubhe.biz.log.utils.LogUtil; | ||||
| import org.dubhe.k8s.api.DistributeTrainApi; | import org.dubhe.k8s.api.DistributeTrainApi; | ||||
| import org.dubhe.k8s.api.ModelServingApi; | import org.dubhe.k8s.api.ModelServingApi; | ||||
| import org.dubhe.k8s.api.NodeApi; | |||||
| import org.dubhe.k8s.api.TrainJobApi; | import org.dubhe.k8s.api.TrainJobApi; | ||||
| import org.dubhe.k8s.domain.PtBaseResult; | import org.dubhe.k8s.domain.PtBaseResult; | ||||
| import org.dubhe.k8s.domain.bo.DistributeTrainBO; | import org.dubhe.k8s.domain.bo.DistributeTrainBO; | ||||
| @@ -38,7 +39,7 @@ import org.dubhe.k8s.domain.bo.ModelServingBO; | |||||
| import org.dubhe.k8s.domain.bo.PtJupyterJobBO; | import org.dubhe.k8s.domain.bo.PtJupyterJobBO; | ||||
| import org.dubhe.k8s.domain.bo.PtMountDirBO; | import org.dubhe.k8s.domain.bo.PtMountDirBO; | ||||
| import org.dubhe.k8s.domain.resource.BizDistributeTrain; | import org.dubhe.k8s.domain.resource.BizDistributeTrain; | ||||
| import org.dubhe.k8s.domain.resource.BizIngressRule; | |||||
| import org.dubhe.k8s.domain.resource.BizServicePort; | |||||
| import org.dubhe.k8s.domain.vo.ModelServingVO; | import org.dubhe.k8s.domain.vo.ModelServingVO; | ||||
| import org.dubhe.k8s.domain.vo.PtJupyterJobVO; | import org.dubhe.k8s.domain.vo.PtJupyterJobVO; | ||||
| import org.dubhe.k8s.utils.K8sNameTool; | import org.dubhe.k8s.utils.K8sNameTool; | ||||
| @@ -88,6 +89,9 @@ public class DeployServingAsyncTask { | |||||
| @Resource | @Resource | ||||
| private ModelServingApi modelServingApi; | private ModelServingApi modelServingApi; | ||||
| @Resource | |||||
| private NodeApi nodeApi; | |||||
| @Resource | @Resource | ||||
| private K8sNameTool k8sNameTool; | private K8sNameTool k8sNameTool; | ||||
| @@ -143,11 +147,14 @@ public class DeployServingAsyncTask { | |||||
| String statusDetailKey = ServingStatusDetailDescUtil.getServingStatusDetailKey(ServingStatusDetailDescUtil.CONTAINER_DEPLOYMENT_EXCEPTION, uniqueName); | String statusDetailKey = ServingStatusDetailDescUtil.getServingStatusDetailKey(ServingStatusDetailDescUtil.CONTAINER_DEPLOYMENT_EXCEPTION, uniqueName); | ||||
| if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { | if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { | ||||
| // 获取pod对应的url,并修改模型部署状态 | // 获取pod对应的url,并修改模型部署状态 | ||||
| List<BizIngressRule> rules = modelServingVO.getBizIngress().getRules(); | |||||
| List<BizServicePort> ports = modelServingVO.getBizService().getPorts(); | |||||
| if (CollectionUtils.isNotEmpty(rules)) { | |||||
| if (CollectionUtils.isNotEmpty(ports)) { | |||||
| //取第一个url | //取第一个url | ||||
| String url = rules.get(NumberConstant.NUMBER_0).getHost(); | |||||
| String url = ""; | |||||
| if (ports.get(NumberConstant.NUMBER_0).getNodePort() != null){ | |||||
| url = nodeApi.getAvailableNodeIp()+":"+ports.get(NumberConstant.NUMBER_0).getNodePort(); | |||||
| } | |||||
| servingModelConfig.setUrl(url); | servingModelConfig.setUrl(url); | ||||
| flag = true; | flag = true; | ||||
| servingInfo.removeStatusDetail(statusDetailKey); | servingInfo.removeStatusDetail(statusDetailKey); | ||||
| @@ -176,7 +183,6 @@ public class DeployServingAsyncTask { | |||||
| servingInfo.putStatusDetail(statusDetailKey, e.getMessage()); | servingInfo.putStatusDetail(statusDetailKey, e.getMessage()); | ||||
| LogUtil.error(LogEnum.SERVING, "User {} create serving failed.The name of serving is {}", user.getUsername(), servingInfo.getName(), e); | LogUtil.error(LogEnum.SERVING, "User {} create serving failed.The name of serving is {}", user.getUsername(), servingInfo.getName(), e); | ||||
| } | } | ||||
| } | } | ||||
| //修改服务状态 | //修改服务状态 | ||||
| @@ -276,7 +282,7 @@ public class DeployServingAsyncTask { | |||||
| return null; | return null; | ||||
| } | } | ||||
| } else { | } else { | ||||
| LogUtil.info(LogEnum.SERVING, "User {} failed to copy the inference script, script {} not exist", user.getUsername(), pyPathList.get(0)); | |||||
| LogUtil.info(LogEnum.SERVING, "User {} failed to copy the inference script, script {} not exist", user.getUsername(), pyPathList); | |||||
| return null; | return null; | ||||
| } | } | ||||
| servingPath = fileStoreApi.formatPath(targetPath + getSourceName(sourcePath)); | servingPath = fileStoreApi.formatPath(targetPath + getSourceName(sourcePath)); | ||||
| @@ -17,13 +17,9 @@ | |||||
| package org.dubhe.serving.utils; | package org.dubhe.serving.utils; | ||||
| import cn.hutool.core.codec.Base64; | |||||
| import cn.hutool.core.io.FileUtil; | |||||
| import io.grpc.ManagedChannel; | import io.grpc.ManagedChannel; | ||||
| import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; | |||||
| import io.grpc.netty.shaded.io.grpc.netty.NegotiationType; | import io.grpc.netty.shaded.io.grpc.netty.NegotiationType; | ||||
| import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; | import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; | ||||
| import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; | |||||
| import org.dubhe.biz.base.constant.NumberConstant; | import org.dubhe.biz.base.constant.NumberConstant; | ||||
| import org.dubhe.biz.base.context.UserContext; | import org.dubhe.biz.base.context.UserContext; | ||||
| import org.dubhe.biz.log.enums.LogEnum; | import org.dubhe.biz.log.enums.LogEnum; | ||||
| @@ -31,12 +27,9 @@ import org.dubhe.biz.log.utils.LogUtil; | |||||
| import org.dubhe.serving.domain.entity.DataInfo; | import org.dubhe.serving.domain.entity.DataInfo; | ||||
| import org.dubhe.serving.proto.Inference; | import org.dubhe.serving.proto.Inference; | ||||
| import org.dubhe.serving.proto.InferenceServiceGrpc; | import org.dubhe.serving.proto.InferenceServiceGrpc; | ||||
| import org.springframework.beans.factory.annotation.Value; | |||||
| import org.springframework.stereotype.Component; | import org.springframework.stereotype.Component; | ||||
| import javax.net.ssl.SSLException; | import javax.net.ssl.SSLException; | ||||
| import java.io.File; | |||||
| import java.io.InputStream; | |||||
| import java.util.List; | import java.util.List; | ||||
| import java.util.concurrent.ConcurrentHashMap; | import java.util.concurrent.ConcurrentHashMap; | ||||
| import java.util.concurrent.TimeUnit; | import java.util.concurrent.TimeUnit; | ||||
| @@ -52,51 +45,20 @@ public class GrpcClient { | |||||
| * <服务id, grpc客户端channel> | * <服务id, grpc客户端channel> | ||||
| */ | */ | ||||
| public static ConcurrentHashMap<Long, ManagedChannel> channelMap = new ConcurrentHashMap<>(); | public static ConcurrentHashMap<Long, ManagedChannel> channelMap = new ConcurrentHashMap<>(); | ||||
| /** | |||||
| * 项目根路径 | |||||
| */ | |||||
| private final static String USER_DIR_SYSTEM_PROPERTY = "user.dir"; | |||||
| /** | |||||
| * k8s crt证书文件名 | |||||
| */ | |||||
| private final static String TLS_CRT = "server.crt"; | |||||
| /** | |||||
| * k8s grpc端口 | |||||
| */ | |||||
| @Value("${k8s.https-port:31365}") | |||||
| private String grpcPort; | |||||
| /** | |||||
| * k8s 模型部署tls证书 crt | |||||
| */ | |||||
| @Value("${k8s.serving.tls-crt}") | |||||
| private String tlsCrt; | |||||
| /** | /** | ||||
| * 创建带tls认证的grpc通道 | |||||
| * 创建grpc通道 | |||||
| * | * | ||||
| * @param url tls证书地址 | |||||
| * @param url | |||||
| * @return ManagedChannel 通道 | * @return ManagedChannel 通道 | ||||
| * @throws SSLException | * @throws SSLException | ||||
| */ | */ | ||||
| public ManagedChannel createTlsChannel(String url) throws Exception { | |||||
| String path = getTlsCrt(); | |||||
| SslContext sslContext = GrpcSslContexts.forClient().trustManager(new File(path)).build(); | |||||
| ManagedChannel channel = NettyChannelBuilder.forAddress(url, Integer.parseInt(grpcPort)).maxInboundMessageSize(NumberConstant.MAX_MESSAGE_LENGTH).negotiationType(NegotiationType.TLS).sslContext(sslContext).build(); | |||||
| public ManagedChannel createChannel(String url) { | |||||
| String[] host = url.split(":"); | |||||
| ManagedChannel channel = NettyChannelBuilder.forAddress(host[0], Integer.parseInt(host[1])).maxInboundMessageSize(NumberConstant.MAX_MESSAGE_LENGTH).negotiationType(NegotiationType.PLAINTEXT).build(); | |||||
| return channel; | return channel; | ||||
| } | } | ||||
| /** | |||||
| * 获取crt证书路径 | |||||
| */ | |||||
| public String getTlsCrt() { | |||||
| String path = System.getProperty(USER_DIR_SYSTEM_PROPERTY) + File.separator.concat(TLS_CRT); | |||||
| if(!FileUtil.exist(path)) { | |||||
| FileUtil.writeUtf8String(Base64.decodeStr(tlsCrt), path); | |||||
| } | |||||
| return path; | |||||
| } | |||||
| /** | /** | ||||
| * 关闭通道 | * 关闭通道 | ||||
| * | * | ||||
| @@ -148,7 +110,7 @@ public class GrpcClient { | |||||
| } | } | ||||
| ManagedChannel channel = null; | ManagedChannel channel = null; | ||||
| try { | try { | ||||
| channel = this.createTlsChannel(url); | |||||
| channel = this.createChannel(url); | |||||
| channelMap.put(servingId, channel); | channelMap.put(servingId, channel); | ||||
| } catch (Exception e) { | } catch (Exception e) { | ||||
| LogUtil.error(LogEnum.SERVING, "An Exception occurred when user {} creating the grpc channel, service id:{}", user.getUsername(), servingId, e); | LogUtil.error(LogEnum.SERVING, "An Exception occurred when user {} creating the grpc channel, service id:{}", user.getUsername(), servingId, e); | ||||
| @@ -169,7 +131,7 @@ public class GrpcClient { | |||||
| } | } | ||||
| ManagedChannel channel = null; | ManagedChannel channel = null; | ||||
| try { | try { | ||||
| channel = this.createTlsChannel(url); | |||||
| channel = this.createChannel(url); | |||||
| channelMap.put(servingId, channel); | channelMap.put(servingId, channel); | ||||
| } catch (Exception e) { | } catch (Exception e) { | ||||
| LogUtil.error(LogEnum.SERVING, "An Exception occurred when getting grpc channel, service id:{}", servingId, e); | LogUtil.error(LogEnum.SERVING, "An Exception occurred when getting grpc channel, service id:{}", servingId, e); | ||||
| @@ -71,13 +71,6 @@ public class PtImage extends BaseEntity { | |||||
| @TableField(value = "image_resource") | @TableField(value = "image_resource") | ||||
| private Integer imageResource; | private Integer imageResource; | ||||
| /** | |||||
| * 镜像状态 | |||||
| */ | |||||
| @TableField(value = "image_status") | |||||
| private Integer imageStatus; | |||||
| /** | /** | ||||
| * 资源拥有者ID | * 资源拥有者ID | ||||
| */ | */ | ||||
| @@ -182,6 +182,6 @@ public class Terminal extends BaseEntity { | |||||
| return null; | return null; | ||||
| } | } | ||||
| String[] strings = imageUrl.split(SymbolConstant.SLASH); | String[] strings = imageUrl.split(SymbolConstant.SLASH); | ||||
| return strings.length > 0 ? strings[MagicNumConstant.ZERO] : null; | |||||
| return strings.length > 0 ? strings[MagicNumConstant.ONE] : null; | |||||
| } | } | ||||
| } | } | ||||
| @@ -16,7 +16,6 @@ | |||||
| */ | */ | ||||
| package org.dubhe.terminal.service.impl; | package org.dubhe.terminal.service.impl; | ||||
| import java.sql.Timestamp; | |||||
| import cn.hutool.core.util.RandomUtil; | import cn.hutool.core.util.RandomUtil; | ||||
| import cn.hutool.core.util.StrUtil; | import cn.hutool.core.util.StrUtil; | ||||
| @@ -38,6 +37,7 @@ import org.dubhe.biz.log.utils.LogUtil; | |||||
| import org.dubhe.biz.permission.base.BaseService; | import org.dubhe.biz.permission.base.BaseService; | ||||
| import org.dubhe.biz.redis.utils.RedisUtils; | import org.dubhe.biz.redis.utils.RedisUtils; | ||||
| import org.dubhe.docker.api.DockerApi; | import org.dubhe.docker.api.DockerApi; | ||||
| import org.dubhe.docker.callback.TerminalPushImageResultCallback; | |||||
| import org.dubhe.docker.config.DockerClientFactory; | import org.dubhe.docker.config.DockerClientFactory; | ||||
| import org.dubhe.docker.enums.DockerOperationEnum; | import org.dubhe.docker.enums.DockerOperationEnum; | ||||
| import org.dubhe.docker.utils.DockerCallbackTool; | import org.dubhe.docker.utils.DockerCallbackTool; | ||||
| @@ -53,7 +53,6 @@ import org.dubhe.k8s.domain.vo.TerminalResourceVO; | |||||
| import org.dubhe.k8s.enums.BusinessLabelServiceNameEnum; | import org.dubhe.k8s.enums.BusinessLabelServiceNameEnum; | ||||
| import org.dubhe.k8s.enums.PodPhaseEnum; | import org.dubhe.k8s.enums.PodPhaseEnum; | ||||
| import org.dubhe.k8s.utils.K8sNameTool; | import org.dubhe.k8s.utils.K8sNameTool; | ||||
| import org.dubhe.docker.callback.TerminalPushImageResultCallback; | |||||
| import org.dubhe.terminal.config.TerminalConfig; | import org.dubhe.terminal.config.TerminalConfig; | ||||
| import org.dubhe.terminal.constant.TerminalConstant; | import org.dubhe.terminal.constant.TerminalConstant; | ||||
| import org.dubhe.terminal.dao.PtImageMapper; | import org.dubhe.terminal.dao.PtImageMapper; | ||||
| @@ -77,6 +76,7 @@ import org.springframework.stereotype.Service; | |||||
| import org.springframework.transaction.annotation.Transactional; | import org.springframework.transaction.annotation.Transactional; | ||||
| import org.springframework.util.CollectionUtils; | import org.springframework.util.CollectionUtils; | ||||
| import java.sql.Timestamp; | |||||
| import java.util.ArrayList; | import java.util.ArrayList; | ||||
| import java.util.Date; | import java.util.Date; | ||||
| import java.util.List; | import java.util.List; | ||||
| @@ -174,38 +174,43 @@ public class TerminalServiceImpl implements TerminalService { | |||||
| @Override | @Override | ||||
| public boolean preserve(TerminalPreserveDTO terminalPreserveDTO) { | public boolean preserve(TerminalPreserveDTO terminalPreserveDTO) { | ||||
| Terminal terminal = terminalMapper.selectById(terminalPreserveDTO.getId()); | Terminal terminal = terminalMapper.selectById(terminalPreserveDTO.getId()); | ||||
| if (terminal == null){ | |||||
| LogUtil.error(LogEnum.TERMINAL,"preserve terminal 数据不存在 terminalPreserveDTO: {}",terminalPreserveDTO); | |||||
| if (checkImageExists(terminal.getImageProject(), terminalPreserveDTO.getImageName(), terminalPreserveDTO.getImageTag())) { | |||||
| throw new BusinessException("请确认镜像版本号是否重复!"); | |||||
| } | |||||
| if (terminal == null) { | |||||
| LogUtil.error(LogEnum.TERMINAL, "preserve terminal 数据不存在 terminalPreserveDTO: {}", terminalPreserveDTO); | |||||
| throw new BusinessException("数据不存在"); | throw new BusinessException("数据不存在"); | ||||
| } | } | ||||
| LambdaQueryWrapper<TerminalInfo> wrapper = new LambdaQueryWrapper<>(); | LambdaQueryWrapper<TerminalInfo> wrapper = new LambdaQueryWrapper<>(); | ||||
| wrapper.eq(TerminalInfo::getTerminalId, terminalPreserveDTO.getId()); | wrapper.eq(TerminalInfo::getTerminalId, terminalPreserveDTO.getId()); | ||||
| List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectList(wrapper); | List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectList(wrapper); | ||||
| if (CollectionUtils.isEmpty(terminalInfoList)){ | |||||
| LogUtil.error(LogEnum.TERMINAL,"preserve terminalInfoList 数据不存在 terminalPreserveDTO: {}",terminalPreserveDTO); | |||||
| if (CollectionUtils.isEmpty(terminalInfoList)) { | |||||
| LogUtil.error(LogEnum.TERMINAL, "preserve terminalInfoList 数据不存在 terminalPreserveDTO: {}", terminalPreserveDTO); | |||||
| throw new BusinessException("数据不存在"); | throw new BusinessException("数据不存在"); | ||||
| } | } | ||||
| TerminalInfo masterTerminalInfo = null; | TerminalInfo masterTerminalInfo = null; | ||||
| for (TerminalInfo terminalInfo : terminalInfoList) { | for (TerminalInfo terminalInfo : terminalInfoList) { | ||||
| if (terminalInfo.isMasterFlag()){ | |||||
| if (terminalInfo.isMasterFlag()) { | |||||
| masterTerminalInfo = terminalInfo; | masterTerminalInfo = terminalInfo; | ||||
| } | } | ||||
| } | } | ||||
| if (masterTerminalInfo == null){ | |||||
| LogUtil.error(LogEnum.TERMINAL,"master 节点不存在 terminalPreserveDTO:{}",terminalPreserveDTO); | |||||
| if (masterTerminalInfo == null) { | |||||
| LogUtil.error(LogEnum.TERMINAL, "master 节点不存在 terminalPreserveDTO:{}", terminalPreserveDTO); | |||||
| throw new BusinessException("master 节点不存在"); | throw new BusinessException("master 节点不存在"); | ||||
| } | } | ||||
| BizPod pod = podApi.getWithResourceName(k8sNameTool.getNamespace(terminal.getCreateUserId()),masterTerminalInfo.getK8sResourceName()); | |||||
| if (pod == null){ | |||||
| LogUtil.error(LogEnum.TERMINAL,"master 容器不存在 terminalPreserveDTO:{}",terminalPreserveDTO); | |||||
| BizPod pod = podApi.getWithResourceName(k8sNameTool.getNamespace(terminal.getCreateUserId()), masterTerminalInfo.getK8sResourceName()); | |||||
| if (pod == null) { | |||||
| LogUtil.error(LogEnum.TERMINAL, "master 容器不存在 terminalPreserveDTO:{}", terminalPreserveDTO); | |||||
| throw new BusinessException("master 容器不存在"); | throw new BusinessException("master 容器不存在"); | ||||
| } | } | ||||
| if (!PodPhaseEnum.RUNNING.getPhase().equals(pod.getPhase()) || pod.getPodIp() == null || StringUtils.isNotEmpty(pod.getContainerStateMessages())){ | |||||
| LogUtil.error(LogEnum.TERMINAL,"master 容器未运行 terminalPreserveDTO:{}",terminalPreserveDTO); | |||||
| if (!PodPhaseEnum.RUNNING.getPhase().equals(pod.getPhase()) || pod.getPodIp() == null || StringUtils.isNotEmpty(pod.getContainerStateMessages())) { | |||||
| LogUtil.error(LogEnum.TERMINAL, "master 容器未运行 terminalPreserveDTO:{}", terminalPreserveDTO); | |||||
| throw new BusinessException("master 容器未运行"); | throw new BusinessException("master 容器未运行"); | ||||
| } | } | ||||
| String containerID = pod.getContainerId(); | String containerID = pod.getContainerId(); | ||||
| @@ -217,14 +222,13 @@ public class TerminalServiceImpl implements TerminalService { | |||||
| terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"commit 镜像..."); | terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"commit 镜像..."); | ||||
| terminalMapper.updateById(terminal); | terminalMapper.updateById(terminal); | ||||
| DockerClient dockerClient = dockerClientFactory.getDockerClient(pod.getHostIP()); | DockerClient dockerClient = dockerClientFactory.getDockerClient(pod.getHostIP()); | ||||
| String newImagePath = terminal.getImageProject()+SymbolConstant.SLASH+terminal.getCreateUserId()+SymbolConstant.SLASH+terminalPreserveDTO.getImageName(); | |||||
| String newImageRepository = terminalConfig.getHarborAddress()+SymbolConstant.SLASH+newImagePath; | |||||
| String newImagePath = terminalConfig.getHarborAddress() + StrUtil.SLASH + terminal.getImageProject() + SymbolConstant.SLASH + terminalPreserveDTO.getImageName(); | |||||
| try { | try { | ||||
| dockerApi.commit(dockerClient,containerID,newImageRepository,terminalPreserveDTO.getImageTag()); | |||||
| dockerApi.commit(dockerClient,containerID,newImagePath,terminalPreserveDTO.getImageTag()); | |||||
| terminal.setStatus(null); | terminal.setStatus(null); | ||||
| terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"push 镜像..."); | terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"push 镜像..."); | ||||
| terminalMapper.updateById(terminal); | terminalMapper.updateById(terminal); | ||||
| boolean pushResult = dockerApi.push(dockerClient,newImageRepository+SymbolConstant.COLON+terminalPreserveDTO.getImageTag(),new TerminalPushImageResultCallback(dockerCallbackTool.getCallbackUrl(SymbolConstant.LOCAL_HOST,terminalConfig.getServerPort(), DockerOperationEnum.PUSH.getType()),terminal.getId(),dockerClient,terminal.getCreateUserId())); | |||||
| boolean pushResult = dockerApi.push(dockerClient,newImagePath+SymbolConstant.COLON+terminalPreserveDTO.getImageTag(),new TerminalPushImageResultCallback(dockerCallbackTool.getCallbackUrl(SymbolConstant.LOCAL_HOST,terminalConfig.getServerPort(), DockerOperationEnum.PUSH.getType()),terminal.getId(),dockerClient,terminal.getCreateUserId())); | |||||
| if (!pushResult){ | if (!pushResult){ | ||||
| LogUtil.error(LogEnum.TERMINAL,"master 推送镜像错误 terminalPreserveDTO:{}",terminalPreserveDTO); | LogUtil.error(LogEnum.TERMINAL,"master 推送镜像错误 terminalPreserveDTO:{}",terminalPreserveDTO); | ||||
| throw new BusinessException("推送镜像错误:"); | throw new BusinessException("推送镜像错误:"); | ||||
| @@ -234,7 +238,7 @@ public class TerminalServiceImpl implements TerminalService { | |||||
| throw new BusinessException("保存容器错误:"+e.getMessage()); | throw new BusinessException("保存容器错误:"+e.getMessage()); | ||||
| } | } | ||||
| terminal.setImageUrl(newImagePath+SymbolConstant.COLON+terminalPreserveDTO.getImageTag()); | |||||
| terminal.setImageUrl(newImagePath + SymbolConstant.COLON + terminalPreserveDTO.getImageTag()); | |||||
| terminal.setImageName(terminalPreserveDTO.getImageName()); | terminal.setImageName(terminalPreserveDTO.getImageName()); | ||||
| terminal.setDescription(terminalPreserveDTO.getImageRemark()); | terminal.setDescription(terminalPreserveDTO.getImageRemark()); | ||||
| terminal.setImageTag(terminalPreserveDTO.getImageTag()); | terminal.setImageTag(terminalPreserveDTO.getImageTag()); | ||||
| @@ -244,13 +248,24 @@ public class TerminalServiceImpl implements TerminalService { | |||||
| return terminalMapper.updateById(terminal) > 0; | return terminalMapper.updateById(terminal) > 0; | ||||
| } | } | ||||
| private boolean checkImageExists(String imageProject, String imageName, String imageTag) { | |||||
| // 拼接镜像的完整路径 | |||||
| String imageUrl = terminalConfig.getHarborAddress() + StrUtil.SLASH + imageProject + SymbolConstant.SLASH + imageName + StrUtil.COLON + imageTag; | |||||
| // 校验该待保存镜像是否存在 | |||||
| LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | |||||
| queryWrapper.eq(PtImage::getImageUrl, imageUrl); | |||||
| queryWrapper.ne(PtImage::getCreateUserId, userContextService.getCurUserId()); | |||||
| return ptImageMapper.selectCount(queryWrapper) > 0; | |||||
| } | |||||
| @Override | @Override | ||||
| @Transactional(rollbackFor = Exception.class) | @Transactional(rollbackFor = Exception.class) | ||||
| public boolean delete(TerminalDTO terminalDTO) { | public boolean delete(TerminalDTO terminalDTO) { | ||||
| try{ | |||||
| try { | |||||
| Terminal terminal = terminalMapper.selectById(terminalDTO.getId()); | Terminal terminal = terminalMapper.selectById(terminalDTO.getId()); | ||||
| if (terminal == null){ | |||||
| LogUtil.error(LogEnum.TERMINAL,"delete 数据不存在 terminalDTO: {}",terminalDTO); | |||||
| if (terminal == null) { | |||||
| LogUtil.error(LogEnum.TERMINAL, "delete 数据不存在 terminalDTO: {}", terminalDTO); | |||||
| throw new BusinessException("数据不存在"); | throw new BusinessException("数据不存在"); | ||||
| } | } | ||||
| terminal.setDeleted(true); | terminal.setDeleted(true); | ||||
| @@ -259,7 +274,7 @@ public class TerminalServiceImpl implements TerminalService { | |||||
| terminalMapper.deleteById(terminal); | terminalMapper.deleteById(terminal); | ||||
| // 删除任务缓存 | // 删除任务缓存 | ||||
| String taskIdentify = (String) redisUtils.get(terminalIdPrefix + String.valueOf(terminal.getId())); | String taskIdentify = (String) redisUtils.get(terminalIdPrefix + String.valueOf(terminal.getId())); | ||||
| if (StringUtils.isNotEmpty(taskIdentify)){ | |||||
| if (StringUtils.isNotEmpty(taskIdentify)) { | |||||
| redisUtils.del(taskIdentify, terminalIdPrefix + String.valueOf(terminal.getId())); | redisUtils.del(taskIdentify, terminalIdPrefix + String.valueOf(terminal.getId())); | ||||
| } | } | ||||
| String namespace = k8sNameTool.getNamespace(terminal.getCreateUserId()); | String namespace = k8sNameTool.getNamespace(terminal.getCreateUserId()); | ||||
| @@ -477,6 +492,13 @@ public class TerminalServiceImpl implements TerminalService { | |||||
| return; | return; | ||||
| } | } | ||||
| Terminal terminal = terminalMapper.selectById(terminalId); | Terminal terminal = terminalMapper.selectById(terminalId); | ||||
| List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectByTerminalId(terminalId); | |||||
| TerminalInfo masterTerminalInfo = null; | |||||
| for (TerminalInfo info : terminalInfoList){ | |||||
| if (info.isMasterFlag()){ | |||||
| masterTerminalInfo = info; | |||||
| } | |||||
| } | |||||
| if (terminal == null){ | if (terminal == null){ | ||||
| LogUtil.error(LogEnum.TERMINAL,"pushImageComplete no terminal found id:{}",terminalId); | LogUtil.error(LogEnum.TERMINAL,"pushImageComplete no terminal found id:{}",terminalId); | ||||
| return; | return; | ||||
| @@ -499,10 +521,11 @@ public class TerminalServiceImpl implements TerminalService { | |||||
| ptImage.setImageTag(terminal.getImageTag()); | ptImage.setImageTag(terminal.getImageTag()); | ||||
| ptImage.setRemark(terminal.getDescription()); | ptImage.setRemark(terminal.getDescription()); | ||||
| ptImage.setImageResource(MagicNumConstant.ZERO); | ptImage.setImageResource(MagicNumConstant.ZERO); | ||||
| ptImage.setImageStatus(MagicNumConstant.ONE); | |||||
| ptImage.setDeleted(false); | ptImage.setDeleted(false); | ||||
| ptImage.setUpdateUserId(userId); | ptImage.setUpdateUserId(userId); | ||||
| ptImage.setUpdateTime(new Timestamp(new java.util.Date().getTime())); | ptImage.setUpdateTime(new Timestamp(new java.util.Date().getTime())); | ||||
| ptImage.setSshUser(masterTerminalInfo.getSshUser()); | |||||
| ptImage.setSshPwd(masterTerminalInfo.getSshPassword()); | |||||
| if (ptImage.getId() != null){ | if (ptImage.getId() != null){ | ||||
| ptImageMapper.updateById(ptImage); | ptImageMapper.updateById(ptImage); | ||||
| @@ -563,7 +586,7 @@ public class TerminalServiceImpl implements TerminalService { | |||||
| terminalBO.setGpuNum(terminalInfo.getGpuNum()); | terminalBO.setGpuNum(terminalInfo.getGpuNum()); | ||||
| terminalBO.setMemNum(terminalInfo.getMemNum()); | terminalBO.setMemNum(terminalInfo.getMemNum()); | ||||
| terminalBO.setCpuNum(terminalInfo.getCpuNum()); | terminalBO.setCpuNum(terminalInfo.getCpuNum()); | ||||
| terminalBO.setImage(terminalConfig.getHarborAddress()+SymbolConstant.SLASH+terminalCreateDTO.getImageUrl()); | |||||
| terminalBO.setImage(terminalCreateDTO.getImageUrl()); | |||||
| terminalBO.setFsMounts(Maps.newHashMap()); | terminalBO.setFsMounts(Maps.newHashMap()); | ||||
| terminalBO.setBusinessLabel(BusinessLabelServiceNameEnum.TERMINAL.getBusinessLabel()); | terminalBO.setBusinessLabel(BusinessLabelServiceNameEnum.TERMINAL.getBusinessLabel()); | ||||
| terminalBO.setTaskIdentifyLabel(taskIdentifyLabel); | terminalBO.setTaskIdentifyLabel(taskIdentifyLabel); | ||||
| @@ -26,7 +26,6 @@ import org.springframework.stereotype.Component; | |||||
| * @description 镜像远程服务调用熔断类 | * @description 镜像远程服务调用熔断类 | ||||
| * @date 2020-12-15 | * @date 2020-12-15 | ||||
| */ | */ | ||||
| @Component | |||||
| public class ImageClientFallback implements ImageClient { | public class ImageClientFallback implements ImageClient { | ||||
| @Override | @Override | ||||
| @@ -1,39 +0,0 @@ | |||||
| /** | |||||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||||
| * | |||||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||||
| * you may not use this file except in compliance with the License. | |||||
| * You may obtain a copy of the License at | |||||
| * | |||||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||||
| * | |||||
| * Unless required by applicable law or agreed to in writing, software | |||||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
| * See the License for the specific language governing permissions and | |||||
| * limitations under the License. | |||||
| * ============================================================= | |||||
| */ | |||||
| package org.dubhe.train.config; | |||||
| import lombok.Data; | |||||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||||
| import org.springframework.stereotype.Component; | |||||
| /** | |||||
| * @description harbor相关配置 | |||||
| * @date 2020-07-17 | |||||
| */ | |||||
| @Data | |||||
| @Component | |||||
| @ConfigurationProperties(prefix = "harbor") | |||||
| public class TrainHarborConfig { | |||||
| private String address; | |||||
| private String username; | |||||
| private String password; | |||||
| private String modelName; | |||||
| } | |||||
| @@ -96,7 +96,6 @@ import org.dubhe.train.client.ModelBranchClient; | |||||
| import org.dubhe.train.client.ModelInfoClient; | import org.dubhe.train.client.ModelInfoClient; | ||||
| import org.dubhe.train.client.NoteBookClient; | import org.dubhe.train.client.NoteBookClient; | ||||
| import org.dubhe.train.client.ResourceSpecsClient; | import org.dubhe.train.client.ResourceSpecsClient; | ||||
| import org.dubhe.train.config.TrainHarborConfig; | |||||
| import org.dubhe.train.config.TrainJobConfig; | import org.dubhe.train.config.TrainJobConfig; | ||||
| import org.dubhe.train.constant.ATlasTrainConstant; | import org.dubhe.train.constant.ATlasTrainConstant; | ||||
| import org.dubhe.train.constant.TrainConstant; | import org.dubhe.train.constant.TrainConstant; | ||||
| @@ -202,9 +201,6 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| @Autowired | @Autowired | ||||
| private TrainJobConfig trainJobConfig; | private TrainJobConfig trainJobConfig; | ||||
| @Autowired | |||||
| private TrainHarborConfig trainHarborConfig; | |||||
| @Autowired | @Autowired | ||||
| private K8sNameTool k8sNameTool; | private K8sNameTool k8sNameTool; | ||||
| @@ -484,7 +480,12 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| public void buildImageAndTagInfo(PtJobParam ptJobParam, PtTrainJobDetailVO ptTrainJobDetailVO) { | public void buildImageAndTagInfo(PtJobParam ptJobParam, PtTrainJobDetailVO ptTrainJobDetailVO) { | ||||
| //image信息拼装 | //image信息拼装 | ||||
| if (StringUtils.isNotBlank(ptJobParam.getImageName())) { | if (StringUtils.isNotBlank(ptJobParam.getImageName())) { | ||||
| String imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); | |||||
| String imageNameSuffix; | |||||
| if (ptJobParam.getImageName().contains(StrUtil.SLASH)) { | |||||
| imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); | |||||
| } else { | |||||
| imageNameSuffix = ptJobParam.getImageName(); | |||||
| } | |||||
| String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); | String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); | ||||
| ptTrainJobDetailVO.setImageName(imageNameSuffixArray[0]); | ptTrainJobDetailVO.setImageName(imageNameSuffixArray[0]); | ||||
| ptTrainJobDetailVO.setImageTag(imageNameSuffixArray[1]); | ptTrainJobDetailVO.setImageTag(imageNameSuffixArray[1]); | ||||
| @@ -563,11 +564,10 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| .setMemNum(ptTrainJobCreateDTO.getMemNum()) | .setMemNum(ptTrainJobCreateDTO.getMemNum()) | ||||
| .setWorkspaceRequest(ptTrainJobCreateDTO.getWorkspaceRequest()) | .setWorkspaceRequest(ptTrainJobCreateDTO.getWorkspaceRequest()) | ||||
| .setTaskIdentify(taskIdentify); | .setTaskIdentify(taskIdentify); | ||||
| //例如: 将harbor.dubhe.ai/notebook/notebook:v1 去掉 harbor地址 | |||||
| String userImageName = trimHarborAddress(ptImageAndAlgorithmVO.getImageName()); | |||||
| String imageNameAndTag = ptTrainJobCreateDTO.getImageName() + StrUtil.COLON + ptTrainJobCreateDTO.getImageTag(); | |||||
| //结果集处理 | //结果集处理 | ||||
| PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, userContextService.getCurUser(), userImageName, trainKey, baseTrainJobDTO); | |||||
| PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, userContextService.getCurUser(), imageNameAndTag, trainKey, baseTrainJobDTO); | |||||
| //添加任务缓存 | //添加任务缓存 | ||||
| resourceCache.addTaskCache(taskIdentify, ptTrainJob.getTrainId(), ptTrainJobCreateDTO.getTrainName(), trainIdPrefix); | resourceCache.addTaskCache(taskIdentify, ptTrainJob.getTrainId(), ptTrainJobCreateDTO.getTrainName(), trainIdPrefix); | ||||
| // 提交job | // 提交job | ||||
| @@ -575,16 +575,6 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| return Collections.singletonList(ptTrainJob.getTrainId()); | return Collections.singletonList(ptTrainJob.getTrainId()); | ||||
| } | } | ||||
| /** | |||||
| * 去掉harbor地址 | |||||
| * | |||||
| * @param imageName | |||||
| * @return | |||||
| */ | |||||
| private String trimHarborAddress(String imageName) { | |||||
| return StringUtils.isBlank(imageName) ? StringUtils.EMPTY : imageName.replace(trainHarborConfig.getAddress() + StrUtil.SLASH, StringUtils.EMPTY); | |||||
| } | |||||
| /** | /** | ||||
| * 构建镜像和算法目录VO 考虑到无算法创建 | * 构建镜像和算法目录VO 考虑到无算法创建 | ||||
| * | * | ||||
| @@ -611,9 +601,8 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| String imageUrl = imageUtil.getImageUrl(ptTrainJobBaseDTO, userContextService.getCurUser()); | String imageUrl = imageUtil.getImageUrl(ptTrainJobBaseDTO, userContextService.getCurUser()); | ||||
| String userImageName = imageUrl.split(StrUtil.SLASH)[0] + StrUtil.SLASH + ptTrainJobBaseDTO.getImageName() + StrUtil.COLON + ptTrainJobBaseDTO.getImageTag(); | String userImageName = imageUrl.split(StrUtil.SLASH)[0] + StrUtil.SLASH + ptTrainJobBaseDTO.getImageName() + StrUtil.COLON + ptTrainJobBaseDTO.getImageTag(); | ||||
| ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobBaseDTO.getAlgorithmId()); | ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobBaseDTO.getAlgorithmId()); | ||||
| String imageName = trainHarborConfig.getAddress() + StrUtil.SLASH + userImageName; | |||||
| ptImageAndAlgorithmVO.setImageName(imageName); | |||||
| ptImageAndAlgorithmVO.setImageUrl(trainHarborConfig.getAddress() + StrUtil.SLASH + imageUrl); | |||||
| ptImageAndAlgorithmVO.setImageName(ptTrainJobBaseDTO.getImageName()); | |||||
| ptImageAndAlgorithmVO.setImageUrl(imageUrl); | |||||
| } | } | ||||
| ptImageAndAlgorithmVO.setRunCommand(ptTrainJobBaseDTO.getRunCommand()); | ptImageAndAlgorithmVO.setRunCommand(ptTrainJobBaseDTO.getRunCommand()); | ||||
| @@ -676,13 +665,13 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| * | * | ||||
| * @param ptTrainJobCreateDTO 创建训练任务DTO | * @param ptTrainJobCreateDTO 创建训练任务DTO | ||||
| * @param currentUser 用户 | * @param currentUser 用户 | ||||
| * @param imageName 镜像名称 | |||||
| * @param imageNameAndTag 镜像名称和tag | |||||
| * @param trainKey 训练key | * @param trainKey 训练key | ||||
| * @param baseTrainJobDTO 基础训练参数 | * @param baseTrainJobDTO 基础训练参数 | ||||
| * @return PtTrain 训练 | * @return PtTrain 训练 | ||||
| */ | */ | ||||
| private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserContext currentUser, | private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserContext currentUser, | ||||
| String imageName, String trainKey, BaseTrainJobDTO baseTrainJobDTO) { | |||||
| String imageNameAndTag, String trainKey, BaseTrainJobDTO baseTrainJobDTO) { | |||||
| // 添加train表 | // 添加train表 | ||||
| PtTrain ptTrain = new PtTrain(); | PtTrain ptTrain = new PtTrain(); | ||||
| ptTrain.setTrainName(ptTrainJobCreateDTO.getTrainName()) | ptTrain.setTrainName(ptTrainJobCreateDTO.getTrainName()) | ||||
| @@ -732,7 +721,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| ptJobParam.setTrainJobId(ptTrainJob.getId()) | ptJobParam.setTrainJobId(ptTrainJob.getId()) | ||||
| .setAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId()) | .setAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId()) | ||||
| .setRunCommand(ptTrainJobCreateDTO.getRunCommand()) | .setRunCommand(ptTrainJobCreateDTO.getRunCommand()) | ||||
| .setImageName(imageName) | |||||
| .setImageName(imageNameAndTag) | |||||
| .setRunParams(ptTrainJobCreateDTO.getRunParams()) | .setRunParams(ptTrainJobCreateDTO.getRunParams()) | ||||
| .setRunParamsNameMap(ptTrainJobCreateDTO.getRunParamsNameMap()) | .setRunParamsNameMap(ptTrainJobCreateDTO.getRunParamsNameMap()) | ||||
| .setCreateUserId(currentUser.getId()); | .setCreateUserId(currentUser.getId()); | ||||
| @@ -1010,9 +999,8 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| .setWorkspaceRequest(ptTrainJobUpdateDTO.getWorkspaceRequest()) | .setWorkspaceRequest(ptTrainJobUpdateDTO.getWorkspaceRequest()) | ||||
| .setTaskIdentify(taskIdentify); | .setTaskIdentify(taskIdentify); | ||||
| String userImageName = trimHarborAddress(ptImageAndAlgorithmVO.getImageName()); | |||||
| //结果集处理 | //结果集处理 | ||||
| PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, userContextService.getCurUser(), existPtTrainJob, userImageName, ptTrain, baseTrainJobDTO); | |||||
| PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, userContextService.getCurUser(), existPtTrainJob, ptTrainJobUpdateDTO.getImageName() + StrUtil.COLON + ptTrainJobUpdateDTO.getImageTag(), ptTrain, baseTrainJobDTO); | |||||
| //提交job | //提交job | ||||
| asyncManager.execute(baseTrainJobDTO, ptTrain.getCreateUserId(), ptImageAndAlgorithmVO, ptTrainJob); | asyncManager.execute(baseTrainJobDTO, ptTrain.getCreateUserId(), ptImageAndAlgorithmVO, ptTrainJob); | ||||
| @@ -1025,13 +1013,13 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| * @param ptTrainJobUpdateDTO 更新训练任务DTO | * @param ptTrainJobUpdateDTO 更新训练任务DTO | ||||
| * @param currentUser 当前用户 | * @param currentUser 当前用户 | ||||
| * @param existPtTrainJob 存在的训练任务 | * @param existPtTrainJob 存在的训练任务 | ||||
| * @param imageName 镜像名称 | |||||
| * @param imageNameAndTag 镜像名称和tag | |||||
| * @param ptTrain 训练 | * @param ptTrain 训练 | ||||
| * @param baseTrainJobDTO 基本训练信息 | * @param baseTrainJobDTO 基本训练信息 | ||||
| * @return PtTrainJob 训练任务 | * @return PtTrainJob 训练任务 | ||||
| */ | */ | ||||
| private PtTrainJob updateTrainJobTableData(PtTrainJobUpdateDTO ptTrainJobUpdateDTO, UserContext | private PtTrainJob updateTrainJobTableData(PtTrainJobUpdateDTO ptTrainJobUpdateDTO, UserContext | ||||
| currentUser, PtTrainJob existPtTrainJob, String imageName, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) { | |||||
| currentUser, PtTrainJob existPtTrainJob, String imageNameAndTag, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) { | |||||
| //检查模型是否合法,合法则保存其路径地址 | //检查模型是否合法,合法则保存其路径地址 | ||||
| checkModelAndSavePath(currentUser, baseTrainJobDTO); | checkModelAndSavePath(currentUser, baseTrainJobDTO); | ||||
| @@ -1070,7 +1058,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||||
| ptJobParam.setTrainJobId(ptTrainJob.getId()) | ptJobParam.setTrainJobId(ptTrainJob.getId()) | ||||
| .setAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId()) | .setAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId()) | ||||
| .setRunCommand(ptTrainJobUpdateDTO.getRunCommand()) | .setRunCommand(ptTrainJobUpdateDTO.getRunCommand()) | ||||
| .setImageName(imageName) | |||||
| .setImageName(imageNameAndTag) | |||||
| .setRunParams(ptTrainJobUpdateDTO.getRunParams()) | .setRunParams(ptTrainJobUpdateDTO.getRunParams()) | ||||
| .setRunParamsNameMap(ptTrainJobUpdateDTO.getRunParamsNameMap()) | .setRunParamsNameMap(ptTrainJobUpdateDTO.getRunParamsNameMap()) | ||||
| .setCreateUserId(ptTrain.getCreateUserId()); | .setCreateUserId(ptTrain.getCreateUserId()); | ||||
| @@ -252,10 +252,10 @@ ALTER TABLE `data_task` ADD COLUMN `file_type` smallint(3) NULL COMMENT '待处 | |||||
| ALTER TABLE `data_task` ADD COLUMN `stop` bit(1) NULL COMMENT '是否停止 0-没有 1-已停止' AFTER `file_type`; | ALTER TABLE `data_task` ADD COLUMN `stop` bit(1) NULL COMMENT '是否停止 0-没有 1-已停止' AFTER `file_type`; | ||||
| ALTER TABLE `data_task` ADD COLUMN `of_record_version` varchar(255) NULL COMMENT '生成ofRecord的版本号' AFTER `stop`; | ALTER TABLE `data_task` ADD COLUMN `of_record_version` varchar(255) NULL COMMENT '生成ofRecord的版本号' AFTER `stop`; | ||||
| -- 医学表 | -- 医学表 | ||||
| ALTER TABLE `data_medicine` ADD COLUMN `stop` bit(1) NULL AFTER `annotate_type`; | ALTER TABLE `data_medicine` ADD COLUMN `stop` bit(1) NULL AFTER `annotate_type`; | ||||
| -- 点云数据集表 | -- 点云数据集表 | ||||
| create table if not exists `pc_dataset` ( | create table if not exists `pc_dataset` ( | ||||
| `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键id', | `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键id', | ||||
| @@ -397,3 +397,5 @@ CREATE TABLE `pt_atlas_camera` ( | |||||
| PRIMARY KEY (`id`), | PRIMARY KEY (`id`), | ||||
| UNIQUE KEY `camera_index_uniq` (`camera_index_code`) COMMENT '设备标识唯一' | UNIQUE KEY `camera_index_uniq` (`camera_index_code`) COMMENT '设备标识唯一' | ||||
| ) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb4 COMMENT='模型炼知视频流管理'; | ) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb4 COMMENT='模型炼知视频流管理'; | ||||
| ALTER TABLE pt_image DROP image_status; | |||||
| @@ -33,11 +33,11 @@ ribbon: | |||||
| dubhe-proxy: | dubhe-proxy: | ||||
| visual: | visual: | ||||
| keyword: visual | keyword: visual | ||||
| server: 127.0.0.1 | |||||
| server: 10.5.26.88 | |||||
| port: 9898 | port: 9898 | ||||
| refine: | refine: | ||||
| keyword: refine | keyword: refine | ||||
| server: 127.0.0.1 | |||||
| server: localhost | |||||
| port: 9797 | port: 9797 | ||||
| ## 测试标识 | ## 测试标识 | ||||
| debug: | debug: | ||||
| @@ -17,7 +17,7 @@ spring: | |||||
| redis: | redis: | ||||
| #数据库索引 | #数据库索引 | ||||
| database: 4 | database: 4 | ||||
| host: 127.0.0.1 | |||||
| host: 10.5.26.88 | |||||
| port: 6379 | port: 6379 | ||||
| password: | password: | ||||
| #连接超时时间 | #连接超时时间 | ||||
| @@ -26,9 +26,9 @@ spring: | |||||
| druid: | druid: | ||||
| db-type: com.alibaba.druid.pool.DruidDataSource | db-type: com.alibaba.druid.pool.DruidDataSource | ||||
| driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | ||||
| url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||||
| url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||||
| username: test | username: test | ||||
| password: test | |||||
| password: zj12345678 | |||||
| # 初始化配置 | # 初始化配置 | ||||
| initial-size: 3 | initial-size: 3 | ||||
| @@ -67,17 +67,17 @@ logging.config: | |||||
| # 文件存储服务配置 | # 文件存储服务配置 | ||||
| storage: | storage: | ||||
| # 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 | # 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 | ||||
| file-store: 127.0.0.1 | |||||
| file-store: 10.5.26.234 | |||||
| # 文件存储服务端 共享目录 | # 文件存储服务端 共享目录 | ||||
| file-store-root-path: /nfs/ | file-store-root-path: /nfs/ | ||||
| file-store-root-windows-path: "Z:" | file-store-root-windows-path: "Z:" | ||||
| # minio配置 | # minio配置 | ||||
| minio: | minio: | ||||
| url: http://127.0.0.1:9000/ | |||||
| url: http://10.5.26.90:9000/ | |||||
| accessKey: admin | accessKey: admin | ||||
| secretKey: 123@abc.com | secretKey: 123@abc.com | ||||
| bucketName: dubhe-prod | |||||
| bucketName: dubhe-cloud-test | |||||
| presignedUrlExpiryTime: 300 | presignedUrlExpiryTime: 300 | ||||
| annotation: /annotation/ | annotation: /annotation/ | ||||
| @@ -121,13 +121,6 @@ feign: | |||||
| okhttp: | okhttp: | ||||
| enabled: false | enabled: false | ||||
| #配置harbor | |||||
| harbor: | |||||
| address: harbor.dubhe.ai | |||||
| username: admin | |||||
| password: Harbor12345 | |||||
| model-name: train | |||||
| #配置dev环境单元测试用户名与密码 | #配置dev环境单元测试用户名与密码 | ||||
| unittest: | unittest: | ||||
| username: admin | username: admin | ||||
| @@ -136,10 +129,10 @@ unittest: | |||||
| model: | model: | ||||
| measuring: | measuring: | ||||
| url: | url: | ||||
| package: http://127.0.0.1:32760/model_measure/package | |||||
| json: http://127.0.0.1:32760/model_measure/measure | |||||
| package: http://10.5.26.90:32760/model_measure/package | |||||
| json: http://10.5.26.90:32760/model_measure/measure | |||||
| converter: | converter: | ||||
| url: http://127.0.0.1:32230/model_convert | |||||
| url: http://10.5.26.90:32230/model_convert | |||||
| user: | user: | ||||
| config: | config: | ||||
| @@ -5,21 +5,17 @@ k8s: | |||||
| # k8s集群配置文件 | # k8s集群配置文件 | ||||
| kubeconfig: kubeconfig_test | kubeconfig: kubeconfig_test | ||||
| # nfs服务暴露的IP地址 如需测试需修改为合适的地址 | # nfs服务暴露的IP地址 如需测试需修改为合适的地址 | ||||
| nfs: 127.0.0.1 | |||||
| nfs: 10.5.26.234 | |||||
| #nfs服务端 共享目录 | #nfs服务端 共享目录 | ||||
| nfs-root-path: /nfs/ | nfs-root-path: /nfs/ | ||||
| nfs-root-windows-path: "Z:" | nfs-root-windows-path: "Z:" | ||||
| # 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 | # 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 | ||||
| file-store: 127.0.0.1 | |||||
| file-store: 10.5.26.234 | |||||
| #文件存储服务端 共享目录 | #文件存储服务端 共享目录 | ||||
| file-store-root-path: /nfs/ | file-store-root-path: /nfs/ | ||||
| file-store-root-windows-path: "Z:" | file-store-root-windows-path: "Z:" | ||||
| # 命名空间关键字 | # 命名空间关键字 | ||||
| namespace: namespace | namespace: namespace | ||||
| # k8s ingress域名 如需测试需修改为合适的域名,注意:需要为此域名配置泛域名解析到 k8s集群master节点的ip | |||||
| host: notebooktest.dubhe.club | |||||
| # k8s ingress-controller 对外port | |||||
| port: 30865 | |||||
| # k8s ingress-controller 对外grpc port | # k8s ingress-controller 对外grpc port | ||||
| https-port: 31287 | https-port: 31287 | ||||
| # k8s 模型部署配置 | # k8s 模型部署配置 | ||||
| @@ -27,12 +23,12 @@ k8s: | |||||
| # 在线服务模型部署后容器域名(k8s ingress域名),解析地址为k8s集群地址,如需测试需修改为合适的域名 | # 在线服务模型部署后容器域名(k8s ingress域名),解析地址为k8s集群地址,如需测试需修改为合适的域名 | ||||
| host: servingtest.dubhe.club | host: servingtest.dubhe.club | ||||
| # tls 证书 crt | # tls 证书 crt | ||||
| tls-crt: | |||||
| tls-crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURYekNDQWtlZ0F3SUJBZ0lKQUp3Z0VCYlhGdk9HTUEwR0NTcUdTSWIzRFFFQkN3VUFNRVl4SVRBZkJnTlYKQkFNTUdDb3VjMlZ5ZG1sdVozUmxjM1F1WkhWaWFHVXVZMngxWWpFaE1COEdBMVVFQ2d3WUtpNXpaWEoyYVc1bgpkR1Z6ZEM1a2RXSm9aUzVqYkhWaU1CNFhEVEl4TURVeE9UQTNNRGN5TTFvWERUTXhNRFV4TnpBM01EY3lNMW93ClJqRWhNQjhHQTFVRUF3d1lLaTV6WlhKMmFXNW5kR1Z6ZEM1a2RXSm9aUzVqYkhWaU1TRXdId1lEVlFRS0RCZ3EKTG5ObGNuWnBibWQwWlhOMExtUjFZbWhsTG1Oc2RXSXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBdwpnZ0VLQW9JQkFRREwxNDBMUXk2dXFHUi9WckVXeVNUdFppVGwvOEo1eGlHenB5WUJtTldUTzVQSTczTHRKbXdkCmFPMGNCNXhBcy96SXlpYjM3b0loR0FQejZNWlBBZjJEeUMxNXJLb3Via3h2Vkswa3hJY0k2L1dsWFUxMjdZZzQKdUllNzJEbFNIak9tbStVQ2JDWkxtc0VlQVI2S3RwNW9lVUFUQVgwOWEyL012ZkdtNU1URWZFSVplTENoL21rVAppZGFKcnl0NThwYVFLZmpJcVdKQnlnUGpJYlFMUlZnSG5mckIzdlB5b3RmMFBZZWVKVHlEa1lNQnNzblMxS044ClY2Qldvb2dzWUxQcCs2VE1wOXBHS3pBdksxdXJMWkFsa1N0SUw1TkppQWJOZ1lMOXN1UVE3OU9aano2c1ZrbWQKait6aHZHSGUrYk9zdlBrYmtqNS9lbzQyQUdQcW9zdTlBZ01CQUFHalVEQk9NQjBHQTFVZERnUVdCQlFzWDlCegpxVEhNN21xMUJrOFRNWXgzY1ZRaFB6QWZCZ05WSFNNRUdEQVdnQlFzWDlCenFUSE03bXExQms4VE1ZeDNjVlFoClB6QU1CZ05WSFJNRUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCc1BJSUpPMm1qUFNJV1lTaloKQmdjbFJoUTRudEkvdjFWczVjb0xOVzlrZUhqUGlCYlhvVzgzZExqSWh5ZmVYSk93Y3lpRnJkaVlZeVV5SEprTAprTmlxajRxVmZuaGkwcUoweDQyN3MzUG40TkZZRnNJYVM2OHA1WlZpUlJSZGNWY0dIMkNiRUxzYW41ZUhkNExYCnlTUUdDZmVkUVdtMEVyWWxDMGJWSER1d3RCa0NzWElPRmJEb0tSMW5wMDc3YlBrbEpIRS9rUi82QVpJR3hFM28KbDd1SDVRWWRsWldoanY5WFZRM1plVnZLUzVkM1pHYjcxZ0dwSEU2b2hmL1IweVhoS1RadUREYkhjK2lFc1dWbQowa24xenlvTUVHSW5nM0N6cVk4aE1tdHhRVStLekJrSEliVzh0UnkrS0MxZ2ZvNktFa01JVGpjK2pwQU4wZ2VDCm1LWWYKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= | |||||
| # tls 证书 key | # tls 证书 key | ||||
| tls-key: | |||||
| tls-key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRREwxNDBMUXk2dXFHUi8KVnJFV3lTVHRaaVRsLzhKNXhpR3pweVlCbU5XVE81UEk3M0x0Sm13ZGFPMGNCNXhBcy96SXlpYjM3b0loR0FQego2TVpQQWYyRHlDMTVyS291Ymt4dlZLMGt4SWNJNi9XbFhVMTI3WWc0dUllNzJEbFNIak9tbStVQ2JDWkxtc0VlCkFSNkt0cDVvZVVBVEFYMDlhMi9NdmZHbTVNVEVmRUlaZUxDaC9ta1RpZGFKcnl0NThwYVFLZmpJcVdKQnlnUGoKSWJRTFJWZ0huZnJCM3ZQeW90ZjBQWWVlSlR5RGtZTUJzc25TMUtOOFY2Qldvb2dzWUxQcCs2VE1wOXBHS3pBdgpLMXVyTFpBbGtTdElMNU5KaUFiTmdZTDlzdVFRNzlPWmp6NnNWa21kait6aHZHSGUrYk9zdlBrYmtqNS9lbzQyCkFHUHFvc3U5QWdNQkFBRUNnZ0VCQUtJRG83WDA0dEN0UkpzMVV6RnlFK2lnUyttR201TGdHSDlLOWRsRkdWT1oKZ2IrdUdhRkY3WjUrZkI3UUthLy9ub1lVcmw1VzhwVE5HcTh0THhreTBzV0FRQ3k2UU5VQWs3M1ZCbnozdXdBcgpVeVhvLzUzcjgxNXB4SEJYems5bmM5UVRpalNmc3R4YWx1MTdKRVJLRzZPYjQ0SjNwNHcrclRDRk0rRmJhTTFsCkxuWUh2bGR4ZnRoMmhBWHdaU2dFYXdCNXRBbTY2YnIxbUc4ZW5saEIvbisrY09iYkZWYnk4OUcyUmVMMjJxdHIKa0R0RHVQOGY1NVdxNVB0a25TRE4zaGoweEYydG9aSC80RHpKOWVibVNMVHpUK1pQSjVjK1dlZ3BNdCtVd3YzbwpNNFdGQnFUUUZSR0MyQVJTaXVFTEdtRUJnVk50VDRmRVhRSmFveGJKeWYwQ2dZRUE2MDRJOXlDZUttY2hjSEJvCmRFeFJ2bnNhcGhwbkJqc3pHazZIWVhhTFI0NGpvalRCWm5ZWVhHYklreDdZaytla1RQYjMwRUpzQjlRRmxpVUUKQ25oZXhCS0pjTk5VcTdWdHFGTXBhT0JKWG9Md0cySmVwVmNhN1MzcFo3THZMWGRQR2wvOXFTTW5pMXROV1kxdwpmTGl1Zzlqb3JCeEJaL250NU43MDJFTWpaL2NDZ1lFQTNjVWV2U3U1VmlHaGNPSmFVSlBoUjdnclViMW9EWDF4CnROdDdXNS9pNkFyNDBsY1FZK2RCcll2VEljRllEL3piZnRWNGFnMDhsLzVHamxQK2grWW01QWNRb1c1Z3BaaHgKd1ZhTFFzRTg4ZUdPY2pLakVqd2krTEZRNE5HRmdXdU9pc1hZWk5GTlBvT1JDbHRUbkRMRnFjS0FNY2tBTEgwVwpRT2poTkJqeGhPc0NnWUFRaHRPVVgrTWNBVkJVOEdBMXd1Sis2WENPdUE0Q1h3Z1EwZkVxUkVRMkMyS3ZVdHMrCmtnN1Y5cFloMXluSkFaMEZsdGNDOXBkVjJXdG5CMFNJWTduc05ZMFhzcEFnMjBaUGF6L2VVTnAyVytYM1ZtcGwKWEgvVXBzUGM5N0ZhMVNWbUtkWE1HbDc5cDdVQUZESVJZSHRKWVdPK0t1SGhKcW14eUlNVDZXdEVNd0tCZ0R0ZAo4WVNpbDlLKzNnRGlGMXRLdXh2LzZWalFZM0o1Q2w5b0FmWGRMMWorMXUwMzhXTk5IUC9nVm56S3pWQTZXR2Z6CnJYQjJhcW9sbjYycVBwRVN3NFozZmJRNVlCWDBZVDlvYzQ5RE81VmsxRVV0MlFtZ241d0RtNnNUYTdIaG9SNzEKSjZDVmh5QWRDRTdGYy9SMGd3V1cwOHFBREZQY2lJQ0gxd0dqUzhSUkFvR0JBTklUNW5yTzNwZmNKYTROZWFZeApKMzd5RzFodUJGcWdRRWpVU3pmbHQ3UWwvNjI3LzNWT2lIRjVqSkhES0JqODBEdzh1UjhoQ2VhWDY2WkJqUmxKCk5vOUEvUm43Q2F4YUl2dkNjWG4rUW5oLy9BVlNHRzU4WWI2UUlQNFVqT1RKOW1GUlFFajN0cm9KNnVpL3FCZTQKUkhXSE44WC91Z242dzZUSGRWNktJbSs4Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K | |||||
| # elasticsearch暴露的服务地址 | # elasticsearch暴露的服务地址 | ||||
| elasticsearch: | elasticsearch: | ||||
| hostlist: ${eshostlist:127.0.0.1:30498} | |||||
| hostlist: ${eshostlist:10.5.26.90:30498} | |||||
| # 日志采集配置信息 | # 日志采集配置信息 | ||||
| log: | log: | ||||
| type: _doc | type: _doc | ||||
| @@ -50,9 +46,9 @@ k8s: | |||||
| # 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址 | # 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址 | ||||
| pod: | pod: | ||||
| metrics: | metrics: | ||||
| grafanaUrl: http://127.0.0.1:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod= | |||||
| grafanaUrl: http://10.5.26.90:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod= | |||||
| prometheus: | prometheus: | ||||
| url: http://127.0.0.1:30003/ | |||||
| url: http://10.5.26.90:30003/ | |||||
| query: api/v1/query | query: api/v1/query | ||||
| query-range: api/v1/query_range | query-range: api/v1/query_range | ||||
| gpu-query-param: sum(DCGM_FI_DEV_GPU_UTIL{pod="pod-name-placeholder"})by(pod,UUID) | gpu-query-param: sum(DCGM_FI_DEV_GPU_UTIL{pod="pod-name-placeholder"})by(pod,UUID) | ||||
| @@ -75,21 +71,40 @@ k8s: | |||||
| cpu: 10 | cpu: 10 | ||||
| memory: 32 | memory: 32 | ||||
| gpu: 2 | gpu: 2 | ||||
| #配置harbor | |||||
| harbor: | |||||
| address: harbor.dubhe.ai | |||||
| username: admin | |||||
| password: Harbor12345 | |||||
| model-name: train | |||||
| # minio配置 | # minio配置 | ||||
| minio: | minio: | ||||
| url: http://127.0.0.1:9000/ | |||||
| url: http://10.5.26.90:9000/ | |||||
| accessKey: admin | accessKey: admin | ||||
| secretKey: 123@abc.com | secretKey: 123@abc.com | ||||
| bucketName: dubhe-prod | |||||
| bucketName: dubhe-cloud-test | |||||
| presignedUrlExpiryTime: 300 | presignedUrlExpiryTime: 300 | ||||
| annotation: /annotation/ | annotation: /annotation/ | ||||
| docker: | docker: | ||||
| remote-api-port: 2375 | remote-api-port: 2375 | ||||
| clusters: | |||||
| - cluster: | |||||
| certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1USXdPREE0TWpnd00xb1hEVE14TVRJd05qQTRNamd3TTFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWVKCkRjTThQUHFlV0tHazE3S2JtN3kwbzkzTGtxb2w5dG1QbVN0YTVyUXZSWU9rOFJYK2QvVEpVaXE2S2c4NXFyMGUKZFZ3SG5vTjg3Q0VFbXhOK2o4OUhlcnY1dDFaRFBHMVRHS3U5N1l1WnV4RCt2R1hGQkhDZ2ViT0s0Zmc0am1iSgpwUmlJeCtUZmlyUjM4TlVhZTBkR1g5c1o1TWF2NkhiSVVEK2hNTm9VU2pyWUZ2bTcxbGtYdlhVWW9wd1FSR3FvCkZiWG5KTFFVakd3UFFRclRCQ0wycFE2c1dGbkpkQ0VBSHhXU0grTUN4cXV0L25SUHhORlFOWGFMZmU3clB3a3MKd3NrVlFlTzgyL205aFZkWnNnNUFnRDJtTS9jS2VybE9acVBGcWFEaFBoYnpkRzF4QjRtcVZmTGJVUUppa3I1SQpYQ1FhTWR2c2FRTnhPcmNPOHQwQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZFek92OGhVUGtRYS90bGNhQTE0OERmcmF5TXRNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBMTFqcDR4V1hyZkRPNDJlblBpdm9HNXZxY3hqbysxaktZV1YxR1NIdHNVdlBNU1UzVAptdVFCQWU2VVRzM0JtSTgzMUQ4aFd4cDlka3BkMlhWbCt6OTZabFIvQ2Qrb3NSWWE1MUpZd2RvMUp4Z29hbUk2CmszOS9sNTRBQVpWMkFPWWNQU0pKNkNzdVcxZ2NTMDlaTlA0djlhK0NIKzY5OVpQMGlhUStEc0FPRFJzUnhRcVMKK08zNUU4K0RzRjBSOWZCYlhvbkxiK05VSHlTQ2pQSnVVT21lS0RYMFpQQm81blNscmh5cUIxbHVNZnhEaUltdgpIU3BscXB4M1duMzJnNEZYOCs3clZBNUZjWkIvNDg3bUZmdG1CTXc3aENidlFJeWY2Ymd6VTUycUdaeFZGTTVjCnUxeEZyRXlXaFA1eElJY09lY1M1RCtJa29ZOHUyT3ozQldEeQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== | |||||
| server: https://10.105.0.6:6443 | |||||
| name: kubernetes | |||||
| contexts: | |||||
| - context: | |||||
| cluster: kubernetes | |||||
| user: kubernetes-admin | |||||
| name: kubernetes-admin@kubernetes | |||||
| current-context: kubernetes-admin@kubernetes | |||||
| kind: Config | |||||
| preferences: {} | |||||
| users: | |||||
| - name: kubernetes-admin | |||||
| user: | |||||
| client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lJQjUxN3hkT3V2cnd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TVRFeU1EZ3dPREk0TUROYUZ3MHlNakV5TURnd09ESTRNRFZhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXlFaURrYVJ1WmVUbVJ4WmsKRkxUbkRPMUxIM1k5dE80WXpXREt4QVA2bFphTUJwZXJWRWpiU1VMOGIzdG9WWFppVnVnaHhvMzRHdjF4dlJKLwpDN3NhWGVaU3EzcTVMcEFhQjBrelRDNFRtbGtrdE03YkJTZFNhMG9SeGJXTUdvVEM0WUVBckJCNS9FRmdUYjRCCnE4aG1USUlmdUcwa05sNmd5a2M4UnhSL3N5Q1lSVjZCTTZTYWFzODR1Tmx3ZU9vV2RaQkQ0UVZQbzJueUI4NGQKUnc0d0w2VEQ5MlUwMG5KcjY3bUtpTjhBNGtBampFbWNrNm1JbHdsUTB2MHR5Uk8zTFBXZFdNazZ5QVIyVEZ6RgpBZCs4dUd5L3VDQlB6UDlVcDFERkI5TDF0Y25sd1lTeTYycGdFWC9wMzl6ZjZ0RFpZRVJwK0Jpd21Eb3RmVmFaClNLam9md0lEQVFBQm8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0h3WURWUjBqQkJnd0ZvQVVUTTYveUZRK1JCcisyVnhvRFhqd04rdHJJeTB3RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFCQ01US2VEV1dHa0VFZWp6dlN3TTFKZit0QUJaYklrd0FoZEt1MjFnUkxyQktzbHQ4Y0NHMG1KCmc3eDhndFJYcW0xRGlEU01DMnlWMW9OWTFSdGlLZVhZT29RbVY2SFh2bEQybWZyY2VzS3VSNHlIbCtlWHcyL1AKb2FLZVFEM21yVm9ZZ1ZMMVNqOWpzcDRBa1hEM1ErUzdKYXN4Y3MvVkR3SmlRQzhGamkzaWZMVFFKQkxCdm1WVwpCZ3QvM0xPZ1lCdWxYRWg3cFlQUzduTVFqNFpxQ0QwWTNJOENicUM0VGlZQjB1aTZlbVRDd3cvRnBNSzNIRFZ4CnZOd3FPRHBaWW5DK0IxSzR5RUk4NGIvM1o3ZXA0MUE1NGk3QkpxTklORUVtRjdGcmpQQy91V1RuYlpzVUNFQjAKWTFUR1Y3czNBUTZMZXNCRFF6UVRzVnBENWFRRk9VYz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= | |||||
| client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBeUVpRGthUnVaZVRtUnhaa0ZMVG5ETzFMSDNZOXRPNFl6V0RLeEFQNmxaYU1CcGVyClZFamJTVUw4YjN0b1ZYWmlWdWdoeG8zNEd2MXh2UkovQzdzYVhlWlNxM3E1THBBYUIwa3pUQzRUbWxra3RNN2IKQlNkU2Ewb1J4YldNR29UQzRZRUFyQkI1L0VGZ1RiNEJxOGhtVElJZnVHMGtObDZneWtjOFJ4Ui9zeUNZUlY2QgpNNlNhYXM4NHVObHdlT29XZFpCRDRRVlBvMm55Qjg0ZFJ3NHdMNlREOTJVMDBuSnI2N21LaU44QTRrQWpqRW1jCms2bUlsd2xRMHYwdHlSTzNMUFdkV01rNnlBUjJURnpGQWQrOHVHeS91Q0JQelA5VXAxREZCOUwxdGNubHdZU3kKNjJwZ0VYL3AzOXpmNnREWllFUnArQml3bURvdGZWYVpTS2pvZndJREFRQUJBb0lCQUgrbHY3ME9TSkpHZmdHbQpvcWlUMTRKa3BuRnA0ZEF6dzdqNXpLRjdTN1VWR3krRWNOeXFCcUM5d2JlbnRvcHBoaW1Qang5R0VtL1pRaWxYCjVZTHJmOVdDMndPUmx2NjNOdStYMXNyaHZ2cXJmL3FBc0JTcnlCcTdQWEo5ejhxQy9OWE9hMGcreEJCaTltYjIKQjRpZGs1MkZmWVFFZzRUbmNLRWJINjdKd00yL2dmZ3BXMGNEd0RiNVc2aHIxclUzNkNGc1AyVTlEYytMVkV5cQpvK3FZZ0RFZ0MxSWxucGVIdzROejJmK3RkbFpLVHN5dzV1ZG5xVm1kWGNkLyszR3NvQVM3Q25DTmQ3a1pKQ1BPClNpRktZSDZZUS93ZmJLZjFibC84eERXU1UvKytQUTZyODJGV21mMFBjd3ExbWV5OXIvUW1kaHc5YndiamkxVDQKdURjSzUva0NnWUVBOURIZi8xZnN4czlqVldoaE94UXFzUFN3UzdDbVZYcHV2S2Q3MjlMRGFIV1VQT3E5Y0l4QQpaV01Ibi9EbEN3U2dxaitPL0FNSDdaQVpPbDV5V2RiZFVEM2I0WWVKLzlubXdvN25ReG9ndTlUdTlNZEVnM0pvCm1wTzVqQTYzYTE2Z0sxRWlmTDJkdllRQkJGQzg2aDJzYlV0WHNGRDV6R3QzTU96bVF6SFFQM3NDZ1lFQTBmY3gKMzlUR0YrZFdVb3MzOXRLaE9MZVpFaWtUSmtqckdrTUdldHpYNEVCRE5oRUUvdWNkYUZCWmJoU3ltZHAwMmtKNgpKQkpvaEJRaEN4Zk9nUXhRdFp2dUt3WUMvTEYrMVF1TG1MaERGV2NEa05Ma1E0RWU5WVRFV1VuNE9NNkJJTmVOCldseklzcncyeHRvaUVoMXZFWW92ODdobEZCQ0E4N0xYSElpa1NjMENnWUFnZ1pNajFueDZhcGo2Z1k2UDRydGMKR3ZaczNQTUhaZWpmekJ1OHcyMm50aDhwak1YeStYaUpCb201VE1Qd2w0a2JvS0pVQWNOSWFHb3pUdWRCOWt3Mwpwa1JpM1R3Tmh1QWsvZ2ppOGROeHJZS3hxdVQvNm5icVBNZnlVUE14bzhNR2dTSnFJSU9pK3JOMzg1ZHlRc2oxCk01WlVyakxTZDJ2Q1k0YXpZeW9waVFLQmdBS0RBb1c2OWVPelVNVU1CVTllbGE4b0F4ekVnUVlrT1N2SFFYeVcKeDc1WEJuYlBIVkF2VTVxNzIxWUZ5VjB0ejlnTWs0bDY3dVVsbUgrWmVVN3g1c1ZGYUVQN2ZtMm5jZXo2aG1EOQpVMUFlTzF5d0tTcmxrSWsvWFZuMEdKUVZaRllRUGhDbXEzLzM0Ry9nakNmTFVsalRYbDk4QlRtSU9RS3hVUUYyCllRREJBb0dBRTB5Q0NzcFE3Y1lYaVFyQnF4VVkxU2N5SjViVm1iaFN5cmgyb3FiUmRMRmxzWHFjUnBIK2tORjYKdXdjUzhZalJodmdQazgyZm1QcXlieE5ZWGxTYWoydGdzSFdpcGNETXZSM1dtSlQzTGpjeXZObDh6cUlkZDNBdQpJT1pNeG0vWWVMemNFOUdteDRsQkhmMnU5MkhiRjRiSCtqeVp3d3RWcHpRdmphaFF6dGs9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== | |||||
| #配置harbor | |||||
| harbor: | |||||
| address: harbor.dubhe.ai | |||||
| username: admin | |||||
| password: Harbor12345 | |||||
| @@ -6,9 +6,9 @@ spring: | |||||
| master: | master: | ||||
| type: com.alibaba.druid.pool.DruidDataSource | type: com.alibaba.druid.pool.DruidDataSource | ||||
| driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | ||||
| url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||||
| url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||||
| username: test | username: test | ||||
| password: test | |||||
| password: zj12345678 | |||||
| props: | props: | ||||
| sql: | sql: | ||||
| show: true | show: true | ||||
| @@ -18,7 +18,7 @@ data: | |||||
| # 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 | # 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 | ||||
| dcm: | dcm: | ||||
| host: 127.0.0.1 | |||||
| host: 10.5.29.100 | |||||
| port: 11112 | port: 11112 | ||||
| # 资源回收 | # 资源回收 | ||||
| @@ -27,13 +27,13 @@ recycle: | |||||
| # 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 | # 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 | ||||
| dcm: | dcm: | ||||
| host: 127.0.0.1 | |||||
| host: 10.5.29.100 | |||||
| port: 11112 | port: 11112 | ||||
| # ES服务地址及端口 | # ES服务地址及端口 | ||||
| es: | es: | ||||
| host: 127.0.0.1 | |||||
| serverPort: 31321 | |||||
| transportPort: 31322 | |||||
| clusterName: kubenets-logging | |||||
| host: 10.5.29.100 | |||||
| serverPort: 9200 | |||||
| transportPort: 9300 | |||||
| clusterName: docker-cluster | |||||
| index: dataset_text_test | index: dataset_text_test | ||||
| @@ -32,11 +32,11 @@ spring: | |||||
| precise-algorithm-class-name: org.dubhe.data.util.MyPreciseShardingAlgorithm | precise-algorithm-class-name: org.dubhe.data.util.MyPreciseShardingAlgorithm | ||||
| k8s: | k8s: | ||||
| # nfs服务暴露的IP地址 如需测试需修改为合适的地址 | # nfs服务暴露的IP地址 如需测试需修改为合适的地址 | ||||
| nfs: 127.0.0.1 | |||||
| nfs: 10.5.26.234 | |||||
| #nfs服务端 共享目录 | #nfs服务端 共享目录 | ||||
| nfs-root-path: /nfs/ | nfs-root-path: /nfs/ | ||||
| nfs-root-windows-path: "Z:" | nfs-root-windows-path: "Z:" | ||||
| file-store: 127.0.0.1 | |||||
| file-store: 10.5.26.234 | |||||
| #文件存储服务端 共享目录 | #文件存储服务端 共享目录 | ||||
| file-store-root-path: /nfs/ | file-store-root-path: /nfs/ | ||||
| file-store-root-windows-path: "Z:" | file-store-root-windows-path: "Z:" | ||||
| @@ -80,8 +80,8 @@ recycle: | |||||
| # ES服务地址及端口 | # ES服务地址及端口 | ||||
| es: | es: | ||||
| host: 127.0.0.1 | |||||
| serverPort: 32321 | |||||
| transportPort: 32322 | |||||
| clusterName: kubenets-logging | |||||
| host: 10.5.29.100 | |||||
| serverPort: 9200 | |||||
| transportPort: 9300 | |||||
| clusterName: docker-cluster | |||||
| index: dataset_text_test | index: dataset_text_test | ||||
| @@ -3,7 +3,7 @@ notebook-specs: | |||||
| # cpu数量(核) | # cpu数量(核) | ||||
| cpu-num: 1 | cpu-num: 1 | ||||
| # gpu数量(核) | # gpu数量(核) | ||||
| gpu-num: 1 | |||||
| gpu-num: 0 | |||||
| # 内存大小(M) | # 内存大小(M) | ||||
| mem-num: 1024 | mem-num: 1024 | ||||
| # 工作空间配额(m) | # 工作空间配额(m) | ||||
| @@ -22,8 +22,8 @@ management: | |||||
| serving: | serving: | ||||
| gateway: | gateway: | ||||
| # 在线服务http接口域名,解析地址为serving网关部署的服务器地址 | |||||
| postfixUrl: .openapi.dubhe.club | |||||
| # 在线服务http接口 host:port,解析地址为serving网关部署的服务地址 | |||||
| postfixUrl: 10.101.12.41:8081 | |||||
| token: | token: | ||||
| secret-key: 1n3512x | secret-key: 1n3512x | ||||
| expire-seconds: 108000 | expire-seconds: 108000 | ||||
| @@ -31,7 +31,6 @@ serving: | |||||
| httpclient: | httpclient: | ||||
| connect-timeout: 1000 | connect-timeout: 1000 | ||||
| response-timeout: 3000 | response-timeout: 3000 | ||||
| #关闭熔断机制 | #关闭熔断机制 | ||||
| feign: | feign: | ||||
| hystrix: | hystrix: | ||||
| @@ -1,52 +0,0 @@ | |||||
| # serving预测上传文件大小限制 | |||||
| spring: | |||||
| servlet: | |||||
| multipart: | |||||
| # 设置单个文件的大小 | |||||
| max-file-size: 10MB | |||||
| # 设置单次请求的文件的总大小 | |||||
| max-request-size: 100MB | |||||
| k8s: | |||||
| # nfs服务暴露的IP地址 如需测试需修改为合适的地址 | |||||
| nfs: 127.0.0.1 | |||||
| #nfs服务端 共享目录 | |||||
| nfs-root-path: /nfs/ | |||||
| nfs-root-windows-path: "Z:" | |||||
| # data模块配置 | |||||
| data: | |||||
| annotation: | |||||
| task: | |||||
| # 自动标注任务分割的文件split size | |||||
| splitSize: 16 | |||||
| # 单位ms | |||||
| retryInterval: 5000 | |||||
| # 任务不更新置为失败的时间单位秒 | |||||
| failTime: 43200 | |||||
| server: | |||||
| # 文件存储服务器用户名 | |||||
| userName: root | |||||
| #数据集训练配置 | |||||
| ptversion: http://127.0.0.1:8000/ | |||||
| minioweb: | |||||
| GetToken: | |||||
| url: minio/webrpc | |||||
| param: | |||||
| id: 2 | |||||
| jsonrpc: 2.0 | |||||
| method: Web.Login | |||||
| zip: | |||||
| url: minio/zip?token= | |||||
| #logback | |||||
| logging.config: | |||||
| classpath:logback-spring-dev.xml | |||||
| # ES服务地址及端口 | |||||
| es: | |||||
| host: 127.0.0.1 | |||||
| serverPort: 32321 | |||||
| transportPort: 32322 | |||||
| index: dataset_text_test | |||||
| @@ -5,7 +5,7 @@ terminal: | |||||
| # 用户workspace目录 | # 用户workspace目录 | ||||
| workspace-dir: "/workspace" | workspace-dir: "/workspace" | ||||
| # ssh主机 | # ssh主机 | ||||
| ssh-host: 127.0.0.1 | |||||
| ssh-host: 10.5.26.91 | |||||
| # 可匿名访问路径 | # 可匿名访问路径 | ||||
| security: | security: | ||||
| @@ -4,149 +4,139 @@ spring: | |||||
| gateway: | gateway: | ||||
| enabled: true | enabled: true | ||||
| routes: | routes: | ||||
| # id唯一表示 路由至admin服务 | |||||
| - id: admin-route | |||||
| # uri=> lb://服务名称 | |||||
| uri: lb://admin | |||||
| # 路由规则 | |||||
| predicates: | |||||
| - Path=/api/v1/admin/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| - id: visual-route | |||||
| # uri=> lb://服务名称 | |||||
| uri: lb://admin | |||||
| # 路由规则 | |||||
| predicates: | |||||
| - Path=/visual/api/** | |||||
| # 路由至auth服务 | |||||
| - id: auth-route | |||||
| uri: lb://auth | |||||
| predicates: | |||||
| - Path=/api/v1/auth/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至notebook服务 | |||||
| - id: notebook-route | |||||
| uri: lb://dubhe-notebook | |||||
| predicates: | |||||
| - Path=/api/v1/notebook/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至镜像管理服务 | |||||
| - id: image-route | |||||
| uri: lb://dubhe-image | |||||
| predicates: | |||||
| - Path=/api/v1/image/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至度量管理服务 | |||||
| - id: measure-route | |||||
| uri: lb://dubhe-measure | |||||
| predicates: | |||||
| - Path=/api/v1/measure/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-k8s服务 | |||||
| - id: dubhe-k8s | |||||
| uri: lb://dubhe-k8s | |||||
| predicates: | |||||
| - Path=/api/v1/k8s/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-model(模型管理)服务 | |||||
| - id: dubhe-model | |||||
| uri: lb://dubhe-model | |||||
| predicates: | |||||
| - Path=/api/v1/model/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-algorithm(算法管理)服务 | |||||
| - id: dubhe-algorithm | |||||
| uri: lb://dubhe-algorithm | |||||
| predicates: | |||||
| - Path=/api/v1/algorithm/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-train(训练管理)服务 | |||||
| - id: dubhe-train | |||||
| uri: lb://dubhe-train | |||||
| predicates: | |||||
| - Path=/api/v1/train/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # id唯一表示 路由至data(数据集)服务 | |||||
| - id: dubhe-data | |||||
| # uri=> lb://服务名称 | |||||
| uri: lb://dubhe-data | |||||
| # 路由规则 | |||||
| predicates: | |||||
| - Path=/api/v1/data/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-optimize(模型优化)服务 | |||||
| - id: dubhe-optimize | |||||
| uri: lb://dubhe-optimize | |||||
| predicates: | |||||
| - Path=/api/v1/optimize/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-serving(云端Serving)服务 | |||||
| - id: dubhe-serving | |||||
| uri: lb://dubhe-serving | |||||
| predicates: | |||||
| - Path=/api/v1/serving/**,/api/v1/batchServing/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 负载均衡样例 | |||||
| - id: weight-route1 | |||||
| uri: lb://demo-client | |||||
| # 路由规则 | |||||
| predicates: | |||||
| # 接口匹配前缀 | |||||
| - Path=/api/v1/w/** | |||||
| # 权重组weight-group1,权重比例 1/4 | |||||
| - Weight=weight-group1, 1 | |||||
| filters: | |||||
| # 根据接口匹配前缀截取 | |||||
| - StripPrefix=3 | |||||
| - id: weight-route2 | |||||
| uri: lb://demo-provider | |||||
| # 路由规则 | |||||
| predicates: | |||||
| - Path=/api/v1/w/** | |||||
| # 权重组weight-group1,权重比例 3/4 | |||||
| - Weight=weight-group1, 3 | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-data-dcm(医学数据集)服务 | |||||
| - id: dubhe-data-dcm | |||||
| uri: lb://dubhe-data-dcm | |||||
| predicates: | |||||
| - Path=/api/v1/dcm/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-terminal 天枢专业版终端 | |||||
| - id: dubhe-terminal | |||||
| uri: lb://dubhe-terminal | |||||
| predicates: | |||||
| - Path=/api/v1/terminal/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-tadl服务 | |||||
| - id: dubhe-tadl | |||||
| uri: lb://dubhe-tadl | |||||
| predicates: | |||||
| - Path=/api/v1/tadl/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-point-cloud服务 | |||||
| - id: dubhe-point-cloud | |||||
| uri: lb://dubhe-point-cloud | |||||
| predicates: | |||||
| - Path=/api/v1/pointCloud/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| server: | |||||
| maxPostSize: -1 | |||||
| maxHttpHeaderSize: 10240000 | |||||
| # id唯一表示 路由至admin服务 | |||||
| - id: admin-route | |||||
| # uri=> lb://服务名称 | |||||
| uri: lb://admin | |||||
| # 路由规则 | |||||
| predicates: | |||||
| - Path=/api/v1/admin/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| - id: visual-route | |||||
| # uri=> lb://服务名称 | |||||
| uri: lb://admin | |||||
| # 路由规则 | |||||
| predicates: | |||||
| - Path=/visual/api/** | |||||
| # 路由至demo-provider服务 | |||||
| - id: nacos-server-route | |||||
| uri: lb://demo-provider | |||||
| predicates: | |||||
| - Path=/api/v1/server/**,/api/v1/service/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至auth服务 | |||||
| - id: auth-route | |||||
| uri: lb://auth | |||||
| predicates: | |||||
| - Path=/api/v1/auth/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至notebook服务 | |||||
| - id: notebook-route | |||||
| uri: lb://dubhe-notebook | |||||
| predicates: | |||||
| - Path=/api/v1/notebook/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至镜像管理服务 | |||||
| - id: image-route | |||||
| uri: lb://dubhe-image | |||||
| predicates: | |||||
| - Path=/api/v1/image/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至度量管理服务 | |||||
| - id: measure-route | |||||
| uri: lb://dubhe-measure | |||||
| predicates: | |||||
| - Path=/api/v1/measure/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-k8s服务 | |||||
| - id: dubhe-k8s | |||||
| uri: lb://dubhe-k8s | |||||
| predicates: | |||||
| - Path=/api/v1/k8s/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-model(模型管理)服务 | |||||
| - id: dubhe-model | |||||
| uri: lb://dubhe-model | |||||
| predicates: | |||||
| - Path=/api/v1/model/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-algorithm(算法管理)服务 | |||||
| - id: dubhe-algorithm | |||||
| uri: lb://dubhe-algorithm | |||||
| predicates: | |||||
| - Path=/api/v1/algorithm/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-train(训练管理)服务 | |||||
| - id: dubhe-train | |||||
| uri: lb://dubhe-train | |||||
| predicates: | |||||
| - Path=/api/v1/train/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # id唯一表示 路由至data(数据集)服务 | |||||
| - id: dubhe-data | |||||
| # uri=> lb://服务名称 | |||||
| uri: lb://dubhe-data | |||||
| # 路由规则 | |||||
| predicates: | |||||
| - Path=/api/v1/data/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-optimize(模型优化)服务 | |||||
| - id: dubhe-optimize | |||||
| uri: lb://dubhe-optimize | |||||
| predicates: | |||||
| - Path=/api/v1/optimize/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-serving(云端Serving)服务 | |||||
| - id: dubhe-serving | |||||
| uri: lb://dubhe-serving | |||||
| predicates: | |||||
| - Path=/api/v1/serving/**,/api/v1/batchServing/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 负载均衡样例 | |||||
| - id: weight-route1 | |||||
| uri: lb://demo-client | |||||
| # 路由规则 | |||||
| predicates: | |||||
| # 接口匹配前缀 | |||||
| - Path=/api/v1/w/** | |||||
| # 权重组weight-group1,权重比例 1/4 | |||||
| - Weight=weight-group1, 1 | |||||
| filters: | |||||
| # 根据接口匹配前缀截取 | |||||
| - StripPrefix=3 | |||||
| - id: weight-route2 | |||||
| uri: lb://demo-provider | |||||
| # 路由规则 | |||||
| predicates: | |||||
| - Path=/api/v1/w/** | |||||
| # 权重组weight-group1,权重比例 3/4 | |||||
| - Weight=weight-group1, 3 | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-data-dcm(医学数据集)服务 | |||||
| - id: dubhe-data-dcm | |||||
| uri: lb://dubhe-data-dcm | |||||
| predicates: | |||||
| - Path=/api/v1/dcm/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| # 路由至dubhe-tadl服务 | |||||
| - id: dubhe-tadl | |||||
| uri: lb://dubhe-tadl | |||||
| predicates: | |||||
| - Path=/api/v1/tadl/** | |||||
| filters: | |||||
| - StripPrefix=3 | |||||
| @@ -3,7 +3,7 @@ spring: | |||||
| redis: | redis: | ||||
| #数据库索引 | #数据库索引 | ||||
| database: 1 | database: 1 | ||||
| host: 127.0.0.1 | |||||
| host: 10.5.26.88 | |||||
| port: 6379 | port: 6379 | ||||
| password: | password: | ||||
| #连接超时时间 | #连接超时时间 | ||||
| @@ -12,9 +12,9 @@ spring: | |||||
| druid: | druid: | ||||
| db-type: com.alibaba.druid.pool.DruidDataSource | db-type: com.alibaba.druid.pool.DruidDataSource | ||||
| driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | ||||
| url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true | |||||
| url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true | |||||
| username: test | username: test | ||||
| password: test | |||||
| password: zj12345678 | |||||
| # 初始化配置 | # 初始化配置 | ||||
| initial-size: 3 | initial-size: 3 | ||||
| @@ -43,11 +43,5 @@ spring: | |||||
| url-pattern: /* | url-pattern: /* | ||||
| exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" | exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" | ||||
| #配置harbor | |||||
| harbor: | |||||
| address: harbor.dubhe.ai | |||||
| username: admin | |||||
| password: Harbor12345 | |||||
| @@ -1,19 +0,0 @@ | |||||
| apiVersion: v1 | |||||
| clusters: | |||||
| - cluster: | |||||
| certificate-authority-data: | |||||
| server: https://127.0.0.1:6443 | |||||
| name: kubernetes | |||||
| contexts: | |||||
| - context: | |||||
| cluster: kubernetes | |||||
| user: kubernetes-admin | |||||
| name: kubernetes-admin@kubernetes | |||||
| current-context: kubernetes-admin@kubernetes | |||||
| kind: Config | |||||
| preferences: {} | |||||
| users: | |||||
| - name: kubernetes-admin | |||||
| user: | |||||
| client-certificate-data: | |||||
| client-key-data: | |||||
| @@ -3,7 +3,7 @@ spring: | |||||
| redis: | redis: | ||||
| #数据库索引 | #数据库索引 | ||||
| database: 1 | database: 1 | ||||
| host: 127.0.0.1 | |||||
| host: 10.5.26.88 | |||||
| port: 6379 | port: 6379 | ||||
| password: | password: | ||||
| #连接超时时间 | #连接超时时间 | ||||
| @@ -16,10 +16,10 @@ model: | |||||
| atlas: | atlas: | ||||
| url: | url: | ||||
| # 流服务地址 | # 流服务地址 | ||||
| rtmp: rtmp://127.0.0.1:1935/live/ | |||||
| rtmp: rtmp://10.5.30.42:1935/live/ | |||||
| # vms接口地址 | # vms接口地址 | ||||
| vms: http://127.0.0.1:8000/api/vms | |||||
| vms: http://10.105.10.51:8000/api/vms | |||||
| # 推理服务地址 | # 推理服务地址 | ||||
| video-serve: http://127.0.0.1:32761/video | |||||
| video-serve: http://10.105.0.6:32761/video | |||||
| # 拉流地址,需要拼接streamId | # 拉流地址,需要拼接streamId | ||||
| stream: http://127.0.0.1:8055/live?port=1935&app=live&stream= | |||||
| stream: http://10.5.30.42:8055/live?port=1935&app=live&stream= | |||||
| @@ -14,16 +14,16 @@ VUE_APP_DATA_API = '/' | |||||
| VUE_APP_VISUAL_API = '/' | VUE_APP_VISUAL_API = '/' | ||||
| # minio | # minio | ||||
| VUE_APP_MINIO_API = 'http://127.0.0.1:9000/minio' | |||||
| VUE_APP_MINIO_API = 'http://10.105.1.132:9000/minio' | |||||
| # atlas | # atlas | ||||
| VUE_APP_ATLAS_HOST = 'http://127.0.0.1' | |||||
| VUE_APP_ATLAS_HOST = 'http://183.129.174.186' | |||||
| # DCM4CHEE | # DCM4CHEE | ||||
| VUE_APP_DCM_API = 'http://pre.dubhe.club/dcm4chee/dcm4chee-arc/aets/DCM4CHEE_ADMIN' | VUE_APP_DCM_API = 'http://pre.dubhe.club/dcm4chee/dcm4chee-arc/aets/DCM4CHEE_ADMIN' | ||||
| # minIO 服务 IP | # minIO 服务 IP | ||||
| VUE_APP_MINIO_ENDPOINT = '127.0.0.1' | |||||
| VUE_APP_MINIO_ENDPOINT = '10.105.1.132' | |||||
| # minIO 服务 端口 | # minIO 服务 端口 | ||||
| VUE_APP_MINIO_PORT = '9000' | VUE_APP_MINIO_PORT = '9000' | ||||
| # 是否开启 SSL | # 是否开启 SSL | ||||
| @@ -5,4 +5,5 @@ dist | |||||
| src/components/Crud | src/components/Crud | ||||
| mock | mock | ||||
| src/views/visual | src/views/visual | ||||
| src/store/modules/Visual | |||||
| src/store/modules/Visual | |||||
| src/utils/VisualUtils | |||||
| @@ -1,12 +1,9 @@ | |||||
| const plugins = [ | |||||
| "@vue/babel-plugin-transform-vue-jsx", | |||||
| "@babel/plugin-proposal-optional-chaining", | |||||
| ]; | |||||
| const plugins = ['@vue/babel-plugin-transform-vue-jsx', '@babel/plugin-proposal-optional-chaining']; | |||||
| // 生产环境移除console | // 生产环境移除console | ||||
| if (process.env.NODE_ENV === "production") { | |||||
| plugins.push("transform-remove-console"); | |||||
| if (process.env.NODE_ENV === 'production') { | |||||
| plugins.push('transform-remove-console'); | |||||
| } | } | ||||
| module.exports = { | module.exports = { | ||||
| plugins, | plugins, | ||||
| presets: [["@vue/app",{ useBuiltIns: "entry" }]], | |||||
| presets: [['@vue/app', { useBuiltIns: 'entry' }]], | |||||
| }; | }; | ||||
| @@ -24,6 +24,18 @@ | |||||
| "lint:style": "stylelint src/**/*.{html,vue,css,sass,scss}", | "lint:style": "stylelint src/**/*.{html,vue,css,sass,scss}", | ||||
| "fix:style": "stylelint src/**/*.{html,vue,css,sass,scss} --fix" | "fix:style": "stylelint src/**/*.{html,vue,css,sass,scss} --fix" | ||||
| }, | }, | ||||
| "husky": { | |||||
| "hooks": { | |||||
| "pre-commit": "lint-staged", | |||||
| "commit-msg": "commitlint -E HUSKY_GIT_PARAMS" | |||||
| } | |||||
| }, | |||||
| "lint-staged": { | |||||
| "src/**/*.{js,vue}": [ | |||||
| "eslint --fix", | |||||
| "git add" | |||||
| ] | |||||
| }, | |||||
| "repository": { | "repository": { | ||||
| "type": "git", | "type": "git", | ||||
| "url": "git@codeup.teambition.com:zhejianglab/dubhe-web.git" | "url": "git@codeup.teambition.com:zhejianglab/dubhe-web.git" | ||||
| @@ -19,7 +19,7 @@ import { API_MODULE_NAME } from '@/config'; | |||||
| export function list(params) { | export function list(params) { | ||||
| return request({ | return request({ | ||||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage/info`, | |||||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage/list`, | |||||
| method: 'get', | method: 'get', | ||||
| params, | params, | ||||
| }); | }); | ||||
| @@ -27,7 +27,7 @@ export function list(params) { | |||||
| export function add(data) { | export function add(data) { | ||||
| return request({ | return request({ | ||||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage/uploadImage`, | |||||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage`, | |||||
| method: 'post', | method: 'post', | ||||
| data, | data, | ||||
| }); | }); | ||||
| @@ -65,14 +65,6 @@ export function getImageTagList(params) { | |||||
| }); | }); | ||||
| } | } | ||||
| export function setPrecast(params) { | |||||
| return request({ | |||||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage/imageResource`, | |||||
| method: 'put', | |||||
| params, | |||||
| }); | |||||
| } | |||||
| // 设置notebook默认镜像-获取默认镜像 | // 设置notebook默认镜像-获取默认镜像 | ||||
| export function getDefaultImage(params) { | export function getDefaultImage(params) { | ||||
| return request({ | return request({ | ||||
| @@ -332,6 +332,10 @@ | |||||
| width: 320px; | width: 320px; | ||||
| } | } | ||||
| .w-400 { | |||||
| width: 400px; | |||||
| } | |||||
| .w-500 { | .w-500 { | ||||
| width: 500px; | width: 500px; | ||||
| } | } | ||||
| @@ -9,6 +9,9 @@ the License. * ============================================================= */ | |||||
| <template> | <template> | ||||
| <el-tooltip v-bind="mergedAttrs"> | <el-tooltip v-bind="mergedAttrs"> | ||||
| <i class="primary f18 vm" :class="[icon]" /> | <i class="primary f18 vm" :class="[icon]" /> | ||||
| <template #content> | |||||
| <slot name="content" /> | |||||
| </template> | |||||
| </el-tooltip> | </el-tooltip> | ||||
| </template> | </template> | ||||
| @@ -541,6 +541,7 @@ import { | |||||
| RESOURCES_MODULE_ENUM, | RESOURCES_MODULE_ENUM, | ||||
| RESOURCES_POOL_TYPE_ENUM, | RESOURCES_POOL_TYPE_ENUM, | ||||
| RESOURCES_POOL_TYPE_MAP, | RESOURCES_POOL_TYPE_MAP, | ||||
| IMAGE_TYPE_ENUM, | |||||
| } from '@/utils'; | } from '@/utils'; | ||||
| import { list as getAlgorithmList, getAlgorithmInfo } from '@/api/algorithm/algorithm'; | import { list as getAlgorithmList, getAlgorithmInfo } from '@/api/algorithm/algorithm'; | ||||
| import { getModelByResource } from '@/api/model/model'; | import { getModelByResource } from '@/api/model/model'; | ||||
| @@ -551,7 +552,7 @@ import { list as getSpecsNames } from '@/api/system/resources'; | |||||
| import { list as getNotebooks } from '@/api/development/notebook'; | import { list as getNotebooks } from '@/api/development/notebook'; | ||||
| import { trainConfig } from '@/config'; | import { trainConfig } from '@/config'; | ||||
| import { NOTEBOOK_STATUS_ENUM } from '@/views/development/utils'; | import { NOTEBOOK_STATUS_ENUM } from '@/views/development/utils'; | ||||
| import { IMAGE_TYPE, TRAINING_TYPE_ENUM } from '@/views/trainingJob/utils'; | |||||
| import { TRAINING_TYPE_ENUM } from '@/views/trainingJob/utils'; | |||||
| import BaseTooltip from '@/components/BaseTooltip'; | import BaseTooltip from '@/components/BaseTooltip'; | ||||
| import DataSourceSelector from './dataSourceSelector'; | import DataSourceSelector from './dataSourceSelector'; | ||||
| @@ -929,7 +930,7 @@ export default { | |||||
| }, | }, | ||||
| async getHarborProjects() { | async getHarborProjects() { | ||||
| this.harborProjectList = await getImageNameList({ | this.harborProjectList = await getImageNameList({ | ||||
| imageTypes: [IMAGE_TYPE.TRAIN], | |||||
| imageTypes: [IMAGE_TYPE_ENUM.TRAIN], | |||||
| }); | }); | ||||
| if ( | if ( | ||||
| this.form.imageName && | this.form.imageName && | ||||
| @@ -39,7 +39,7 @@ const covertSVG2Image = (node, name, width, height, type = 'png') => { | |||||
| const context = canvas.getContext('2d'); | const context = canvas.getContext('2d'); | ||||
| context.fillStyle = '#fff'; | context.fillStyle = '#fff'; | ||||
| context.fillRect(0, 0, 10000, 10000); | context.fillRect(0, 0, 10000, 10000); | ||||
| image.onload = function() { | |||||
| image.onload = () => { | |||||
| context.drawImage(image, 0, 0); | context.drawImage(image, 0, 0); | ||||
| const a = document.createElement('a'); | const a = document.createElement('a'); | ||||
| a.download = `${name}.${type}`; | a.download = `${name}.${type}`; | ||||
| @@ -133,6 +133,24 @@ export const K8S_BUSINESS_LABEL_MAP = { | |||||
| [K8S_BUSINESS_LABEL_ENUM.POINT_CLOUD]: '点云数据集', | [K8S_BUSINESS_LABEL_ENUM.POINT_CLOUD]: '点云数据集', | ||||
| }; | }; | ||||
| // 镜像分类枚举 | |||||
| export const IMAGE_TYPE_ENUM = { | |||||
| NOTEBOOK: 0, // notebook镜像类型 | |||||
| TRAIN: 1, // 训练镜像和预置镜像类型 | |||||
| SERVING: 2, // serving镜像 | |||||
| TERMINAL: 3, // 终端镜像 | |||||
| POINTCLOUD: 4, // 点云镜像 | |||||
| DATASETMARKED: 5, // 数据标注镜像 | |||||
| }; | |||||
| export const IMAGE_TYPE_MAP = { | |||||
| [IMAGE_TYPE_ENUM.NOTEBOOK]: 'Notebook镜像', | |||||
| [IMAGE_TYPE_ENUM.TRAIN]: '训练镜像', | |||||
| [IMAGE_TYPE_ENUM.SERVING]: 'Serving镜像', | |||||
| [IMAGE_TYPE_ENUM.TERMINAL]: '终端镜像', | |||||
| [IMAGE_TYPE_ENUM.POINTCLOUD]: '点云镜像', | |||||
| [IMAGE_TYPE_ENUM.DATASETMARKED]: '数据标注镜像', | |||||
| }; | |||||
| // 默认进度条颜色 | // 默认进度条颜色 | ||||
| export const defaultProcessColors = [ | export const defaultProcessColors = [ | ||||
| { color: '#909399', percentage: 40 }, | { color: '#909399', percentage: 40 }, | ||||
| @@ -20,7 +20,7 @@ import ZIP from './zip'; | |||||
| const pMap = require('p-map'); | const pMap = require('p-map'); | ||||
| //streamSaver.mitm = 'https://static.zhejianglab.com/mitm.html'; | |||||
| streamSaver.mitm = 'https://static.zhejianglab.com/mitm.html'; | |||||
| // 默认名字解析 | // 默认名字解析 | ||||
| const defaultName = (file) => file.name; | const defaultName = (file) => file.name; | ||||
| @@ -171,13 +171,13 @@ import { | |||||
| invalidFileNameChar, | invalidFileNameChar, | ||||
| defaultProcessColors, | defaultProcessColors, | ||||
| ALGORITHM_RESOURCE_ENUM, | ALGORITHM_RESOURCE_ENUM, | ||||
| IMAGE_TYPE_ENUM, | |||||
| } from '@/utils'; | } from '@/utils'; | ||||
| import UploadInline from '@/components/UploadForm/inline'; | import UploadInline from '@/components/UploadForm/inline'; | ||||
| import UploadProgress from '@/components/UploadProgress'; | import UploadProgress from '@/components/UploadProgress'; | ||||
| import { useMapGetters } from '@/hooks'; | import { useMapGetters } from '@/hooks'; | ||||
| import { algorithmConfig } from '@/config'; | import { algorithmConfig } from '@/config'; | ||||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | ||||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||||
| const defaultForm = { | const defaultForm = { | ||||
| id: null, | id: null, | ||||
| @@ -289,7 +289,7 @@ export default { | |||||
| }; | }; | ||||
| // 获取镜像名列表 | // 获取镜像名列表 | ||||
| const getImageNames = async (keepValue = false) => { | const getImageNames = async (keepValue = false) => { | ||||
| state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.TRAIN] }); | |||||
| state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.TRAIN] }); | |||||
| if (!keepValue || !form.imageName) { | if (!keepValue || !form.imageName) { | ||||
| form.imageTag = null; | form.imageTag = null; | ||||
| } else if (!state.imageNameList.includes(form.imageName)) { | } else if (!state.imageNameList.includes(form.imageName)) { | ||||
| @@ -199,14 +199,13 @@ | |||||
| <script> | <script> | ||||
| import { mapGetters } from 'vuex'; | import { mapGetters } from 'vuex'; | ||||
| import { getUniqueId, invalidFileNameChar, RESOURCES_MODULE_ENUM } from '@/utils'; | |||||
| import { getUniqueId, invalidFileNameChar, RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils'; | |||||
| import { getServingModel } from '@/api/model/model'; | import { getServingModel } from '@/api/model/model'; | ||||
| import { list as getModelVersions } from '@/api/model/modelVersion'; | import { list as getModelVersions } from '@/api/model/modelVersion'; | ||||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | ||||
| import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; | import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; | ||||
| import { list as getSpecsNames } from '@/api/system/resources'; | import { list as getSpecsNames } from '@/api/system/resources'; | ||||
| import { validateNameWithHyphen } from '@/utils/validate'; | import { validateNameWithHyphen } from '@/utils/validate'; | ||||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||||
| import RunParamForm from '@/components/Training/runParamForm'; | import RunParamForm from '@/components/Training/runParamForm'; | ||||
| import BaseModal from '@/components/BaseModal'; | import BaseModal from '@/components/BaseModal'; | ||||
| import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; | import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; | ||||
| @@ -541,7 +540,7 @@ export default { | |||||
| // 镜像选择 | // 镜像选择 | ||||
| async getImageNames(keepValue = false) { | async getImageNames(keepValue = false) { | ||||
| this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.SERVING] }); | |||||
| this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.SERVING] }); | |||||
| if (!keepValue || !this.form.imageName) { | if (!keepValue || !this.form.imageName) { | ||||
| this.form.imageTag = null; | this.form.imageTag = null; | ||||
| } else if (!this.imageNameList.includes(this.form.imageName)) { | } else if (!this.imageNameList.includes(this.form.imageName)) { | ||||
| @@ -204,8 +204,7 @@ import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||||
| import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; | import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; | ||||
| import { list as getSpecsNames } from '@/api/system/resources'; | import { list as getSpecsNames } from '@/api/system/resources'; | ||||
| import { servingConfig } from '@/config'; | import { servingConfig } from '@/config'; | ||||
| import { RESOURCES_MODULE_ENUM } from '@/utils'; | |||||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||||
| import { RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils'; | |||||
| import RunParamForm from '@/components/Training/runParamForm'; | import RunParamForm from '@/components/Training/runParamForm'; | ||||
| import BaseModal from '@/components/BaseModal'; | import BaseModal from '@/components/BaseModal'; | ||||
| import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; | import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; | ||||
| @@ -540,7 +539,7 @@ export default { | |||||
| // 镜像选择 | // 镜像选择 | ||||
| // 获取镜像名称列表 | // 获取镜像名称列表 | ||||
| async getImageNames(keepValue = false) { | async getImageNames(keepValue = false) { | ||||
| this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.SERVING] }); | |||||
| this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.SERVING] }); | |||||
| if (!keepValue || !this.form.imageName) { | if (!keepValue || !this.form.imageName) { | ||||
| this.form.imageTag = null; | this.form.imageTag = null; | ||||
| } else if (!this.imageNameList.includes(this.form.imageName)) { | } else if (!this.imageNameList.includes(this.form.imageName)) { | ||||
| @@ -554,7 +553,7 @@ export default { | |||||
| async getImageTags(imageName, keepValue = false) { | async getImageTags(imageName, keepValue = false) { | ||||
| this.imageTagList = await getImageTagList({ | this.imageTagList = await getImageTagList({ | ||||
| imageName, | imageName, | ||||
| imageTypes: [IMAGE_TYPE.SERVING], | |||||
| imageTypes: [IMAGE_TYPE_ENUM.SERVING], | |||||
| }); | }); | ||||
| if (keepValue && this.form.imageTag) { | if (keepValue && this.form.imageTag) { | ||||
| if (!this.imageTagList.some((image) => image.imageTag === this.form.imageTag)) { | if (!this.imageTagList.some((image) => image.imageTag === this.form.imageTag)) { | ||||
| @@ -134,8 +134,12 @@ import { list as listBranchModel } from '@/api/model/modelVersion'; | |||||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | ||||
| import { list as listResourceSpec } from '@/api/system/resources'; | import { list as listResourceSpec } from '@/api/system/resources'; | ||||
| import { types } from '@/utils/validate'; | import { types } from '@/utils/validate'; | ||||
| import { RESOURCES_POOL_TYPE_ENUM, RESOURCES_MODULE_ENUM, modelTypeSymbol } from '@/utils/constant'; | |||||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||||
| import { | |||||
| RESOURCES_POOL_TYPE_ENUM, | |||||
| RESOURCES_MODULE_ENUM, | |||||
| modelTypeSymbol, | |||||
| IMAGE_TYPE_ENUM, | |||||
| } from '@/utils/constant'; | |||||
| const initialForm = { | const initialForm = { | ||||
| name: undefined, | name: undefined, | ||||
| @@ -246,7 +250,7 @@ export default { | |||||
| }; | }; | ||||
| const queryImages = () => { | const queryImages = () => { | ||||
| return getImageNameList({ imageTypes: [IMAGE_TYPE.DATASETMARKED] }); | |||||
| return getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.DATASETMARKED] }); | |||||
| }; | }; | ||||
| const handleModelParentChange = () => { | const handleModelParentChange = () => { | ||||
| @@ -143,8 +143,12 @@ import { list as listBranchModel } from '@/api/model/modelVersion'; | |||||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | ||||
| import { list as listResourceSpec } from '@/api/system/resources'; | import { list as listResourceSpec } from '@/api/system/resources'; | ||||
| import { types } from '@/utils/validate'; | import { types } from '@/utils/validate'; | ||||
| import { RESOURCES_POOL_TYPE_ENUM, RESOURCES_MODULE_ENUM, modelTypeSymbol } from '@/utils/constant'; | |||||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||||
| import { | |||||
| RESOURCES_POOL_TYPE_ENUM, | |||||
| RESOURCES_MODULE_ENUM, | |||||
| modelTypeSymbol, | |||||
| IMAGE_TYPE_ENUM, | |||||
| } from '@/utils/constant'; | |||||
| const initialForm = { | const initialForm = { | ||||
| name: undefined, | name: undefined, | ||||
| @@ -283,7 +287,7 @@ export default { | |||||
| // 查询镜像列表 | // 查询镜像列表 | ||||
| const queryImages = () => { | const queryImages = () => { | ||||
| return getImageNameList({ imageTypes: [IMAGE_TYPE.DATASETMARKED] }); | |||||
| return getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.DATASETMARKED] }); | |||||
| }; | }; | ||||
| // 查询镜像版本列表 | // 查询镜像版本列表 | ||||
| @@ -163,8 +163,8 @@ import { | |||||
| ALGORITHM_RESOURCE_ENUM, | ALGORITHM_RESOURCE_ENUM, | ||||
| MODEL_RESOURCE_ENUM, | MODEL_RESOURCE_ENUM, | ||||
| RESOURCES_POOL_TYPE_ENUM, | RESOURCES_POOL_TYPE_ENUM, | ||||
| IMAGE_TYPE_ENUM, | |||||
| } from '@/utils'; | } from '@/utils'; | ||||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||||
| const initFormState = { | const initFormState = { | ||||
| modelId: null, | modelId: null, | ||||
| @@ -285,7 +285,7 @@ export default { | |||||
| return Promise.reject(); | return Promise.reject(); | ||||
| } | } | ||||
| return getImageTagList({ | return getImageTagList({ | ||||
| imageTypes: IMAGE_TYPE.POINTCLOUD, | |||||
| imageTypes: IMAGE_TYPE_ENUM.POINTCLOUD, | |||||
| imageName: modelState.imageName, | imageName: modelState.imageName, | ||||
| }).then((res) => { | }).then((res) => { | ||||
| state.imageTagList = res; | state.imageTagList = res; | ||||
| @@ -294,7 +294,7 @@ export default { | |||||
| // 镜像选择 | // 镜像选择 | ||||
| const getImageNames = async () => { | const getImageNames = async () => { | ||||
| state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.POINTCLOUD] }); | |||||
| state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.POINTCLOUD] }); | |||||
| if ( | if ( | ||||
| modelState.imageName && | modelState.imageName && | ||||
| !state.imageNameList.some((image) => image === modelState.imageName) | !state.imageNameList.some((image) => image === modelState.imageName) | ||||
| @@ -122,14 +122,13 @@ | |||||
| </template> | </template> | ||||
| <script> | <script> | ||||
| import { validateNameWithHyphen, RESOURCES_MODULE_ENUM } from '@/utils'; | |||||
| import { validateNameWithHyphen, RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils'; | |||||
| import BaseModal from '@/components/BaseModal'; | import BaseModal from '@/components/BaseModal'; | ||||
| import InfoSelect from '@/components/InfoSelect'; | import InfoSelect from '@/components/InfoSelect'; | ||||
| import { getPublishedDatasets, getDatasetVersions } from '@/api/preparation/dataset'; | import { getPublishedDatasets, getDatasetVersions } from '@/api/preparation/dataset'; | ||||
| import { add as addNotebook } from '@/api/development/notebook'; | import { add as addNotebook } from '@/api/development/notebook'; | ||||
| import { list as getSpecsNames } from '@/api/system/resources'; | import { list as getSpecsNames } from '@/api/system/resources'; | ||||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage/index'; | import { getImageNameList, getImageTagList } from '@/api/trainingImage/index'; | ||||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||||
| const defaultForm = { | const defaultForm = { | ||||
| noteBookName: null, | noteBookName: null, | ||||
| @@ -208,7 +207,7 @@ export default { | |||||
| } | } | ||||
| }, | }, | ||||
| async getHarborProjects() { | async getHarborProjects() { | ||||
| this.harborProjectList = await getImageNameList({ imageTypes: [IMAGE_TYPE.NOTEBOOK] }); | |||||
| this.harborProjectList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.NOTEBOOK] }); | |||||
| if ( | if ( | ||||
| this.form.imageName && | this.form.imageName && | ||||
| !this.harborProjectList.some((project) => project === this.form.imageName) | !this.harborProjectList.some((project) => project === this.form.imageName) | ||||
| @@ -237,7 +236,7 @@ export default { | |||||
| } | } | ||||
| return getImageTagList({ | return getImageTagList({ | ||||
| imageName: this.form.imageName, | imageName: this.form.imageName, | ||||
| imageTypes: [IMAGE_TYPE.NOTEBOOK], | |||||
| imageTypes: [IMAGE_TYPE_ENUM.NOTEBOOK], | |||||
| }).then((res) => { | }).then((res) => { | ||||
| this.harborImageList = res; | this.harborImageList = res; | ||||
| }); | }); | ||||
| @@ -48,7 +48,7 @@ | |||||
| <el-table | <el-table | ||||
| v-if="prefabricate" | v-if="prefabricate" | ||||
| ref="table" | ref="table" | ||||
| v-loading="crud.loading || disableEdit" | |||||
| v-loading="crud.loading" | |||||
| :data="crud.data" | :data="crud.data" | ||||
| highlight-current-row | highlight-current-row | ||||
| @selection-change="crud.selectionChangeHandler" | @selection-change="crud.selectionChangeHandler" | ||||
| @@ -57,21 +57,6 @@ | |||||
| <el-table-column v-if="!isPreset" prop="id" label="ID" sortable="custom" width="80px" /> | <el-table-column v-if="!isPreset" prop="id" label="ID" sortable="custom" width="80px" /> | ||||
| <el-table-column prop="imageName" label="镜像名称" sortable="custom" /> | <el-table-column prop="imageName" label="镜像名称" sortable="custom" /> | ||||
| <el-table-column prop="imageTag" label="镜像版本号" sortable="custom" /> | <el-table-column prop="imageTag" label="镜像版本号" sortable="custom" /> | ||||
| <el-table-column prop="imageStatus" label="状态" width="130px"> | |||||
| <template #header> | |||||
| <dropdown-header | |||||
| title="状态" | |||||
| :list="imageStatusList" | |||||
| :filtered="Boolean(localQuery.imageStatus)" | |||||
| @command="filterStatus" | |||||
| /> | |||||
| </template> | |||||
| <template slot-scope="scope"> | |||||
| <el-tag effect="plain" :type="map[scope.row.imageStatus]"> | |||||
| {{ statusMap[scope.row.imageStatus] }} | |||||
| </el-tag> | |||||
| </template> | |||||
| </el-table-column> | |||||
| <el-table-column prop="imageTypes" label="镜像用途" width="180px"> | <el-table-column prop="imageTypes" label="镜像用途" width="180px"> | ||||
| <template #header> | <template #header> | ||||
| <dropdown-header | <dropdown-header | ||||
| @@ -91,31 +76,8 @@ | |||||
| <span>{{ parseTime(scope.row.createTime) }}</span> | <span>{{ parseTime(scope.row.createTime) }}</span> | ||||
| </template> | </template> | ||||
| </el-table-column> | </el-table-column> | ||||
| <el-table-column v-if="isNotebook" prop="imageResource" label="是否为默认" align="center"> | |||||
| <template slot-scope="scope"> | |||||
| <i | |||||
| :class="resourceObj(scope.row.imageResource).icon" | |||||
| :style="{ color: resourceObj(scope.row.imageResource).color, fontSize: '20px' }" | |||||
| ></i> | |||||
| </template> | |||||
| </el-table-column> | |||||
| <el-table-column v-if="isAdmin || isCustom" label="操作" width="200px" fixed="right"> | <el-table-column v-if="isAdmin || isCustom" label="操作" width="200px" fixed="right"> | ||||
| <template slot-scope="scope"> | <template slot-scope="scope"> | ||||
| <el-tooltip | |||||
| v-if="isAdmin && isNotebook" | |||||
| effect="dark" | |||||
| content="设为在线编辑算法时创建nootbook的默认镜像" | |||||
| placement="top" | |||||
| > | |||||
| <el-button | |||||
| :id="`doPrecast_` + scope.$index" | |||||
| :disabled="Boolean(scope.row.imageResource)" | |||||
| type="text" | |||||
| @click.stop="doPrecast(scope.row.id)" | |||||
| > | |||||
| {{ resourceObj(scope.row.imageResource).butText }} | |||||
| </el-button> | |||||
| </el-tooltip> | |||||
| <el-button | <el-button | ||||
| v-if="(hasPermission('training:image:edit') && isCustom) || (isPreset && isAdmin)" | v-if="(hasPermission('training:image:edit') && isCustom) || (isPreset && isAdmin)" | ||||
| :id="`doEdit_` + scope.$index" | :id="`doEdit_` + scope.$index" | ||||
| @@ -143,15 +105,13 @@ | |||||
| :visible="crud.status.cu > 0" | :visible="crud.status.cu > 0" | ||||
| :title="crud.status.title" | :title="crud.status.title" | ||||
| :loading="crud.status.cu === 2" | :loading="crud.status.cu === 2" | ||||
| :disabled="loading" | |||||
| width="600px" | width="600px" | ||||
| @open="onDialogOpen" | @open="onDialogOpen" | ||||
| @close="onDialogClose" | |||||
| @cancel="crud.cancelCU" | @cancel="crud.cancelCU" | ||||
| @ok="crud.submitCU" | @ok="crud.submitCU" | ||||
| > | > | ||||
| <el-form ref="form" :model="form" :rules="rules" label-width="120px"> | <el-form ref="form" :model="form" :rules="rules" label-width="120px"> | ||||
| <el-form-item v-if="isEdit && isAdmin" label="镜像类别" prop="imageResource"> | |||||
| <el-form-item v-if="isFormAdd && isAdmin" label="镜像类别" prop="imageResource"> | |||||
| <el-radio-group v-model="form.imageResource" @change="imageResourceChange"> | <el-radio-group v-model="form.imageResource" @change="imageResourceChange"> | ||||
| <el-radio :label="Number(IMAGE_RESOURCE_ENUM.CUSTOM)" border class="mr-0" | <el-radio :label="Number(IMAGE_RESOURCE_ENUM.CUSTOM)" border class="mr-0" | ||||
| >我的镜像</el-radio | >我的镜像</el-radio | ||||
| @@ -159,21 +119,20 @@ | |||||
| <el-radio :label="Number(IMAGE_RESOURCE_ENUM.PRESET)" border>预置镜像</el-radio> | <el-radio :label="Number(IMAGE_RESOURCE_ENUM.PRESET)" border>预置镜像</el-radio> | ||||
| </el-radio-group> | </el-radio-group> | ||||
| </el-form-item> | </el-form-item> | ||||
| <el-form-item v-if="isEdit" label="镜像名称" prop="imageName"> | |||||
| <el-form-item label="镜像名称" prop="imageName"> | |||||
| <el-autocomplete | <el-autocomplete | ||||
| ref="imageName" | ref="imageName" | ||||
| v-model="form.imageName" | v-model="form.imageName" | ||||
| class="inline-input" | |||||
| class="inline-input w-400" | |||||
| :fetch-suggestions="querySearchAsync" | :fetch-suggestions="querySearchAsync" | ||||
| placeholder="请选择或输入镜像名称" | placeholder="请选择或输入镜像名称" | ||||
| style="width: 400px;" | |||||
| ></el-autocomplete> | ></el-autocomplete> | ||||
| </el-form-item> | </el-form-item> | ||||
| <el-form-item label="镜像用途"> | <el-form-item label="镜像用途"> | ||||
| <el-select | <el-select | ||||
| v-model="form.imageTypes" | v-model="form.imageTypes" | ||||
| placeholder="请选择或输入镜像用途" | placeholder="请选择或输入镜像用途" | ||||
| style="width: 400px;" | |||||
| class="w-400" | |||||
| multiple | multiple | ||||
| filterable | filterable | ||||
| allow-create | allow-create | ||||
| @@ -187,36 +146,18 @@ | |||||
| /> | /> | ||||
| </el-select> | </el-select> | ||||
| </el-form-item> | </el-form-item> | ||||
| <el-form-item v-if="isEdit" ref="imagePath" label="镜像文件路径" prop="imagePath"> | |||||
| <upload-inline | |||||
| v-if="crud.status.cu > 0" | |||||
| ref="upload" | |||||
| action="fakeApi" | |||||
| accept=".zip,.tar,.rar,.gz" | |||||
| list-type="text" | |||||
| :acceptSize="imageConfig.uploadFileAcceptSize" | |||||
| :acceptSizeFormat="uploadSizeFomatter" | |||||
| :params="uploadParams" | |||||
| :show-file-count="false" | |||||
| :auto-upload="true" | |||||
| :filters="uploadFilters" | |||||
| :limit="1" | |||||
| :on-remove="onFileRemove" | |||||
| @uploadStart="uploadStart" | |||||
| @uploadSuccess="uploadSuccess" | |||||
| @uploadError="uploadError" | |||||
| /> | |||||
| <upload-progress | |||||
| v-if="loading" | |||||
| :progress="progress" | |||||
| :color="customColors" | |||||
| :status="status" | |||||
| :size="size" | |||||
| @onSetProgress="onSetProgress" | |||||
| /> | |||||
| <el-form-item ref="imageUrl" label="镜像地址" prop="imageUrl"> | |||||
| <el-input v-model="form.imageUrl" placeholder="请输入镜像地址" class="w-400" /> | |||||
| <BaseTooltip icon="el-icon-warning" class="c-info"> | |||||
| <template #content> | |||||
| <div> | |||||
| 镜像地址标准格式:镜像仓库域名/命名空间/镜像名:镜像版本号<br />示例:registry.cn-hangzhou.aliyuncs.com/enlin/notebook:v1 | |||||
| </div> | |||||
| </template> | |||||
| </BaseTooltip> | |||||
| </el-form-item> | </el-form-item> | ||||
| <el-form-item v-if="isEdit" label="镜像版本号" prop="imageTag"> | |||||
| <el-input id="imageTag" v-model="form.imageTag" style="width: 400px;" /> | |||||
| <el-form-item label="镜像版本号" prop="imageTag"> | |||||
| <el-input id="imageTag" v-model="form.imageTag" class="w-400" /> | |||||
| </el-form-item> | </el-form-item> | ||||
| <el-form-item label="描述" prop="remark"> | <el-form-item label="描述" prop="remark"> | ||||
| <el-input | <el-input | ||||
| @@ -227,19 +168,19 @@ | |||||
| maxlength="1024" | maxlength="1024" | ||||
| show-word-limit | show-word-limit | ||||
| placeholder | placeholder | ||||
| style="width: 400px;" | |||||
| class="w-400" | |||||
| /> | /> | ||||
| </el-form-item> | </el-form-item> | ||||
| </el-form> | </el-form> | ||||
| </BaseModal> | </BaseModal> | ||||
| <!--Notebook默认镜像设置表单--> | <!--Notebook默认镜像设置表单--> | ||||
| <BaseModal | <BaseModal | ||||
| :visible.sync="formVisible" | |||||
| :visible.sync="notebookFormVisible" | |||||
| title="Notebook默认镜像设置" | title="Notebook默认镜像设置" | ||||
| :loading="formSubmitting" | |||||
| :loading="notebookFormSubmitting" | |||||
| width="800px" | width="800px" | ||||
| @cancel="formVisible = false" | |||||
| @ok="onSubmitForm" | |||||
| @cancel="notebookFormVisible = false" | |||||
| @ok="onSubmitNotebookForm" | |||||
| > | > | ||||
| <el-form | <el-form | ||||
| ref="noteBookFormRef" | ref="noteBookFormRef" | ||||
| @@ -253,7 +194,7 @@ | |||||
| id="defaultImage" | id="defaultImage" | ||||
| v-model="noteBookForm.defaultImage" | v-model="noteBookForm.defaultImage" | ||||
| placeholder="请选择镜像" | placeholder="请选择镜像" | ||||
| style="width: 400px;" | |||||
| class="w-400" | |||||
| filterable | filterable | ||||
| allow-create | allow-create | ||||
| default-first-option | default-first-option | ||||
| @@ -283,8 +224,6 @@ | |||||
| <script> | <script> | ||||
| import { mapGetters } from 'vuex'; | import { mapGetters } from 'vuex'; | ||||
| // eslint-disable-next-line import/no-extraneous-dependencies | |||||
| import { debounce } from 'throttle-debounce'; | |||||
| import cdOperation from '@crud/CD.operation'; | import cdOperation from '@crud/CD.operation'; | ||||
| import rrOperation from '@crud/RR.operation'; | import rrOperation from '@crud/RR.operation'; | ||||
| @@ -292,40 +231,36 @@ import pagination from '@crud/Pagination'; | |||||
| import CRUD, { presenter, header, form, crud } from '@crud/crud'; | import CRUD, { presenter, header, form, crud } from '@crud/crud'; | ||||
| import trainingImageApi, { | import trainingImageApi, { | ||||
| del, | del, | ||||
| setPrecast, | |||||
| setDefaultImage, | setDefaultImage, | ||||
| getDefaultImage, | getDefaultImage, | ||||
| getImageNameList, | getImageNameList, | ||||
| getImageTagList, | getImageTagList, | ||||
| } from '@/api/trainingImage/index'; | } from '@/api/trainingImage/index'; | ||||
| import { | import { | ||||
| getUniqueId, | |||||
| uploadSizeFomatter, | |||||
| invalidFileNameChar, | |||||
| ADMIN_ROLE_ID, | ADMIN_ROLE_ID, | ||||
| hasPermission, | hasPermission, | ||||
| validateImageName, | validateImageName, | ||||
| validateImageTag, | validateImageTag, | ||||
| IMAGE_TYPE_ENUM, | |||||
| IMAGE_TYPE_MAP, | |||||
| } from '@/utils'; | } from '@/utils'; | ||||
| import BaseModal from '@/components/BaseModal'; | import BaseModal from '@/components/BaseModal'; | ||||
| import UploadInline from '@/components/UploadForm/inline'; | |||||
| import BaseTooltip from '@/components/BaseTooltip'; | |||||
| import DropdownHeader from '@/components/DropdownHeader'; | import DropdownHeader from '@/components/DropdownHeader'; | ||||
| import UploadProgress from '@/components/UploadProgress'; | |||||
| import { imageConfig } from '@/config'; | import { imageConfig } from '@/config'; | ||||
| import { IMAGE_RESOURCE_ENUM, IMAGE_TYPE } from '../trainingJob/utils'; | |||||
| import { IMAGE_RESOURCE_ENUM } from '../trainingJob/utils'; | |||||
| const defaultForm = { | const defaultForm = { | ||||
| imagePath: null, | |||||
| imageUrl: null, | |||||
| imageTag: null, | imageTag: null, | ||||
| remark: null, | remark: null, | ||||
| imageTypes: Number(IMAGE_TYPE), | |||||
| imageTypes: Number(IMAGE_TYPE_ENUM), | |||||
| imageResource: Number(IMAGE_RESOURCE_ENUM.CUSTOM), | imageResource: Number(IMAGE_RESOURCE_ENUM.CUSTOM), | ||||
| imageName: null, | imageName: null, | ||||
| }; | }; | ||||
| const defaultQuery = { | const defaultQuery = { | ||||
| imageStatus: null, | |||||
| imageNameOrId: null, | imageNameOrId: null, | ||||
| imageTypes: null, | imageTypes: null, | ||||
| }; | }; | ||||
| @@ -334,28 +269,27 @@ export default { | |||||
| name: 'TrainingImage', | name: 'TrainingImage', | ||||
| components: { | components: { | ||||
| BaseModal, | BaseModal, | ||||
| BaseTooltip, | |||||
| pagination, | pagination, | ||||
| cdOperation, | cdOperation, | ||||
| rrOperation, | rrOperation, | ||||
| UploadInline, | |||||
| DropdownHeader, | DropdownHeader, | ||||
| UploadProgress, | |||||
| }, | }, | ||||
| cruds() { | cruds() { | ||||
| return CRUD({ | return CRUD({ | ||||
| title: '镜像', | title: '镜像', | ||||
| crudMethod: { ...trainingImageApi }, | crudMethod: { ...trainingImageApi }, | ||||
| optShow: { | optShow: { | ||||
| add: imageConfig.allowUploadImage && hasPermission('training:image:upload'), | |||||
| add: imageConfig.allowUploadImage && hasPermission('training:image:save'), | |||||
| del: false, | del: false, | ||||
| }, | }, | ||||
| queryOnPresenterCreated: false, | queryOnPresenterCreated: false, | ||||
| props: { | props: { | ||||
| optText: { | optText: { | ||||
| add: '上传镜像', | |||||
| add: '创建镜像', | |||||
| }, | }, | ||||
| optTitle: { | optTitle: { | ||||
| add: '上传', | |||||
| add: '创建', | |||||
| }, | }, | ||||
| }, | }, | ||||
| }); | }); | ||||
| @@ -365,24 +299,6 @@ export default { | |||||
| return { | return { | ||||
| active: IMAGE_RESOURCE_ENUM.CUSTOM, | active: IMAGE_RESOURCE_ENUM.CUSTOM, | ||||
| localQuery: { ...defaultQuery }, | localQuery: { ...defaultQuery }, | ||||
| map: { | |||||
| 0: 'info', | |||||
| 1: 'success', | |||||
| 2: 'danger', | |||||
| }, | |||||
| statusMap: { | |||||
| 0: '制作中', | |||||
| 1: '制作成功', | |||||
| 2: '制作失败', | |||||
| }, | |||||
| imageTypesMap: { | |||||
| 0: 'Notebook镜像', | |||||
| 1: '训练镜像', | |||||
| 2: 'Serving镜像', | |||||
| 3: '终端镜像', | |||||
| 4: '点云镜像', | |||||
| 5: '数据标注镜像', | |||||
| }, | |||||
| rules: { | rules: { | ||||
| imageTypes: [{ required: true, message: '请选择镜像类型', trigger: 'change' }], | imageTypes: [{ required: true, message: '请选择镜像类型', trigger: 'change' }], | ||||
| imageResource: [{ required: true, message: '请选择镜像来源', trigger: 'change' }], | imageResource: [{ required: true, message: '请选择镜像来源', trigger: 'change' }], | ||||
| @@ -390,7 +306,7 @@ export default { | |||||
| { required: true, message: '请选择项目名称', trigger: 'change' }, | { required: true, message: '请选择项目名称', trigger: 'change' }, | ||||
| { validator: validateImageName, trigger: ['blur', 'change'] }, | { validator: validateImageName, trigger: ['blur', 'change'] }, | ||||
| ], | ], | ||||
| imagePath: [{ required: true, message: '请输入镜像路径', trigger: ['blur', 'manual'] }], | |||||
| imageUrl: [{ required: true, message: '请输入镜像地址', trigger: ['blur', 'manual'] }], | |||||
| imageTag: [ | imageTag: [ | ||||
| { required: true, message: '请输入镜像版本号', trigger: 'blur' }, | { required: true, message: '请输入镜像版本号', trigger: 'blur' }, | ||||
| { validator: validateImageTag, trigger: ['blur', 'change'] }, | { validator: validateImageTag, trigger: ['blur', 'change'] }, | ||||
| @@ -401,32 +317,17 @@ export default { | |||||
| id: [{ required: true, message: '请选择默认镜像版本', trigger: 'blur' }], | id: [{ required: true, message: '请选择默认镜像版本', trigger: 'blur' }], | ||||
| }, | }, | ||||
| harborProjectList: [], | harborProjectList: [], | ||||
| drawer: false, | |||||
| uploadParams: { | |||||
| objectPath: null, // 对象存储路径 | |||||
| }, | |||||
| progress: 0, | |||||
| size: 0, | |||||
| customColors: [ | |||||
| { color: '#909399', percentage: 40 }, | |||||
| { color: '#e6a23c', percentage: 80 }, | |||||
| { color: '#67c23a', percentage: 100 }, | |||||
| ], | |||||
| disableEdit: false, | |||||
| loading: false, | |||||
| isEdit: false, | |||||
| prefabricate: true, | prefabricate: true, | ||||
| // 以下为配置参数及常量参数 | // 以下为配置参数及常量参数 | ||||
| imageConfig, | |||||
| IMAGE_RESOURCE_ENUM, | IMAGE_RESOURCE_ENUM, | ||||
| IMAGE_TYPE, | |||||
| uploadFilters: [invalidFileNameChar], | |||||
| // 设置notebook默认镜像相关参数 | // 设置notebook默认镜像相关参数 | ||||
| noteBookImages: [], | noteBookImages: [], | ||||
| noteBookTags: [], | noteBookTags: [], | ||||
| noteBookForm: { defaultImage: '', defaultTag: '', id: '' }, | noteBookForm: { defaultImage: '', defaultTag: '', id: '' }, | ||||
| formVisible: false, | |||||
| formSubmitting: false, | |||||
| formType: 'add', | |||||
| notebookFormVisible: false, | |||||
| notebookFormSubmitting: false, | |||||
| }; | }; | ||||
| }, | }, | ||||
| computed: { | computed: { | ||||
| @@ -441,14 +342,8 @@ export default { | |||||
| isPreset() { | isPreset() { | ||||
| return this.active === IMAGE_RESOURCE_ENUM.PRESET; | return this.active === IMAGE_RESOURCE_ENUM.PRESET; | ||||
| }, | }, | ||||
| isNotebook() { | |||||
| return this.active === IMAGE_RESOURCE_ENUM.NOTEBOOK; | |||||
| }, | |||||
| isTerminal() { | |||||
| return this.active === IMAGE_RESOURCE_ENUM.TERMINAL; | |||||
| }, | |||||
| disableAdd() { | disableAdd() { | ||||
| if (this.isAdmin) return this.isTerminal; // 管理员只有在终端镜像处无法点击上传 | |||||
| if (this.isAdmin) return false; // 管理员可以创建我的镜像和预置镜像 | |||||
| return !this.isCustom; // 其他角色只有在我的镜像处可以点击上传 | return !this.isCustom; // 其他角色只有在我的镜像处可以点击上传 | ||||
| }, | }, | ||||
| operationProps() { | operationProps() { | ||||
| @@ -456,37 +351,24 @@ export default { | |||||
| disabled: this.disableAdd, | disabled: this.disableAdd, | ||||
| }; | }; | ||||
| }, | }, | ||||
| imageStatusList() { | |||||
| const arr = [{ label: '全部', value: null }]; | |||||
| for (const key in this.statusMap) { | |||||
| arr.push({ label: this.statusMap[key], value: key }); | |||||
| } | |||||
| return arr; | |||||
| }, | |||||
| imageTypesList() { | imageTypesList() { | ||||
| const arr = [{ label: '全部', value: null }]; | const arr = [{ label: '全部', value: null }]; | ||||
| for (const key in this.imageTypesMap) { | |||||
| arr.push({ label: this.imageTypesMap[key], value: +key }); | |||||
| for (const key in IMAGE_TYPE_MAP) { | |||||
| arr.push({ label: IMAGE_TYPE_MAP[key], value: +key }); | |||||
| } | } | ||||
| return arr; | return arr; | ||||
| }, | }, | ||||
| status() { | |||||
| return this.progress === 100 ? 'success' : null; | |||||
| isFormAdd() { | |||||
| return this.formType === 'add'; | |||||
| }, | }, | ||||
| }, | }, | ||||
| mounted() { | mounted() { | ||||
| this.crud.refresh(); | this.crud.refresh(); | ||||
| this.refetch = debounce(3000, this.crud.refresh); | |||||
| this.updateImagePath(); | |||||
| }, | }, | ||||
| methods: { | methods: { | ||||
| hasPermission, | hasPermission, | ||||
| getImageTypes(imageTypes) { | getImageTypes(imageTypes) { | ||||
| const usageStr = []; | |||||
| imageTypes.forEach((item) => { | |||||
| usageStr.push(this.imageTypesMap[item]); | |||||
| }); | |||||
| return usageStr.join(','); | |||||
| return imageTypes.map((type) => IMAGE_TYPE_MAP[type]).join(','); | |||||
| }, | }, | ||||
| getNoteBookTags() { | getNoteBookTags() { | ||||
| this.noteBookForm.defaultTag = null; | this.noteBookForm.defaultTag = null; | ||||
| @@ -497,7 +379,7 @@ export default { | |||||
| } | } | ||||
| return getImageTagList({ | return getImageTagList({ | ||||
| imageName: this.noteBookForm.defaultImage, | imageName: this.noteBookForm.defaultImage, | ||||
| imageTypes: IMAGE_TYPE.NOTEBOOK, | |||||
| imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK, | |||||
| imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | ||||
| }).then((res) => { | }).then((res) => { | ||||
| this.noteBookTags = res; | this.noteBookTags = res; | ||||
| @@ -513,46 +395,11 @@ export default { | |||||
| this.prefabricate = true; | this.prefabricate = true; | ||||
| }); | }); | ||||
| }, | }, | ||||
| onFileRemove() { | |||||
| this.form.imagePath = null; | |||||
| this.loading = false; | |||||
| this.$refs.imagePath.validate('manual'); | |||||
| }, | |||||
| uploadStart(files) { | |||||
| this.updateImagePath(); | |||||
| [this.loading, this.size, this.progress] = [true, files.size, 0]; | |||||
| }, | |||||
| onSetProgress(val) { | |||||
| this.progress += val; | |||||
| }, | |||||
| uploadSuccess(res) { | |||||
| this.progress = 100; | |||||
| setTimeout(() => { | |||||
| this.loading = false; | |||||
| }, 1000); | |||||
| if (this.loading) { | |||||
| this.form.imagePath = res[0].data.objectName; | |||||
| this.$refs.imagePath.validate('manual'); | |||||
| } | |||||
| }, | |||||
| uploadError() { | |||||
| this.$message({ | |||||
| message: '上传文件失败', | |||||
| type: 'error', | |||||
| }); | |||||
| this.loading = false; | |||||
| }, | |||||
| // hook | // hook | ||||
| [CRUD.HOOK.afterRefresh]() { | |||||
| this.checkStatus(); | |||||
| }, | |||||
| [CRUD.HOOK.beforeToAdd]() { | [CRUD.HOOK.beforeToAdd]() { | ||||
| this.isEdit = true; | |||||
| this.formType = 'add'; | this.formType = 'add'; | ||||
| if (this.isPreset) { | if (this.isPreset) { | ||||
| this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.PRESET); | this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.PRESET); | ||||
| } else if (this.isNotebook) { | |||||
| this.form.imageTypes = IMAGE_TYPE.NOTEBOOK; | |||||
| } | } | ||||
| }, | }, | ||||
| [CRUD.HOOK.beforeRefresh]() { | [CRUD.HOOK.beforeRefresh]() { | ||||
| @@ -567,7 +414,7 @@ export default { | |||||
| } | } | ||||
| }, | }, | ||||
| [CRUD.HOOK.beforeToEdit]() { | [CRUD.HOOK.beforeToEdit]() { | ||||
| this.isEdit = false; | |||||
| this.formType = 'edit'; | |||||
| }, | }, | ||||
| async querySearchAsync(queryString, cb) { | async querySearchAsync(queryString, cb) { | ||||
| let { harborProjectList } = this; | let { harborProjectList } = this; | ||||
| @@ -597,24 +444,9 @@ export default { | |||||
| this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.CUSTOM); | this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.CUSTOM); | ||||
| this.getImageNameList(); | this.getImageNameList(); | ||||
| }, | }, | ||||
| onDialogClose() { | |||||
| if (this.isEdit) { | |||||
| this.$refs.upload.formRef.reset(); | |||||
| } | |||||
| this.loading = false; | |||||
| }, | |||||
| async onDialogOpen() { | async onDialogOpen() { | ||||
| this.getImageNameList(); | this.getImageNameList(); | ||||
| }, | }, | ||||
| checkStatus() { | |||||
| if (this.crud.data.some((item) => [0].includes(item.imageStatus))) { | |||||
| this.refetch(); | |||||
| } | |||||
| }, | |||||
| filterStatus(status) { | |||||
| this.localQuery.imageStatus = status; | |||||
| this.crud.toQuery(); | |||||
| }, | |||||
| filterImageTypes(imageTypes) { | filterImageTypes(imageTypes) { | ||||
| this.localQuery.imageTypes = imageTypes; | this.localQuery.imageTypes = imageTypes; | ||||
| this.crud.toQuery(); | this.crud.toQuery(); | ||||
| @@ -622,9 +454,6 @@ export default { | |||||
| resetQuery() { | resetQuery() { | ||||
| this.localQuery = { ...defaultQuery }; | this.localQuery = { ...defaultQuery }; | ||||
| }, | }, | ||||
| updateImagePath() { | |||||
| this.uploadParams.objectPath = `upload-temp/${this.user.id}/${getUniqueId()}`; | |||||
| }, | |||||
| async doEdit(imageObj) { | async doEdit(imageObj) { | ||||
| const dataObj = { | const dataObj = { | ||||
| ids: [imageObj.id], | ids: [imageObj.id], | ||||
| @@ -642,21 +471,6 @@ export default { | |||||
| this.crud.refresh(); | this.crud.refresh(); | ||||
| }); | }); | ||||
| }, | }, | ||||
| doPrecast(i) { | |||||
| setPrecast({ id: i }).then(() => { | |||||
| this.$message({ | |||||
| message: '设置成功', | |||||
| type: 'success', | |||||
| }); | |||||
| this.crud.refresh(); | |||||
| }); | |||||
| }, | |||||
| resourceObj(resource) { | |||||
| return resource | |||||
| ? { icon: 'el-icon-circle-check', color: '#67C23A', butText: '当前为默认' } | |||||
| : { icon: 'el-icon-circle-close', color: '#F56C6C', butText: '设为默认' }; | |||||
| }, | |||||
| uploadSizeFomatter, | |||||
| async doSetDefaultImage() { | async doSetDefaultImage() { | ||||
| // 获取默认镜像 | // 获取默认镜像 | ||||
| @@ -666,20 +480,20 @@ export default { | |||||
| this.noteBookForm.id = defaultImage.length ? defaultImage[0].id : ''; | this.noteBookForm.id = defaultImage.length ? defaultImage[0].id : ''; | ||||
| // 获取镜像列表 | // 获取镜像列表 | ||||
| this.noteBookImages = await getImageNameList({ | this.noteBookImages = await getImageNameList({ | ||||
| imageTypes: IMAGE_TYPE.NOTEBOOK, | |||||
| imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK, | |||||
| imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | ||||
| }); | }); | ||||
| if (this.noteBookForm.defaultImage) { | if (this.noteBookForm.defaultImage) { | ||||
| this.noteBookTags = await getImageTagList({ | this.noteBookTags = await getImageTagList({ | ||||
| imageName: this.noteBookForm.defaultImage, | imageName: this.noteBookForm.defaultImage, | ||||
| imageTypes: IMAGE_TYPE.NOTEBOOK, | |||||
| imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK, | |||||
| imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | ||||
| }); | }); | ||||
| } else { | } else { | ||||
| this.noteBookTags = []; | this.noteBookTags = []; | ||||
| } | } | ||||
| this.formVisible = true; | |||||
| this.notebookFormVisible = true; | |||||
| this.$nextTick(() => { | this.$nextTick(() => { | ||||
| this.clearValidate(); | this.clearValidate(); | ||||
| }); | }); | ||||
| @@ -690,17 +504,17 @@ export default { | |||||
| validateField(field) { | validateField(field) { | ||||
| this.$refs.noteBookFormRef.validateField(field); | this.$refs.noteBookFormRef.validateField(field); | ||||
| }, | }, | ||||
| onSubmitForm() { | |||||
| onSubmitNotebookForm() { | |||||
| this.$refs.noteBookFormRef.validate((valid) => { | this.$refs.noteBookFormRef.validate((valid) => { | ||||
| if (valid) { | if (valid) { | ||||
| this.formSubmitting = true; | |||||
| this.notebookFormSubmitting = true; | |||||
| setDefaultImage({ id: this.noteBookForm.id }) | setDefaultImage({ id: this.noteBookForm.id }) | ||||
| .then(() => { | .then(() => { | ||||
| this.formVisible = false; | |||||
| this.notebookFormVisible = false; | |||||
| this.crud.toQuery(); | this.crud.toQuery(); | ||||
| }) | }) | ||||
| .finally(() => { | .finally(() => { | ||||
| this.formSubmitting = false; | |||||
| this.notebookFormSubmitting = false; | |||||
| }); | }); | ||||
| } | } | ||||
| }); | }); | ||||
| @@ -96,15 +96,6 @@ export const ATLAS_ALGORITHM_TYPE_ENUM = { | |||||
| DDRL: 4, // ddrl | DDRL: 4, // ddrl | ||||
| }; | }; | ||||
| export const IMAGE_TYPE = { | |||||
| NOTEBOOK: 0, // notebook镜像类型 | |||||
| TRAIN: 1, // 训练镜像和预置镜像类型 | |||||
| SERVING: 2, // serving镜像 | |||||
| TERMINAL: 3, // 终端镜像 | |||||
| POINTCLOUD: 4, // 点云镜像 | |||||
| DATASETMARKED: 5, // 数据标注镜像 | |||||
| }; | |||||
| // 训练可视化列表页查询项 | // 训练可视化列表页查询项 | ||||
| export const trainVisualQueryFormItems = [ | export const trainVisualQueryFormItems = [ | ||||
| { | { | ||||