| @@ -11,8 +11,8 @@ Nacos + Fegin + Gateway + (Spring Security + JWT + OAuth2) | |||
| 初始化sql位置 /sql | |||
| **地址:** 127.0.0.1:3306 | |||
| **用户名:** test **密码:** test | |||
| **地址:** 127.0.0.1 | |||
| **用户名:** test **密码:** zj12345678 | |||
| ### Nacos | |||
| @@ -27,7 +27,6 @@ import java.util.Map; | |||
| * @description Feign 熔断处理类 | |||
| * @date 2020-11-04 | |||
| */ | |||
| @Component | |||
| public class AuthServiceFallback implements AuthServiceClient { | |||
| @@ -16,6 +16,7 @@ | |||
| */ | |||
| package org.dubhe.admin.service.impl; | |||
| import cn.hutool.core.collection.CollUtil; | |||
| import cn.hutool.core.util.ObjectUtil; | |||
| import cn.hutool.core.util.StrUtil; | |||
| import cn.hutool.crypto.asymmetric.KeyType; | |||
| @@ -704,6 +705,9 @@ public class UserServiceImpl extends ServiceImpl<UserMapper, User> implements Us | |||
| Oauth2TokenDTO userDto = restResult.getData(); | |||
| UserDTO user = findByName(authUserDTO.getUsername()); | |||
| Set<String> permissions = this.queryPermissionByUserId(user.getId()); | |||
| if (CollUtil.isEmpty(permissions)) { | |||
| throw new BusinessException(BaseErrorCodeEnum.SYSTEM_ROLE_NOT_EXISTS); | |||
| } | |||
| // 返回 token 与 用户信息 | |||
| authInfo.put("token", userDto.getTokenHead() + userDto.getToken()); | |||
| authInfo.put("user", user); | |||
| @@ -14,7 +14,6 @@ | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.admin; | |||
| import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | |||
| @@ -14,7 +14,6 @@ | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.auth; | |||
| import org.junit.jupiter.api.Test; | |||
| @@ -1,37 +0,0 @@ | |||
| /** | |||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.biz.base.constant; | |||
| import lombok.Data; | |||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||
| import org.springframework.stereotype.Component; | |||
| /** | |||
| * @description harbor相关配置 | |||
| * @date 2020-07-17 | |||
| */ | |||
| @Data | |||
| @Component | |||
| @ConfigurationProperties(prefix = "harbor") | |||
| public class HarborProperties { | |||
| private String address; | |||
| private String username; | |||
| private String password; | |||
| } | |||
| @@ -110,7 +110,7 @@ public final class Permissions { | |||
| * 镜像管理 | |||
| */ | |||
| public static final String IMAGE = "hasAuthority('ROLE_training:image')"; | |||
| public static final String IMAGE_UPLOAD = "hasAuthority('ROLE_training:image:upload')"; | |||
| public static final String IMAGE_SAVE = "hasAuthority('ROLE_training:image:save')"; | |||
| public static final String IMAGE_EDIT = "hasAuthority('ROLE_training:image:edit')"; | |||
| public static final String IMAGE_DELETE = "hasAuthority('ROLE_training:image:delete')"; | |||
| public static final String IMAGE_EDIT_DEFAULT = "hasAuthority('ROLE_training:image:editDefault')"; | |||
| @@ -126,9 +126,6 @@ public final class StringConstant { | |||
| public static final String CACHE_TASK_ID ="task_id"; | |||
| public static final String CACHE_TASK_NAME ="task_name"; | |||
| public static final String DEFAULT_IMAGE_PROJECT ="dubhe"; | |||
| /** | |||
| * python命令行参数格式 | |||
| */ | |||
| @@ -54,8 +54,8 @@ public enum BaseErrorCodeEnum implements ErrorCode { | |||
| SYSTEM_USER_EMAIL_NOT_EXISTS(20013, "该邮箱未注册!"), | |||
| SYSTEM_USER_CANNOT_DELETE(20014, "系统默认用户不可删除!"), | |||
| SYSTEM_ROLE_CANNOT_DELETE(20015, "系统默认角色不可删除!"), | |||
| DATASET_ADMIN_PERMISSION_ERROR(1310,"无此权限,请联系管理员"), | |||
| SYSTEM_ROLE_NOT_EXISTS(20016, "用户绑定角色异常,请联系管理员!"), | |||
| DATASET_ADMIN_PERMISSION_ERROR(1310, "无此权限,请联系管理员"), | |||
| ; | |||
| @@ -1,53 +0,0 @@ | |||
| /** | |||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.biz.base.enums; | |||
| /** | |||
| * @description 镜像运行状态枚举 | |||
| * @date 2020-07-15 | |||
| **/ | |||
| public enum ImageStateEnum { | |||
| MAKING(0, "制作中"), | |||
| SUCCESS(1, "制作成功"), | |||
| FAIL(2, "制作失败"); | |||
| /** | |||
| * 编码 | |||
| */ | |||
| private Integer code; | |||
| /** | |||
| * 描述 | |||
| */ | |||
| private String description; | |||
| ImageStateEnum(int code, String description) { | |||
| this.code = code; | |||
| this.description = description; | |||
| } | |||
| public Integer getCode() { | |||
| return code; | |||
| } | |||
| public String getDescription() { | |||
| return description; | |||
| } | |||
| } | |||
| @@ -1,10 +1,12 @@ | |||
| spring: | |||
| cloud: | |||
| nacos: | |||
| username: nacos | |||
| password: Tianshu | |||
| context-path: /nacos | |||
| config: | |||
| namespace: dubhe-prod | |||
| server-addr: 127.0.0.1:8848 | |||
| server-addr: 10.105.1.132:8848 | |||
| discovery: | |||
| namespace: dubhe-prod | |||
| server-addr: 127.0.0.1:8848 | |||
| server-addr: 10.105.1.132:8848 | |||
| @@ -1,10 +0,0 @@ | |||
| spring: | |||
| cloud: | |||
| nacos: | |||
| context-path: /nacos | |||
| config: | |||
| namespace: dubhe-prod | |||
| server-addr: 127.0.0.1:8848 | |||
| discovery: | |||
| namespace: dubhe-prod | |||
| server-addr: 127.0.0.1:8848 | |||
| @@ -207,4 +207,11 @@ public interface NodeApi { | |||
| * @return | |||
| */ | |||
| List<BizTaint> geBizTaintListByUserId(); | |||
| /** | |||
| * 获取一个可用node的ip | |||
| * | |||
| * @return String | |||
| */ | |||
| String getAvailableNodeIp(); | |||
| } | |||
| @@ -389,14 +389,11 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
| StatefulSet statefulSet = deployStatefulSet(); | |||
| //部署svc | |||
| Service service = deployService(); | |||
| //部署ingress | |||
| Ingress ingress = deployIngress(); | |||
| if (delayDelete != null && delayDelete > ZERO) { | |||
| taskYamlBO.append(secret); | |||
| taskYamlBO.append(statefulSet); | |||
| taskYamlBO.append(service); | |||
| taskYamlBO.append(ingress); | |||
| long stopUnixTime = System.currentTimeMillis() / THOUSAND_LONG + delayDelete * SIXTY_LONG; | |||
| Timestamp stopDisplayTime = new Timestamp(stopUnixTime * THOUSAND_LONG); | |||
| @@ -412,7 +409,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
| k8sTaskService.createOrUpdateTask(k8sTask); | |||
| } | |||
| return new PtJupyterDeployVO(secret, statefulSet, service, ingress); | |||
| return new PtJupyterDeployVO(secret, statefulSet, service, null); | |||
| } | |||
| /** | |||
| @@ -440,7 +437,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
| .build(); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}", secretName); | |||
| secret = client.secrets().create(secret); | |||
| secret = client.secrets().inNamespace(namespace).create(secret); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", secretName); | |||
| } | |||
| @@ -647,7 +644,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
| .build(); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", statefulSetName, YamlUtils.dumpAsYaml(statefulSet)); | |||
| resourceIisolationApi.addIisolationInfo(statefulSet); | |||
| statefulSet = client.apps().statefulSets().create(statefulSet); | |||
| statefulSet = client.apps().statefulSets().inNamespace(namespace).create(statefulSet); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", statefulSetName); | |||
| return statefulSet; | |||
| } | |||
| @@ -678,22 +675,23 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
| .withTargetPort(new IntOrString(CONTAINER_PORT)) | |||
| .withName(CONTAINER_NAME) | |||
| .endPort() | |||
| .withClusterIP("None") | |||
| .withSelector(podLabels) | |||
| .withType("NodePort") | |||
| .endSpec() | |||
| .build(); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", svcName, YamlUtils.dumpAsYaml(svc)); | |||
| svc = client.services().create(svc); | |||
| svc = client.services().inNamespace(svc.getMetadata().getNamespace()).create(svc); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", svcName); | |||
| return svc; | |||
| } | |||
| /** | |||
| * 部署ingress | |||
| * 部署ingress,弃用泛域名,暴露方式由ingress改为 service nodePort | |||
| * | |||
| * @return Ingress 类 | |||
| */ | |||
| @Deprecated | |||
| private Ingress deployIngress() { | |||
| Ingress ingress = null; | |||
| IngressList list = client.extensions().ingresses().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(baseName)).list(); | |||
| @@ -727,7 +725,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
| .endSpec() | |||
| .build(); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", ingressName, YamlUtils.dumpAsYaml(ingress)); | |||
| ingress = client.extensions().ingresses().create(ingress); | |||
| ingress = client.extensions().ingresses().inNamespace(namespace).create(ingress); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", ingressName); | |||
| return ingress; | |||
| } | |||
| @@ -65,6 +65,7 @@ import org.dubhe.k8s.enums.K8sKindEnum; | |||
| import org.dubhe.k8s.enums.K8sResponseEnum; | |||
| import org.dubhe.k8s.enums.LimitsOfResourcesEnum; | |||
| import org.dubhe.k8s.enums.RestartPolicyEnum; | |||
| import org.dubhe.k8s.enums.ServiceTypeENum; | |||
| import org.dubhe.k8s.enums.ShellCommandEnum; | |||
| import org.dubhe.k8s.utils.BizConvertUtils; | |||
| import org.dubhe.k8s.utils.K8sUtils; | |||
| @@ -102,15 +103,6 @@ public class ModelServingApiImpl implements ModelServingApi { | |||
| @Autowired | |||
| private ResourceIisolationApi resourceIisolationApi; | |||
| @Value("${k8s.serving.host}") | |||
| String servingHost; | |||
| @Value("${k8s.serving.tls-crt}") | |||
| String servingTlsCrt; | |||
| @Value("${k8s.serving.tls-key}") | |||
| String servingTlsKey; | |||
| private static final String MODEL_SERVING_MAX_UPLOAD_SIZE = "100m"; | |||
| public ModelServingApiImpl(K8sUtils k8sUtils) { | |||
| @@ -144,7 +136,6 @@ public class ModelServingApiImpl implements ModelServingApi { | |||
| //名称生成 | |||
| String deploymentName = StrUtil.format(K8sParamConstants.RESOURCE_NAME_TEMPLATE, bo.getResourceName(), RandomUtil.randomString(MagicNumConstant.EIGHT)); | |||
| String svcName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.SVC_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE)); | |||
| String ingressName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.INGRESS_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE)); | |||
| //标签生成 | |||
| Map<String, String> baseLabels = LabelUtils.getBaseLabels(bo.getResourceName(), bo.getBusinessLabel()); | |||
| @@ -164,45 +155,12 @@ public class ModelServingApiImpl implements ModelServingApi { | |||
| if (bo.getGrpcPort() != null) { | |||
| buildServiceBO.addPort(ResourceBuildUtils.buildServicePort(bo.getGrpcPort(), bo.getGrpcPort(), SymbolConstant.GRPC)); | |||
| } | |||
| buildServiceBO.setType(ServiceTypeENum.NODE_PORT.getType()); | |||
| Service service = ResourceBuildUtils.buildService(buildServiceBO); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); | |||
| Service serviceResult = client.services().create(service); | |||
| //部署ingress | |||
| BuildIngressBO buildIngressBO = new BuildIngressBO(bo.getNamespace(), ingressName, baseLabels); | |||
| if (StringUtils.isNotEmpty(buildIngressBO.getMaxUploadSize())) { | |||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_PROXY_BODY_SIZE_KEY, buildIngressBO.getMaxUploadSize()); | |||
| } | |||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_READ_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN)); | |||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_NEXT_UPSTREAM_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN)); | |||
| if (bo.getHttpPort() != null) { | |||
| String httpHost = RandomUtil.randomString(MagicNumConstant.SIX) + SymbolConstant.DOT + servingHost; | |||
| buildIngressBO.addIngressRule(ResourceBuildUtils.buildIngressRule(httpHost, svcName, SymbolConstant.HTTP)); | |||
| } | |||
| Secret secretResult = null; | |||
| if (bo.getGrpcPort() != null) { | |||
| String secretName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), SymbolConstant.TOKEN, RandomUtil.randomString(MagicNumConstant.FIVE)); | |||
| Map<String, String> data = new HashMap<String, String>(MagicNumConstant.FOUR) { | |||
| { | |||
| put(K8sParamConstants.SECRET_TLS_TLS_CRT, servingTlsCrt); | |||
| put(K8sParamConstants.SECRET_TLS_TLS_KEY, servingTlsKey); | |||
| } | |||
| }; | |||
| Secret secret = ResourceBuildUtils.buildTlsSecret(bo.getNamespace(), secretName, baseLabels, data); | |||
| secretResult = client.secrets().create(secret); | |||
| String grpcHost = RandomUtil.randomString(MagicNumConstant.SIX) + SymbolConstant.DOT + servingHost; | |||
| buildIngressBO.addIngressRule(ResourceBuildUtils.buildIngressRule(grpcHost, svcName, SymbolConstant.GRPC)); | |||
| buildIngressBO.addIngressTLS(ResourceBuildUtils.buildIngressTLS(secretName, grpcHost)); | |||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_CLASS_KEY, StringConstant.NGINX_LOWERCASE); | |||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_SSL_REDIRECT_KEY, StringConstant.TRUE_LOWERCASE); | |||
| buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_BACKEND_PROTOCOL_KEY, StringConstant.GRPC_CAPITALIZE); | |||
| } | |||
| Ingress ingress = ResourceBuildUtils.buildIngress(buildIngressBO); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", ingressName, YamlUtils.dumpAsYaml(ingress)); | |||
| Ingress ingressResult = client.extensions().ingresses().create(ingress); | |||
| Service serviceResult = client.services().inNamespace(bo.getNamespace()).create(service); | |||
| return new ModelServingVO(BizConvertUtils.toBizSecret(secretResult), BizConvertUtils.toBizService(serviceResult), BizConvertUtils.toBizDeployment(deploymentResult), BizConvertUtils.toBizIngress(ingressResult)); | |||
| return new ModelServingVO(null, BizConvertUtils.toBizService(serviceResult), BizConvertUtils.toBizDeployment(deploymentResult), null); | |||
| } catch (KubernetesClientException e) { | |||
| LogUtil.error(LogEnum.BIZ_K8S, "ModelOptJobApiImpl.create error, param:{} error:", bo, e); | |||
| return new ModelServingVO().error(String.valueOf(e.getCode()), e.getMessage()); | |||
| @@ -41,6 +41,7 @@ import org.dubhe.k8s.constant.K8sLabelConstants; | |||
| import org.dubhe.k8s.constant.K8sParamConstants; | |||
| import org.dubhe.k8s.domain.PtBaseResult; | |||
| import org.dubhe.k8s.domain.resource.BizNode; | |||
| import org.dubhe.k8s.domain.resource.BizNodeAddress; | |||
| import org.dubhe.k8s.domain.resource.BizTaint; | |||
| import org.dubhe.k8s.domain.vo.PtNodeMetricsVO; | |||
| import org.dubhe.k8s.enums.K8sResponseEnum; | |||
| @@ -560,6 +561,25 @@ public class NodeApiImpl implements NodeApi { | |||
| return geBizTaintListByUserId(userContextService.getCurUserId()); | |||
| } | |||
| /** | |||
| * 获取一个可用node的ip | |||
| * | |||
| * @return String | |||
| */ | |||
| @Override | |||
| public String getAvailableNodeIp(){ | |||
| List<BizNode> nodes = listAll(); | |||
| for (BizNode node : nodes){ | |||
| if (node.getReady() && !node.isUnschedulable()){ | |||
| for (BizNodeAddress address : node.getAddresses()){ | |||
| if (K8sParamConstants.INTERNAL_IP.equals(address.getType())){ | |||
| return address.getAddress(); | |||
| } | |||
| } | |||
| } | |||
| } | |||
| return ""; | |||
| } | |||
| /** | |||
| * 查询节点内存资源是否可分配 | |||
| @@ -400,13 +400,16 @@ public class PodApiImpl implements PodApi { | |||
| public String getUrlByResourceName(String namespace, String resourceName) { | |||
| LogUtil.info(LogEnum.BIZ_K8S,"Start GetUrlByResourceName {} {}",namespace,resourceName); | |||
| PtJupyterDeployVO info = jupyterResourceApi.get(namespace, resourceName); | |||
| if (info != null && info.getIngressInfo() != null && CollectionUtil.isNotEmpty(info.getIngressInfo().getRules())) { | |||
| if (info != null && info.getServiceInfo() != null && CollectionUtil.isNotEmpty(info.getServiceInfo().getPorts())) { | |||
| String token = getTokenByResourceName(namespace, resourceName); | |||
| if (StringUtils.isBlank(token)) { | |||
| LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter Notebook token not generated,[namespace]={}, [resourceName]={}", namespace, resourceName); | |||
| return ""; | |||
| } | |||
| String url = StrUtil.format(POD_URL, info.getIngressInfo().getRules().get(0).getHost(), k8sUtils.getPort(), token); | |||
| BizPod pod = getWithResourceName(namespace,resourceName); | |||
| String url = StrUtil.format(POD_URL, pod.getHostIP(), info.getServiceInfo().getPorts().get(0).getNodePort(), token); | |||
| return validateJupyterUrl(url); | |||
| } | |||
| LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter statefulset not created,[namespace]={}, [resourceName]={}",namespace,resourceName); | |||
| @@ -148,7 +148,7 @@ public class TerminalApiImpl implements TerminalApi { | |||
| } | |||
| Service service = ResourceBuildUtils.buildService(buildServiceBO); | |||
| LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); | |||
| Service serviceResult = client.services().create(service); | |||
| Service serviceResult = client.services().inNamespace(service.getMetadata().getNamespace()).create(service); | |||
| return new TerminalResourceVO(BizConvertUtils.toBizDeployment(deploymentResult),BizConvertUtils.toBizService(serviceResult)); | |||
| }catch (KubernetesClientException e) { | |||
| LogUtil.error(LogEnum.BIZ_K8S, "TerminalApiImpl.create error, param:{} error:", bo, e); | |||
| @@ -101,4 +101,5 @@ public class K8sParamConstants { | |||
| public static final String WAITING_REASON_CONTAINER_CREATING = "ContainerCreating"; | |||
| public final static String INTERNAL_IP = "InternalIP"; | |||
| } | |||
| @@ -57,16 +57,21 @@ public class BaseK8sDeploymentCallbackCreateDTO { | |||
| @NotNull(message = "replicas 不能为空!") | |||
| private Integer replicas; | |||
| @ApiModelProperty(required = true, value = "action") | |||
| @NotBlank(message = "action 不能为空!") | |||
| private String action; | |||
| public BaseK8sDeploymentCallbackCreateDTO() { | |||
| } | |||
| public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas) { | |||
| public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas,String action) { | |||
| this.namespace = namespace; | |||
| this.resourceName = resourceName; | |||
| this.deploymentName = deploymentName; | |||
| this.readyReplicas = readyReplicas; | |||
| this.replicas = replicas; | |||
| this.action = action; | |||
| } | |||
| @Override | |||
| @@ -77,6 +82,7 @@ public class BaseK8sDeploymentCallbackCreateDTO { | |||
| ", deploymentName='" + deploymentName + '\'' + | |||
| ", readyReplicas='" + readyReplicas + '\'' + | |||
| ", replicas='" + replicas + '\'' + | |||
| ", action='" + action + '\'' + | |||
| '}'; | |||
| } | |||
| } | |||
| @@ -30,6 +30,7 @@ import org.dubhe.k8s.annotation.K8sField; | |||
| import org.dubhe.k8s.constant.K8sParamConstants; | |||
| import org.dubhe.k8s.domain.PtBaseResult; | |||
| import org.dubhe.k8s.domain.resource.BizContainer; | |||
| import org.dubhe.k8s.domain.resource.BizServicePort; | |||
| import org.dubhe.k8s.utils.MappingUtils; | |||
| import java.util.List; | |||
| @@ -105,6 +106,8 @@ public class PtJupyterDeployVO extends PtBaseResult<PtJupyterDeployVO> { | |||
| private String namespace; | |||
| @K8sField("metadata:uid") | |||
| private String uid; | |||
| @K8sField("spec:ports") | |||
| private List<BizServicePort> ports; | |||
| } | |||
| @Data | |||
| @@ -33,10 +33,6 @@ public class ClusterProperties { | |||
| private String nfs; | |||
| private String host; | |||
| private String port; | |||
| private String kubeconfig; | |||
| /** | |||
| @@ -218,7 +218,9 @@ public class BizConvertUtils { | |||
| */ | |||
| public static BizIngress toBizIngress(Ingress ingress) { | |||
| BizIngress bizIngress = MappingUtils.mappingTo(ingress, BizIngress.class); | |||
| bizIngress.getRules().forEach(BizIngressRule::takeServicePort); | |||
| if (bizIngress != null){ | |||
| bizIngress.getRules().forEach(BizIngressRule::takeServicePort); | |||
| } | |||
| return bizIngress; | |||
| } | |||
| @@ -98,8 +98,6 @@ public class K8sUtils implements ApplicationContextAware { | |||
| nfs = clusterProperties.getNfs(); | |||
| host = clusterProperties.getHost(); | |||
| port = clusterProperties.getPort(); | |||
| //打印集群信息 | |||
| LogUtil.info(LogEnum.BIZ_K8S, "ApiVersion : {}", client.getApiVersion()); | |||
| @@ -69,11 +69,6 @@ public enum RecycleResourceEnum { | |||
| */ | |||
| MEASURE_RECYCLE_FILE("measureRecycleFile", "度量文件回收"), | |||
| /** | |||
| * 镜像回收 | |||
| */ | |||
| IMAGE_RECYCLE_FILE("imageRecycleFile", "镜像回收"), | |||
| /** | |||
| * 算法文件回收 | |||
| */ | |||
| @@ -14,7 +14,6 @@ | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.task; | |||
| import org.junit.jupiter.api.Test; | |||
| @@ -1,36 +0,0 @@ | |||
| /** | |||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.data.config; | |||
| import lombok.Data; | |||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||
| import org.springframework.stereotype.Component; | |||
| @Data | |||
| @Component | |||
| @ConfigurationProperties(prefix = "harbor") | |||
| public class DataHarborConfig { | |||
| private String address; | |||
| private String username; | |||
| private String password; | |||
| private String modelName; | |||
| } | |||
| @@ -28,9 +28,15 @@ import com.baomidou.mybatisplus.extension.service.IService; | |||
| import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; | |||
| import org.apache.commons.compress.utils.Lists; | |||
| import org.dubhe.biz.base.constant.MagicNumConstant; | |||
| import org.dubhe.biz.base.constant.SymbolConstant; | |||
| import org.dubhe.biz.base.context.UserContext; | |||
| import org.dubhe.biz.base.dto.*; | |||
| import org.dubhe.biz.base.dto.PtImageIdDTO; | |||
| import org.dubhe.biz.base.dto.PtImageIdsDTO; | |||
| import org.dubhe.biz.base.dto.PtModelBranchQueryByIdDTO; | |||
| import org.dubhe.biz.base.dto.PtModelBranchQueryByIdsDTO; | |||
| import org.dubhe.biz.base.dto.TrainAlgorithmSelectAllBatchIdDTO; | |||
| import org.dubhe.biz.base.dto.TrainAlgorithmSelectByIdDTO; | |||
| import org.dubhe.biz.base.dto.UserDTO; | |||
| import org.dubhe.biz.base.dto.UserSmallDTO; | |||
| import org.dubhe.biz.base.enums.BizEnum; | |||
| import org.dubhe.biz.base.enums.ResourcesPoolTypeEnum; | |||
| import org.dubhe.biz.base.exception.BusinessException; | |||
| @@ -49,13 +55,15 @@ import org.dubhe.cloud.authconfig.service.AdminClient; | |||
| import org.dubhe.data.client.AlgorithmClient; | |||
| import org.dubhe.data.client.ImageClient; | |||
| import org.dubhe.data.client.ModelClient; | |||
| import org.dubhe.data.config.DataHarborConfig; | |||
| import org.dubhe.data.config.DataRedisConfig; | |||
| import org.dubhe.data.constant.AutoLabelModelServiceStatusEnum; | |||
| import org.dubhe.data.constant.Constant; | |||
| import org.dubhe.data.constant.ErrorEnum; | |||
| import org.dubhe.data.dao.AutoLabelModelServiceMapper; | |||
| import org.dubhe.data.domain.dto.*; | |||
| import org.dubhe.data.domain.dto.AutoLabelModelServiceCreateDTO; | |||
| import org.dubhe.data.domain.dto.AutoLabelModelServiceQueryDTO; | |||
| import org.dubhe.data.domain.dto.AutoLabelModelServiceUpdateDTO; | |||
| import org.dubhe.data.domain.dto.DataK8sDeploymentCallbackCreateDTO; | |||
| import org.dubhe.data.domain.entity.AutoLabelModelService; | |||
| import org.dubhe.data.domain.vo.AutoLabelModelServicePodVO; | |||
| import org.dubhe.data.domain.vo.AutoLabelModelServiceVO; | |||
| @@ -75,9 +83,16 @@ import org.springframework.beans.factory.annotation.Autowired; | |||
| import org.springframework.beans.factory.annotation.Value; | |||
| import org.springframework.dao.DuplicateKeyException; | |||
| import org.springframework.stereotype.Service; | |||
| import javax.annotation.Resource; | |||
| import java.sql.Timestamp; | |||
| import java.util.*; | |||
| import java.util.ArrayList; | |||
| import java.util.Arrays; | |||
| import java.util.HashMap; | |||
| import java.util.List; | |||
| import java.util.Map; | |||
| import java.util.Objects; | |||
| import java.util.Set; | |||
| import java.util.concurrent.TimeUnit; | |||
| import java.util.function.Function; | |||
| import java.util.stream.Collectors; | |||
| @@ -116,9 +131,6 @@ public class AutoLabelModelServiceServiceImpl extends ServiceImpl<AutoLabelModel | |||
| @Resource | |||
| private UserContextService userContextService; | |||
| @Resource | |||
| private DataHarborConfig dataHarborConfig; | |||
| @Resource | |||
| private DataRedisConfig dataRedisConfig; | |||
| @@ -197,7 +209,7 @@ public class AutoLabelModelServiceServiceImpl extends ServiceImpl<AutoLabelModel | |||
| modelServiceBO.setResourceName(k8sNameTool.generateResourceName(BizEnum.DATA, autoLabelModelService.getId().toString())); | |||
| modelServiceBO.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.DATA)); | |||
| modelServiceBO.setReplicas(autoLabelModelService.getInstanceNum()); | |||
| modelServiceBO.setImage(dataHarborConfig.getAddress() + SymbolConstant.SLASH + ptImageVO.getImageUrl()); | |||
| modelServiceBO.setImage(ptImageVO.getImageUrl()); | |||
| modelServiceBO.setTaskIdentifyLabel(taskIdentify); | |||
| String command = String.format(Constant.MODEL_SERVICE_COMMAND, modelServiceId, | |||
| @@ -1,142 +0,0 @@ | |||
| /** | |||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.image.async; | |||
| import cn.hutool.core.util.StrUtil; | |||
| import org.dubhe.biz.base.constant.HarborProperties; | |||
| import org.dubhe.biz.base.constant.StringConstant; | |||
| import org.dubhe.biz.base.enums.ImageStateEnum; | |||
| import org.dubhe.biz.base.exception.BusinessException; | |||
| import org.dubhe.biz.base.utils.StringUtils; | |||
| import org.dubhe.biz.file.utils.IOUtil; | |||
| import org.dubhe.biz.log.enums.LogEnum; | |||
| import org.dubhe.biz.log.utils.LogUtil; | |||
| import org.dubhe.harbor.api.HarborApi; | |||
| import org.dubhe.image.dao.PtImageMapper; | |||
| import org.dubhe.image.domain.entity.PtImage; | |||
| import org.springframework.beans.factory.annotation.Autowired; | |||
| import org.springframework.scheduling.annotation.Async; | |||
| import org.springframework.stereotype.Component; | |||
| import java.io.BufferedReader; | |||
| import java.io.InputStreamReader; | |||
| /** | |||
| * @description 异步推送镜像到harbor | |||
| * @date 2020-07-10 | |||
| */ | |||
| @Component | |||
| public class HarborImagePushAsync { | |||
| @Autowired | |||
| private HarborProperties harborProperties; | |||
| @Autowired | |||
| private HarborApi harborApi; | |||
| @Autowired | |||
| private PtImageMapper ptImageMapper; | |||
| /** | |||
| * 组装shell脚本执行命令 | |||
| * | |||
| * @param imagePath | |||
| * @param imageNameandTag | |||
| **/ | |||
| @Async | |||
| public void execShell(String imagePath, String imageNameandTag, PtImage ptImage) { | |||
| try { | |||
| String imageResource = harborProperties.getAddress() + StrUtil.SLASH + StringConstant.DEFAULT_IMAGE_PROJECT | |||
| + StrUtil.SLASH + imageNameandTag; | |||
| String cmdStr = "docker login --username=" + harborProperties.getUsername() + " " + harborProperties.getAddress() + " --password=" + harborProperties.getPassword() + " ; docker " + | |||
| "load < " + imagePath + " |awk '{print $3}' |xargs -I str docker tag str " + imageResource + " ; docker push " + imageResource + "; docker rmi " + imageResource; | |||
| String[] cmd = {"/bin/bash", "-c", cmdStr}; | |||
| LogUtil.info(LogEnum.IMAGE, "镜像上传执行脚本参数:{}", cmd); | |||
| Process process = Runtime.getRuntime().exec(cmd); | |||
| if (checkImagePushIsOk(ptImage, process)) { | |||
| updateImageStatus(ptImage, ImageStateEnum.SUCCESS.getCode()); | |||
| } else { | |||
| updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); | |||
| } | |||
| } catch (Exception e) { | |||
| LogUtil.error(LogEnum.IMAGE, "upload image failed,exception {}", e); | |||
| updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); | |||
| throw new BusinessException("上传镜像异常!"); | |||
| } | |||
| } | |||
| /** | |||
| * 更新镜像上传状态 | |||
| * | |||
| * @param ptImage | |||
| * @param status | |||
| **/ | |||
| public void updateImageStatus(PtImage ptImage, Integer status) { | |||
| ptImage.setImageStatus(status); | |||
| ptImageMapper.updateById(ptImage); | |||
| } | |||
| /** | |||
| * 校验镜像是否上传成功 | |||
| * | |||
| * @param ptImage 镜像信息 | |||
| * @param process process对象 | |||
| * @return 是否上传成功 | |||
| */ | |||
| public boolean checkImagePushIsOk(PtImage ptImage, Process process) { | |||
| //读取标准输出流 | |||
| BufferedReader brOut = new BufferedReader(new InputStreamReader(process.getInputStream())); | |||
| //读取标准错误流 | |||
| BufferedReader brErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); | |||
| String line; | |||
| StringBuilder outMessage = new StringBuilder(); | |||
| StringBuilder errMessage = new StringBuilder(); | |||
| boolean isPushOk = true; | |||
| try { | |||
| while ((line = brOut.readLine()) != null) { | |||
| outMessage.append(line); | |||
| } | |||
| if (StringUtils.isNotEmpty(outMessage)) { | |||
| LogUtil.info(LogEnum.IMAGE, "upload image with shell standard output message {}", outMessage.toString()); | |||
| } | |||
| while ((line = brErr.readLine()) != null) { | |||
| errMessage.append(line); | |||
| } | |||
| if (StringUtils.isNotEmpty(errMessage)) { | |||
| LogUtil.error(LogEnum.IMAGE, "upload image with shell standard output message {}", errMessage.toString()); | |||
| } | |||
| Integer status = process.waitFor(); | |||
| LogUtil.info(LogEnum.IMAGE, "上传镜像状态:{}", status); | |||
| if (status == null) { | |||
| if (!harborApi.isExistImage(ptImage.getImageUrl())) { | |||
| isPushOk = false; | |||
| } | |||
| } else if (status != 0) { | |||
| isPushOk = false; | |||
| } | |||
| } catch (Exception e) { | |||
| LogUtil.error(LogEnum.IMAGE, "upload image failed,exception {}", e); | |||
| return false; | |||
| } finally { | |||
| IOUtil.close(brErr, brOut); | |||
| } | |||
| return isPushOk; | |||
| } | |||
| } | |||
| @@ -41,9 +41,6 @@ public class PtImageQueryDTO extends PageQueryBase implements Serializable { | |||
| @ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") | |||
| private Integer imageResource; | |||
| @ApiModelProperty(value = "镜像状态,0为制作中,1位制作成功,2位制作失败") | |||
| private Integer imageStatus; | |||
| @ApiModelProperty(value = "镜像名称或id") | |||
| private String imageNameOrId; | |||
| @@ -38,7 +38,7 @@ import java.util.List; | |||
| */ | |||
| @Data | |||
| @Accessors(chain = true) | |||
| public class PtImageUploadDTO implements Serializable { | |||
| public class PtImageSaveDTO implements Serializable { | |||
| private static final long serialVersionUID = 1L; | |||
| @@ -50,9 +50,9 @@ public class PtImageUploadDTO implements Serializable { | |||
| @FlagValidator(value = {"0", "1"}, message = "无效镜像来源") | |||
| private Integer imageResource; | |||
| @ApiModelProperty(value = "镜像文件路径", required = true) | |||
| @NotBlank(message = "镜像文件路径不能为空") | |||
| private String imagePath; | |||
| @ApiModelProperty(value = "镜像地址", required = true) | |||
| @NotBlank(message = "镜像地址不能为空") | |||
| private String imageUrl; | |||
| @ApiModelProperty(value = "镜像名称", required = true) | |||
| @NotBlank(message = "源镜像名称不能为空") | |||
| @@ -20,10 +20,13 @@ import io.swagger.annotations.ApiModelProperty; | |||
| import lombok.Data; | |||
| import lombok.experimental.Accessors; | |||
| import org.dubhe.biz.base.constant.MagicNumConstant; | |||
| import org.dubhe.biz.base.constant.StringConstant; | |||
| import org.hibernate.validator.constraints.Length; | |||
| import javax.validation.constraints.NotBlank; | |||
| import javax.validation.constraints.NotEmpty; | |||
| import javax.validation.constraints.NotNull; | |||
| import javax.validation.constraints.Pattern; | |||
| import java.io.Serializable; | |||
| import java.util.List; | |||
| @@ -39,14 +42,29 @@ public class PtImageUpdateDTO implements Serializable { | |||
| @ApiModelProperty(value = "id", required = true) | |||
| @NotNull(message = "镜像id不能为空") | |||
| private List<Long> ids; | |||
| @ApiModelProperty("镜像描述") | |||
| @Length(max = MagicNumConstant.BINARY_TEN_EXP, message = "镜像描述-输入长度不能超过1024个字符") | |||
| private String remark; | |||
| private Long id; | |||
| @ApiModelProperty(value = "镜像用途", required = true) | |||
| @NotEmpty(message = "镜像用途(0:notebook , 1:train , 2:serving, 3:terminal, 4:point-cloud)不能为空") | |||
| private List<Integer> imageTypes; | |||
| @ApiModelProperty(value = "镜像路径", required = true) | |||
| @NotBlank(message = "镜像路径不能为空") | |||
| private String imageUrl; | |||
| @ApiModelProperty(value = "镜像名称", required = true) | |||
| @NotBlank(message = "源镜像名称不能为空") | |||
| @Length(min = MagicNumConstant.ONE, max = MagicNumConstant.SIXTY_FOUR, message = "镜像名称长度在1-64个字符") | |||
| @Pattern(regexp = StringConstant.REGEXP_NAME, message = "镜像名称支持字母、数字、英文横杠和下划线") | |||
| private String imageName; | |||
| @ApiModelProperty(value = "镜像版本号", required = true) | |||
| @NotBlank(message = "镜像版本号不能为空") | |||
| @Length(max = MagicNumConstant.SIXTY_FOUR, message = "镜像版本号长度在1-64个字符") | |||
| @Pattern(regexp = StringConstant.REGEXP_TAG, message = "镜像版本号支持字母、数字、英文横杠、英文.号和下划线") | |||
| private String imageTag; | |||
| @ApiModelProperty("镜像描述") | |||
| @Length(max = MagicNumConstant.BINARY_TEN_EXP, message = "镜像描述-输入长度不能超过1024个字符") | |||
| private String remark; | |||
| } | |||
| @@ -80,13 +80,6 @@ public class PtImage extends BaseEntity { | |||
| @TableField(value = "image_resource") | |||
| private Integer imageResource; | |||
| /** | |||
| * 镜像状态 | |||
| */ | |||
| @TableField(value = "image_status") | |||
| private Integer imageStatus; | |||
| /** | |||
| * 资源拥有者ID | |||
| */ | |||
| @@ -41,9 +41,6 @@ public class PtImageQueryVO implements Serializable { | |||
| @ApiModelProperty("镜像版本") | |||
| private String imageTag; | |||
| @ApiModelProperty("镜像状态(0:上传中,1:上传成功,2:上传失败)") | |||
| private Integer imageStatus; | |||
| @ApiModelProperty("备注") | |||
| private String remark; | |||
| @@ -58,4 +55,7 @@ public class PtImageQueryVO implements Serializable { | |||
| @ApiModelProperty("镜像用途") | |||
| private List<Integer> imageTypes; | |||
| @ApiModelProperty("镜像地址") | |||
| private String imageUrl; | |||
| } | |||
| @@ -24,12 +24,25 @@ import org.dubhe.biz.base.dto.PtImageIdDTO; | |||
| import org.dubhe.biz.base.dto.PtImageIdsDTO; | |||
| import org.dubhe.biz.base.vo.DataResponseBody; | |||
| import org.dubhe.biz.base.vo.PtImageVO; | |||
| import org.dubhe.image.domain.dto.*; | |||
| import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryImageDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryNameDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryUrlDTO; | |||
| import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||
| import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||
| import org.dubhe.image.service.PtImageService; | |||
| import org.springframework.beans.factory.annotation.Autowired; | |||
| import org.springframework.security.access.prepost.PreAuthorize; | |||
| import org.springframework.validation.annotation.Validated; | |||
| import org.springframework.web.bind.annotation.*; | |||
| import org.springframework.web.bind.annotation.DeleteMapping; | |||
| import org.springframework.web.bind.annotation.GetMapping; | |||
| import org.springframework.web.bind.annotation.PostMapping; | |||
| import org.springframework.web.bind.annotation.PutMapping; | |||
| import org.springframework.web.bind.annotation.RequestBody; | |||
| import org.springframework.web.bind.annotation.RequestMapping; | |||
| import org.springframework.web.bind.annotation.RequestParam; | |||
| import org.springframework.web.bind.annotation.RestController; | |||
| import java.util.List; | |||
| @@ -45,7 +58,7 @@ public class PtImageController { | |||
| @Autowired | |||
| private PtImageService ptImageService; | |||
| @GetMapping("/info") | |||
| @GetMapping("/list") | |||
| @ApiOperation("查询镜像") | |||
| @PreAuthorize(Permissions.IMAGE) | |||
| public DataResponseBody getImage(PtImageQueryDTO ptImageQueryDTO) { | |||
| @@ -58,11 +71,11 @@ public class PtImageController { | |||
| return new DataResponseBody(ptImageService.searchImages(ptImageQueryImageDTO)); | |||
| } | |||
| @PostMapping("uploadImage") | |||
| @ApiOperation("上传镜像包到harbor") | |||
| @PreAuthorize(Permissions.IMAGE_UPLOAD) | |||
| public DataResponseBody uploadImage(@Validated @RequestBody PtImageUploadDTO ptImageUploadDTO) { | |||
| ptImageService.uploadImage(ptImageUploadDTO); | |||
| @PostMapping | |||
| @ApiOperation("保存镜像信息") | |||
| @PreAuthorize(Permissions.IMAGE_SAVE) | |||
| public DataResponseBody saveImageInfo(@Validated @RequestBody PtImageSaveDTO ptImageSaveDTO) { | |||
| ptImageService.saveImageInfo(ptImageSaveDTO); | |||
| return new DataResponseBody(); | |||
| } | |||
| @@ -18,9 +18,14 @@ | |||
| package org.dubhe.image.service; | |||
| import org.dubhe.biz.base.vo.PtImageVO; | |||
| import org.dubhe.image.domain.dto.*; | |||
| import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryImageDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryNameDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryUrlDTO; | |||
| import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||
| import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||
| import org.dubhe.image.domain.entity.PtImage; | |||
| import org.dubhe.recycle.domain.dto.RecycleCreateDTO; | |||
| import java.util.List; | |||
| import java.util.Map; | |||
| @@ -42,11 +47,11 @@ public interface PtImageService { | |||
| /** | |||
| * 上传镜像到harbor | |||
| * 保存镜像信息 | |||
| * | |||
| * @param ptImageUploadDTO 上传条件 | |||
| * @param ptImageSaveDTO 镜像信息DTO | |||
| */ | |||
| void uploadImage(PtImageUploadDTO ptImageUploadDTO); | |||
| void saveImageInfo(PtImageSaveDTO ptImageSaveDTO); | |||
| /** | |||
| @@ -98,13 +103,6 @@ public interface PtImageService { | |||
| */ | |||
| String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO); | |||
| /** | |||
| * 镜像回收还原 | |||
| * | |||
| * @param dto 还原DTO对象 | |||
| */ | |||
| void recycleRollback(RecycleCreateDTO dto); | |||
| /** | |||
| * 获取终端镜像列表 | |||
| * | |||
| @@ -27,12 +27,14 @@ import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; | |||
| import com.baomidou.mybatisplus.core.metadata.IPage; | |||
| import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | |||
| import org.apache.commons.compress.utils.Lists; | |||
| import org.dubhe.biz.base.constant.*; | |||
| import org.dubhe.biz.base.constant.MagicNumConstant; | |||
| import org.dubhe.biz.base.constant.NumberConstant; | |||
| import org.dubhe.biz.base.constant.ResponseCode; | |||
| import org.dubhe.biz.base.constant.StringConstant; | |||
| import org.dubhe.biz.base.context.DataContext; | |||
| import org.dubhe.biz.base.context.UserContext; | |||
| import org.dubhe.biz.base.enums.DatasetTypeEnum; | |||
| import org.dubhe.biz.base.enums.ImageSourceEnum; | |||
| import org.dubhe.biz.base.enums.ImageStateEnum; | |||
| import org.dubhe.biz.base.enums.ImageTypeEnum; | |||
| import org.dubhe.biz.base.exception.BusinessException; | |||
| import org.dubhe.biz.base.service.UserContextService; | |||
| @@ -40,31 +42,34 @@ import org.dubhe.biz.base.utils.ReflectionUtils; | |||
| import org.dubhe.biz.base.utils.StringUtils; | |||
| import org.dubhe.biz.base.vo.PtImageVO; | |||
| import org.dubhe.biz.db.utils.PageUtil; | |||
| import org.dubhe.biz.file.config.NfsConfig; | |||
| import org.dubhe.biz.log.enums.LogEnum; | |||
| import org.dubhe.biz.log.utils.LogUtil; | |||
| import org.dubhe.biz.permission.annotation.DataPermissionMethod; | |||
| import org.dubhe.biz.permission.base.BaseService; | |||
| import org.dubhe.image.async.HarborImagePushAsync; | |||
| import org.dubhe.image.dao.PtImageMapper; | |||
| import org.dubhe.image.domain.dto.*; | |||
| import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryImageDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryNameDTO; | |||
| import org.dubhe.image.domain.dto.PtImageQueryUrlDTO; | |||
| import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||
| import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||
| import org.dubhe.image.domain.entity.PtImage; | |||
| import org.dubhe.image.domain.vo.PtImageQueryVO; | |||
| import org.dubhe.image.service.PtImageService; | |||
| import org.dubhe.recycle.config.RecycleConfig; | |||
| import org.dubhe.recycle.domain.dto.RecycleCreateDTO; | |||
| import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO; | |||
| import org.dubhe.recycle.enums.RecycleModuleEnum; | |||
| import org.dubhe.recycle.enums.RecycleResourceEnum; | |||
| import org.dubhe.recycle.enums.RecycleTypeEnum; | |||
| import org.dubhe.recycle.service.RecycleService; | |||
| import org.dubhe.recycle.utils.RecycleTool; | |||
| import org.springframework.beans.BeanUtils; | |||
| import org.springframework.beans.factory.annotation.Autowired; | |||
| import org.springframework.stereotype.Service; | |||
| import org.springframework.transaction.annotation.Transactional; | |||
| import java.util.*; | |||
| import java.util.ArrayList; | |||
| import java.util.Comparator; | |||
| import java.util.HashSet; | |||
| import java.util.List; | |||
| import java.util.Map; | |||
| import java.util.Objects; | |||
| import java.util.Set; | |||
| import java.util.TreeSet; | |||
| import java.util.stream.Collectors; | |||
| /** | |||
| @@ -77,25 +82,9 @@ public class PtImageServiceImpl implements PtImageService { | |||
| @Autowired | |||
| private PtImageMapper ptImageMapper; | |||
| @Autowired | |||
| private NfsConfig nfsConfig; | |||
| @Autowired | |||
| private HarborImagePushAsync imagePushAsync; | |||
| @Autowired | |||
| private HarborProperties harborProperties; | |||
| @Autowired | |||
| private UserContextService userContextService; | |||
| @Autowired | |||
| private RecycleService recycleService; | |||
| @Autowired | |||
| private RecycleConfig recycleConfig; | |||
| public final static List<String> FIELD_NAMES; | |||
| @@ -119,9 +108,7 @@ public class PtImageServiceImpl implements PtImageService { | |||
| QueryWrapper<PtImage> query = new QueryWrapper<>(); | |||
| query.eq("deleted", NumberConstant.NUMBER_0); | |||
| if (ptImageQueryDTO.getImageStatus() != null) { | |||
| query.eq("image_status", ptImageQueryDTO.getImageStatus()); | |||
| } | |||
| if (ptImageQueryDTO.getImageResource() != null) { | |||
| query.eq("image_resource", ptImageQueryDTO.getImageResource()); | |||
| } | |||
| @@ -166,67 +153,55 @@ public class PtImageServiceImpl implements PtImageService { | |||
| } | |||
| /** | |||
| * 上传镜像到harbor | |||
| * 保存镜像信息 | |||
| * | |||
| * @param ptImageUploadDTO 上传条件 | |||
| * @param ptImageSaveDTO 镜像信息DTO | |||
| */ | |||
| @Override | |||
| @Transactional(rollbackFor = Exception.class) | |||
| public void uploadImage(PtImageUploadDTO ptImageUploadDTO) { | |||
| public void saveImageInfo(PtImageSaveDTO ptImageSaveDTO) { | |||
| UserContext user = userContextService.getCurUser(); | |||
| //普通用户不支持上传预置镜像 | |||
| if (ImageSourceEnum.PRE.getCode().equals(ptImageUploadDTO.getImageResource()) && | |||
| validImageInfo(ptImageSaveDTO.getImageUrl(), ptImageSaveDTO.getImageName(), ptImageSaveDTO.getImageTag()); | |||
| //普通用户不支持预置镜像信息的保存 | |||
| if (ImageSourceEnum.PRE.getCode().equals(ptImageSaveDTO.getImageResource()) && | |||
| !BaseService.isAdmin(user)) { | |||
| throw new BusinessException(ResponseCode.UNAUTHORIZED, "普通用户不支持上传预置镜像!"); | |||
| throw new BusinessException(ResponseCode.UNAUTHORIZED, "普通用户不支持保存预置镜像!"); | |||
| } | |||
| //校验用户自定义镜像不能和预置镜像重名 | |||
| List<PtImage> resList = checkUploadImage(ptImageUploadDTO, null, ImageSourceEnum.PRE.getCode()); | |||
| List<PtImage> resList = checkSaveImage(ptImageSaveDTO, null, ImageSourceEnum.PRE.getCode()); | |||
| if (CollUtil.isNotEmpty(resList)) { | |||
| throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复,请重新上传!"); | |||
| throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复!"); | |||
| } | |||
| //同一用户上传镜像的(userId+imageName+imageTag)存在的情况下是不能重复上传的 | |||
| List<PtImage> imageList = checkUploadImage(ptImageUploadDTO, user, ImageSourceEnum.MINE.getCode()); | |||
| if (CollUtil.isNotEmpty(imageList) && ImageStateEnum.SUCCESS.getCode().equals(imageList.get(0).getImageStatus())) { | |||
| throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在,不允许重复上传!"); | |||
| List<PtImage> imageList = checkSaveImage(ptImageSaveDTO, user, ImageSourceEnum.MINE.getCode()); | |||
| if (CollUtil.isNotEmpty(imageList)) { | |||
| throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在!"); | |||
| } | |||
| String harborImagePath = StringConstant.DEFAULT_IMAGE_PROJECT + StrUtil.SLASH + ptImageUploadDTO.getImageName() + StrUtil.DASHED + user.getId() + | |||
| StrUtil.COLON + ptImageUploadDTO.getImageTag(); | |||
| //存储镜像信息 | |||
| PtImage ptImage = new PtImage(); | |||
| ptImage.setImageName(ptImageUploadDTO.getImageName()) | |||
| .setImageUrl(harborImagePath) | |||
| .setImageResource(ptImageUploadDTO.getImageResource()) | |||
| .setImageStatus(ImageStateEnum.MAKING.getCode()) | |||
| .setRemark(ptImageUploadDTO.getRemark()) | |||
| .setImageTag(ptImageUploadDTO.getImageTag()) | |||
| ptImage.setImageName(ptImageSaveDTO.getImageName()) | |||
| .setImageUrl(ptImageSaveDTO.getImageUrl()) | |||
| .setImageResource(ptImageSaveDTO.getImageResource()) | |||
| .setRemark(ptImageSaveDTO.getRemark()) | |||
| .setImageTag(ptImageSaveDTO.getImageTag()) | |||
| .setCreateUserId(user.getId()); | |||
| if (ImageSourceEnum.PRE.getCode().equals(ptImageUploadDTO.getImageResource())) { | |||
| if (ImageSourceEnum.PRE.getCode().equals(ptImageSaveDTO.getImageResource())) { | |||
| ptImage.setOriginUserId(MagicNumConstant.ZERO_LONG); | |||
| } else { | |||
| ptImage.setOriginUserId(user.getId()); | |||
| } | |||
| int count = ptImageMapper.insert(ptImage); | |||
| if (count < 1) { | |||
| imagePushAsync.updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); | |||
| throw new BusinessException("内部错误!"); | |||
| } | |||
| for (Integer imageType : ptImageUploadDTO.getImageTypes()) { | |||
| ptImageMapper.insert(ptImage); | |||
| for (Integer imageType : ptImageSaveDTO.getImageTypes()) { | |||
| ptImageMapper.insertImageType(ptImage.getId(), imageType); | |||
| } | |||
| //shell脚本上传镜像 | |||
| try { | |||
| String imagePath = nfsConfig.getRootDir() + nfsConfig.getBucket().substring(1) + ptImageUploadDTO.getImagePath(); | |||
| String imageNameAndTag = ptImageUploadDTO.getImageName() + StrUtil.DASHED + user.getId() + StrUtil.COLON + ptImageUploadDTO.getImageTag(); | |||
| imagePushAsync.execShell(imagePath, imageNameAndTag, ptImage); | |||
| } catch (Exception e) { | |||
| LogUtil.error(LogEnum.IMAGE, "Image upload exception {}", e); | |||
| throw new BusinessException("镜像上传失败!"); | |||
| } | |||
| } | |||
| /** | |||
| @@ -240,9 +215,8 @@ public class PtImageServiceImpl implements PtImageService { | |||
| public List<PtImage> searchImages(PtImageQueryImageDTO ptImageQueryImageDTO) { | |||
| LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | |||
| queryWrapper.eq(PtImage::getImageName, ptImageQueryImageDTO.getImageName()) | |||
| .eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()) | |||
| .eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | |||
| List<PtImage> ptImages = new ArrayList<>(); | |||
| List<PtImage> ptImages; | |||
| if (ptImageQueryImageDTO.getImageResource() != null) { | |||
| queryWrapper.eq(PtImage::getImageResource, ptImageQueryImageDTO.getImageResource()); | |||
| } | |||
| @@ -252,18 +226,13 @@ public class PtImageServiceImpl implements PtImageService { | |||
| ptImages = ptImageMapper.getImagesByTypes(queryWrapper, ptImageQueryImageDTO.getImageTypes()); | |||
| } | |||
| List<PtImage> list = new ArrayList<>(); | |||
| if (CollUtil.isEmpty(ptImages)) { | |||
| throw new BusinessException(ResponseCode.BADREQUEST, "未查询到镜像信息!"); | |||
| } | |||
| ptImages = ptImages.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> | |||
| new TreeSet<>(Comparator.comparing(PtImage::getImageTag))), ArrayList::new)); | |||
| ptImages.stream().forEach(ptImage -> { | |||
| ptImage.setImageUrl(harborProperties.getAddress() + StrUtil.SLASH + ptImage.getImageUrl()); | |||
| list.add(ptImage); | |||
| }); | |||
| return list; | |||
| return ptImages; | |||
| } | |||
| @@ -282,37 +251,9 @@ public class PtImageServiceImpl implements PtImageService { | |||
| //删除本地镜像 | |||
| imageList.forEach(image -> { | |||
| ptImageMapper.deleteById(image.getId()); | |||
| //创建镜像回收任务 | |||
| createRecycleTask(image); | |||
| }); | |||
| } | |||
| /** | |||
| * 文件定时清理 | |||
| * | |||
| * @param ptImage 镜像实体对象 | |||
| */ | |||
| private void createRecycleTask(PtImage ptImage) { | |||
| String imageUrl = ImageStateEnum.SUCCESS.getCode().equals(ptImage.getImageStatus()) ? ptImage.getImageUrl() : ""; | |||
| RecycleCreateDTO recycleCreateDTO = RecycleCreateDTO.builder() | |||
| .recycleModule(RecycleModuleEnum.BIZ_IMAGE.getValue()) | |||
| .recycleDelayDate(recycleConfig.getImageValid()) | |||
| .recycleNote(RecycleTool.generateRecycleNote("删除镜像", ptImage.getImageName(), ptImage.getId())) | |||
| .recycleCustom(RecycleResourceEnum.IMAGE_RECYCLE_FILE.getClassName()) | |||
| .restoreCustom(RecycleResourceEnum.IMAGE_RECYCLE_FILE.getClassName()) | |||
| .remark(String.valueOf(ptImage.getId())) | |||
| .build(); | |||
| recycleCreateDTO.addRecycleDetailCreateDTO(RecycleDetailCreateDTO.builder() | |||
| .recycleCondition(imageUrl) | |||
| .recycleType(RecycleTypeEnum.FILE.getCode()) | |||
| .recycleNote(RecycleTool.generateRecycleNote("删除镜像", ptImage.getImageName(), ptImage.getId())) | |||
| .remark(String.valueOf(ptImage.getId())) | |||
| .build() | |||
| ); | |||
| recycleService.createRecycleTask(recycleCreateDTO); | |||
| } | |||
| /** | |||
| * 修改镜像信息 | |||
| * | |||
| @@ -324,29 +265,33 @@ public class PtImageServiceImpl implements PtImageService { | |||
| public void updateTrainImage(PtImageUpdateDTO imageUpdateDTO) { | |||
| UserContext curUser = userContextService.getCurUser(); | |||
| List<PtImage> imageList = ptImageMapper.selectList(new LambdaQueryWrapper<PtImage>() | |||
| .in(PtImage::getId, imageUpdateDTO.getIds())); | |||
| PtImage image = ptImageMapper.selectById(imageUpdateDTO.getId()); | |||
| if (CollUtil.isEmpty(imageList)) { | |||
| throw new BusinessException("内部错误"); | |||
| if (image == null) { | |||
| throw new BusinessException("镜像不存在!"); | |||
| } | |||
| for (PtImage image : imageList) { | |||
| //非管理员禁止修改预置镜像 | |||
| if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource()) && !BaseService.isAdmin(curUser)) { | |||
| throw new BusinessException("非管理员无权限修改预置镜像信息"); | |||
| } | |||
| image.setRemark(imageUpdateDTO.getRemark()); | |||
| ptImageMapper.updateById(image); | |||
| List<Integer> imageTypes = ptImageMapper.selectImageType(image.getId()); | |||
| for (Integer imageType : imageUpdateDTO.getImageTypes()) { | |||
| if (!CollectionUtil.contains(imageTypes, imageType)) { | |||
| ptImageMapper.insertImageType(image.getId(), imageType); | |||
| } | |||
| validImageInfo(imageUpdateDTO.getImageUrl(), imageUpdateDTO.getImageName(), imageUpdateDTO.getImageTag()); | |||
| //非管理员禁止修改预置镜像 | |||
| if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource()) && !BaseService.isAdmin(curUser)) { | |||
| throw new BusinessException("非管理员无权限修改预置镜像信息"); | |||
| } | |||
| image.setImageTypes(imageUpdateDTO.getImageTypes()) | |||
| .setImageName(imageUpdateDTO.getImageName()) | |||
| .setImageUrl(imageUpdateDTO.getImageUrl()) | |||
| .setImageTag(imageUpdateDTO.getImageTag()) | |||
| .setRemark(imageUpdateDTO.getRemark()); | |||
| ptImageMapper.updateById(image); | |||
| List<Integer> imageTypes = ptImageMapper.selectImageType(image.getId()); | |||
| for (Integer imageType : imageUpdateDTO.getImageTypes()) { | |||
| if (!CollectionUtil.contains(imageTypes, imageType)) { | |||
| ptImageMapper.insertImageType(image.getId(), imageType); | |||
| } | |||
| for (Integer imageType : imageTypes) { | |||
| if (!CollectionUtil.contains(imageUpdateDTO.getImageTypes(), imageType)) { | |||
| ptImageMapper.deleteImageType(image.getId(), imageType); | |||
| } | |||
| } | |||
| for (Integer imageType : imageTypes) { | |||
| if (!CollectionUtil.contains(imageUpdateDTO.getImageTypes(), imageType)) { | |||
| ptImageMapper.deleteImageType(image.getId(), imageType); | |||
| } | |||
| } | |||
| } | |||
| @@ -362,10 +307,8 @@ public class PtImageServiceImpl implements PtImageService { | |||
| //从会话中获取用户信息 | |||
| UserContext user = userContextService.getCurUser(); | |||
| QueryWrapper<PtImage> queryWrapper = new QueryWrapper<>(); | |||
| queryWrapper.eq("image_status", ImageStateEnum.SUCCESS.getCode()); | |||
| queryWrapper.eq("deleted", NumberConstant.NUMBER_0); | |||
| if (!BaseService.isAdmin(user)) { | |||
| queryWrapper.in("origin_user_id", user.getId(), 0L); | |||
| } | |||
| if (ptImageQueryNameDTO.getImageResource() != null) { | |||
| @@ -403,12 +346,8 @@ public class PtImageServiceImpl implements PtImageService { | |||
| throw new BusinessException(ResponseCode.BADREQUEST, "该镜像不存在或镜像用途不支持!"); | |||
| } | |||
| //仅支持[制作成功]状态镜像设置为默认镜像 | |||
| if (!ImageStateEnum.SUCCESS.getCode().equals(image.getImageStatus())) { | |||
| throw new BusinessException(ResponseCode.BADREQUEST, "仅支持[制作成功]状态镜像设置为默认镜像!"); | |||
| } | |||
| if (image.getImageResource() != ImageSourceEnum.PRE.getCode()) { | |||
| if (!image.getImageResource().equals(ImageSourceEnum.PRE.getCode())) { | |||
| throw new BusinessException(ResponseCode.BADREQUEST, "非预制镜像不能设置为默认镜像!"); | |||
| } | |||
| @@ -434,7 +373,7 @@ public class PtImageServiceImpl implements PtImageService { | |||
| /** | |||
| * 获取镜像URL | |||
| * | |||
| * @param imageQueryUrlDTO 查询镜像路径DTO | |||
| * @param imageQueryUrlDTO 查询镜像地址DTO | |||
| * @return String 镜像url | |||
| */ | |||
| @Override | |||
| @@ -453,7 +392,6 @@ public class PtImageServiceImpl implements PtImageService { | |||
| if (imageQueryUrlDTO.getIsDefault() != null) { | |||
| queryWrapper.eq(PtImage::getIsDefault, imageQueryUrlDTO.getIsDefault()); | |||
| } | |||
| queryWrapper.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()); | |||
| queryWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | |||
| List<PtImage> imageList = new ArrayList<>(); | |||
| if (CollectionUtil.isNotEmpty(imageQueryUrlDTO.getImageTypes())) { | |||
| @@ -470,23 +408,13 @@ public class PtImageServiceImpl implements PtImageService { | |||
| return imageUrl; | |||
| } | |||
| /** | |||
| * 自定义镜像回收的还原实现 | |||
| * | |||
| * @param dto 资源回收DTO对象 | |||
| */ | |||
| @Override | |||
| public void recycleRollback(RecycleCreateDTO dto) { | |||
| String imageId = dto.getRemark(); | |||
| ptImageMapper.updateDeletedById(Long.valueOf(imageId), false); | |||
| } | |||
| @Override | |||
| @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) | |||
| public List<PtImage> getTerminalImageList() { | |||
| UserContext user = userContextService.getCurUser(); | |||
| LambdaQueryWrapper<PtImage> queryTerminalWrapper = new LambdaQueryWrapper<>(); | |||
| queryTerminalWrapper.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()).eq(PtImage::getDeleted, NumberConstant.NUMBER_0);; | |||
| queryTerminalWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | |||
| ; | |||
| if (user != null && !BaseService.isAdmin()) { | |||
| queryTerminalWrapper.and(wrapper -> wrapper.eq(PtImage::getCreateUserId, user.getId()).or().eq(PtImage::getImageResource, ImageSourceEnum.PRE.getCode())) | |||
| .and(wrapper -> wrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0)); | |||
| @@ -509,17 +437,16 @@ public class PtImageServiceImpl implements PtImageService { | |||
| /** | |||
| * @param ptImageUploadDTO 镜像上传逻辑校验 | |||
| * @param ptImageSaveDTO 镜像保存逻辑校验 | |||
| * @param user 用户 | |||
| * @return List<PtImage> 镜像列表 | |||
| **/ | |||
| @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) | |||
| private List<PtImage> checkUploadImage(PtImageUploadDTO ptImageUploadDTO, UserContext user, Integer source) { | |||
| private List<PtImage> checkSaveImage(PtImageSaveDTO ptImageSaveDTO, UserContext user, Integer source) { | |||
| LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | |||
| queryWrapper.eq(PtImage::getImageName, ptImageUploadDTO.getImageName()) | |||
| .eq(PtImage::getImageTag, ptImageUploadDTO.getImageTag()) | |||
| queryWrapper.eq(PtImage::getImageUrl, ptImageSaveDTO.getImageUrl()) | |||
| .eq(PtImage::getImageResource, source); | |||
| if (user != null) { | |||
| @@ -549,7 +476,7 @@ public class PtImageServiceImpl implements PtImageService { | |||
| ptImages.stream().forEach( | |||
| ptImage -> { | |||
| PtImageVO ptImageVO =new PtImageVO(); | |||
| PtImageVO ptImageVO = new PtImageVO(); | |||
| ptImageVO.setId(ptImage.getId()); | |||
| ptImageVO.setName(ptImage.getImageName()); | |||
| ptImageVO.setTag(ptImage.getImageTag()); | |||
| @@ -558,4 +485,24 @@ public class PtImageServiceImpl implements PtImageService { | |||
| ); | |||
| return ptImageVOS; | |||
| } | |||
| } | |||
| /** | |||
| * 校验镜像地址与镜像名称、tag是否匹配 | |||
| * | |||
| * @param imageUrl 镜像地址 | |||
| * @param imageName 镜像名称 | |||
| * @param imageTag 镜像tag | |||
| */ | |||
| private void validImageInfo(String imageUrl, String imageName, String imageTag) { | |||
| // 标准镜像地址:镜像地址域名/命名空间/镜像名:镜像tag | |||
| // 分解镜像地址 | |||
| String[] split = imageUrl.substring(imageUrl.lastIndexOf(StrUtil.SLASH) + 1).split(StrUtil.COLON); | |||
| if (!split[0].equals(imageName)) { | |||
| throw new BusinessException("镜像名称不匹配"); | |||
| } | |||
| if (!split[1].equals(imageTag)) { | |||
| throw new BusinessException("镜像版本号不匹配"); | |||
| } | |||
| } | |||
| } | |||
| @@ -1,77 +0,0 @@ | |||
| /** | |||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.image.service.task; | |||
| import cn.hutool.core.util.StrUtil; | |||
| import org.dubhe.biz.base.constant.HarborProperties; | |||
| import org.dubhe.biz.log.enums.LogEnum; | |||
| import org.dubhe.biz.log.utils.LogUtil; | |||
| import org.dubhe.harbor.api.HarborApi; | |||
| import org.dubhe.image.service.PtImageService; | |||
| import org.dubhe.recycle.domain.dto.RecycleCreateDTO; | |||
| import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO; | |||
| import org.dubhe.recycle.global.AbstractGlobalRecycle; | |||
| import org.springframework.beans.factory.annotation.Autowired; | |||
| import org.springframework.cloud.context.config.annotation.RefreshScope; | |||
| import org.springframework.stereotype.Component; | |||
| /** | |||
| * @description 镜像资源回收/还原 | |||
| * @date 2021-03-23 | |||
| */ | |||
| @RefreshScope | |||
| @Component(value = "imageRecycleFile") | |||
| public class ImageRecycleFile extends AbstractGlobalRecycle { | |||
| @Autowired | |||
| private HarborApi harborApi; | |||
| @Autowired | |||
| private HarborProperties harborProperties; | |||
| @Autowired | |||
| private PtImageService ptImageService; | |||
| /** | |||
| * 自定义回收镜像实现 | |||
| * | |||
| * @param detail 数据清理详情参数 | |||
| * @param dto 资源回收创建对象 | |||
| * @return true 继续执行,false 中断任务详情回收(本次无法执行完毕,创建新任务到下次执行) | |||
| */ | |||
| @Override | |||
| protected boolean clearDetail(RecycleDetailCreateDTO detail, RecycleCreateDTO dto) throws Exception { | |||
| LogUtil.info(LogEnum.IMAGE, "image custom recycle file,params:{}", detail); | |||
| if (StrUtil.isNotBlank(detail.getRecycleCondition())) { | |||
| String imageUrl = harborProperties.getAddress() + StrUtil.SLASH + detail.getRecycleCondition(); | |||
| LogUtil.info(LogEnum.IMAGE, "delete harbor image url:{}", imageUrl); | |||
| //同步删除harbor镜像 | |||
| harborApi.deleteImageByTag(imageUrl); | |||
| } | |||
| return true; | |||
| } | |||
| /** | |||
| * 数据还原 | |||
| * | |||
| * @param dto 数据清理参数 | |||
| */ | |||
| @Override | |||
| protected void rollback(RecycleCreateDTO dto) { | |||
| ptImageService.recycleRollback(dto); | |||
| } | |||
| } | |||
| @@ -21,8 +21,8 @@ import com.alibaba.fastjson.JSON; | |||
| import org.dubhe.biz.base.constant.AuthConst; | |||
| import org.dubhe.cloud.unittest.base.BaseTest; | |||
| import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||
| import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||
| import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||
| import org.dubhe.image.domain.dto.PtImageUploadDTO; | |||
| import org.junit.Test; | |||
| import org.junit.runner.RunWith; | |||
| import org.springframework.boot.test.context.SpringBootTest; | |||
| @@ -78,9 +78,9 @@ public class PtImageTest extends BaseTest { | |||
| */ | |||
| @Test | |||
| public void uploadImageTest() throws Exception { | |||
| PtImageUploadDTO ptImageUploadDTO = new PtImageUploadDTO(); | |||
| PtImageSaveDTO ptImageUploadDTO = new PtImageSaveDTO(); | |||
| ptImageUploadDTO.setImageName("mysql"); | |||
| ptImageUploadDTO.setImagePath("F:/mysql.tar"); | |||
| ptImageUploadDTO.setImageUrl("F:/mysql.tar"); | |||
| ptImageUploadDTO.setImageTag("5.7"); | |||
| ptImageUploadDTO.setRemark("测试上传镜像"); | |||
| @@ -94,7 +94,7 @@ public class PtImageTest extends BaseTest { | |||
| @Test | |||
| public void updateImageTest() throws Exception { | |||
| PtImageUpdateDTO imageUpdateDTO = new PtImageUpdateDTO(); | |||
| imageUpdateDTO.setIds(Arrays.asList()); | |||
| imageUpdateDTO.setId(1L); | |||
| imageUpdateDTO.setRemark(""); | |||
| mockMvcTest(MockMvcRequestBuilders.put("/ptImage"), JSON.toJSONString(imageUpdateDTO), | |||
| MockMvcResultMatchers.status().is2xxSuccessful(), 200); | |||
| @@ -75,7 +75,7 @@ public class DeploymentCallback extends Observable { | |||
| setChanged(); | |||
| notifyObservers(deployment); | |||
| if (StringUtils.isNotEmpty(businessLabel)){ | |||
| BaseK8sDeploymentCallbackCreateDTO baseK8sDeploymentCallbackCreateDTO = new BaseK8sDeploymentCallbackCreateDTO(deployment.getNamespace(), deployment.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),deployment.getName(), deployment.getReadyReplicas() == null?0:deployment.getReadyReplicas(), deployment.getReplicas() == null?0:deployment.getReplicas()); | |||
| BaseK8sDeploymentCallbackCreateDTO baseK8sDeploymentCallbackCreateDTO = new BaseK8sDeploymentCallbackCreateDTO(deployment.getNamespace(), deployment.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),deployment.getName(), deployment.getReadyReplicas() == null?0:deployment.getReadyReplicas(), deployment.getReplicas() == null?0:deployment.getReplicas(),watcherActionEnum.getAction()); | |||
| String url = k8sCallBackTool.getDeploymentCallbackUrl(businessLabel); | |||
| String token = k8sCallBackTool.generateToken(); | |||
| @@ -17,6 +17,7 @@ | |||
| package org.dubhe.dubhek8s.observer; | |||
| import org.dubhe.biz.base.constant.MagicNumConstant; | |||
| import org.dubhe.biz.base.enums.BizEnum; | |||
| import org.dubhe.biz.base.utils.SpringContextHolder; | |||
| import org.dubhe.biz.log.enums.LogEnum; | |||
| @@ -54,8 +55,18 @@ public class TrainJobObserver implements Observer { | |||
| BizPod pod = (BizPod)arg; | |||
| boolean trainJobFailed = PodPhaseEnum.FAILED.getPhase().equals(pod.getPhase()) && BizEnum.ALGORITHM.getBizCode().equals(pod.getBusinessLabel()) && SpringContextHolder.getActiveProfile().equals(pod.getLabel(K8sLabelConstants.PLATFORM_RUNTIME_ENV)); | |||
| if (trainJobFailed){ | |||
| LogUtil.warn(LogEnum.BIZ_K8S,"delete failed train job resourceName {};phase {};podName {}",pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),pod.getPhase(),pod.getName()); | |||
| trainJobApi.delete(pod.getNamespace(),pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE)); | |||
| new Thread(new Runnable(){ | |||
| @Override | |||
| public void run(){ | |||
| try { | |||
| Thread.sleep(MagicNumConstant.ONE_MINUTE); | |||
| } catch (InterruptedException e) { | |||
| LogUtil.error(LogEnum.BIZ_K8S,"TrainJobObserver update error {}",e.getMessage(),e); | |||
| } | |||
| LogUtil.warn(LogEnum.BIZ_K8S,"delete failed train job resourceName {};phase {};podName {}",pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),pod.getPhase(),pod.getName()); | |||
| trainJobApi.delete(pod.getNamespace(),pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE)); | |||
| } | |||
| }).start(); | |||
| } | |||
| } | |||
| } | |||
| @@ -79,7 +79,7 @@ public class SystemNodeServiceImpl implements SystemNodeService { | |||
| @Resource | |||
| private AdminClient adminClient; | |||
| private final static String INTERNAL_IP = "InternalIP"; | |||
| /** | |||
| @@ -231,7 +231,7 @@ public class SystemNodeServiceImpl implements SystemNodeService { | |||
| nodeDTO.setUid(node.getUid()); | |||
| nodeDTO.setName(node.getName()); | |||
| node.getAddresses().stream().forEach(bizNodeAddress -> { | |||
| if (INTERNAL_IP.equals(bizNodeAddress.getType())) { | |||
| if (K8sParamConstants.INTERNAL_IP.equals(bizNodeAddress.getType())) { | |||
| nodeDTO.setIp(bizNodeAddress.getAddress()); | |||
| return; | |||
| } | |||
| @@ -141,7 +141,7 @@ public class PtAtlasCameraServiceImpl implements PtAtlasCameraService { | |||
| ResponseEntity<Map> exchange = restTemplate.exchange(atlasUrlProperties.getVms() + "/checkCameraUrl", HttpMethod.POST, requestEntity, Map.class, param); | |||
| Map<String, String> statusMap = (Map) exchange.getBody().get("data"); | |||
| if (statusMap.containsKey("cameraUrl")) { | |||
| ptAtlasCamera.setHlsUrl(statusMap.get("cameraUrl").replace("127.0.0.1", "127.0.0.1")); | |||
| ptAtlasCamera.setHlsUrl(statusMap.get("cameraUrl").replace("172.18.26.2", "10.105.10.51")); | |||
| ptAtlasCameraMapper.updateById(ptAtlasCamera); | |||
| } | |||
| } catch (Exception e) { | |||
| @@ -21,13 +21,10 @@ import cn.hutool.core.collection.CollUtil; | |||
| import cn.hutool.core.date.DateBetween; | |||
| import cn.hutool.core.date.DateUnit; | |||
| import cn.hutool.core.util.RandomUtil; | |||
| import cn.hutool.core.util.StrUtil; | |||
| import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; | |||
| import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; | |||
| import com.baomidou.mybatisplus.core.metadata.IPage; | |||
| import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | |||
| import org.dubhe.biz.base.utils.StringUtils; | |||
| import org.dubhe.biz.base.constant.HarborProperties; | |||
| import org.dubhe.biz.base.constant.MagicNumConstant; | |||
| import org.dubhe.biz.base.constant.NumberConstant; | |||
| import org.dubhe.biz.base.constant.StringConstant; | |||
| @@ -45,8 +42,10 @@ import org.dubhe.biz.base.service.UserContextService; | |||
| import org.dubhe.biz.base.utils.HttpUtils; | |||
| import org.dubhe.biz.base.utils.NumberUtil; | |||
| import org.dubhe.biz.base.utils.ResultUtil; | |||
| import org.dubhe.biz.base.utils.StringUtils; | |||
| import org.dubhe.biz.base.vo.DataResponseBody; | |||
| import org.dubhe.biz.base.vo.DatasetVO; | |||
| import org.dubhe.biz.base.vo.NoteBookVO; | |||
| import org.dubhe.biz.db.utils.PageUtil; | |||
| import org.dubhe.biz.db.utils.WrapperHelp; | |||
| import org.dubhe.biz.file.api.FileStoreApi; | |||
| @@ -79,7 +78,6 @@ import org.dubhe.notebook.enums.NoteBookStatusEnum; | |||
| import org.dubhe.notebook.service.NoteBookService; | |||
| import org.dubhe.notebook.service.ProcessNotebookCommand; | |||
| import org.dubhe.notebook.utils.NotebookUtil; | |||
| import org.dubhe.biz.base.vo.NoteBookVO; | |||
| import org.springframework.beans.BeanUtils; | |||
| import org.springframework.beans.factory.annotation.Autowired; | |||
| import org.springframework.beans.factory.annotation.Qualifier; | |||
| @@ -89,7 +87,13 @@ import org.springframework.transaction.annotation.Transactional; | |||
| import org.springframework.util.CollectionUtils; | |||
| import java.io.File; | |||
| import java.util.*; | |||
| import java.util.ArrayList; | |||
| import java.util.Arrays; | |||
| import java.util.Collections; | |||
| import java.util.Date; | |||
| import java.util.List; | |||
| import java.util.Map; | |||
| import java.util.Set; | |||
| import java.util.stream.Collectors; | |||
| /** | |||
| @@ -127,9 +131,6 @@ public class NoteBookServiceImpl implements NoteBookService { | |||
| @Autowired | |||
| private ImageClient imageClient; | |||
| @Autowired | |||
| private HarborProperties harborProperties; | |||
| @Autowired | |||
| @Qualifier("hostFileStoreApiImpl") | |||
| private FileStoreApi fileStoreApi; | |||
| @@ -223,7 +224,7 @@ public class NoteBookServiceImpl implements NoteBookService { | |||
| throw new BusinessException(ImageTypeEnum.NOTEBOOK.getCode() + "未配置默认镜像!"); | |||
| } | |||
| return harborProperties.getAddress() + StrUtil.SLASH + imageUrl; | |||
| return imageUrl; | |||
| } | |||
| /** | |||
| @@ -1,39 +0,0 @@ | |||
| /** | |||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.pointcloud.config; | |||
| import lombok.Data; | |||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||
| import org.springframework.stereotype.Component; | |||
| /** | |||
| * @description harbor相关配置 | |||
| * @date 2020-07-17 | |||
| */ | |||
| @Data | |||
| @Component | |||
| @ConfigurationProperties(prefix = "harbor") | |||
| public class PointCloudHarborConfig { | |||
| private String address; | |||
| private String username; | |||
| private String password; | |||
| private String modelName; | |||
| } | |||
| @@ -66,7 +66,6 @@ import org.dubhe.pointcloud.client.DataClient; | |||
| import org.dubhe.pointcloud.client.ImageClient; | |||
| import org.dubhe.pointcloud.client.ModelClient; | |||
| import org.dubhe.pointcloud.common.Constant; | |||
| import org.dubhe.pointcloud.config.PointCloudHarborConfig; | |||
| import org.dubhe.pointcloud.dao.PcAnnotationDetailMapper; | |||
| import org.dubhe.pointcloud.dao.PcDatasetFileMapper; | |||
| import org.dubhe.pointcloud.dao.PcDatasetMapper; | |||
| @@ -141,9 +140,6 @@ public class PcDatasetServiceImpl implements PcDatasetService { | |||
| @Resource | |||
| private DeployAsyncTask deployAsyncTask; | |||
| @Resource | |||
| private PointCloudHarborConfig pointCloudHarborConfig; | |||
| @Resource | |||
| private ImageClient imageClient; | |||
| @@ -652,7 +648,7 @@ public class PcDatasetServiceImpl implements PcDatasetService { | |||
| if (!dataResponseBody.succeed()) { | |||
| throw new BusinessException(ErrorEnum.CALL_IMAGE_SERVER_FAIL); | |||
| } | |||
| return pointCloudHarborConfig.getAddress() + SymbolConstant.SLASH + dataResponseBody.getData(); | |||
| return dataResponseBody.getData(); | |||
| } | |||
| /** | |||
| @@ -18,8 +18,10 @@ | |||
| package org.dubhe.servinggateway.config; | |||
| import lombok.extern.slf4j.Slf4j; | |||
| import org.dubhe.biz.base.constant.MagicNumConstant; | |||
| import org.dubhe.biz.base.constant.NumberConstant; | |||
| import org.dubhe.biz.base.constant.SymbolConstant; | |||
| import org.dubhe.biz.base.utils.StringUtils; | |||
| import org.dubhe.biz.log.enums.LogEnum; | |||
| import org.dubhe.biz.log.utils.LogUtil; | |||
| import org.dubhe.servinggateway.constant.GatewayConstant; | |||
| @@ -38,8 +40,8 @@ import org.springframework.web.util.UriComponentsBuilder; | |||
| import reactor.core.publisher.Mono; | |||
| import javax.annotation.Resource; | |||
| import java.util.ArrayList; | |||
| import java.util.Arrays; | |||
| import java.util.Collections; | |||
| import java.util.HashMap; | |||
| import java.util.List; | |||
| import java.util.Map; | |||
| @@ -55,21 +57,12 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
| private ApplicationEventPublisher publisher; | |||
| @Value("${serving.gateway.postfixUrl}") | |||
| private String postfixUrl; | |||
| @Resource | |||
| private RedisRouteDefinitionRepository routeDefinitionRepository; | |||
| @Resource | |||
| private GatewayRouteService gatewayRouteService; | |||
| /** | |||
| * 模型服务部署使用的端口 | |||
| */ | |||
| @Value("${k8s.port}") | |||
| private String httpPort; | |||
| @Override | |||
| public void run(String... args) { | |||
| this.loadRouteConfig(); | |||
| @@ -87,6 +80,9 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
| LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin load route config"); | |||
| List<GatewayRouteQueryVO> activeRoutes = gatewayRouteService.findActiveRoutes(); | |||
| for (GatewayRouteQueryVO activeRoute : activeRoutes) { | |||
| if (StringUtils.isEmpty(activeRoute.getUri())){ | |||
| continue; | |||
| } | |||
| RouteDefinition definition = this.convert2RouteDefinition(activeRoute); | |||
| routeDefinitionRepository.save(Mono.just(definition)).subscribe(); | |||
| } | |||
| @@ -103,7 +99,19 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
| public void saveRouteByRouteId(Long id) { | |||
| LogUtil.info(LogEnum.SERVING_GATEWAY, "Deal save route event"); | |||
| GatewayRouteQueryVO gatewayRouteQueryVO = gatewayRouteService.findActiveById(id); | |||
| if (Objects.nonNull(gatewayRouteQueryVO)) { | |||
| //查不到数据进行重试 | |||
| int count = 0; | |||
| while (gatewayRouteQueryVO == null && count < MagicNumConstant.THREE){ | |||
| LogUtil.info(LogEnum.SERVING_GATEWAY, "gatewayRouteQueryVO is null retry:"+count +" id="+id); | |||
| gatewayRouteQueryVO = gatewayRouteService.findActiveById(id); | |||
| count++; | |||
| try { | |||
| Thread.sleep(MagicNumConstant.FIVE_HUNDRED); | |||
| } catch (InterruptedException e) { | |||
| e.printStackTrace(); | |||
| } | |||
| } | |||
| if (Objects.nonNull(gatewayRouteQueryVO) && StringUtils.isNotEmpty(gatewayRouteQueryVO.getUri())) { | |||
| routeDefinitionRepository.save(Mono.just(this.convert2RouteDefinition(gatewayRouteQueryVO))).subscribe(); | |||
| } | |||
| // 发布刷新路由事件 | |||
| @@ -131,17 +139,19 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
| * @return 网关所需的RouteDefinition | |||
| */ | |||
| private RouteDefinition convert2RouteDefinition(GatewayRouteQueryVO gatewayRouteQueryVO) { | |||
| if(StringUtils.isEmpty(gatewayRouteQueryVO.getUri())){ | |||
| return null; | |||
| } | |||
| RouteDefinition definition = new RouteDefinition(); | |||
| // 设置路由基础信息 | |||
| definition.setId(GatewayConstant.ROUTE_PREFIX + gatewayRouteQueryVO.getId()); | |||
| definition.setUri(UriComponentsBuilder.fromHttpUrl(SymbolConstant.HTTP_SLASH + gatewayRouteQueryVO.getUri() + | |||
| SymbolConstant.COLON + httpPort).build().toUri()); | |||
| definition.setUri(UriComponentsBuilder.fromHttpUrl(SymbolConstant.HTTP_SLASH + gatewayRouteQueryVO.getUri()).build().toUri()); | |||
| // 定义url匹配规则的断言 | |||
| // 举例 {abc}.dubhe.ai 匹配{abc}部分 | |||
| PredicateDefinition pathPredicate = new PredicateDefinition(); | |||
| pathPredicate.setName("Host"); | |||
| pathPredicate.setName("Path"); | |||
| Map<String, String> predicateParams = new HashMap<>(NumberConstant.NUMBER_8); | |||
| predicateParams.put("pattern", gatewayRouteQueryVO.getPatternPath() + postfixUrl); | |||
| predicateParams.put("pattern", "/"+gatewayRouteQueryVO.getPatternPath() + "/**"); | |||
| pathPredicate.setArgs(predicateParams); | |||
| // 定义权重断言 | |||
| // 根据配置的权重信息进行分流 | |||
| @@ -152,11 +162,21 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
| weightParams.put("weight.weight", gatewayRouteQueryVO.getWeight()); | |||
| weightPredicate.setArgs(weightParams); | |||
| definition.setPredicates(Arrays.asList(pathPredicate, weightPredicate)); | |||
| List<FilterDefinition> filters =new ArrayList<>(); | |||
| // 传入自定义的监控指标过滤器 | |||
| FilterDefinition filterDefinition = new FilterDefinition(); | |||
| // name设置为定义的MetricsGatewayFilterFactory中GatewayFilterFactory前部分 | |||
| filterDefinition.setName("Metrics"); | |||
| definition.setFilters(Collections.singletonList(filterDefinition)); | |||
| filters.add(filterDefinition); | |||
| //去除一级url转发 | |||
| FilterDefinition filterDefinitionUrl = new FilterDefinition(); | |||
| filterDefinitionUrl.setName("StripPrefix"); | |||
| filterDefinitionUrl.addArg("parts","1"); | |||
| filters.add(filterDefinitionUrl); | |||
| definition.setFilters(filters); | |||
| // 塞入自定义数据,供之后filter使用 | |||
| HashMap<String, Object> metadata = new HashMap<>(NumberConstant.NUMBER_4); | |||
| metadata.put("servingInfoId", gatewayRouteQueryVO.getServiceInfoId()); | |||
| @@ -120,7 +120,7 @@ public class MetricsGatewayFilterFactory extends AbstractGatewayFilterFactory<Me | |||
| DataBufferUtils.join(body) | |||
| .doOnNext(dataBuffer -> { | |||
| // 判断该请求是否为推理请求 | |||
| boolean isInference = exchange.getRequest().getPath().toString().startsWith(GatewayConstant.INFERENCE_INTERFACE_NAME); | |||
| boolean isInference = exchange.getRequest().getPath().toString().endsWith(GatewayConstant.INFERENCE_INTERFACE_NAME); | |||
| if (isInference) { | |||
| LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin deal inference request filter"); | |||
| // 调用请求是否成功 | |||
| @@ -7,7 +7,7 @@ | |||
| select smc.id id, si.id serviceInfoId, si.uuid patternPath, smc.release_rate weight, smc.url uri | |||
| from serving_model_config smc | |||
| left join serving_info si on si.id = smc.serving_id | |||
| where si.`status` = 2 and si.deleted = 0 and smc.ready_replicas > 0 and smc.deleted = 0 | |||
| where si.deleted = 0 and smc.deleted = 0 | |||
| and smc.id = #{id} | |||
| </select> | |||
| @@ -1,39 +0,0 @@ | |||
| /** | |||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.serving.config; | |||
| import lombok.Data; | |||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||
| import org.springframework.stereotype.Component; | |||
| /** | |||
| * @description harbor相关配置 | |||
| * @date 2020-07-17 | |||
| */ | |||
| @Data | |||
| @Component | |||
| @ConfigurationProperties(prefix = "harbor") | |||
| public class TrainHarborConfig { | |||
| private String address; | |||
| private String username; | |||
| private String password; | |||
| private String modelName; | |||
| } | |||
| @@ -71,7 +71,6 @@ import org.dubhe.serving.client.AlgorithmClient; | |||
| import org.dubhe.serving.client.ImageClient; | |||
| import org.dubhe.serving.client.ModelBranchClient; | |||
| import org.dubhe.serving.client.ModelInfoClient; | |||
| import org.dubhe.serving.config.TrainHarborConfig; | |||
| import org.dubhe.serving.constant.ServingConstant; | |||
| import org.dubhe.serving.dao.BatchServingMapper; | |||
| import org.dubhe.serving.domain.dto.*; | |||
| @@ -135,8 +134,6 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat | |||
| @Resource | |||
| private ImageClient imageClient; | |||
| @Resource | |||
| private TrainHarborConfig trainHarborConfig; | |||
| @Resource | |||
| private AlgorithmClient algorithmClient; | |||
| @Autowired | |||
| private RedisUtils redisUtils; | |||
| @@ -238,7 +235,7 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat | |||
| BeanUtils.copyProperties(batchServingCreateDTO, batchServing); | |||
| String imageUrl = getImageUrl(batchServingCreateDTO.getImageName(), batchServingCreateDTO.getImageTag()); | |||
| batchServing.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + imageUrl); | |||
| batchServing.setImage(imageUrl); | |||
| PtModelInfoQueryVO ptModelInfoQueryVO = getPtModelInfo(batchServingCreateDTO.getModelId()); | |||
| batchServing.setModelAddress(ptModelInfoQueryVO.getModelAddress()); | |||
| if (ptModelInfoQueryVO.getFrameType() > NumberConstant.NUMBER_4) { | |||
| @@ -421,7 +418,7 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat | |||
| BeanUtils.copyProperties(batchServingUpdateDTO, batchServing); | |||
| String imageUrl = getImageUrl(batchServingUpdateDTO.getImageName(), batchServingUpdateDTO.getImageTag()); | |||
| batchServing.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + imageUrl); | |||
| batchServing.setImage(imageUrl); | |||
| checkScriptPath(batchServing); | |||
| checkResourceType(batchServing.getFrameType(), batchServing.getResourcesPoolType()); | |||
| batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); | |||
| @@ -18,6 +18,7 @@ | |||
| package org.dubhe.serving.service.impl; | |||
| import cn.hutool.core.bean.BeanUtil; | |||
| import com.alibaba.fastjson.JSON; | |||
| import com.alibaba.fastjson.JSONObject; | |||
| import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; | |||
| import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; | |||
| @@ -56,11 +57,16 @@ import org.dubhe.biz.permission.annotation.DataPermissionMethod; | |||
| import org.dubhe.biz.permission.base.BaseService; | |||
| import org.dubhe.biz.redis.utils.RedisUtils; | |||
| import org.dubhe.k8s.api.MetricsApi; | |||
| import org.dubhe.k8s.api.ModelServingApi; | |||
| import org.dubhe.k8s.api.NodeApi; | |||
| import org.dubhe.k8s.cache.ResourceCache; | |||
| import org.dubhe.k8s.domain.dto.PodQueryDTO; | |||
| import org.dubhe.k8s.domain.resource.BizServicePort; | |||
| import org.dubhe.k8s.domain.vo.ModelServingVO; | |||
| import org.dubhe.k8s.domain.vo.PodVO; | |||
| import org.dubhe.k8s.domain.vo.PtPodsVO; | |||
| import org.dubhe.k8s.enums.PodPhaseEnum; | |||
| import org.dubhe.k8s.enums.WatcherActionEnum; | |||
| import org.dubhe.k8s.service.PodService; | |||
| import org.dubhe.k8s.utils.K8sNameTool; | |||
| import org.dubhe.recycle.config.RecycleConfig; | |||
| @@ -75,7 +81,6 @@ import org.dubhe.serving.client.AlgorithmClient; | |||
| import org.dubhe.serving.client.ImageClient; | |||
| import org.dubhe.serving.client.ModelBranchClient; | |||
| import org.dubhe.serving.client.ModelInfoClient; | |||
| import org.dubhe.serving.config.TrainHarborConfig; | |||
| import org.dubhe.serving.constant.ServingConstant; | |||
| import org.dubhe.serving.dao.ServingInfoMapper; | |||
| import org.dubhe.serving.dao.ServingModelConfigMapper; | |||
| @@ -178,8 +183,6 @@ public class ServingServiceImpl implements ServingService { | |||
| @Resource | |||
| private ImageClient imageClient; | |||
| @Resource | |||
| private TrainHarborConfig trainHarborConfig; | |||
| @Resource | |||
| private AlgorithmClient algorithmClient; | |||
| @Resource | |||
| private RecycleConfig recycleConfig; | |||
| @@ -191,6 +194,10 @@ public class ServingServiceImpl implements ServingService { | |||
| private ResourceCache resourceCache; | |||
| @Value("Task:Serving:" + "${spring.profiles.active}_serving_id_") | |||
| private String servingIdPrefix; | |||
| @Resource | |||
| private ModelServingApi modelServingApi; | |||
| @Resource | |||
| private NodeApi nodeApi; | |||
| private final static List<String> FILE_NAMES; | |||
| @@ -265,7 +272,7 @@ public class ServingServiceImpl implements ServingService { | |||
| List<ServingInfoQueryVO> queryList = servingInfos.getRecords().stream().map(servingInfo -> { | |||
| ServingInfoQueryVO servingInfoQueryVO = new ServingInfoQueryVO(); | |||
| BeanUtils.copyProperties(servingInfo, servingInfoQueryVO); | |||
| servingInfoQueryVO.setUrl(servingInfo.getUuid() + GATEWAY_URI_POSTFIX); | |||
| servingInfoQueryVO.setUrl(GATEWAY_URI_POSTFIX+"/"+servingInfo.getUuid()); | |||
| Map<String, String> statistics = servingLuaScriptService.countCallsByServingInfoId(servingInfo.getId()); | |||
| servingInfoQueryVO.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO)); | |||
| servingInfoQueryVO.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO)); | |||
| @@ -419,7 +426,7 @@ public class ServingServiceImpl implements ServingService { | |||
| throw new BusinessException(ServingErrorEnum.CALL_IMAGE_SERVER_FAIL); | |||
| } | |||
| servingModelConfig | |||
| .setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + dataResponseBody.getData()); | |||
| .setImage(dataResponseBody.getData()); | |||
| // 校验模型文件是否存在 | |||
| String path = k8sNameTool.getAbsolutePath(servingModelConfig.getModelAddress()); | |||
| @@ -933,8 +940,8 @@ public class ServingServiceImpl implements ServingService { | |||
| predictParamVO.setOutputs(outputs); | |||
| predictParamVO.setOther(other); | |||
| } else if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | |||
| String url = "http://" + servingInfo.getUuid() + GATEWAY_URI_POSTFIX | |||
| + ServingConstant.INFERENCE_INTERFACE_NAME; | |||
| String url = "http://" + GATEWAY_URI_POSTFIX | |||
| +"/"+ servingInfo.getUuid()+ ServingConstant.INFERENCE_INTERFACE_NAME; | |||
| predictParamVO.setUrl(url); | |||
| Map<String, String> inputs = new HashMap<>(); | |||
| inputs.put("files", "File"); | |||
| @@ -1079,6 +1086,7 @@ public class ServingServiceImpl implements ServingService { | |||
| */ | |||
| @Override | |||
| public boolean servingDeploymentCallback(ServingK8sDeploymentCallbackCreateDTO req) { | |||
| LogUtil.info(LogEnum.BIZ_K8S,"servingDeploymentCallback:{}", JSON.toJSONString(req)); | |||
| // 根据namespace和podName找到模型配置 | |||
| String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.SERVING, req.getResourceName()); | |||
| if (StringUtils.isBlank(resourceInfo)) { | |||
| @@ -1105,13 +1113,13 @@ public class ServingServiceImpl implements ServingService { | |||
| return false; | |||
| } | |||
| // 增加发送路由信息 | |||
| if (req.getReadyReplicas() > NumberConstant.NUMBER_0 | |||
| if (WatcherActionEnum.ADDED.getAction().equals(req.getAction()) | |||
| && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | |||
| this.notifyUpdateServingRoute(Collections.singletonList(servingModelConfig.getId()), | |||
| Collections.emptyList()); | |||
| } | |||
| // 增加删除路由信息 | |||
| if (req.getReadyReplicas() == NumberConstant.NUMBER_0 | |||
| if (WatcherActionEnum.DELETED.getAction().equals(req.getAction()) | |||
| && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | |||
| this.notifyUpdateServingRoute(Collections.emptyList(), | |||
| Collections.singletonList(servingModelConfig.getId())); | |||
| @@ -1148,8 +1156,29 @@ public class ServingServiceImpl implements ServingService { | |||
| @Transactional(rollbackFor = Exception.class) | |||
| public boolean updateByCallback(ServingK8sDeploymentCallbackCreateDTO req, ServingModelConfig servingModelConfig, | |||
| ServingInfo servingInfo) { | |||
| //更新url | |||
| if (StringUtils.isEmpty(servingModelConfig.getUrl())){ | |||
| //查询 | |||
| ModelServingVO modelServingVO = modelServingApi.get(req.getNamespace(),req.getResourceName()); | |||
| if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { | |||
| // 获取pod对应的url,并修改模型部署状态 | |||
| List<BizServicePort> ports = modelServingVO.getBizService().getPorts(); | |||
| if (CollectionUtils.isNotEmpty(ports)) { | |||
| //取第一个url | |||
| String url = ""; | |||
| if (ports.get(NumberConstant.NUMBER_0).getNodePort() != null) { | |||
| url = nodeApi.getAvailableNodeIp() + ":" + ports.get(NumberConstant.NUMBER_0).getNodePort(); | |||
| } | |||
| servingModelConfig.setUrl(url); | |||
| } | |||
| } | |||
| } | |||
| // 更新当前模型配置有效节点数 | |||
| servingModelConfig.setReadyReplicas(req.getReadyReplicas()); | |||
| servingModelConfig.setResourceInfo(null); | |||
| int result = servingModelConfigMapper.updateById(servingModelConfig); | |||
| if (result < NumberConstant.NUMBER_1) { | |||
| return true; | |||
| @@ -31,6 +31,7 @@ import org.dubhe.biz.log.enums.LogEnum; | |||
| import org.dubhe.biz.log.utils.LogUtil; | |||
| import org.dubhe.k8s.api.DistributeTrainApi; | |||
| import org.dubhe.k8s.api.ModelServingApi; | |||
| import org.dubhe.k8s.api.NodeApi; | |||
| import org.dubhe.k8s.api.TrainJobApi; | |||
| import org.dubhe.k8s.domain.PtBaseResult; | |||
| import org.dubhe.k8s.domain.bo.DistributeTrainBO; | |||
| @@ -38,7 +39,7 @@ import org.dubhe.k8s.domain.bo.ModelServingBO; | |||
| import org.dubhe.k8s.domain.bo.PtJupyterJobBO; | |||
| import org.dubhe.k8s.domain.bo.PtMountDirBO; | |||
| import org.dubhe.k8s.domain.resource.BizDistributeTrain; | |||
| import org.dubhe.k8s.domain.resource.BizIngressRule; | |||
| import org.dubhe.k8s.domain.resource.BizServicePort; | |||
| import org.dubhe.k8s.domain.vo.ModelServingVO; | |||
| import org.dubhe.k8s.domain.vo.PtJupyterJobVO; | |||
| import org.dubhe.k8s.utils.K8sNameTool; | |||
| @@ -88,6 +89,9 @@ public class DeployServingAsyncTask { | |||
| @Resource | |||
| private ModelServingApi modelServingApi; | |||
| @Resource | |||
| private NodeApi nodeApi; | |||
| @Resource | |||
| private K8sNameTool k8sNameTool; | |||
| @@ -143,11 +147,14 @@ public class DeployServingAsyncTask { | |||
| String statusDetailKey = ServingStatusDetailDescUtil.getServingStatusDetailKey(ServingStatusDetailDescUtil.CONTAINER_DEPLOYMENT_EXCEPTION, uniqueName); | |||
| if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { | |||
| // 获取pod对应的url,并修改模型部署状态 | |||
| List<BizIngressRule> rules = modelServingVO.getBizIngress().getRules(); | |||
| List<BizServicePort> ports = modelServingVO.getBizService().getPorts(); | |||
| if (CollectionUtils.isNotEmpty(rules)) { | |||
| if (CollectionUtils.isNotEmpty(ports)) { | |||
| //取第一个url | |||
| String url = rules.get(NumberConstant.NUMBER_0).getHost(); | |||
| String url = ""; | |||
| if (ports.get(NumberConstant.NUMBER_0).getNodePort() != null){ | |||
| url = nodeApi.getAvailableNodeIp()+":"+ports.get(NumberConstant.NUMBER_0).getNodePort(); | |||
| } | |||
| servingModelConfig.setUrl(url); | |||
| flag = true; | |||
| servingInfo.removeStatusDetail(statusDetailKey); | |||
| @@ -176,7 +183,6 @@ public class DeployServingAsyncTask { | |||
| servingInfo.putStatusDetail(statusDetailKey, e.getMessage()); | |||
| LogUtil.error(LogEnum.SERVING, "User {} create serving failed.The name of serving is {}", user.getUsername(), servingInfo.getName(), e); | |||
| } | |||
| } | |||
| //修改服务状态 | |||
| @@ -276,7 +282,7 @@ public class DeployServingAsyncTask { | |||
| return null; | |||
| } | |||
| } else { | |||
| LogUtil.info(LogEnum.SERVING, "User {} failed to copy the inference script, script {} not exist", user.getUsername(), pyPathList.get(0)); | |||
| LogUtil.info(LogEnum.SERVING, "User {} failed to copy the inference script, script {} not exist", user.getUsername(), pyPathList); | |||
| return null; | |||
| } | |||
| servingPath = fileStoreApi.formatPath(targetPath + getSourceName(sourcePath)); | |||
| @@ -17,13 +17,9 @@ | |||
| package org.dubhe.serving.utils; | |||
| import cn.hutool.core.codec.Base64; | |||
| import cn.hutool.core.io.FileUtil; | |||
| import io.grpc.ManagedChannel; | |||
| import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; | |||
| import io.grpc.netty.shaded.io.grpc.netty.NegotiationType; | |||
| import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; | |||
| import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; | |||
| import org.dubhe.biz.base.constant.NumberConstant; | |||
| import org.dubhe.biz.base.context.UserContext; | |||
| import org.dubhe.biz.log.enums.LogEnum; | |||
| @@ -31,12 +27,9 @@ import org.dubhe.biz.log.utils.LogUtil; | |||
| import org.dubhe.serving.domain.entity.DataInfo; | |||
| import org.dubhe.serving.proto.Inference; | |||
| import org.dubhe.serving.proto.InferenceServiceGrpc; | |||
| import org.springframework.beans.factory.annotation.Value; | |||
| import org.springframework.stereotype.Component; | |||
| import javax.net.ssl.SSLException; | |||
| import java.io.File; | |||
| import java.io.InputStream; | |||
| import java.util.List; | |||
| import java.util.concurrent.ConcurrentHashMap; | |||
| import java.util.concurrent.TimeUnit; | |||
| @@ -52,51 +45,20 @@ public class GrpcClient { | |||
| * <服务id, grpc客户端channel> | |||
| */ | |||
| public static ConcurrentHashMap<Long, ManagedChannel> channelMap = new ConcurrentHashMap<>(); | |||
| /** | |||
| * 项目根路径 | |||
| */ | |||
| private final static String USER_DIR_SYSTEM_PROPERTY = "user.dir"; | |||
| /** | |||
| * k8s crt证书文件名 | |||
| */ | |||
| private final static String TLS_CRT = "server.crt"; | |||
| /** | |||
| * k8s grpc端口 | |||
| */ | |||
| @Value("${k8s.https-port:31365}") | |||
| private String grpcPort; | |||
| /** | |||
| * k8s 模型部署tls证书 crt | |||
| */ | |||
| @Value("${k8s.serving.tls-crt}") | |||
| private String tlsCrt; | |||
| /** | |||
| * 创建带tls认证的grpc通道 | |||
| * 创建grpc通道 | |||
| * | |||
| * @param url tls证书地址 | |||
| * @param url | |||
| * @return ManagedChannel 通道 | |||
| * @throws SSLException | |||
| */ | |||
| public ManagedChannel createTlsChannel(String url) throws Exception { | |||
| String path = getTlsCrt(); | |||
| SslContext sslContext = GrpcSslContexts.forClient().trustManager(new File(path)).build(); | |||
| ManagedChannel channel = NettyChannelBuilder.forAddress(url, Integer.parseInt(grpcPort)).maxInboundMessageSize(NumberConstant.MAX_MESSAGE_LENGTH).negotiationType(NegotiationType.TLS).sslContext(sslContext).build(); | |||
| public ManagedChannel createChannel(String url) { | |||
| String[] host = url.split(":"); | |||
| ManagedChannel channel = NettyChannelBuilder.forAddress(host[0], Integer.parseInt(host[1])).maxInboundMessageSize(NumberConstant.MAX_MESSAGE_LENGTH).negotiationType(NegotiationType.PLAINTEXT).build(); | |||
| return channel; | |||
| } | |||
| /** | |||
| * 获取crt证书路径 | |||
| */ | |||
| public String getTlsCrt() { | |||
| String path = System.getProperty(USER_DIR_SYSTEM_PROPERTY) + File.separator.concat(TLS_CRT); | |||
| if(!FileUtil.exist(path)) { | |||
| FileUtil.writeUtf8String(Base64.decodeStr(tlsCrt), path); | |||
| } | |||
| return path; | |||
| } | |||
| /** | |||
| * 关闭通道 | |||
| * | |||
| @@ -148,7 +110,7 @@ public class GrpcClient { | |||
| } | |||
| ManagedChannel channel = null; | |||
| try { | |||
| channel = this.createTlsChannel(url); | |||
| channel = this.createChannel(url); | |||
| channelMap.put(servingId, channel); | |||
| } catch (Exception e) { | |||
| LogUtil.error(LogEnum.SERVING, "An Exception occurred when user {} creating the grpc channel, service id:{}", user.getUsername(), servingId, e); | |||
| @@ -169,7 +131,7 @@ public class GrpcClient { | |||
| } | |||
| ManagedChannel channel = null; | |||
| try { | |||
| channel = this.createTlsChannel(url); | |||
| channel = this.createChannel(url); | |||
| channelMap.put(servingId, channel); | |||
| } catch (Exception e) { | |||
| LogUtil.error(LogEnum.SERVING, "An Exception occurred when getting grpc channel, service id:{}", servingId, e); | |||
| @@ -71,13 +71,6 @@ public class PtImage extends BaseEntity { | |||
| @TableField(value = "image_resource") | |||
| private Integer imageResource; | |||
| /** | |||
| * 镜像状态 | |||
| */ | |||
| @TableField(value = "image_status") | |||
| private Integer imageStatus; | |||
| /** | |||
| * 资源拥有者ID | |||
| */ | |||
| @@ -182,6 +182,6 @@ public class Terminal extends BaseEntity { | |||
| return null; | |||
| } | |||
| String[] strings = imageUrl.split(SymbolConstant.SLASH); | |||
| return strings.length > 0 ? strings[MagicNumConstant.ZERO] : null; | |||
| return strings.length > 0 ? strings[MagicNumConstant.ONE] : null; | |||
| } | |||
| } | |||
| @@ -16,7 +16,6 @@ | |||
| */ | |||
| package org.dubhe.terminal.service.impl; | |||
| import java.sql.Timestamp; | |||
| import cn.hutool.core.util.RandomUtil; | |||
| import cn.hutool.core.util.StrUtil; | |||
| @@ -38,6 +37,7 @@ import org.dubhe.biz.log.utils.LogUtil; | |||
| import org.dubhe.biz.permission.base.BaseService; | |||
| import org.dubhe.biz.redis.utils.RedisUtils; | |||
| import org.dubhe.docker.api.DockerApi; | |||
| import org.dubhe.docker.callback.TerminalPushImageResultCallback; | |||
| import org.dubhe.docker.config.DockerClientFactory; | |||
| import org.dubhe.docker.enums.DockerOperationEnum; | |||
| import org.dubhe.docker.utils.DockerCallbackTool; | |||
| @@ -53,7 +53,6 @@ import org.dubhe.k8s.domain.vo.TerminalResourceVO; | |||
| import org.dubhe.k8s.enums.BusinessLabelServiceNameEnum; | |||
| import org.dubhe.k8s.enums.PodPhaseEnum; | |||
| import org.dubhe.k8s.utils.K8sNameTool; | |||
| import org.dubhe.docker.callback.TerminalPushImageResultCallback; | |||
| import org.dubhe.terminal.config.TerminalConfig; | |||
| import org.dubhe.terminal.constant.TerminalConstant; | |||
| import org.dubhe.terminal.dao.PtImageMapper; | |||
| @@ -77,6 +76,7 @@ import org.springframework.stereotype.Service; | |||
| import org.springframework.transaction.annotation.Transactional; | |||
| import org.springframework.util.CollectionUtils; | |||
| import java.sql.Timestamp; | |||
| import java.util.ArrayList; | |||
| import java.util.Date; | |||
| import java.util.List; | |||
| @@ -174,38 +174,43 @@ public class TerminalServiceImpl implements TerminalService { | |||
| @Override | |||
| public boolean preserve(TerminalPreserveDTO terminalPreserveDTO) { | |||
| Terminal terminal = terminalMapper.selectById(terminalPreserveDTO.getId()); | |||
| if (terminal == null){ | |||
| LogUtil.error(LogEnum.TERMINAL,"preserve terminal 数据不存在 terminalPreserveDTO: {}",terminalPreserveDTO); | |||
| if (checkImageExists(terminal.getImageProject(), terminalPreserveDTO.getImageName(), terminalPreserveDTO.getImageTag())) { | |||
| throw new BusinessException("请确认镜像版本号是否重复!"); | |||
| } | |||
| if (terminal == null) { | |||
| LogUtil.error(LogEnum.TERMINAL, "preserve terminal 数据不存在 terminalPreserveDTO: {}", terminalPreserveDTO); | |||
| throw new BusinessException("数据不存在"); | |||
| } | |||
| LambdaQueryWrapper<TerminalInfo> wrapper = new LambdaQueryWrapper<>(); | |||
| wrapper.eq(TerminalInfo::getTerminalId, terminalPreserveDTO.getId()); | |||
| List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectList(wrapper); | |||
| if (CollectionUtils.isEmpty(terminalInfoList)){ | |||
| LogUtil.error(LogEnum.TERMINAL,"preserve terminalInfoList 数据不存在 terminalPreserveDTO: {}",terminalPreserveDTO); | |||
| if (CollectionUtils.isEmpty(terminalInfoList)) { | |||
| LogUtil.error(LogEnum.TERMINAL, "preserve terminalInfoList 数据不存在 terminalPreserveDTO: {}", terminalPreserveDTO); | |||
| throw new BusinessException("数据不存在"); | |||
| } | |||
| TerminalInfo masterTerminalInfo = null; | |||
| for (TerminalInfo terminalInfo : terminalInfoList) { | |||
| if (terminalInfo.isMasterFlag()){ | |||
| if (terminalInfo.isMasterFlag()) { | |||
| masterTerminalInfo = terminalInfo; | |||
| } | |||
| } | |||
| if (masterTerminalInfo == null){ | |||
| LogUtil.error(LogEnum.TERMINAL,"master 节点不存在 terminalPreserveDTO:{}",terminalPreserveDTO); | |||
| if (masterTerminalInfo == null) { | |||
| LogUtil.error(LogEnum.TERMINAL, "master 节点不存在 terminalPreserveDTO:{}", terminalPreserveDTO); | |||
| throw new BusinessException("master 节点不存在"); | |||
| } | |||
| BizPod pod = podApi.getWithResourceName(k8sNameTool.getNamespace(terminal.getCreateUserId()),masterTerminalInfo.getK8sResourceName()); | |||
| if (pod == null){ | |||
| LogUtil.error(LogEnum.TERMINAL,"master 容器不存在 terminalPreserveDTO:{}",terminalPreserveDTO); | |||
| BizPod pod = podApi.getWithResourceName(k8sNameTool.getNamespace(terminal.getCreateUserId()), masterTerminalInfo.getK8sResourceName()); | |||
| if (pod == null) { | |||
| LogUtil.error(LogEnum.TERMINAL, "master 容器不存在 terminalPreserveDTO:{}", terminalPreserveDTO); | |||
| throw new BusinessException("master 容器不存在"); | |||
| } | |||
| if (!PodPhaseEnum.RUNNING.getPhase().equals(pod.getPhase()) || pod.getPodIp() == null || StringUtils.isNotEmpty(pod.getContainerStateMessages())){ | |||
| LogUtil.error(LogEnum.TERMINAL,"master 容器未运行 terminalPreserveDTO:{}",terminalPreserveDTO); | |||
| if (!PodPhaseEnum.RUNNING.getPhase().equals(pod.getPhase()) || pod.getPodIp() == null || StringUtils.isNotEmpty(pod.getContainerStateMessages())) { | |||
| LogUtil.error(LogEnum.TERMINAL, "master 容器未运行 terminalPreserveDTO:{}", terminalPreserveDTO); | |||
| throw new BusinessException("master 容器未运行"); | |||
| } | |||
| String containerID = pod.getContainerId(); | |||
| @@ -217,14 +222,13 @@ public class TerminalServiceImpl implements TerminalService { | |||
| terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"commit 镜像..."); | |||
| terminalMapper.updateById(terminal); | |||
| DockerClient dockerClient = dockerClientFactory.getDockerClient(pod.getHostIP()); | |||
| String newImagePath = terminal.getImageProject()+SymbolConstant.SLASH+terminal.getCreateUserId()+SymbolConstant.SLASH+terminalPreserveDTO.getImageName(); | |||
| String newImageRepository = terminalConfig.getHarborAddress()+SymbolConstant.SLASH+newImagePath; | |||
| String newImagePath = terminalConfig.getHarborAddress() + StrUtil.SLASH + terminal.getImageProject() + SymbolConstant.SLASH + terminalPreserveDTO.getImageName(); | |||
| try { | |||
| dockerApi.commit(dockerClient,containerID,newImageRepository,terminalPreserveDTO.getImageTag()); | |||
| dockerApi.commit(dockerClient,containerID,newImagePath,terminalPreserveDTO.getImageTag()); | |||
| terminal.setStatus(null); | |||
| terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"push 镜像..."); | |||
| terminalMapper.updateById(terminal); | |||
| boolean pushResult = dockerApi.push(dockerClient,newImageRepository+SymbolConstant.COLON+terminalPreserveDTO.getImageTag(),new TerminalPushImageResultCallback(dockerCallbackTool.getCallbackUrl(SymbolConstant.LOCAL_HOST,terminalConfig.getServerPort(), DockerOperationEnum.PUSH.getType()),terminal.getId(),dockerClient,terminal.getCreateUserId())); | |||
| boolean pushResult = dockerApi.push(dockerClient,newImagePath+SymbolConstant.COLON+terminalPreserveDTO.getImageTag(),new TerminalPushImageResultCallback(dockerCallbackTool.getCallbackUrl(SymbolConstant.LOCAL_HOST,terminalConfig.getServerPort(), DockerOperationEnum.PUSH.getType()),terminal.getId(),dockerClient,terminal.getCreateUserId())); | |||
| if (!pushResult){ | |||
| LogUtil.error(LogEnum.TERMINAL,"master 推送镜像错误 terminalPreserveDTO:{}",terminalPreserveDTO); | |||
| throw new BusinessException("推送镜像错误:"); | |||
| @@ -234,7 +238,7 @@ public class TerminalServiceImpl implements TerminalService { | |||
| throw new BusinessException("保存容器错误:"+e.getMessage()); | |||
| } | |||
| terminal.setImageUrl(newImagePath+SymbolConstant.COLON+terminalPreserveDTO.getImageTag()); | |||
| terminal.setImageUrl(newImagePath + SymbolConstant.COLON + terminalPreserveDTO.getImageTag()); | |||
| terminal.setImageName(terminalPreserveDTO.getImageName()); | |||
| terminal.setDescription(terminalPreserveDTO.getImageRemark()); | |||
| terminal.setImageTag(terminalPreserveDTO.getImageTag()); | |||
| @@ -244,13 +248,24 @@ public class TerminalServiceImpl implements TerminalService { | |||
| return terminalMapper.updateById(terminal) > 0; | |||
| } | |||
| private boolean checkImageExists(String imageProject, String imageName, String imageTag) { | |||
| // 拼接镜像的完整路径 | |||
| String imageUrl = terminalConfig.getHarborAddress() + StrUtil.SLASH + imageProject + SymbolConstant.SLASH + imageName + StrUtil.COLON + imageTag; | |||
| // 校验该待保存镜像是否存在 | |||
| LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | |||
| queryWrapper.eq(PtImage::getImageUrl, imageUrl); | |||
| queryWrapper.ne(PtImage::getCreateUserId, userContextService.getCurUserId()); | |||
| return ptImageMapper.selectCount(queryWrapper) > 0; | |||
| } | |||
| @Override | |||
| @Transactional(rollbackFor = Exception.class) | |||
| public boolean delete(TerminalDTO terminalDTO) { | |||
| try{ | |||
| try { | |||
| Terminal terminal = terminalMapper.selectById(terminalDTO.getId()); | |||
| if (terminal == null){ | |||
| LogUtil.error(LogEnum.TERMINAL,"delete 数据不存在 terminalDTO: {}",terminalDTO); | |||
| if (terminal == null) { | |||
| LogUtil.error(LogEnum.TERMINAL, "delete 数据不存在 terminalDTO: {}", terminalDTO); | |||
| throw new BusinessException("数据不存在"); | |||
| } | |||
| terminal.setDeleted(true); | |||
| @@ -259,7 +274,7 @@ public class TerminalServiceImpl implements TerminalService { | |||
| terminalMapper.deleteById(terminal); | |||
| // 删除任务缓存 | |||
| String taskIdentify = (String) redisUtils.get(terminalIdPrefix + String.valueOf(terminal.getId())); | |||
| if (StringUtils.isNotEmpty(taskIdentify)){ | |||
| if (StringUtils.isNotEmpty(taskIdentify)) { | |||
| redisUtils.del(taskIdentify, terminalIdPrefix + String.valueOf(terminal.getId())); | |||
| } | |||
| String namespace = k8sNameTool.getNamespace(terminal.getCreateUserId()); | |||
| @@ -477,6 +492,13 @@ public class TerminalServiceImpl implements TerminalService { | |||
| return; | |||
| } | |||
| Terminal terminal = terminalMapper.selectById(terminalId); | |||
| List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectByTerminalId(terminalId); | |||
| TerminalInfo masterTerminalInfo = null; | |||
| for (TerminalInfo info : terminalInfoList){ | |||
| if (info.isMasterFlag()){ | |||
| masterTerminalInfo = info; | |||
| } | |||
| } | |||
| if (terminal == null){ | |||
| LogUtil.error(LogEnum.TERMINAL,"pushImageComplete no terminal found id:{}",terminalId); | |||
| return; | |||
| @@ -499,10 +521,11 @@ public class TerminalServiceImpl implements TerminalService { | |||
| ptImage.setImageTag(terminal.getImageTag()); | |||
| ptImage.setRemark(terminal.getDescription()); | |||
| ptImage.setImageResource(MagicNumConstant.ZERO); | |||
| ptImage.setImageStatus(MagicNumConstant.ONE); | |||
| ptImage.setDeleted(false); | |||
| ptImage.setUpdateUserId(userId); | |||
| ptImage.setUpdateTime(new Timestamp(new java.util.Date().getTime())); | |||
| ptImage.setSshUser(masterTerminalInfo.getSshUser()); | |||
| ptImage.setSshPwd(masterTerminalInfo.getSshPassword()); | |||
| if (ptImage.getId() != null){ | |||
| ptImageMapper.updateById(ptImage); | |||
| @@ -563,7 +586,7 @@ public class TerminalServiceImpl implements TerminalService { | |||
| terminalBO.setGpuNum(terminalInfo.getGpuNum()); | |||
| terminalBO.setMemNum(terminalInfo.getMemNum()); | |||
| terminalBO.setCpuNum(terminalInfo.getCpuNum()); | |||
| terminalBO.setImage(terminalConfig.getHarborAddress()+SymbolConstant.SLASH+terminalCreateDTO.getImageUrl()); | |||
| terminalBO.setImage(terminalCreateDTO.getImageUrl()); | |||
| terminalBO.setFsMounts(Maps.newHashMap()); | |||
| terminalBO.setBusinessLabel(BusinessLabelServiceNameEnum.TERMINAL.getBusinessLabel()); | |||
| terminalBO.setTaskIdentifyLabel(taskIdentifyLabel); | |||
| @@ -26,7 +26,6 @@ import org.springframework.stereotype.Component; | |||
| * @description 镜像远程服务调用熔断类 | |||
| * @date 2020-12-15 | |||
| */ | |||
| @Component | |||
| public class ImageClientFallback implements ImageClient { | |||
| @Override | |||
| @@ -1,39 +0,0 @@ | |||
| /** | |||
| * Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
| * | |||
| * Licensed under the Apache License, Version 2.0 (the "License"); | |||
| * you may not use this file except in compliance with the License. | |||
| * You may obtain a copy of the License at | |||
| * | |||
| * http://www.apache.org/licenses/LICENSE-2.0 | |||
| * | |||
| * Unless required by applicable law or agreed to in writing, software | |||
| * distributed under the License is distributed on an "AS IS" BASIS, | |||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
| * See the License for the specific language governing permissions and | |||
| * limitations under the License. | |||
| * ============================================================= | |||
| */ | |||
| package org.dubhe.train.config; | |||
| import lombok.Data; | |||
| import org.springframework.boot.context.properties.ConfigurationProperties; | |||
| import org.springframework.stereotype.Component; | |||
| /** | |||
| * @description harbor相关配置 | |||
| * @date 2020-07-17 | |||
| */ | |||
| @Data | |||
| @Component | |||
| @ConfigurationProperties(prefix = "harbor") | |||
| public class TrainHarborConfig { | |||
| private String address; | |||
| private String username; | |||
| private String password; | |||
| private String modelName; | |||
| } | |||
| @@ -96,7 +96,6 @@ import org.dubhe.train.client.ModelBranchClient; | |||
| import org.dubhe.train.client.ModelInfoClient; | |||
| import org.dubhe.train.client.NoteBookClient; | |||
| import org.dubhe.train.client.ResourceSpecsClient; | |||
| import org.dubhe.train.config.TrainHarborConfig; | |||
| import org.dubhe.train.config.TrainJobConfig; | |||
| import org.dubhe.train.constant.ATlasTrainConstant; | |||
| import org.dubhe.train.constant.TrainConstant; | |||
| @@ -202,9 +201,6 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| @Autowired | |||
| private TrainJobConfig trainJobConfig; | |||
| @Autowired | |||
| private TrainHarborConfig trainHarborConfig; | |||
| @Autowired | |||
| private K8sNameTool k8sNameTool; | |||
| @@ -484,7 +480,12 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| public void buildImageAndTagInfo(PtJobParam ptJobParam, PtTrainJobDetailVO ptTrainJobDetailVO) { | |||
| //image信息拼装 | |||
| if (StringUtils.isNotBlank(ptJobParam.getImageName())) { | |||
| String imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); | |||
| String imageNameSuffix; | |||
| if (ptJobParam.getImageName().contains(StrUtil.SLASH)) { | |||
| imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); | |||
| } else { | |||
| imageNameSuffix = ptJobParam.getImageName(); | |||
| } | |||
| String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); | |||
| ptTrainJobDetailVO.setImageName(imageNameSuffixArray[0]); | |||
| ptTrainJobDetailVO.setImageTag(imageNameSuffixArray[1]); | |||
| @@ -563,11 +564,10 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| .setMemNum(ptTrainJobCreateDTO.getMemNum()) | |||
| .setWorkspaceRequest(ptTrainJobCreateDTO.getWorkspaceRequest()) | |||
| .setTaskIdentify(taskIdentify); | |||
| //例如: 将harbor.dubhe.ai/notebook/notebook:v1 去掉 harbor地址 | |||
| String userImageName = trimHarborAddress(ptImageAndAlgorithmVO.getImageName()); | |||
| String imageNameAndTag = ptTrainJobCreateDTO.getImageName() + StrUtil.COLON + ptTrainJobCreateDTO.getImageTag(); | |||
| //结果集处理 | |||
| PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, userContextService.getCurUser(), userImageName, trainKey, baseTrainJobDTO); | |||
| PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, userContextService.getCurUser(), imageNameAndTag, trainKey, baseTrainJobDTO); | |||
| //添加任务缓存 | |||
| resourceCache.addTaskCache(taskIdentify, ptTrainJob.getTrainId(), ptTrainJobCreateDTO.getTrainName(), trainIdPrefix); | |||
| // 提交job | |||
| @@ -575,16 +575,6 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| return Collections.singletonList(ptTrainJob.getTrainId()); | |||
| } | |||
| /** | |||
| * 去掉harbor地址 | |||
| * | |||
| * @param imageName | |||
| * @return | |||
| */ | |||
| private String trimHarborAddress(String imageName) { | |||
| return StringUtils.isBlank(imageName) ? StringUtils.EMPTY : imageName.replace(trainHarborConfig.getAddress() + StrUtil.SLASH, StringUtils.EMPTY); | |||
| } | |||
| /** | |||
| * 构建镜像和算法目录VO 考虑到无算法创建 | |||
| * | |||
| @@ -611,9 +601,8 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| String imageUrl = imageUtil.getImageUrl(ptTrainJobBaseDTO, userContextService.getCurUser()); | |||
| String userImageName = imageUrl.split(StrUtil.SLASH)[0] + StrUtil.SLASH + ptTrainJobBaseDTO.getImageName() + StrUtil.COLON + ptTrainJobBaseDTO.getImageTag(); | |||
| ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobBaseDTO.getAlgorithmId()); | |||
| String imageName = trainHarborConfig.getAddress() + StrUtil.SLASH + userImageName; | |||
| ptImageAndAlgorithmVO.setImageName(imageName); | |||
| ptImageAndAlgorithmVO.setImageUrl(trainHarborConfig.getAddress() + StrUtil.SLASH + imageUrl); | |||
| ptImageAndAlgorithmVO.setImageName(ptTrainJobBaseDTO.getImageName()); | |||
| ptImageAndAlgorithmVO.setImageUrl(imageUrl); | |||
| } | |||
| ptImageAndAlgorithmVO.setRunCommand(ptTrainJobBaseDTO.getRunCommand()); | |||
| @@ -676,13 +665,13 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| * | |||
| * @param ptTrainJobCreateDTO 创建训练任务DTO | |||
| * @param currentUser 用户 | |||
| * @param imageName 镜像名称 | |||
| * @param imageNameAndTag 镜像名称和tag | |||
| * @param trainKey 训练key | |||
| * @param baseTrainJobDTO 基础训练参数 | |||
| * @return PtTrain 训练 | |||
| */ | |||
| private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserContext currentUser, | |||
| String imageName, String trainKey, BaseTrainJobDTO baseTrainJobDTO) { | |||
| String imageNameAndTag, String trainKey, BaseTrainJobDTO baseTrainJobDTO) { | |||
| // 添加train表 | |||
| PtTrain ptTrain = new PtTrain(); | |||
| ptTrain.setTrainName(ptTrainJobCreateDTO.getTrainName()) | |||
| @@ -732,7 +721,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| ptJobParam.setTrainJobId(ptTrainJob.getId()) | |||
| .setAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId()) | |||
| .setRunCommand(ptTrainJobCreateDTO.getRunCommand()) | |||
| .setImageName(imageName) | |||
| .setImageName(imageNameAndTag) | |||
| .setRunParams(ptTrainJobCreateDTO.getRunParams()) | |||
| .setRunParamsNameMap(ptTrainJobCreateDTO.getRunParamsNameMap()) | |||
| .setCreateUserId(currentUser.getId()); | |||
| @@ -1010,9 +999,8 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| .setWorkspaceRequest(ptTrainJobUpdateDTO.getWorkspaceRequest()) | |||
| .setTaskIdentify(taskIdentify); | |||
| String userImageName = trimHarborAddress(ptImageAndAlgorithmVO.getImageName()); | |||
| //结果集处理 | |||
| PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, userContextService.getCurUser(), existPtTrainJob, userImageName, ptTrain, baseTrainJobDTO); | |||
| PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, userContextService.getCurUser(), existPtTrainJob, ptTrainJobUpdateDTO.getImageName() + StrUtil.COLON + ptTrainJobUpdateDTO.getImageTag(), ptTrain, baseTrainJobDTO); | |||
| //提交job | |||
| asyncManager.execute(baseTrainJobDTO, ptTrain.getCreateUserId(), ptImageAndAlgorithmVO, ptTrainJob); | |||
| @@ -1025,13 +1013,13 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| * @param ptTrainJobUpdateDTO 更新训练任务DTO | |||
| * @param currentUser 当前用户 | |||
| * @param existPtTrainJob 存在的训练任务 | |||
| * @param imageName 镜像名称 | |||
| * @param imageNameAndTag 镜像名称和tag | |||
| * @param ptTrain 训练 | |||
| * @param baseTrainJobDTO 基本训练信息 | |||
| * @return PtTrainJob 训练任务 | |||
| */ | |||
| private PtTrainJob updateTrainJobTableData(PtTrainJobUpdateDTO ptTrainJobUpdateDTO, UserContext | |||
| currentUser, PtTrainJob existPtTrainJob, String imageName, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) { | |||
| currentUser, PtTrainJob existPtTrainJob, String imageNameAndTag, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) { | |||
| //检查模型是否合法,合法则保存其路径地址 | |||
| checkModelAndSavePath(currentUser, baseTrainJobDTO); | |||
| @@ -1070,7 +1058,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
| ptJobParam.setTrainJobId(ptTrainJob.getId()) | |||
| .setAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId()) | |||
| .setRunCommand(ptTrainJobUpdateDTO.getRunCommand()) | |||
| .setImageName(imageName) | |||
| .setImageName(imageNameAndTag) | |||
| .setRunParams(ptTrainJobUpdateDTO.getRunParams()) | |||
| .setRunParamsNameMap(ptTrainJobUpdateDTO.getRunParamsNameMap()) | |||
| .setCreateUserId(ptTrain.getCreateUserId()); | |||
| @@ -252,10 +252,10 @@ ALTER TABLE `data_task` ADD COLUMN `file_type` smallint(3) NULL COMMENT '待处 | |||
| ALTER TABLE `data_task` ADD COLUMN `stop` bit(1) NULL COMMENT '是否停止 0-没有 1-已停止' AFTER `file_type`; | |||
| ALTER TABLE `data_task` ADD COLUMN `of_record_version` varchar(255) NULL COMMENT '生成ofRecord的版本号' AFTER `stop`; | |||
| -- 医学表 | |||
| ALTER TABLE `data_medicine` ADD COLUMN `stop` bit(1) NULL AFTER `annotate_type`; | |||
| -- 点云数据集表 | |||
| create table if not exists `pc_dataset` ( | |||
| `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键id', | |||
| @@ -397,3 +397,5 @@ CREATE TABLE `pt_atlas_camera` ( | |||
| PRIMARY KEY (`id`), | |||
| UNIQUE KEY `camera_index_uniq` (`camera_index_code`) COMMENT '设备标识唯一' | |||
| ) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb4 COMMENT='模型炼知视频流管理'; | |||
| ALTER TABLE pt_image DROP image_status; | |||
| @@ -33,11 +33,11 @@ ribbon: | |||
| dubhe-proxy: | |||
| visual: | |||
| keyword: visual | |||
| server: 127.0.0.1 | |||
| server: 10.5.26.88 | |||
| port: 9898 | |||
| refine: | |||
| keyword: refine | |||
| server: 127.0.0.1 | |||
| server: localhost | |||
| port: 9797 | |||
| ## 测试标识 | |||
| debug: | |||
| @@ -17,7 +17,7 @@ spring: | |||
| redis: | |||
| #数据库索引 | |||
| database: 4 | |||
| host: 127.0.0.1 | |||
| host: 10.5.26.88 | |||
| port: 6379 | |||
| password: | |||
| #连接超时时间 | |||
| @@ -26,9 +26,9 @@ spring: | |||
| druid: | |||
| db-type: com.alibaba.druid.pool.DruidDataSource | |||
| driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | |||
| url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||
| url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||
| username: test | |||
| password: test | |||
| password: zj12345678 | |||
| # 初始化配置 | |||
| initial-size: 3 | |||
| @@ -67,17 +67,17 @@ logging.config: | |||
| # 文件存储服务配置 | |||
| storage: | |||
| # 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 | |||
| file-store: 127.0.0.1 | |||
| file-store: 10.5.26.234 | |||
| # 文件存储服务端 共享目录 | |||
| file-store-root-path: /nfs/ | |||
| file-store-root-windows-path: "Z:" | |||
| # minio配置 | |||
| minio: | |||
| url: http://127.0.0.1:9000/ | |||
| url: http://10.5.26.90:9000/ | |||
| accessKey: admin | |||
| secretKey: 123@abc.com | |||
| bucketName: dubhe-prod | |||
| bucketName: dubhe-cloud-test | |||
| presignedUrlExpiryTime: 300 | |||
| annotation: /annotation/ | |||
| @@ -121,13 +121,6 @@ feign: | |||
| okhttp: | |||
| enabled: false | |||
| #配置harbor | |||
| harbor: | |||
| address: harbor.dubhe.ai | |||
| username: admin | |||
| password: Harbor12345 | |||
| model-name: train | |||
| #配置dev环境单元测试用户名与密码 | |||
| unittest: | |||
| username: admin | |||
| @@ -136,10 +129,10 @@ unittest: | |||
| model: | |||
| measuring: | |||
| url: | |||
| package: http://127.0.0.1:32760/model_measure/package | |||
| json: http://127.0.0.1:32760/model_measure/measure | |||
| package: http://10.5.26.90:32760/model_measure/package | |||
| json: http://10.5.26.90:32760/model_measure/measure | |||
| converter: | |||
| url: http://127.0.0.1:32230/model_convert | |||
| url: http://10.5.26.90:32230/model_convert | |||
| user: | |||
| config: | |||
| @@ -5,21 +5,17 @@ k8s: | |||
| # k8s集群配置文件 | |||
| kubeconfig: kubeconfig_test | |||
| # nfs服务暴露的IP地址 如需测试需修改为合适的地址 | |||
| nfs: 127.0.0.1 | |||
| nfs: 10.5.26.234 | |||
| #nfs服务端 共享目录 | |||
| nfs-root-path: /nfs/ | |||
| nfs-root-windows-path: "Z:" | |||
| # 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 | |||
| file-store: 127.0.0.1 | |||
| file-store: 10.5.26.234 | |||
| #文件存储服务端 共享目录 | |||
| file-store-root-path: /nfs/ | |||
| file-store-root-windows-path: "Z:" | |||
| # 命名空间关键字 | |||
| namespace: namespace | |||
| # k8s ingress域名 如需测试需修改为合适的域名,注意:需要为此域名配置泛域名解析到 k8s集群master节点的ip | |||
| host: notebooktest.dubhe.club | |||
| # k8s ingress-controller 对外port | |||
| port: 30865 | |||
| # k8s ingress-controller 对外grpc port | |||
| https-port: 31287 | |||
| # k8s 模型部署配置 | |||
| @@ -27,12 +23,12 @@ k8s: | |||
| # 在线服务模型部署后容器域名(k8s ingress域名),解析地址为k8s集群地址,如需测试需修改为合适的域名 | |||
| host: servingtest.dubhe.club | |||
| # tls 证书 crt | |||
| tls-crt: | |||
| tls-crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURYekNDQWtlZ0F3SUJBZ0lKQUp3Z0VCYlhGdk9HTUEwR0NTcUdTSWIzRFFFQkN3VUFNRVl4SVRBZkJnTlYKQkFNTUdDb3VjMlZ5ZG1sdVozUmxjM1F1WkhWaWFHVXVZMngxWWpFaE1COEdBMVVFQ2d3WUtpNXpaWEoyYVc1bgpkR1Z6ZEM1a2RXSm9aUzVqYkhWaU1CNFhEVEl4TURVeE9UQTNNRGN5TTFvWERUTXhNRFV4TnpBM01EY3lNMW93ClJqRWhNQjhHQTFVRUF3d1lLaTV6WlhKMmFXNW5kR1Z6ZEM1a2RXSm9aUzVqYkhWaU1TRXdId1lEVlFRS0RCZ3EKTG5ObGNuWnBibWQwWlhOMExtUjFZbWhsTG1Oc2RXSXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBdwpnZ0VLQW9JQkFRREwxNDBMUXk2dXFHUi9WckVXeVNUdFppVGwvOEo1eGlHenB5WUJtTldUTzVQSTczTHRKbXdkCmFPMGNCNXhBcy96SXlpYjM3b0loR0FQejZNWlBBZjJEeUMxNXJLb3Via3h2Vkswa3hJY0k2L1dsWFUxMjdZZzQKdUllNzJEbFNIak9tbStVQ2JDWkxtc0VlQVI2S3RwNW9lVUFUQVgwOWEyL012ZkdtNU1URWZFSVplTENoL21rVAppZGFKcnl0NThwYVFLZmpJcVdKQnlnUGpJYlFMUlZnSG5mckIzdlB5b3RmMFBZZWVKVHlEa1lNQnNzblMxS044ClY2Qldvb2dzWUxQcCs2VE1wOXBHS3pBdksxdXJMWkFsa1N0SUw1TkppQWJOZ1lMOXN1UVE3OU9aano2c1ZrbWQKait6aHZHSGUrYk9zdlBrYmtqNS9lbzQyQUdQcW9zdTlBZ01CQUFHalVEQk9NQjBHQTFVZERnUVdCQlFzWDlCegpxVEhNN21xMUJrOFRNWXgzY1ZRaFB6QWZCZ05WSFNNRUdEQVdnQlFzWDlCenFUSE03bXExQms4VE1ZeDNjVlFoClB6QU1CZ05WSFJNRUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCc1BJSUpPMm1qUFNJV1lTaloKQmdjbFJoUTRudEkvdjFWczVjb0xOVzlrZUhqUGlCYlhvVzgzZExqSWh5ZmVYSk93Y3lpRnJkaVlZeVV5SEprTAprTmlxajRxVmZuaGkwcUoweDQyN3MzUG40TkZZRnNJYVM2OHA1WlZpUlJSZGNWY0dIMkNiRUxzYW41ZUhkNExYCnlTUUdDZmVkUVdtMEVyWWxDMGJWSER1d3RCa0NzWElPRmJEb0tSMW5wMDc3YlBrbEpIRS9rUi82QVpJR3hFM28KbDd1SDVRWWRsWldoanY5WFZRM1plVnZLUzVkM1pHYjcxZ0dwSEU2b2hmL1IweVhoS1RadUREYkhjK2lFc1dWbQowa24xenlvTUVHSW5nM0N6cVk4aE1tdHhRVStLekJrSEliVzh0UnkrS0MxZ2ZvNktFa01JVGpjK2pwQU4wZ2VDCm1LWWYKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= | |||
| # tls 证书 key | |||
| tls-key: | |||
| tls-key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRREwxNDBMUXk2dXFHUi8KVnJFV3lTVHRaaVRsLzhKNXhpR3pweVlCbU5XVE81UEk3M0x0Sm13ZGFPMGNCNXhBcy96SXlpYjM3b0loR0FQego2TVpQQWYyRHlDMTVyS291Ymt4dlZLMGt4SWNJNi9XbFhVMTI3WWc0dUllNzJEbFNIak9tbStVQ2JDWkxtc0VlCkFSNkt0cDVvZVVBVEFYMDlhMi9NdmZHbTVNVEVmRUlaZUxDaC9ta1RpZGFKcnl0NThwYVFLZmpJcVdKQnlnUGoKSWJRTFJWZ0huZnJCM3ZQeW90ZjBQWWVlSlR5RGtZTUJzc25TMUtOOFY2Qldvb2dzWUxQcCs2VE1wOXBHS3pBdgpLMXVyTFpBbGtTdElMNU5KaUFiTmdZTDlzdVFRNzlPWmp6NnNWa21kait6aHZHSGUrYk9zdlBrYmtqNS9lbzQyCkFHUHFvc3U5QWdNQkFBRUNnZ0VCQUtJRG83WDA0dEN0UkpzMVV6RnlFK2lnUyttR201TGdHSDlLOWRsRkdWT1oKZ2IrdUdhRkY3WjUrZkI3UUthLy9ub1lVcmw1VzhwVE5HcTh0THhreTBzV0FRQ3k2UU5VQWs3M1ZCbnozdXdBcgpVeVhvLzUzcjgxNXB4SEJYems5bmM5UVRpalNmc3R4YWx1MTdKRVJLRzZPYjQ0SjNwNHcrclRDRk0rRmJhTTFsCkxuWUh2bGR4ZnRoMmhBWHdaU2dFYXdCNXRBbTY2YnIxbUc4ZW5saEIvbisrY09iYkZWYnk4OUcyUmVMMjJxdHIKa0R0RHVQOGY1NVdxNVB0a25TRE4zaGoweEYydG9aSC80RHpKOWVibVNMVHpUK1pQSjVjK1dlZ3BNdCtVd3YzbwpNNFdGQnFUUUZSR0MyQVJTaXVFTEdtRUJnVk50VDRmRVhRSmFveGJKeWYwQ2dZRUE2MDRJOXlDZUttY2hjSEJvCmRFeFJ2bnNhcGhwbkJqc3pHazZIWVhhTFI0NGpvalRCWm5ZWVhHYklreDdZaytla1RQYjMwRUpzQjlRRmxpVUUKQ25oZXhCS0pjTk5VcTdWdHFGTXBhT0JKWG9Md0cySmVwVmNhN1MzcFo3THZMWGRQR2wvOXFTTW5pMXROV1kxdwpmTGl1Zzlqb3JCeEJaL250NU43MDJFTWpaL2NDZ1lFQTNjVWV2U3U1VmlHaGNPSmFVSlBoUjdnclViMW9EWDF4CnROdDdXNS9pNkFyNDBsY1FZK2RCcll2VEljRllEL3piZnRWNGFnMDhsLzVHamxQK2grWW01QWNRb1c1Z3BaaHgKd1ZhTFFzRTg4ZUdPY2pLakVqd2krTEZRNE5HRmdXdU9pc1hZWk5GTlBvT1JDbHRUbkRMRnFjS0FNY2tBTEgwVwpRT2poTkJqeGhPc0NnWUFRaHRPVVgrTWNBVkJVOEdBMXd1Sis2WENPdUE0Q1h3Z1EwZkVxUkVRMkMyS3ZVdHMrCmtnN1Y5cFloMXluSkFaMEZsdGNDOXBkVjJXdG5CMFNJWTduc05ZMFhzcEFnMjBaUGF6L2VVTnAyVytYM1ZtcGwKWEgvVXBzUGM5N0ZhMVNWbUtkWE1HbDc5cDdVQUZESVJZSHRKWVdPK0t1SGhKcW14eUlNVDZXdEVNd0tCZ0R0ZAo4WVNpbDlLKzNnRGlGMXRLdXh2LzZWalFZM0o1Q2w5b0FmWGRMMWorMXUwMzhXTk5IUC9nVm56S3pWQTZXR2Z6CnJYQjJhcW9sbjYycVBwRVN3NFozZmJRNVlCWDBZVDlvYzQ5RE81VmsxRVV0MlFtZ241d0RtNnNUYTdIaG9SNzEKSjZDVmh5QWRDRTdGYy9SMGd3V1cwOHFBREZQY2lJQ0gxd0dqUzhSUkFvR0JBTklUNW5yTzNwZmNKYTROZWFZeApKMzd5RzFodUJGcWdRRWpVU3pmbHQ3UWwvNjI3LzNWT2lIRjVqSkhES0JqODBEdzh1UjhoQ2VhWDY2WkJqUmxKCk5vOUEvUm43Q2F4YUl2dkNjWG4rUW5oLy9BVlNHRzU4WWI2UUlQNFVqT1RKOW1GUlFFajN0cm9KNnVpL3FCZTQKUkhXSE44WC91Z242dzZUSGRWNktJbSs4Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K | |||
| # elasticsearch暴露的服务地址 | |||
| elasticsearch: | |||
| hostlist: ${eshostlist:127.0.0.1:30498} | |||
| hostlist: ${eshostlist:10.5.26.90:30498} | |||
| # 日志采集配置信息 | |||
| log: | |||
| type: _doc | |||
| @@ -50,9 +46,9 @@ k8s: | |||
| # 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址 | |||
| pod: | |||
| metrics: | |||
| grafanaUrl: http://127.0.0.1:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod= | |||
| grafanaUrl: http://10.5.26.90:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod= | |||
| prometheus: | |||
| url: http://127.0.0.1:30003/ | |||
| url: http://10.5.26.90:30003/ | |||
| query: api/v1/query | |||
| query-range: api/v1/query_range | |||
| gpu-query-param: sum(DCGM_FI_DEV_GPU_UTIL{pod="pod-name-placeholder"})by(pod,UUID) | |||
| @@ -75,21 +71,40 @@ k8s: | |||
| cpu: 10 | |||
| memory: 32 | |||
| gpu: 2 | |||
| #配置harbor | |||
| harbor: | |||
| address: harbor.dubhe.ai | |||
| username: admin | |||
| password: Harbor12345 | |||
| model-name: train | |||
| # minio配置 | |||
| minio: | |||
| url: http://127.0.0.1:9000/ | |||
| url: http://10.5.26.90:9000/ | |||
| accessKey: admin | |||
| secretKey: 123@abc.com | |||
| bucketName: dubhe-prod | |||
| bucketName: dubhe-cloud-test | |||
| presignedUrlExpiryTime: 300 | |||
| annotation: /annotation/ | |||
| docker: | |||
| remote-api-port: 2375 | |||
| clusters: | |||
| - cluster: | |||
| certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1USXdPREE0TWpnd00xb1hEVE14TVRJd05qQTRNamd3TTFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWVKCkRjTThQUHFlV0tHazE3S2JtN3kwbzkzTGtxb2w5dG1QbVN0YTVyUXZSWU9rOFJYK2QvVEpVaXE2S2c4NXFyMGUKZFZ3SG5vTjg3Q0VFbXhOK2o4OUhlcnY1dDFaRFBHMVRHS3U5N1l1WnV4RCt2R1hGQkhDZ2ViT0s0Zmc0am1iSgpwUmlJeCtUZmlyUjM4TlVhZTBkR1g5c1o1TWF2NkhiSVVEK2hNTm9VU2pyWUZ2bTcxbGtYdlhVWW9wd1FSR3FvCkZiWG5KTFFVakd3UFFRclRCQ0wycFE2c1dGbkpkQ0VBSHhXU0grTUN4cXV0L25SUHhORlFOWGFMZmU3clB3a3MKd3NrVlFlTzgyL205aFZkWnNnNUFnRDJtTS9jS2VybE9acVBGcWFEaFBoYnpkRzF4QjRtcVZmTGJVUUppa3I1SQpYQ1FhTWR2c2FRTnhPcmNPOHQwQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZFek92OGhVUGtRYS90bGNhQTE0OERmcmF5TXRNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBMTFqcDR4V1hyZkRPNDJlblBpdm9HNXZxY3hqbysxaktZV1YxR1NIdHNVdlBNU1UzVAptdVFCQWU2VVRzM0JtSTgzMUQ4aFd4cDlka3BkMlhWbCt6OTZabFIvQ2Qrb3NSWWE1MUpZd2RvMUp4Z29hbUk2CmszOS9sNTRBQVpWMkFPWWNQU0pKNkNzdVcxZ2NTMDlaTlA0djlhK0NIKzY5OVpQMGlhUStEc0FPRFJzUnhRcVMKK08zNUU4K0RzRjBSOWZCYlhvbkxiK05VSHlTQ2pQSnVVT21lS0RYMFpQQm81blNscmh5cUIxbHVNZnhEaUltdgpIU3BscXB4M1duMzJnNEZYOCs3clZBNUZjWkIvNDg3bUZmdG1CTXc3aENidlFJeWY2Ymd6VTUycUdaeFZGTTVjCnUxeEZyRXlXaFA1eElJY09lY1M1RCtJa29ZOHUyT3ozQldEeQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== | |||
| server: https://10.105.0.6:6443 | |||
| name: kubernetes | |||
| contexts: | |||
| - context: | |||
| cluster: kubernetes | |||
| user: kubernetes-admin | |||
| name: kubernetes-admin@kubernetes | |||
| current-context: kubernetes-admin@kubernetes | |||
| kind: Config | |||
| preferences: {} | |||
| users: | |||
| - name: kubernetes-admin | |||
| user: | |||
| client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lJQjUxN3hkT3V2cnd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TVRFeU1EZ3dPREk0TUROYUZ3MHlNakV5TURnd09ESTRNRFZhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXlFaURrYVJ1WmVUbVJ4WmsKRkxUbkRPMUxIM1k5dE80WXpXREt4QVA2bFphTUJwZXJWRWpiU1VMOGIzdG9WWFppVnVnaHhvMzRHdjF4dlJKLwpDN3NhWGVaU3EzcTVMcEFhQjBrelRDNFRtbGtrdE03YkJTZFNhMG9SeGJXTUdvVEM0WUVBckJCNS9FRmdUYjRCCnE4aG1USUlmdUcwa05sNmd5a2M4UnhSL3N5Q1lSVjZCTTZTYWFzODR1Tmx3ZU9vV2RaQkQ0UVZQbzJueUI4NGQKUnc0d0w2VEQ5MlUwMG5KcjY3bUtpTjhBNGtBampFbWNrNm1JbHdsUTB2MHR5Uk8zTFBXZFdNazZ5QVIyVEZ6RgpBZCs4dUd5L3VDQlB6UDlVcDFERkI5TDF0Y25sd1lTeTYycGdFWC9wMzl6ZjZ0RFpZRVJwK0Jpd21Eb3RmVmFaClNLam9md0lEQVFBQm8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0h3WURWUjBqQkJnd0ZvQVVUTTYveUZRK1JCcisyVnhvRFhqd04rdHJJeTB3RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFCQ01US2VEV1dHa0VFZWp6dlN3TTFKZit0QUJaYklrd0FoZEt1MjFnUkxyQktzbHQ4Y0NHMG1KCmc3eDhndFJYcW0xRGlEU01DMnlWMW9OWTFSdGlLZVhZT29RbVY2SFh2bEQybWZyY2VzS3VSNHlIbCtlWHcyL1AKb2FLZVFEM21yVm9ZZ1ZMMVNqOWpzcDRBa1hEM1ErUzdKYXN4Y3MvVkR3SmlRQzhGamkzaWZMVFFKQkxCdm1WVwpCZ3QvM0xPZ1lCdWxYRWg3cFlQUzduTVFqNFpxQ0QwWTNJOENicUM0VGlZQjB1aTZlbVRDd3cvRnBNSzNIRFZ4CnZOd3FPRHBaWW5DK0IxSzR5RUk4NGIvM1o3ZXA0MUE1NGk3QkpxTklORUVtRjdGcmpQQy91V1RuYlpzVUNFQjAKWTFUR1Y3czNBUTZMZXNCRFF6UVRzVnBENWFRRk9VYz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= | |||
| client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBeUVpRGthUnVaZVRtUnhaa0ZMVG5ETzFMSDNZOXRPNFl6V0RLeEFQNmxaYU1CcGVyClZFamJTVUw4YjN0b1ZYWmlWdWdoeG8zNEd2MXh2UkovQzdzYVhlWlNxM3E1THBBYUIwa3pUQzRUbWxra3RNN2IKQlNkU2Ewb1J4YldNR29UQzRZRUFyQkI1L0VGZ1RiNEJxOGhtVElJZnVHMGtObDZneWtjOFJ4Ui9zeUNZUlY2QgpNNlNhYXM4NHVObHdlT29XZFpCRDRRVlBvMm55Qjg0ZFJ3NHdMNlREOTJVMDBuSnI2N21LaU44QTRrQWpqRW1jCms2bUlsd2xRMHYwdHlSTzNMUFdkV01rNnlBUjJURnpGQWQrOHVHeS91Q0JQelA5VXAxREZCOUwxdGNubHdZU3kKNjJwZ0VYL3AzOXpmNnREWllFUnArQml3bURvdGZWYVpTS2pvZndJREFRQUJBb0lCQUgrbHY3ME9TSkpHZmdHbQpvcWlUMTRKa3BuRnA0ZEF6dzdqNXpLRjdTN1VWR3krRWNOeXFCcUM5d2JlbnRvcHBoaW1Qang5R0VtL1pRaWxYCjVZTHJmOVdDMndPUmx2NjNOdStYMXNyaHZ2cXJmL3FBc0JTcnlCcTdQWEo5ejhxQy9OWE9hMGcreEJCaTltYjIKQjRpZGs1MkZmWVFFZzRUbmNLRWJINjdKd00yL2dmZ3BXMGNEd0RiNVc2aHIxclUzNkNGc1AyVTlEYytMVkV5cQpvK3FZZ0RFZ0MxSWxucGVIdzROejJmK3RkbFpLVHN5dzV1ZG5xVm1kWGNkLyszR3NvQVM3Q25DTmQ3a1pKQ1BPClNpRktZSDZZUS93ZmJLZjFibC84eERXU1UvKytQUTZyODJGV21mMFBjd3ExbWV5OXIvUW1kaHc5YndiamkxVDQKdURjSzUva0NnWUVBOURIZi8xZnN4czlqVldoaE94UXFzUFN3UzdDbVZYcHV2S2Q3MjlMRGFIV1VQT3E5Y0l4QQpaV01Ibi9EbEN3U2dxaitPL0FNSDdaQVpPbDV5V2RiZFVEM2I0WWVKLzlubXdvN25ReG9ndTlUdTlNZEVnM0pvCm1wTzVqQTYzYTE2Z0sxRWlmTDJkdllRQkJGQzg2aDJzYlV0WHNGRDV6R3QzTU96bVF6SFFQM3NDZ1lFQTBmY3gKMzlUR0YrZFdVb3MzOXRLaE9MZVpFaWtUSmtqckdrTUdldHpYNEVCRE5oRUUvdWNkYUZCWmJoU3ltZHAwMmtKNgpKQkpvaEJRaEN4Zk9nUXhRdFp2dUt3WUMvTEYrMVF1TG1MaERGV2NEa05Ma1E0RWU5WVRFV1VuNE9NNkJJTmVOCldseklzcncyeHRvaUVoMXZFWW92ODdobEZCQ0E4N0xYSElpa1NjMENnWUFnZ1pNajFueDZhcGo2Z1k2UDRydGMKR3ZaczNQTUhaZWpmekJ1OHcyMm50aDhwak1YeStYaUpCb201VE1Qd2w0a2JvS0pVQWNOSWFHb3pUdWRCOWt3Mwpwa1JpM1R3Tmh1QWsvZ2ppOGROeHJZS3hxdVQvNm5icVBNZnlVUE14bzhNR2dTSnFJSU9pK3JOMzg1ZHlRc2oxCk01WlVyakxTZDJ2Q1k0YXpZeW9waVFLQmdBS0RBb1c2OWVPelVNVU1CVTllbGE4b0F4ekVnUVlrT1N2SFFYeVcKeDc1WEJuYlBIVkF2VTVxNzIxWUZ5VjB0ejlnTWs0bDY3dVVsbUgrWmVVN3g1c1ZGYUVQN2ZtMm5jZXo2aG1EOQpVMUFlTzF5d0tTcmxrSWsvWFZuMEdKUVZaRllRUGhDbXEzLzM0Ry9nakNmTFVsalRYbDk4QlRtSU9RS3hVUUYyCllRREJBb0dBRTB5Q0NzcFE3Y1lYaVFyQnF4VVkxU2N5SjViVm1iaFN5cmgyb3FiUmRMRmxzWHFjUnBIK2tORjYKdXdjUzhZalJodmdQazgyZm1QcXlieE5ZWGxTYWoydGdzSFdpcGNETXZSM1dtSlQzTGpjeXZObDh6cUlkZDNBdQpJT1pNeG0vWWVMemNFOUdteDRsQkhmMnU5MkhiRjRiSCtqeVp3d3RWcHpRdmphaFF6dGs9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== | |||
| #配置harbor | |||
| harbor: | |||
| address: harbor.dubhe.ai | |||
| username: admin | |||
| password: Harbor12345 | |||
| @@ -6,9 +6,9 @@ spring: | |||
| master: | |||
| type: com.alibaba.druid.pool.DruidDataSource | |||
| driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | |||
| url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||
| url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||
| username: test | |||
| password: test | |||
| password: zj12345678 | |||
| props: | |||
| sql: | |||
| show: true | |||
| @@ -18,7 +18,7 @@ data: | |||
| # 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 | |||
| dcm: | |||
| host: 127.0.0.1 | |||
| host: 10.5.29.100 | |||
| port: 11112 | |||
| # 资源回收 | |||
| @@ -27,13 +27,13 @@ recycle: | |||
| # 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 | |||
| dcm: | |||
| host: 127.0.0.1 | |||
| host: 10.5.29.100 | |||
| port: 11112 | |||
| # ES服务地址及端口 | |||
| es: | |||
| host: 127.0.0.1 | |||
| serverPort: 31321 | |||
| transportPort: 31322 | |||
| clusterName: kubenets-logging | |||
| host: 10.5.29.100 | |||
| serverPort: 9200 | |||
| transportPort: 9300 | |||
| clusterName: docker-cluster | |||
| index: dataset_text_test | |||
| @@ -32,11 +32,11 @@ spring: | |||
| precise-algorithm-class-name: org.dubhe.data.util.MyPreciseShardingAlgorithm | |||
| k8s: | |||
| # nfs服务暴露的IP地址 如需测试需修改为合适的地址 | |||
| nfs: 127.0.0.1 | |||
| nfs: 10.5.26.234 | |||
| #nfs服务端 共享目录 | |||
| nfs-root-path: /nfs/ | |||
| nfs-root-windows-path: "Z:" | |||
| file-store: 127.0.0.1 | |||
| file-store: 10.5.26.234 | |||
| #文件存储服务端 共享目录 | |||
| file-store-root-path: /nfs/ | |||
| file-store-root-windows-path: "Z:" | |||
| @@ -80,8 +80,8 @@ recycle: | |||
| # ES服务地址及端口 | |||
| es: | |||
| host: 127.0.0.1 | |||
| serverPort: 32321 | |||
| transportPort: 32322 | |||
| clusterName: kubenets-logging | |||
| host: 10.5.29.100 | |||
| serverPort: 9200 | |||
| transportPort: 9300 | |||
| clusterName: docker-cluster | |||
| index: dataset_text_test | |||
| @@ -3,7 +3,7 @@ notebook-specs: | |||
| # cpu数量(核) | |||
| cpu-num: 1 | |||
| # gpu数量(核) | |||
| gpu-num: 1 | |||
| gpu-num: 0 | |||
| # 内存大小(M) | |||
| mem-num: 1024 | |||
| # 工作空间配额(m) | |||
| @@ -22,8 +22,8 @@ management: | |||
| serving: | |||
| gateway: | |||
| # 在线服务http接口域名,解析地址为serving网关部署的服务器地址 | |||
| postfixUrl: .openapi.dubhe.club | |||
| # 在线服务http接口 host:port,解析地址为serving网关部署的服务地址 | |||
| postfixUrl: 10.101.12.41:8081 | |||
| token: | |||
| secret-key: 1n3512x | |||
| expire-seconds: 108000 | |||
| @@ -31,7 +31,6 @@ serving: | |||
| httpclient: | |||
| connect-timeout: 1000 | |||
| response-timeout: 3000 | |||
| #关闭熔断机制 | |||
| feign: | |||
| hystrix: | |||
| @@ -1,52 +0,0 @@ | |||
| # serving预测上传文件大小限制 | |||
| spring: | |||
| servlet: | |||
| multipart: | |||
| # 设置单个文件的大小 | |||
| max-file-size: 10MB | |||
| # 设置单次请求的文件的总大小 | |||
| max-request-size: 100MB | |||
| k8s: | |||
| # nfs服务暴露的IP地址 如需测试需修改为合适的地址 | |||
| nfs: 127.0.0.1 | |||
| #nfs服务端 共享目录 | |||
| nfs-root-path: /nfs/ | |||
| nfs-root-windows-path: "Z:" | |||
| # data模块配置 | |||
| data: | |||
| annotation: | |||
| task: | |||
| # 自动标注任务分割的文件split size | |||
| splitSize: 16 | |||
| # 单位ms | |||
| retryInterval: 5000 | |||
| # 任务不更新置为失败的时间单位秒 | |||
| failTime: 43200 | |||
| server: | |||
| # 文件存储服务器用户名 | |||
| userName: root | |||
| #数据集训练配置 | |||
| ptversion: http://127.0.0.1:8000/ | |||
| minioweb: | |||
| GetToken: | |||
| url: minio/webrpc | |||
| param: | |||
| id: 2 | |||
| jsonrpc: 2.0 | |||
| method: Web.Login | |||
| zip: | |||
| url: minio/zip?token= | |||
| #logback | |||
| logging.config: | |||
| classpath:logback-spring-dev.xml | |||
| # ES服务地址及端口 | |||
| es: | |||
| host: 127.0.0.1 | |||
| serverPort: 32321 | |||
| transportPort: 32322 | |||
| index: dataset_text_test | |||
| @@ -5,7 +5,7 @@ terminal: | |||
| # 用户workspace目录 | |||
| workspace-dir: "/workspace" | |||
| # ssh主机 | |||
| ssh-host: 127.0.0.1 | |||
| ssh-host: 10.5.26.91 | |||
| # 可匿名访问路径 | |||
| security: | |||
| @@ -4,149 +4,139 @@ spring: | |||
| gateway: | |||
| enabled: true | |||
| routes: | |||
| # id唯一表示 路由至admin服务 | |||
| - id: admin-route | |||
| # uri=> lb://服务名称 | |||
| uri: lb://admin | |||
| # 路由规则 | |||
| predicates: | |||
| - Path=/api/v1/admin/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| - id: visual-route | |||
| # uri=> lb://服务名称 | |||
| uri: lb://admin | |||
| # 路由规则 | |||
| predicates: | |||
| - Path=/visual/api/** | |||
| # 路由至auth服务 | |||
| - id: auth-route | |||
| uri: lb://auth | |||
| predicates: | |||
| - Path=/api/v1/auth/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至notebook服务 | |||
| - id: notebook-route | |||
| uri: lb://dubhe-notebook | |||
| predicates: | |||
| - Path=/api/v1/notebook/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至镜像管理服务 | |||
| - id: image-route | |||
| uri: lb://dubhe-image | |||
| predicates: | |||
| - Path=/api/v1/image/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至度量管理服务 | |||
| - id: measure-route | |||
| uri: lb://dubhe-measure | |||
| predicates: | |||
| - Path=/api/v1/measure/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-k8s服务 | |||
| - id: dubhe-k8s | |||
| uri: lb://dubhe-k8s | |||
| predicates: | |||
| - Path=/api/v1/k8s/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-model(模型管理)服务 | |||
| - id: dubhe-model | |||
| uri: lb://dubhe-model | |||
| predicates: | |||
| - Path=/api/v1/model/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-algorithm(算法管理)服务 | |||
| - id: dubhe-algorithm | |||
| uri: lb://dubhe-algorithm | |||
| predicates: | |||
| - Path=/api/v1/algorithm/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-train(训练管理)服务 | |||
| - id: dubhe-train | |||
| uri: lb://dubhe-train | |||
| predicates: | |||
| - Path=/api/v1/train/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # id唯一表示 路由至data(数据集)服务 | |||
| - id: dubhe-data | |||
| # uri=> lb://服务名称 | |||
| uri: lb://dubhe-data | |||
| # 路由规则 | |||
| predicates: | |||
| - Path=/api/v1/data/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-optimize(模型优化)服务 | |||
| - id: dubhe-optimize | |||
| uri: lb://dubhe-optimize | |||
| predicates: | |||
| - Path=/api/v1/optimize/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-serving(云端Serving)服务 | |||
| - id: dubhe-serving | |||
| uri: lb://dubhe-serving | |||
| predicates: | |||
| - Path=/api/v1/serving/**,/api/v1/batchServing/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 负载均衡样例 | |||
| - id: weight-route1 | |||
| uri: lb://demo-client | |||
| # 路由规则 | |||
| predicates: | |||
| # 接口匹配前缀 | |||
| - Path=/api/v1/w/** | |||
| # 权重组weight-group1,权重比例 1/4 | |||
| - Weight=weight-group1, 1 | |||
| filters: | |||
| # 根据接口匹配前缀截取 | |||
| - StripPrefix=3 | |||
| - id: weight-route2 | |||
| uri: lb://demo-provider | |||
| # 路由规则 | |||
| predicates: | |||
| - Path=/api/v1/w/** | |||
| # 权重组weight-group1,权重比例 3/4 | |||
| - Weight=weight-group1, 3 | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-data-dcm(医学数据集)服务 | |||
| - id: dubhe-data-dcm | |||
| uri: lb://dubhe-data-dcm | |||
| predicates: | |||
| - Path=/api/v1/dcm/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-terminal 天枢专业版终端 | |||
| - id: dubhe-terminal | |||
| uri: lb://dubhe-terminal | |||
| predicates: | |||
| - Path=/api/v1/terminal/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-tadl服务 | |||
| - id: dubhe-tadl | |||
| uri: lb://dubhe-tadl | |||
| predicates: | |||
| - Path=/api/v1/tadl/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-point-cloud服务 | |||
| - id: dubhe-point-cloud | |||
| uri: lb://dubhe-point-cloud | |||
| predicates: | |||
| - Path=/api/v1/pointCloud/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| server: | |||
| maxPostSize: -1 | |||
| maxHttpHeaderSize: 10240000 | |||
| # id唯一表示 路由至admin服务 | |||
| - id: admin-route | |||
| # uri=> lb://服务名称 | |||
| uri: lb://admin | |||
| # 路由规则 | |||
| predicates: | |||
| - Path=/api/v1/admin/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| - id: visual-route | |||
| # uri=> lb://服务名称 | |||
| uri: lb://admin | |||
| # 路由规则 | |||
| predicates: | |||
| - Path=/visual/api/** | |||
| # 路由至demo-provider服务 | |||
| - id: nacos-server-route | |||
| uri: lb://demo-provider | |||
| predicates: | |||
| - Path=/api/v1/server/**,/api/v1/service/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至auth服务 | |||
| - id: auth-route | |||
| uri: lb://auth | |||
| predicates: | |||
| - Path=/api/v1/auth/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至notebook服务 | |||
| - id: notebook-route | |||
| uri: lb://dubhe-notebook | |||
| predicates: | |||
| - Path=/api/v1/notebook/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至镜像管理服务 | |||
| - id: image-route | |||
| uri: lb://dubhe-image | |||
| predicates: | |||
| - Path=/api/v1/image/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至度量管理服务 | |||
| - id: measure-route | |||
| uri: lb://dubhe-measure | |||
| predicates: | |||
| - Path=/api/v1/measure/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-k8s服务 | |||
| - id: dubhe-k8s | |||
| uri: lb://dubhe-k8s | |||
| predicates: | |||
| - Path=/api/v1/k8s/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-model(模型管理)服务 | |||
| - id: dubhe-model | |||
| uri: lb://dubhe-model | |||
| predicates: | |||
| - Path=/api/v1/model/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-algorithm(算法管理)服务 | |||
| - id: dubhe-algorithm | |||
| uri: lb://dubhe-algorithm | |||
| predicates: | |||
| - Path=/api/v1/algorithm/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-train(训练管理)服务 | |||
| - id: dubhe-train | |||
| uri: lb://dubhe-train | |||
| predicates: | |||
| - Path=/api/v1/train/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # id唯一表示 路由至data(数据集)服务 | |||
| - id: dubhe-data | |||
| # uri=> lb://服务名称 | |||
| uri: lb://dubhe-data | |||
| # 路由规则 | |||
| predicates: | |||
| - Path=/api/v1/data/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-optimize(模型优化)服务 | |||
| - id: dubhe-optimize | |||
| uri: lb://dubhe-optimize | |||
| predicates: | |||
| - Path=/api/v1/optimize/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-serving(云端Serving)服务 | |||
| - id: dubhe-serving | |||
| uri: lb://dubhe-serving | |||
| predicates: | |||
| - Path=/api/v1/serving/**,/api/v1/batchServing/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 负载均衡样例 | |||
| - id: weight-route1 | |||
| uri: lb://demo-client | |||
| # 路由规则 | |||
| predicates: | |||
| # 接口匹配前缀 | |||
| - Path=/api/v1/w/** | |||
| # 权重组weight-group1,权重比例 1/4 | |||
| - Weight=weight-group1, 1 | |||
| filters: | |||
| # 根据接口匹配前缀截取 | |||
| - StripPrefix=3 | |||
| - id: weight-route2 | |||
| uri: lb://demo-provider | |||
| # 路由规则 | |||
| predicates: | |||
| - Path=/api/v1/w/** | |||
| # 权重组weight-group1,权重比例 3/4 | |||
| - Weight=weight-group1, 3 | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-data-dcm(医学数据集)服务 | |||
| - id: dubhe-data-dcm | |||
| uri: lb://dubhe-data-dcm | |||
| predicates: | |||
| - Path=/api/v1/dcm/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| # 路由至dubhe-tadl服务 | |||
| - id: dubhe-tadl | |||
| uri: lb://dubhe-tadl | |||
| predicates: | |||
| - Path=/api/v1/tadl/** | |||
| filters: | |||
| - StripPrefix=3 | |||
| @@ -3,7 +3,7 @@ spring: | |||
| redis: | |||
| #数据库索引 | |||
| database: 1 | |||
| host: 127.0.0.1 | |||
| host: 10.5.26.88 | |||
| port: 6379 | |||
| password: | |||
| #连接超时时间 | |||
| @@ -12,9 +12,9 @@ spring: | |||
| druid: | |||
| db-type: com.alibaba.druid.pool.DruidDataSource | |||
| driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | |||
| url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true | |||
| url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true | |||
| username: test | |||
| password: test | |||
| password: zj12345678 | |||
| # 初始化配置 | |||
| initial-size: 3 | |||
| @@ -43,11 +43,5 @@ spring: | |||
| url-pattern: /* | |||
| exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" | |||
| #配置harbor | |||
| harbor: | |||
| address: harbor.dubhe.ai | |||
| username: admin | |||
| password: Harbor12345 | |||
| @@ -1,19 +0,0 @@ | |||
| apiVersion: v1 | |||
| clusters: | |||
| - cluster: | |||
| certificate-authority-data: | |||
| server: https://127.0.0.1:6443 | |||
| name: kubernetes | |||
| contexts: | |||
| - context: | |||
| cluster: kubernetes | |||
| user: kubernetes-admin | |||
| name: kubernetes-admin@kubernetes | |||
| current-context: kubernetes-admin@kubernetes | |||
| kind: Config | |||
| preferences: {} | |||
| users: | |||
| - name: kubernetes-admin | |||
| user: | |||
| client-certificate-data: | |||
| client-key-data: | |||
| @@ -3,7 +3,7 @@ spring: | |||
| redis: | |||
| #数据库索引 | |||
| database: 1 | |||
| host: 127.0.0.1 | |||
| host: 10.5.26.88 | |||
| port: 6379 | |||
| password: | |||
| #连接超时时间 | |||
| @@ -16,10 +16,10 @@ model: | |||
| atlas: | |||
| url: | |||
| # 流服务地址 | |||
| rtmp: rtmp://127.0.0.1:1935/live/ | |||
| rtmp: rtmp://10.5.30.42:1935/live/ | |||
| # vms接口地址 | |||
| vms: http://127.0.0.1:8000/api/vms | |||
| vms: http://10.105.10.51:8000/api/vms | |||
| # 推理服务地址 | |||
| video-serve: http://127.0.0.1:32761/video | |||
| video-serve: http://10.105.0.6:32761/video | |||
| # 拉流地址,需要拼接streamId | |||
| stream: http://127.0.0.1:8055/live?port=1935&app=live&stream= | |||
| stream: http://10.5.30.42:8055/live?port=1935&app=live&stream= | |||
| @@ -14,16 +14,16 @@ VUE_APP_DATA_API = '/' | |||
| VUE_APP_VISUAL_API = '/' | |||
| # minio | |||
| VUE_APP_MINIO_API = 'http://127.0.0.1:9000/minio' | |||
| VUE_APP_MINIO_API = 'http://10.105.1.132:9000/minio' | |||
| # atlas | |||
| VUE_APP_ATLAS_HOST = 'http://127.0.0.1' | |||
| VUE_APP_ATLAS_HOST = 'http://183.129.174.186' | |||
| # DCM4CHEE | |||
| VUE_APP_DCM_API = 'http://pre.dubhe.club/dcm4chee/dcm4chee-arc/aets/DCM4CHEE_ADMIN' | |||
| # minIO 服务 IP | |||
| VUE_APP_MINIO_ENDPOINT = '127.0.0.1' | |||
| VUE_APP_MINIO_ENDPOINT = '10.105.1.132' | |||
| # minIO 服务 端口 | |||
| VUE_APP_MINIO_PORT = '9000' | |||
| # 是否开启 SSL | |||
| @@ -5,4 +5,5 @@ dist | |||
| src/components/Crud | |||
| mock | |||
| src/views/visual | |||
| src/store/modules/Visual | |||
| src/store/modules/Visual | |||
| src/utils/VisualUtils | |||
| @@ -1,12 +1,9 @@ | |||
| const plugins = [ | |||
| "@vue/babel-plugin-transform-vue-jsx", | |||
| "@babel/plugin-proposal-optional-chaining", | |||
| ]; | |||
| const plugins = ['@vue/babel-plugin-transform-vue-jsx', '@babel/plugin-proposal-optional-chaining']; | |||
| // 生产环境移除console | |||
| if (process.env.NODE_ENV === "production") { | |||
| plugins.push("transform-remove-console"); | |||
| if (process.env.NODE_ENV === 'production') { | |||
| plugins.push('transform-remove-console'); | |||
| } | |||
| module.exports = { | |||
| plugins, | |||
| presets: [["@vue/app",{ useBuiltIns: "entry" }]], | |||
| presets: [['@vue/app', { useBuiltIns: 'entry' }]], | |||
| }; | |||
| @@ -24,6 +24,18 @@ | |||
| "lint:style": "stylelint src/**/*.{html,vue,css,sass,scss}", | |||
| "fix:style": "stylelint src/**/*.{html,vue,css,sass,scss} --fix" | |||
| }, | |||
| "husky": { | |||
| "hooks": { | |||
| "pre-commit": "lint-staged", | |||
| "commit-msg": "commitlint -E HUSKY_GIT_PARAMS" | |||
| } | |||
| }, | |||
| "lint-staged": { | |||
| "src/**/*.{js,vue}": [ | |||
| "eslint --fix", | |||
| "git add" | |||
| ] | |||
| }, | |||
| "repository": { | |||
| "type": "git", | |||
| "url": "git@codeup.teambition.com:zhejianglab/dubhe-web.git" | |||
| @@ -19,7 +19,7 @@ import { API_MODULE_NAME } from '@/config'; | |||
| export function list(params) { | |||
| return request({ | |||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage/info`, | |||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage/list`, | |||
| method: 'get', | |||
| params, | |||
| }); | |||
| @@ -27,7 +27,7 @@ export function list(params) { | |||
| export function add(data) { | |||
| return request({ | |||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage/uploadImage`, | |||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage`, | |||
| method: 'post', | |||
| data, | |||
| }); | |||
| @@ -65,14 +65,6 @@ export function getImageTagList(params) { | |||
| }); | |||
| } | |||
| export function setPrecast(params) { | |||
| return request({ | |||
| url: `/${API_MODULE_NAME.IMAGE}/ptImage/imageResource`, | |||
| method: 'put', | |||
| params, | |||
| }); | |||
| } | |||
| // 设置notebook默认镜像-获取默认镜像 | |||
| export function getDefaultImage(params) { | |||
| return request({ | |||
| @@ -332,6 +332,10 @@ | |||
| width: 320px; | |||
| } | |||
| .w-400 { | |||
| width: 400px; | |||
| } | |||
| .w-500 { | |||
| width: 500px; | |||
| } | |||
| @@ -9,6 +9,9 @@ the License. * ============================================================= */ | |||
| <template> | |||
| <el-tooltip v-bind="mergedAttrs"> | |||
| <i class="primary f18 vm" :class="[icon]" /> | |||
| <template #content> | |||
| <slot name="content" /> | |||
| </template> | |||
| </el-tooltip> | |||
| </template> | |||
| @@ -541,6 +541,7 @@ import { | |||
| RESOURCES_MODULE_ENUM, | |||
| RESOURCES_POOL_TYPE_ENUM, | |||
| RESOURCES_POOL_TYPE_MAP, | |||
| IMAGE_TYPE_ENUM, | |||
| } from '@/utils'; | |||
| import { list as getAlgorithmList, getAlgorithmInfo } from '@/api/algorithm/algorithm'; | |||
| import { getModelByResource } from '@/api/model/model'; | |||
| @@ -551,7 +552,7 @@ import { list as getSpecsNames } from '@/api/system/resources'; | |||
| import { list as getNotebooks } from '@/api/development/notebook'; | |||
| import { trainConfig } from '@/config'; | |||
| import { NOTEBOOK_STATUS_ENUM } from '@/views/development/utils'; | |||
| import { IMAGE_TYPE, TRAINING_TYPE_ENUM } from '@/views/trainingJob/utils'; | |||
| import { TRAINING_TYPE_ENUM } from '@/views/trainingJob/utils'; | |||
| import BaseTooltip from '@/components/BaseTooltip'; | |||
| import DataSourceSelector from './dataSourceSelector'; | |||
| @@ -929,7 +930,7 @@ export default { | |||
| }, | |||
| async getHarborProjects() { | |||
| this.harborProjectList = await getImageNameList({ | |||
| imageTypes: [IMAGE_TYPE.TRAIN], | |||
| imageTypes: [IMAGE_TYPE_ENUM.TRAIN], | |||
| }); | |||
| if ( | |||
| this.form.imageName && | |||
| @@ -39,7 +39,7 @@ const covertSVG2Image = (node, name, width, height, type = 'png') => { | |||
| const context = canvas.getContext('2d'); | |||
| context.fillStyle = '#fff'; | |||
| context.fillRect(0, 0, 10000, 10000); | |||
| image.onload = function() { | |||
| image.onload = () => { | |||
| context.drawImage(image, 0, 0); | |||
| const a = document.createElement('a'); | |||
| a.download = `${name}.${type}`; | |||
| @@ -133,6 +133,24 @@ export const K8S_BUSINESS_LABEL_MAP = { | |||
| [K8S_BUSINESS_LABEL_ENUM.POINT_CLOUD]: '点云数据集', | |||
| }; | |||
| // 镜像分类枚举 | |||
| export const IMAGE_TYPE_ENUM = { | |||
| NOTEBOOK: 0, // notebook镜像类型 | |||
| TRAIN: 1, // 训练镜像和预置镜像类型 | |||
| SERVING: 2, // serving镜像 | |||
| TERMINAL: 3, // 终端镜像 | |||
| POINTCLOUD: 4, // 点云镜像 | |||
| DATASETMARKED: 5, // 数据标注镜像 | |||
| }; | |||
| export const IMAGE_TYPE_MAP = { | |||
| [IMAGE_TYPE_ENUM.NOTEBOOK]: 'Notebook镜像', | |||
| [IMAGE_TYPE_ENUM.TRAIN]: '训练镜像', | |||
| [IMAGE_TYPE_ENUM.SERVING]: 'Serving镜像', | |||
| [IMAGE_TYPE_ENUM.TERMINAL]: '终端镜像', | |||
| [IMAGE_TYPE_ENUM.POINTCLOUD]: '点云镜像', | |||
| [IMAGE_TYPE_ENUM.DATASETMARKED]: '数据标注镜像', | |||
| }; | |||
| // 默认进度条颜色 | |||
| export const defaultProcessColors = [ | |||
| { color: '#909399', percentage: 40 }, | |||
| @@ -20,7 +20,7 @@ import ZIP from './zip'; | |||
| const pMap = require('p-map'); | |||
| //streamSaver.mitm = 'https://static.zhejianglab.com/mitm.html'; | |||
| streamSaver.mitm = 'https://static.zhejianglab.com/mitm.html'; | |||
| // 默认名字解析 | |||
| const defaultName = (file) => file.name; | |||
| @@ -171,13 +171,13 @@ import { | |||
| invalidFileNameChar, | |||
| defaultProcessColors, | |||
| ALGORITHM_RESOURCE_ENUM, | |||
| IMAGE_TYPE_ENUM, | |||
| } from '@/utils'; | |||
| import UploadInline from '@/components/UploadForm/inline'; | |||
| import UploadProgress from '@/components/UploadProgress'; | |||
| import { useMapGetters } from '@/hooks'; | |||
| import { algorithmConfig } from '@/config'; | |||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
| const defaultForm = { | |||
| id: null, | |||
| @@ -289,7 +289,7 @@ export default { | |||
| }; | |||
| // 获取镜像名列表 | |||
| const getImageNames = async (keepValue = false) => { | |||
| state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.TRAIN] }); | |||
| state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.TRAIN] }); | |||
| if (!keepValue || !form.imageName) { | |||
| form.imageTag = null; | |||
| } else if (!state.imageNameList.includes(form.imageName)) { | |||
| @@ -199,14 +199,13 @@ | |||
| <script> | |||
| import { mapGetters } from 'vuex'; | |||
| import { getUniqueId, invalidFileNameChar, RESOURCES_MODULE_ENUM } from '@/utils'; | |||
| import { getUniqueId, invalidFileNameChar, RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils'; | |||
| import { getServingModel } from '@/api/model/model'; | |||
| import { list as getModelVersions } from '@/api/model/modelVersion'; | |||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
| import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; | |||
| import { list as getSpecsNames } from '@/api/system/resources'; | |||
| import { validateNameWithHyphen } from '@/utils/validate'; | |||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
| import RunParamForm from '@/components/Training/runParamForm'; | |||
| import BaseModal from '@/components/BaseModal'; | |||
| import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; | |||
| @@ -541,7 +540,7 @@ export default { | |||
| // 镜像选择 | |||
| async getImageNames(keepValue = false) { | |||
| this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.SERVING] }); | |||
| this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.SERVING] }); | |||
| if (!keepValue || !this.form.imageName) { | |||
| this.form.imageTag = null; | |||
| } else if (!this.imageNameList.includes(this.form.imageName)) { | |||
| @@ -204,8 +204,7 @@ import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
| import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; | |||
| import { list as getSpecsNames } from '@/api/system/resources'; | |||
| import { servingConfig } from '@/config'; | |||
| import { RESOURCES_MODULE_ENUM } from '@/utils'; | |||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
| import { RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils'; | |||
| import RunParamForm from '@/components/Training/runParamForm'; | |||
| import BaseModal from '@/components/BaseModal'; | |||
| import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; | |||
| @@ -540,7 +539,7 @@ export default { | |||
| // 镜像选择 | |||
| // 获取镜像名称列表 | |||
| async getImageNames(keepValue = false) { | |||
| this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.SERVING] }); | |||
| this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.SERVING] }); | |||
| if (!keepValue || !this.form.imageName) { | |||
| this.form.imageTag = null; | |||
| } else if (!this.imageNameList.includes(this.form.imageName)) { | |||
| @@ -554,7 +553,7 @@ export default { | |||
| async getImageTags(imageName, keepValue = false) { | |||
| this.imageTagList = await getImageTagList({ | |||
| imageName, | |||
| imageTypes: [IMAGE_TYPE.SERVING], | |||
| imageTypes: [IMAGE_TYPE_ENUM.SERVING], | |||
| }); | |||
| if (keepValue && this.form.imageTag) { | |||
| if (!this.imageTagList.some((image) => image.imageTag === this.form.imageTag)) { | |||
| @@ -134,8 +134,12 @@ import { list as listBranchModel } from '@/api/model/modelVersion'; | |||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
| import { list as listResourceSpec } from '@/api/system/resources'; | |||
| import { types } from '@/utils/validate'; | |||
| import { RESOURCES_POOL_TYPE_ENUM, RESOURCES_MODULE_ENUM, modelTypeSymbol } from '@/utils/constant'; | |||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
| import { | |||
| RESOURCES_POOL_TYPE_ENUM, | |||
| RESOURCES_MODULE_ENUM, | |||
| modelTypeSymbol, | |||
| IMAGE_TYPE_ENUM, | |||
| } from '@/utils/constant'; | |||
| const initialForm = { | |||
| name: undefined, | |||
| @@ -246,7 +250,7 @@ export default { | |||
| }; | |||
| const queryImages = () => { | |||
| return getImageNameList({ imageTypes: [IMAGE_TYPE.DATASETMARKED] }); | |||
| return getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.DATASETMARKED] }); | |||
| }; | |||
| const handleModelParentChange = () => { | |||
| @@ -143,8 +143,12 @@ import { list as listBranchModel } from '@/api/model/modelVersion'; | |||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
| import { list as listResourceSpec } from '@/api/system/resources'; | |||
| import { types } from '@/utils/validate'; | |||
| import { RESOURCES_POOL_TYPE_ENUM, RESOURCES_MODULE_ENUM, modelTypeSymbol } from '@/utils/constant'; | |||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
| import { | |||
| RESOURCES_POOL_TYPE_ENUM, | |||
| RESOURCES_MODULE_ENUM, | |||
| modelTypeSymbol, | |||
| IMAGE_TYPE_ENUM, | |||
| } from '@/utils/constant'; | |||
| const initialForm = { | |||
| name: undefined, | |||
| @@ -283,7 +287,7 @@ export default { | |||
| // 查询镜像列表 | |||
| const queryImages = () => { | |||
| return getImageNameList({ imageTypes: [IMAGE_TYPE.DATASETMARKED] }); | |||
| return getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.DATASETMARKED] }); | |||
| }; | |||
| // 查询镜像版本列表 | |||
| @@ -163,8 +163,8 @@ import { | |||
| ALGORITHM_RESOURCE_ENUM, | |||
| MODEL_RESOURCE_ENUM, | |||
| RESOURCES_POOL_TYPE_ENUM, | |||
| IMAGE_TYPE_ENUM, | |||
| } from '@/utils'; | |||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
| const initFormState = { | |||
| modelId: null, | |||
| @@ -285,7 +285,7 @@ export default { | |||
| return Promise.reject(); | |||
| } | |||
| return getImageTagList({ | |||
| imageTypes: IMAGE_TYPE.POINTCLOUD, | |||
| imageTypes: IMAGE_TYPE_ENUM.POINTCLOUD, | |||
| imageName: modelState.imageName, | |||
| }).then((res) => { | |||
| state.imageTagList = res; | |||
| @@ -294,7 +294,7 @@ export default { | |||
| // 镜像选择 | |||
| const getImageNames = async () => { | |||
| state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.POINTCLOUD] }); | |||
| state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.POINTCLOUD] }); | |||
| if ( | |||
| modelState.imageName && | |||
| !state.imageNameList.some((image) => image === modelState.imageName) | |||
| @@ -122,14 +122,13 @@ | |||
| </template> | |||
| <script> | |||
| import { validateNameWithHyphen, RESOURCES_MODULE_ENUM } from '@/utils'; | |||
| import { validateNameWithHyphen, RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils'; | |||
| import BaseModal from '@/components/BaseModal'; | |||
| import InfoSelect from '@/components/InfoSelect'; | |||
| import { getPublishedDatasets, getDatasetVersions } from '@/api/preparation/dataset'; | |||
| import { add as addNotebook } from '@/api/development/notebook'; | |||
| import { list as getSpecsNames } from '@/api/system/resources'; | |||
| import { getImageNameList, getImageTagList } from '@/api/trainingImage/index'; | |||
| import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
| const defaultForm = { | |||
| noteBookName: null, | |||
| @@ -208,7 +207,7 @@ export default { | |||
| } | |||
| }, | |||
| async getHarborProjects() { | |||
| this.harborProjectList = await getImageNameList({ imageTypes: [IMAGE_TYPE.NOTEBOOK] }); | |||
| this.harborProjectList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.NOTEBOOK] }); | |||
| if ( | |||
| this.form.imageName && | |||
| !this.harborProjectList.some((project) => project === this.form.imageName) | |||
| @@ -237,7 +236,7 @@ export default { | |||
| } | |||
| return getImageTagList({ | |||
| imageName: this.form.imageName, | |||
| imageTypes: [IMAGE_TYPE.NOTEBOOK], | |||
| imageTypes: [IMAGE_TYPE_ENUM.NOTEBOOK], | |||
| }).then((res) => { | |||
| this.harborImageList = res; | |||
| }); | |||
| @@ -48,7 +48,7 @@ | |||
| <el-table | |||
| v-if="prefabricate" | |||
| ref="table" | |||
| v-loading="crud.loading || disableEdit" | |||
| v-loading="crud.loading" | |||
| :data="crud.data" | |||
| highlight-current-row | |||
| @selection-change="crud.selectionChangeHandler" | |||
| @@ -57,21 +57,6 @@ | |||
| <el-table-column v-if="!isPreset" prop="id" label="ID" sortable="custom" width="80px" /> | |||
| <el-table-column prop="imageName" label="镜像名称" sortable="custom" /> | |||
| <el-table-column prop="imageTag" label="镜像版本号" sortable="custom" /> | |||
| <el-table-column prop="imageStatus" label="状态" width="130px"> | |||
| <template #header> | |||
| <dropdown-header | |||
| title="状态" | |||
| :list="imageStatusList" | |||
| :filtered="Boolean(localQuery.imageStatus)" | |||
| @command="filterStatus" | |||
| /> | |||
| </template> | |||
| <template slot-scope="scope"> | |||
| <el-tag effect="plain" :type="map[scope.row.imageStatus]"> | |||
| {{ statusMap[scope.row.imageStatus] }} | |||
| </el-tag> | |||
| </template> | |||
| </el-table-column> | |||
| <el-table-column prop="imageTypes" label="镜像用途" width="180px"> | |||
| <template #header> | |||
| <dropdown-header | |||
| @@ -91,31 +76,8 @@ | |||
| <span>{{ parseTime(scope.row.createTime) }}</span> | |||
| </template> | |||
| </el-table-column> | |||
| <el-table-column v-if="isNotebook" prop="imageResource" label="是否为默认" align="center"> | |||
| <template slot-scope="scope"> | |||
| <i | |||
| :class="resourceObj(scope.row.imageResource).icon" | |||
| :style="{ color: resourceObj(scope.row.imageResource).color, fontSize: '20px' }" | |||
| ></i> | |||
| </template> | |||
| </el-table-column> | |||
| <el-table-column v-if="isAdmin || isCustom" label="操作" width="200px" fixed="right"> | |||
| <template slot-scope="scope"> | |||
| <el-tooltip | |||
| v-if="isAdmin && isNotebook" | |||
| effect="dark" | |||
| content="设为在线编辑算法时创建nootbook的默认镜像" | |||
| placement="top" | |||
| > | |||
| <el-button | |||
| :id="`doPrecast_` + scope.$index" | |||
| :disabled="Boolean(scope.row.imageResource)" | |||
| type="text" | |||
| @click.stop="doPrecast(scope.row.id)" | |||
| > | |||
| {{ resourceObj(scope.row.imageResource).butText }} | |||
| </el-button> | |||
| </el-tooltip> | |||
| <el-button | |||
| v-if="(hasPermission('training:image:edit') && isCustom) || (isPreset && isAdmin)" | |||
| :id="`doEdit_` + scope.$index" | |||
| @@ -143,15 +105,13 @@ | |||
| :visible="crud.status.cu > 0" | |||
| :title="crud.status.title" | |||
| :loading="crud.status.cu === 2" | |||
| :disabled="loading" | |||
| width="600px" | |||
| @open="onDialogOpen" | |||
| @close="onDialogClose" | |||
| @cancel="crud.cancelCU" | |||
| @ok="crud.submitCU" | |||
| > | |||
| <el-form ref="form" :model="form" :rules="rules" label-width="120px"> | |||
| <el-form-item v-if="isEdit && isAdmin" label="镜像类别" prop="imageResource"> | |||
| <el-form-item v-if="isFormAdd && isAdmin" label="镜像类别" prop="imageResource"> | |||
| <el-radio-group v-model="form.imageResource" @change="imageResourceChange"> | |||
| <el-radio :label="Number(IMAGE_RESOURCE_ENUM.CUSTOM)" border class="mr-0" | |||
| >我的镜像</el-radio | |||
| @@ -159,21 +119,20 @@ | |||
| <el-radio :label="Number(IMAGE_RESOURCE_ENUM.PRESET)" border>预置镜像</el-radio> | |||
| </el-radio-group> | |||
| </el-form-item> | |||
| <el-form-item v-if="isEdit" label="镜像名称" prop="imageName"> | |||
| <el-form-item label="镜像名称" prop="imageName"> | |||
| <el-autocomplete | |||
| ref="imageName" | |||
| v-model="form.imageName" | |||
| class="inline-input" | |||
| class="inline-input w-400" | |||
| :fetch-suggestions="querySearchAsync" | |||
| placeholder="请选择或输入镜像名称" | |||
| style="width: 400px;" | |||
| ></el-autocomplete> | |||
| </el-form-item> | |||
| <el-form-item label="镜像用途"> | |||
| <el-select | |||
| v-model="form.imageTypes" | |||
| placeholder="请选择或输入镜像用途" | |||
| style="width: 400px;" | |||
| class="w-400" | |||
| multiple | |||
| filterable | |||
| allow-create | |||
| @@ -187,36 +146,18 @@ | |||
| /> | |||
| </el-select> | |||
| </el-form-item> | |||
| <el-form-item v-if="isEdit" ref="imagePath" label="镜像文件路径" prop="imagePath"> | |||
| <upload-inline | |||
| v-if="crud.status.cu > 0" | |||
| ref="upload" | |||
| action="fakeApi" | |||
| accept=".zip,.tar,.rar,.gz" | |||
| list-type="text" | |||
| :acceptSize="imageConfig.uploadFileAcceptSize" | |||
| :acceptSizeFormat="uploadSizeFomatter" | |||
| :params="uploadParams" | |||
| :show-file-count="false" | |||
| :auto-upload="true" | |||
| :filters="uploadFilters" | |||
| :limit="1" | |||
| :on-remove="onFileRemove" | |||
| @uploadStart="uploadStart" | |||
| @uploadSuccess="uploadSuccess" | |||
| @uploadError="uploadError" | |||
| /> | |||
| <upload-progress | |||
| v-if="loading" | |||
| :progress="progress" | |||
| :color="customColors" | |||
| :status="status" | |||
| :size="size" | |||
| @onSetProgress="onSetProgress" | |||
| /> | |||
| <el-form-item ref="imageUrl" label="镜像地址" prop="imageUrl"> | |||
| <el-input v-model="form.imageUrl" placeholder="请输入镜像地址" class="w-400" /> | |||
| <BaseTooltip icon="el-icon-warning" class="c-info"> | |||
| <template #content> | |||
| <div> | |||
| 镜像地址标准格式:镜像仓库域名/命名空间/镜像名:镜像版本号<br />示例:registry.cn-hangzhou.aliyuncs.com/enlin/notebook:v1 | |||
| </div> | |||
| </template> | |||
| </BaseTooltip> | |||
| </el-form-item> | |||
| <el-form-item v-if="isEdit" label="镜像版本号" prop="imageTag"> | |||
| <el-input id="imageTag" v-model="form.imageTag" style="width: 400px;" /> | |||
| <el-form-item label="镜像版本号" prop="imageTag"> | |||
| <el-input id="imageTag" v-model="form.imageTag" class="w-400" /> | |||
| </el-form-item> | |||
| <el-form-item label="描述" prop="remark"> | |||
| <el-input | |||
| @@ -227,19 +168,19 @@ | |||
| maxlength="1024" | |||
| show-word-limit | |||
| placeholder | |||
| style="width: 400px;" | |||
| class="w-400" | |||
| /> | |||
| </el-form-item> | |||
| </el-form> | |||
| </BaseModal> | |||
| <!--Notebook默认镜像设置表单--> | |||
| <BaseModal | |||
| :visible.sync="formVisible" | |||
| :visible.sync="notebookFormVisible" | |||
| title="Notebook默认镜像设置" | |||
| :loading="formSubmitting" | |||
| :loading="notebookFormSubmitting" | |||
| width="800px" | |||
| @cancel="formVisible = false" | |||
| @ok="onSubmitForm" | |||
| @cancel="notebookFormVisible = false" | |||
| @ok="onSubmitNotebookForm" | |||
| > | |||
| <el-form | |||
| ref="noteBookFormRef" | |||
| @@ -253,7 +194,7 @@ | |||
| id="defaultImage" | |||
| v-model="noteBookForm.defaultImage" | |||
| placeholder="请选择镜像" | |||
| style="width: 400px;" | |||
| class="w-400" | |||
| filterable | |||
| allow-create | |||
| default-first-option | |||
| @@ -283,8 +224,6 @@ | |||
| <script> | |||
| import { mapGetters } from 'vuex'; | |||
| // eslint-disable-next-line import/no-extraneous-dependencies | |||
| import { debounce } from 'throttle-debounce'; | |||
| import cdOperation from '@crud/CD.operation'; | |||
| import rrOperation from '@crud/RR.operation'; | |||
| @@ -292,40 +231,36 @@ import pagination from '@crud/Pagination'; | |||
| import CRUD, { presenter, header, form, crud } from '@crud/crud'; | |||
| import trainingImageApi, { | |||
| del, | |||
| setPrecast, | |||
| setDefaultImage, | |||
| getDefaultImage, | |||
| getImageNameList, | |||
| getImageTagList, | |||
| } from '@/api/trainingImage/index'; | |||
| import { | |||
| getUniqueId, | |||
| uploadSizeFomatter, | |||
| invalidFileNameChar, | |||
| ADMIN_ROLE_ID, | |||
| hasPermission, | |||
| validateImageName, | |||
| validateImageTag, | |||
| IMAGE_TYPE_ENUM, | |||
| IMAGE_TYPE_MAP, | |||
| } from '@/utils'; | |||
| import BaseModal from '@/components/BaseModal'; | |||
| import UploadInline from '@/components/UploadForm/inline'; | |||
| import BaseTooltip from '@/components/BaseTooltip'; | |||
| import DropdownHeader from '@/components/DropdownHeader'; | |||
| import UploadProgress from '@/components/UploadProgress'; | |||
| import { imageConfig } from '@/config'; | |||
| import { IMAGE_RESOURCE_ENUM, IMAGE_TYPE } from '../trainingJob/utils'; | |||
| import { IMAGE_RESOURCE_ENUM } from '../trainingJob/utils'; | |||
| const defaultForm = { | |||
| imagePath: null, | |||
| imageUrl: null, | |||
| imageTag: null, | |||
| remark: null, | |||
| imageTypes: Number(IMAGE_TYPE), | |||
| imageTypes: Number(IMAGE_TYPE_ENUM), | |||
| imageResource: Number(IMAGE_RESOURCE_ENUM.CUSTOM), | |||
| imageName: null, | |||
| }; | |||
| const defaultQuery = { | |||
| imageStatus: null, | |||
| imageNameOrId: null, | |||
| imageTypes: null, | |||
| }; | |||
| @@ -334,28 +269,27 @@ export default { | |||
| name: 'TrainingImage', | |||
| components: { | |||
| BaseModal, | |||
| BaseTooltip, | |||
| pagination, | |||
| cdOperation, | |||
| rrOperation, | |||
| UploadInline, | |||
| DropdownHeader, | |||
| UploadProgress, | |||
| }, | |||
| cruds() { | |||
| return CRUD({ | |||
| title: '镜像', | |||
| crudMethod: { ...trainingImageApi }, | |||
| optShow: { | |||
| add: imageConfig.allowUploadImage && hasPermission('training:image:upload'), | |||
| add: imageConfig.allowUploadImage && hasPermission('training:image:save'), | |||
| del: false, | |||
| }, | |||
| queryOnPresenterCreated: false, | |||
| props: { | |||
| optText: { | |||
| add: '上传镜像', | |||
| add: '创建镜像', | |||
| }, | |||
| optTitle: { | |||
| add: '上传', | |||
| add: '创建', | |||
| }, | |||
| }, | |||
| }); | |||
| @@ -365,24 +299,6 @@ export default { | |||
| return { | |||
| active: IMAGE_RESOURCE_ENUM.CUSTOM, | |||
| localQuery: { ...defaultQuery }, | |||
| map: { | |||
| 0: 'info', | |||
| 1: 'success', | |||
| 2: 'danger', | |||
| }, | |||
| statusMap: { | |||
| 0: '制作中', | |||
| 1: '制作成功', | |||
| 2: '制作失败', | |||
| }, | |||
| imageTypesMap: { | |||
| 0: 'Notebook镜像', | |||
| 1: '训练镜像', | |||
| 2: 'Serving镜像', | |||
| 3: '终端镜像', | |||
| 4: '点云镜像', | |||
| 5: '数据标注镜像', | |||
| }, | |||
| rules: { | |||
| imageTypes: [{ required: true, message: '请选择镜像类型', trigger: 'change' }], | |||
| imageResource: [{ required: true, message: '请选择镜像来源', trigger: 'change' }], | |||
| @@ -390,7 +306,7 @@ export default { | |||
| { required: true, message: '请选择项目名称', trigger: 'change' }, | |||
| { validator: validateImageName, trigger: ['blur', 'change'] }, | |||
| ], | |||
| imagePath: [{ required: true, message: '请输入镜像路径', trigger: ['blur', 'manual'] }], | |||
| imageUrl: [{ required: true, message: '请输入镜像地址', trigger: ['blur', 'manual'] }], | |||
| imageTag: [ | |||
| { required: true, message: '请输入镜像版本号', trigger: 'blur' }, | |||
| { validator: validateImageTag, trigger: ['blur', 'change'] }, | |||
| @@ -401,32 +317,17 @@ export default { | |||
| id: [{ required: true, message: '请选择默认镜像版本', trigger: 'blur' }], | |||
| }, | |||
| harborProjectList: [], | |||
| drawer: false, | |||
| uploadParams: { | |||
| objectPath: null, // 对象存储路径 | |||
| }, | |||
| progress: 0, | |||
| size: 0, | |||
| customColors: [ | |||
| { color: '#909399', percentage: 40 }, | |||
| { color: '#e6a23c', percentage: 80 }, | |||
| { color: '#67c23a', percentage: 100 }, | |||
| ], | |||
| disableEdit: false, | |||
| loading: false, | |||
| isEdit: false, | |||
| prefabricate: true, | |||
| // 以下为配置参数及常量参数 | |||
| imageConfig, | |||
| IMAGE_RESOURCE_ENUM, | |||
| IMAGE_TYPE, | |||
| uploadFilters: [invalidFileNameChar], | |||
| // 设置notebook默认镜像相关参数 | |||
| noteBookImages: [], | |||
| noteBookTags: [], | |||
| noteBookForm: { defaultImage: '', defaultTag: '', id: '' }, | |||
| formVisible: false, | |||
| formSubmitting: false, | |||
| formType: 'add', | |||
| notebookFormVisible: false, | |||
| notebookFormSubmitting: false, | |||
| }; | |||
| }, | |||
| computed: { | |||
| @@ -441,14 +342,8 @@ export default { | |||
| isPreset() { | |||
| return this.active === IMAGE_RESOURCE_ENUM.PRESET; | |||
| }, | |||
| isNotebook() { | |||
| return this.active === IMAGE_RESOURCE_ENUM.NOTEBOOK; | |||
| }, | |||
| isTerminal() { | |||
| return this.active === IMAGE_RESOURCE_ENUM.TERMINAL; | |||
| }, | |||
| disableAdd() { | |||
| if (this.isAdmin) return this.isTerminal; // 管理员只有在终端镜像处无法点击上传 | |||
| if (this.isAdmin) return false; // 管理员可以创建我的镜像和预置镜像 | |||
| return !this.isCustom; // 其他角色只有在我的镜像处可以点击上传 | |||
| }, | |||
| operationProps() { | |||
| @@ -456,37 +351,24 @@ export default { | |||
| disabled: this.disableAdd, | |||
| }; | |||
| }, | |||
| imageStatusList() { | |||
| const arr = [{ label: '全部', value: null }]; | |||
| for (const key in this.statusMap) { | |||
| arr.push({ label: this.statusMap[key], value: key }); | |||
| } | |||
| return arr; | |||
| }, | |||
| imageTypesList() { | |||
| const arr = [{ label: '全部', value: null }]; | |||
| for (const key in this.imageTypesMap) { | |||
| arr.push({ label: this.imageTypesMap[key], value: +key }); | |||
| for (const key in IMAGE_TYPE_MAP) { | |||
| arr.push({ label: IMAGE_TYPE_MAP[key], value: +key }); | |||
| } | |||
| return arr; | |||
| }, | |||
| status() { | |||
| return this.progress === 100 ? 'success' : null; | |||
| isFormAdd() { | |||
| return this.formType === 'add'; | |||
| }, | |||
| }, | |||
| mounted() { | |||
| this.crud.refresh(); | |||
| this.refetch = debounce(3000, this.crud.refresh); | |||
| this.updateImagePath(); | |||
| }, | |||
| methods: { | |||
| hasPermission, | |||
| getImageTypes(imageTypes) { | |||
| const usageStr = []; | |||
| imageTypes.forEach((item) => { | |||
| usageStr.push(this.imageTypesMap[item]); | |||
| }); | |||
| return usageStr.join(','); | |||
| return imageTypes.map((type) => IMAGE_TYPE_MAP[type]).join(','); | |||
| }, | |||
| getNoteBookTags() { | |||
| this.noteBookForm.defaultTag = null; | |||
| @@ -497,7 +379,7 @@ export default { | |||
| } | |||
| return getImageTagList({ | |||
| imageName: this.noteBookForm.defaultImage, | |||
| imageTypes: IMAGE_TYPE.NOTEBOOK, | |||
| imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK, | |||
| imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | |||
| }).then((res) => { | |||
| this.noteBookTags = res; | |||
| @@ -513,46 +395,11 @@ export default { | |||
| this.prefabricate = true; | |||
| }); | |||
| }, | |||
| onFileRemove() { | |||
| this.form.imagePath = null; | |||
| this.loading = false; | |||
| this.$refs.imagePath.validate('manual'); | |||
| }, | |||
| uploadStart(files) { | |||
| this.updateImagePath(); | |||
| [this.loading, this.size, this.progress] = [true, files.size, 0]; | |||
| }, | |||
| onSetProgress(val) { | |||
| this.progress += val; | |||
| }, | |||
| uploadSuccess(res) { | |||
| this.progress = 100; | |||
| setTimeout(() => { | |||
| this.loading = false; | |||
| }, 1000); | |||
| if (this.loading) { | |||
| this.form.imagePath = res[0].data.objectName; | |||
| this.$refs.imagePath.validate('manual'); | |||
| } | |||
| }, | |||
| uploadError() { | |||
| this.$message({ | |||
| message: '上传文件失败', | |||
| type: 'error', | |||
| }); | |||
| this.loading = false; | |||
| }, | |||
| // hook | |||
| [CRUD.HOOK.afterRefresh]() { | |||
| this.checkStatus(); | |||
| }, | |||
| [CRUD.HOOK.beforeToAdd]() { | |||
| this.isEdit = true; | |||
| this.formType = 'add'; | |||
| if (this.isPreset) { | |||
| this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.PRESET); | |||
| } else if (this.isNotebook) { | |||
| this.form.imageTypes = IMAGE_TYPE.NOTEBOOK; | |||
| } | |||
| }, | |||
| [CRUD.HOOK.beforeRefresh]() { | |||
| @@ -567,7 +414,7 @@ export default { | |||
| } | |||
| }, | |||
| [CRUD.HOOK.beforeToEdit]() { | |||
| this.isEdit = false; | |||
| this.formType = 'edit'; | |||
| }, | |||
| async querySearchAsync(queryString, cb) { | |||
| let { harborProjectList } = this; | |||
| @@ -597,24 +444,9 @@ export default { | |||
| this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.CUSTOM); | |||
| this.getImageNameList(); | |||
| }, | |||
| onDialogClose() { | |||
| if (this.isEdit) { | |||
| this.$refs.upload.formRef.reset(); | |||
| } | |||
| this.loading = false; | |||
| }, | |||
| async onDialogOpen() { | |||
| this.getImageNameList(); | |||
| }, | |||
| checkStatus() { | |||
| if (this.crud.data.some((item) => [0].includes(item.imageStatus))) { | |||
| this.refetch(); | |||
| } | |||
| }, | |||
| filterStatus(status) { | |||
| this.localQuery.imageStatus = status; | |||
| this.crud.toQuery(); | |||
| }, | |||
| filterImageTypes(imageTypes) { | |||
| this.localQuery.imageTypes = imageTypes; | |||
| this.crud.toQuery(); | |||
| @@ -622,9 +454,6 @@ export default { | |||
| resetQuery() { | |||
| this.localQuery = { ...defaultQuery }; | |||
| }, | |||
| updateImagePath() { | |||
| this.uploadParams.objectPath = `upload-temp/${this.user.id}/${getUniqueId()}`; | |||
| }, | |||
| async doEdit(imageObj) { | |||
| const dataObj = { | |||
| ids: [imageObj.id], | |||
| @@ -642,21 +471,6 @@ export default { | |||
| this.crud.refresh(); | |||
| }); | |||
| }, | |||
| doPrecast(i) { | |||
| setPrecast({ id: i }).then(() => { | |||
| this.$message({ | |||
| message: '设置成功', | |||
| type: 'success', | |||
| }); | |||
| this.crud.refresh(); | |||
| }); | |||
| }, | |||
| resourceObj(resource) { | |||
| return resource | |||
| ? { icon: 'el-icon-circle-check', color: '#67C23A', butText: '当前为默认' } | |||
| : { icon: 'el-icon-circle-close', color: '#F56C6C', butText: '设为默认' }; | |||
| }, | |||
| uploadSizeFomatter, | |||
| async doSetDefaultImage() { | |||
| // 获取默认镜像 | |||
| @@ -666,20 +480,20 @@ export default { | |||
| this.noteBookForm.id = defaultImage.length ? defaultImage[0].id : ''; | |||
| // 获取镜像列表 | |||
| this.noteBookImages = await getImageNameList({ | |||
| imageTypes: IMAGE_TYPE.NOTEBOOK, | |||
| imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK, | |||
| imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | |||
| }); | |||
| if (this.noteBookForm.defaultImage) { | |||
| this.noteBookTags = await getImageTagList({ | |||
| imageName: this.noteBookForm.defaultImage, | |||
| imageTypes: IMAGE_TYPE.NOTEBOOK, | |||
| imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK, | |||
| imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | |||
| }); | |||
| } else { | |||
| this.noteBookTags = []; | |||
| } | |||
| this.formVisible = true; | |||
| this.notebookFormVisible = true; | |||
| this.$nextTick(() => { | |||
| this.clearValidate(); | |||
| }); | |||
| @@ -690,17 +504,17 @@ export default { | |||
| validateField(field) { | |||
| this.$refs.noteBookFormRef.validateField(field); | |||
| }, | |||
| onSubmitForm() { | |||
| onSubmitNotebookForm() { | |||
| this.$refs.noteBookFormRef.validate((valid) => { | |||
| if (valid) { | |||
| this.formSubmitting = true; | |||
| this.notebookFormSubmitting = true; | |||
| setDefaultImage({ id: this.noteBookForm.id }) | |||
| .then(() => { | |||
| this.formVisible = false; | |||
| this.notebookFormVisible = false; | |||
| this.crud.toQuery(); | |||
| }) | |||
| .finally(() => { | |||
| this.formSubmitting = false; | |||
| this.notebookFormSubmitting = false; | |||
| }); | |||
| } | |||
| }); | |||
| @@ -96,15 +96,6 @@ export const ATLAS_ALGORITHM_TYPE_ENUM = { | |||
| DDRL: 4, // ddrl | |||
| }; | |||
| export const IMAGE_TYPE = { | |||
| NOTEBOOK: 0, // notebook镜像类型 | |||
| TRAIN: 1, // 训练镜像和预置镜像类型 | |||
| SERVING: 2, // serving镜像 | |||
| TERMINAL: 3, // 终端镜像 | |||
| POINTCLOUD: 4, // 点云镜像 | |||
| DATASETMARKED: 5, // 数据标注镜像 | |||
| }; | |||
| // 训练可视化列表页查询项 | |||
| export const trainVisualQueryFormItems = [ | |||
| { | |||