@@ -11,8 +11,8 @@ Nacos + Fegin + Gateway + (Spring Security + JWT + OAuth2) | |||
初始化sql位置 /sql | |||
**地址:** 127.0.0.1:3306 | |||
**用户名:** test **密码:** test | |||
**地址:** 127.0.0.1 | |||
**用户名:** test **密码:** zj12345678 | |||
### Nacos | |||
@@ -27,7 +27,6 @@ import java.util.Map; | |||
* @description Feign 熔断处理类 | |||
* @date 2020-11-04 | |||
*/ | |||
@Component | |||
public class AuthServiceFallback implements AuthServiceClient { | |||
@@ -16,6 +16,7 @@ | |||
*/ | |||
package org.dubhe.admin.service.impl; | |||
import cn.hutool.core.collection.CollUtil; | |||
import cn.hutool.core.util.ObjectUtil; | |||
import cn.hutool.core.util.StrUtil; | |||
import cn.hutool.crypto.asymmetric.KeyType; | |||
@@ -704,6 +705,9 @@ public class UserServiceImpl extends ServiceImpl<UserMapper, User> implements Us | |||
Oauth2TokenDTO userDto = restResult.getData(); | |||
UserDTO user = findByName(authUserDTO.getUsername()); | |||
Set<String> permissions = this.queryPermissionByUserId(user.getId()); | |||
if (CollUtil.isEmpty(permissions)) { | |||
throw new BusinessException(BaseErrorCodeEnum.SYSTEM_ROLE_NOT_EXISTS); | |||
} | |||
// 返回 token 与 用户信息 | |||
authInfo.put("token", userDto.getTokenHead() + userDto.getToken()); | |||
authInfo.put("user", user); | |||
@@ -14,7 +14,6 @@ | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.admin; | |||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | |||
@@ -14,7 +14,6 @@ | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.auth; | |||
import org.junit.jupiter.api.Test; | |||
@@ -1,37 +0,0 @@ | |||
/** | |||
* Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.biz.base.constant; | |||
import lombok.Data; | |||
import org.springframework.boot.context.properties.ConfigurationProperties; | |||
import org.springframework.stereotype.Component; | |||
/** | |||
* @description harbor相关配置 | |||
* @date 2020-07-17 | |||
*/ | |||
@Data | |||
@Component | |||
@ConfigurationProperties(prefix = "harbor") | |||
public class HarborProperties { | |||
private String address; | |||
private String username; | |||
private String password; | |||
} |
@@ -110,7 +110,7 @@ public final class Permissions { | |||
* 镜像管理 | |||
*/ | |||
public static final String IMAGE = "hasAuthority('ROLE_training:image')"; | |||
public static final String IMAGE_UPLOAD = "hasAuthority('ROLE_training:image:upload')"; | |||
public static final String IMAGE_SAVE = "hasAuthority('ROLE_training:image:save')"; | |||
public static final String IMAGE_EDIT = "hasAuthority('ROLE_training:image:edit')"; | |||
public static final String IMAGE_DELETE = "hasAuthority('ROLE_training:image:delete')"; | |||
public static final String IMAGE_EDIT_DEFAULT = "hasAuthority('ROLE_training:image:editDefault')"; | |||
@@ -126,9 +126,6 @@ public final class StringConstant { | |||
public static final String CACHE_TASK_ID ="task_id"; | |||
public static final String CACHE_TASK_NAME ="task_name"; | |||
public static final String DEFAULT_IMAGE_PROJECT ="dubhe"; | |||
/** | |||
* python命令行参数格式 | |||
*/ | |||
@@ -54,8 +54,8 @@ public enum BaseErrorCodeEnum implements ErrorCode { | |||
SYSTEM_USER_EMAIL_NOT_EXISTS(20013, "该邮箱未注册!"), | |||
SYSTEM_USER_CANNOT_DELETE(20014, "系统默认用户不可删除!"), | |||
SYSTEM_ROLE_CANNOT_DELETE(20015, "系统默认角色不可删除!"), | |||
DATASET_ADMIN_PERMISSION_ERROR(1310,"无此权限,请联系管理员"), | |||
SYSTEM_ROLE_NOT_EXISTS(20016, "用户绑定角色异常,请联系管理员!"), | |||
DATASET_ADMIN_PERMISSION_ERROR(1310, "无此权限,请联系管理员"), | |||
; | |||
@@ -1,53 +0,0 @@ | |||
/** | |||
* Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.biz.base.enums; | |||
/** | |||
* @description 镜像运行状态枚举 | |||
* @date 2020-07-15 | |||
**/ | |||
public enum ImageStateEnum { | |||
MAKING(0, "制作中"), | |||
SUCCESS(1, "制作成功"), | |||
FAIL(2, "制作失败"); | |||
/** | |||
* 编码 | |||
*/ | |||
private Integer code; | |||
/** | |||
* 描述 | |||
*/ | |||
private String description; | |||
ImageStateEnum(int code, String description) { | |||
this.code = code; | |||
this.description = description; | |||
} | |||
public Integer getCode() { | |||
return code; | |||
} | |||
public String getDescription() { | |||
return description; | |||
} | |||
} |
@@ -1,10 +1,12 @@ | |||
spring: | |||
cloud: | |||
nacos: | |||
username: nacos | |||
password: Tianshu | |||
context-path: /nacos | |||
config: | |||
namespace: dubhe-prod | |||
server-addr: 127.0.0.1:8848 | |||
server-addr: 10.105.1.132:8848 | |||
discovery: | |||
namespace: dubhe-prod | |||
server-addr: 127.0.0.1:8848 | |||
server-addr: 10.105.1.132:8848 |
@@ -1,10 +0,0 @@ | |||
spring: | |||
cloud: | |||
nacos: | |||
context-path: /nacos | |||
config: | |||
namespace: dubhe-prod | |||
server-addr: 127.0.0.1:8848 | |||
discovery: | |||
namespace: dubhe-prod | |||
server-addr: 127.0.0.1:8848 |
@@ -207,4 +207,11 @@ public interface NodeApi { | |||
* @return | |||
*/ | |||
List<BizTaint> geBizTaintListByUserId(); | |||
/** | |||
* 获取一个可用node的ip | |||
* | |||
* @return String | |||
*/ | |||
String getAvailableNodeIp(); | |||
} |
@@ -389,14 +389,11 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
StatefulSet statefulSet = deployStatefulSet(); | |||
//部署svc | |||
Service service = deployService(); | |||
//部署ingress | |||
Ingress ingress = deployIngress(); | |||
if (delayDelete != null && delayDelete > ZERO) { | |||
taskYamlBO.append(secret); | |||
taskYamlBO.append(statefulSet); | |||
taskYamlBO.append(service); | |||
taskYamlBO.append(ingress); | |||
long stopUnixTime = System.currentTimeMillis() / THOUSAND_LONG + delayDelete * SIXTY_LONG; | |||
Timestamp stopDisplayTime = new Timestamp(stopUnixTime * THOUSAND_LONG); | |||
@@ -412,7 +409,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
k8sTaskService.createOrUpdateTask(k8sTask); | |||
} | |||
return new PtJupyterDeployVO(secret, statefulSet, service, ingress); | |||
return new PtJupyterDeployVO(secret, statefulSet, service, null); | |||
} | |||
/** | |||
@@ -440,7 +437,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
.build(); | |||
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}", secretName); | |||
secret = client.secrets().create(secret); | |||
secret = client.secrets().inNamespace(namespace).create(secret); | |||
LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", secretName); | |||
} | |||
@@ -647,7 +644,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
.build(); | |||
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", statefulSetName, YamlUtils.dumpAsYaml(statefulSet)); | |||
resourceIisolationApi.addIisolationInfo(statefulSet); | |||
statefulSet = client.apps().statefulSets().create(statefulSet); | |||
statefulSet = client.apps().statefulSets().inNamespace(namespace).create(statefulSet); | |||
LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", statefulSetName); | |||
return statefulSet; | |||
} | |||
@@ -678,22 +675,23 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
.withTargetPort(new IntOrString(CONTAINER_PORT)) | |||
.withName(CONTAINER_NAME) | |||
.endPort() | |||
.withClusterIP("None") | |||
.withSelector(podLabels) | |||
.withType("NodePort") | |||
.endSpec() | |||
.build(); | |||
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", svcName, YamlUtils.dumpAsYaml(svc)); | |||
svc = client.services().create(svc); | |||
svc = client.services().inNamespace(svc.getMetadata().getNamespace()).create(svc); | |||
LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", svcName); | |||
return svc; | |||
} | |||
/** | |||
* 部署ingress | |||
* 部署ingress,弃用泛域名,暴露方式由ingress改为 service nodePort | |||
* | |||
* @return Ingress 类 | |||
*/ | |||
@Deprecated | |||
private Ingress deployIngress() { | |||
Ingress ingress = null; | |||
IngressList list = client.extensions().ingresses().inNamespace(namespace).withLabels(LabelUtils.withEnvResourceName(baseName)).list(); | |||
@@ -727,7 +725,7 @@ public class JupyterResourceApiImpl implements JupyterResourceApi { | |||
.endSpec() | |||
.build(); | |||
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml info is : {}", ingressName, YamlUtils.dumpAsYaml(ingress)); | |||
ingress = client.extensions().ingresses().create(ingress); | |||
ingress = client.extensions().ingresses().inNamespace(namespace).create(ingress); | |||
LogUtil.info(LogEnum.BIZ_K8S, "{} deployed successfully", ingressName); | |||
return ingress; | |||
} | |||
@@ -65,6 +65,7 @@ import org.dubhe.k8s.enums.K8sKindEnum; | |||
import org.dubhe.k8s.enums.K8sResponseEnum; | |||
import org.dubhe.k8s.enums.LimitsOfResourcesEnum; | |||
import org.dubhe.k8s.enums.RestartPolicyEnum; | |||
import org.dubhe.k8s.enums.ServiceTypeENum; | |||
import org.dubhe.k8s.enums.ShellCommandEnum; | |||
import org.dubhe.k8s.utils.BizConvertUtils; | |||
import org.dubhe.k8s.utils.K8sUtils; | |||
@@ -102,15 +103,6 @@ public class ModelServingApiImpl implements ModelServingApi { | |||
@Autowired | |||
private ResourceIisolationApi resourceIisolationApi; | |||
@Value("${k8s.serving.host}") | |||
String servingHost; | |||
@Value("${k8s.serving.tls-crt}") | |||
String servingTlsCrt; | |||
@Value("${k8s.serving.tls-key}") | |||
String servingTlsKey; | |||
private static final String MODEL_SERVING_MAX_UPLOAD_SIZE = "100m"; | |||
public ModelServingApiImpl(K8sUtils k8sUtils) { | |||
@@ -144,7 +136,6 @@ public class ModelServingApiImpl implements ModelServingApi { | |||
//名称生成 | |||
String deploymentName = StrUtil.format(K8sParamConstants.RESOURCE_NAME_TEMPLATE, bo.getResourceName(), RandomUtil.randomString(MagicNumConstant.EIGHT)); | |||
String svcName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.SVC_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE)); | |||
String ingressName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), K8sParamConstants.INGRESS_SUFFIX, RandomUtil.randomString(MagicNumConstant.FIVE)); | |||
//标签生成 | |||
Map<String, String> baseLabels = LabelUtils.getBaseLabels(bo.getResourceName(), bo.getBusinessLabel()); | |||
@@ -164,45 +155,12 @@ public class ModelServingApiImpl implements ModelServingApi { | |||
if (bo.getGrpcPort() != null) { | |||
buildServiceBO.addPort(ResourceBuildUtils.buildServicePort(bo.getGrpcPort(), bo.getGrpcPort(), SymbolConstant.GRPC)); | |||
} | |||
buildServiceBO.setType(ServiceTypeENum.NODE_PORT.getType()); | |||
Service service = ResourceBuildUtils.buildService(buildServiceBO); | |||
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); | |||
Service serviceResult = client.services().create(service); | |||
//部署ingress | |||
BuildIngressBO buildIngressBO = new BuildIngressBO(bo.getNamespace(), ingressName, baseLabels); | |||
if (StringUtils.isNotEmpty(buildIngressBO.getMaxUploadSize())) { | |||
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_PROXY_BODY_SIZE_KEY, buildIngressBO.getMaxUploadSize()); | |||
} | |||
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_READ_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN)); | |||
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_NEXT_UPSTREAM_TIMEOUT_KEY, String.valueOf(MagicNumConstant.TEN)); | |||
if (bo.getHttpPort() != null) { | |||
String httpHost = RandomUtil.randomString(MagicNumConstant.SIX) + SymbolConstant.DOT + servingHost; | |||
buildIngressBO.addIngressRule(ResourceBuildUtils.buildIngressRule(httpHost, svcName, SymbolConstant.HTTP)); | |||
} | |||
Secret secretResult = null; | |||
if (bo.getGrpcPort() != null) { | |||
String secretName = StrUtil.format(K8sParamConstants.SUB_RESOURCE_NAME_TEMPLATE, bo.getResourceName(), SymbolConstant.TOKEN, RandomUtil.randomString(MagicNumConstant.FIVE)); | |||
Map<String, String> data = new HashMap<String, String>(MagicNumConstant.FOUR) { | |||
{ | |||
put(K8sParamConstants.SECRET_TLS_TLS_CRT, servingTlsCrt); | |||
put(K8sParamConstants.SECRET_TLS_TLS_KEY, servingTlsKey); | |||
} | |||
}; | |||
Secret secret = ResourceBuildUtils.buildTlsSecret(bo.getNamespace(), secretName, baseLabels, data); | |||
secretResult = client.secrets().create(secret); | |||
String grpcHost = RandomUtil.randomString(MagicNumConstant.SIX) + SymbolConstant.DOT + servingHost; | |||
buildIngressBO.addIngressRule(ResourceBuildUtils.buildIngressRule(grpcHost, svcName, SymbolConstant.GRPC)); | |||
buildIngressBO.addIngressTLS(ResourceBuildUtils.buildIngressTLS(secretName, grpcHost)); | |||
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_CLASS_KEY, StringConstant.NGINX_LOWERCASE); | |||
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_SSL_REDIRECT_KEY, StringConstant.TRUE_LOWERCASE); | |||
buildIngressBO.putAnnotation(K8sParamConstants.INGRESS_BACKEND_PROTOCOL_KEY, StringConstant.GRPC_CAPITALIZE); | |||
} | |||
Ingress ingress = ResourceBuildUtils.buildIngress(buildIngressBO); | |||
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", ingressName, YamlUtils.dumpAsYaml(ingress)); | |||
Ingress ingressResult = client.extensions().ingresses().create(ingress); | |||
Service serviceResult = client.services().inNamespace(bo.getNamespace()).create(service); | |||
return new ModelServingVO(BizConvertUtils.toBizSecret(secretResult), BizConvertUtils.toBizService(serviceResult), BizConvertUtils.toBizDeployment(deploymentResult), BizConvertUtils.toBizIngress(ingressResult)); | |||
return new ModelServingVO(null, BizConvertUtils.toBizService(serviceResult), BizConvertUtils.toBizDeployment(deploymentResult), null); | |||
} catch (KubernetesClientException e) { | |||
LogUtil.error(LogEnum.BIZ_K8S, "ModelOptJobApiImpl.create error, param:{} error:", bo, e); | |||
return new ModelServingVO().error(String.valueOf(e.getCode()), e.getMessage()); | |||
@@ -41,6 +41,7 @@ import org.dubhe.k8s.constant.K8sLabelConstants; | |||
import org.dubhe.k8s.constant.K8sParamConstants; | |||
import org.dubhe.k8s.domain.PtBaseResult; | |||
import org.dubhe.k8s.domain.resource.BizNode; | |||
import org.dubhe.k8s.domain.resource.BizNodeAddress; | |||
import org.dubhe.k8s.domain.resource.BizTaint; | |||
import org.dubhe.k8s.domain.vo.PtNodeMetricsVO; | |||
import org.dubhe.k8s.enums.K8sResponseEnum; | |||
@@ -560,6 +561,25 @@ public class NodeApiImpl implements NodeApi { | |||
return geBizTaintListByUserId(userContextService.getCurUserId()); | |||
} | |||
/** | |||
* 获取一个可用node的ip | |||
* | |||
* @return String | |||
*/ | |||
@Override | |||
public String getAvailableNodeIp(){ | |||
List<BizNode> nodes = listAll(); | |||
for (BizNode node : nodes){ | |||
if (node.getReady() && !node.isUnschedulable()){ | |||
for (BizNodeAddress address : node.getAddresses()){ | |||
if (K8sParamConstants.INTERNAL_IP.equals(address.getType())){ | |||
return address.getAddress(); | |||
} | |||
} | |||
} | |||
} | |||
return ""; | |||
} | |||
/** | |||
* 查询节点内存资源是否可分配 | |||
@@ -400,13 +400,16 @@ public class PodApiImpl implements PodApi { | |||
public String getUrlByResourceName(String namespace, String resourceName) { | |||
LogUtil.info(LogEnum.BIZ_K8S,"Start GetUrlByResourceName {} {}",namespace,resourceName); | |||
PtJupyterDeployVO info = jupyterResourceApi.get(namespace, resourceName); | |||
if (info != null && info.getIngressInfo() != null && CollectionUtil.isNotEmpty(info.getIngressInfo().getRules())) { | |||
if (info != null && info.getServiceInfo() != null && CollectionUtil.isNotEmpty(info.getServiceInfo().getPorts())) { | |||
String token = getTokenByResourceName(namespace, resourceName); | |||
if (StringUtils.isBlank(token)) { | |||
LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter Notebook token not generated,[namespace]={}, [resourceName]={}", namespace, resourceName); | |||
return ""; | |||
} | |||
String url = StrUtil.format(POD_URL, info.getIngressInfo().getRules().get(0).getHost(), k8sUtils.getPort(), token); | |||
BizPod pod = getWithResourceName(namespace,resourceName); | |||
String url = StrUtil.format(POD_URL, pod.getHostIP(), info.getServiceInfo().getPorts().get(0).getNodePort(), token); | |||
return validateJupyterUrl(url); | |||
} | |||
LogUtil.info(LogEnum.BIZ_K8S, "GetUrlByResourceName Jupyter statefulset not created,[namespace]={}, [resourceName]={}",namespace,resourceName); | |||
@@ -148,7 +148,7 @@ public class TerminalApiImpl implements TerminalApi { | |||
} | |||
Service service = ResourceBuildUtils.buildService(buildServiceBO); | |||
LogUtil.info(LogEnum.BIZ_K8S, "Ready to deploy {}, yaml信息为{}", svcName, YamlUtils.dumpAsYaml(service)); | |||
Service serviceResult = client.services().create(service); | |||
Service serviceResult = client.services().inNamespace(service.getMetadata().getNamespace()).create(service); | |||
return new TerminalResourceVO(BizConvertUtils.toBizDeployment(deploymentResult),BizConvertUtils.toBizService(serviceResult)); | |||
}catch (KubernetesClientException e) { | |||
LogUtil.error(LogEnum.BIZ_K8S, "TerminalApiImpl.create error, param:{} error:", bo, e); | |||
@@ -101,4 +101,5 @@ public class K8sParamConstants { | |||
public static final String WAITING_REASON_CONTAINER_CREATING = "ContainerCreating"; | |||
public final static String INTERNAL_IP = "InternalIP"; | |||
} |
@@ -57,16 +57,21 @@ public class BaseK8sDeploymentCallbackCreateDTO { | |||
@NotNull(message = "replicas 不能为空!") | |||
private Integer replicas; | |||
@ApiModelProperty(required = true, value = "action") | |||
@NotBlank(message = "action 不能为空!") | |||
private String action; | |||
public BaseK8sDeploymentCallbackCreateDTO() { | |||
} | |||
public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas) { | |||
public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas,String action) { | |||
this.namespace = namespace; | |||
this.resourceName = resourceName; | |||
this.deploymentName = deploymentName; | |||
this.readyReplicas = readyReplicas; | |||
this.replicas = replicas; | |||
this.action = action; | |||
} | |||
@Override | |||
@@ -77,6 +82,7 @@ public class BaseK8sDeploymentCallbackCreateDTO { | |||
", deploymentName='" + deploymentName + '\'' + | |||
", readyReplicas='" + readyReplicas + '\'' + | |||
", replicas='" + replicas + '\'' + | |||
", action='" + action + '\'' + | |||
'}'; | |||
} | |||
} |
@@ -30,6 +30,7 @@ import org.dubhe.k8s.annotation.K8sField; | |||
import org.dubhe.k8s.constant.K8sParamConstants; | |||
import org.dubhe.k8s.domain.PtBaseResult; | |||
import org.dubhe.k8s.domain.resource.BizContainer; | |||
import org.dubhe.k8s.domain.resource.BizServicePort; | |||
import org.dubhe.k8s.utils.MappingUtils; | |||
import java.util.List; | |||
@@ -105,6 +106,8 @@ public class PtJupyterDeployVO extends PtBaseResult<PtJupyterDeployVO> { | |||
private String namespace; | |||
@K8sField("metadata:uid") | |||
private String uid; | |||
@K8sField("spec:ports") | |||
private List<BizServicePort> ports; | |||
} | |||
@Data | |||
@@ -33,10 +33,6 @@ public class ClusterProperties { | |||
private String nfs; | |||
private String host; | |||
private String port; | |||
private String kubeconfig; | |||
/** | |||
@@ -218,7 +218,9 @@ public class BizConvertUtils { | |||
*/ | |||
public static BizIngress toBizIngress(Ingress ingress) { | |||
BizIngress bizIngress = MappingUtils.mappingTo(ingress, BizIngress.class); | |||
bizIngress.getRules().forEach(BizIngressRule::takeServicePort); | |||
if (bizIngress != null){ | |||
bizIngress.getRules().forEach(BizIngressRule::takeServicePort); | |||
} | |||
return bizIngress; | |||
} | |||
@@ -98,8 +98,6 @@ public class K8sUtils implements ApplicationContextAware { | |||
nfs = clusterProperties.getNfs(); | |||
host = clusterProperties.getHost(); | |||
port = clusterProperties.getPort(); | |||
//打印集群信息 | |||
LogUtil.info(LogEnum.BIZ_K8S, "ApiVersion : {}", client.getApiVersion()); | |||
@@ -69,11 +69,6 @@ public enum RecycleResourceEnum { | |||
*/ | |||
MEASURE_RECYCLE_FILE("measureRecycleFile", "度量文件回收"), | |||
/** | |||
* 镜像回收 | |||
*/ | |||
IMAGE_RECYCLE_FILE("imageRecycleFile", "镜像回收"), | |||
/** | |||
* 算法文件回收 | |||
*/ | |||
@@ -14,7 +14,6 @@ | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.task; | |||
import org.junit.jupiter.api.Test; | |||
@@ -1,36 +0,0 @@ | |||
/** | |||
* Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.data.config; | |||
import lombok.Data; | |||
import org.springframework.boot.context.properties.ConfigurationProperties; | |||
import org.springframework.stereotype.Component; | |||
@Data | |||
@Component | |||
@ConfigurationProperties(prefix = "harbor") | |||
public class DataHarborConfig { | |||
private String address; | |||
private String username; | |||
private String password; | |||
private String modelName; | |||
} |
@@ -28,9 +28,15 @@ import com.baomidou.mybatisplus.extension.service.IService; | |||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; | |||
import org.apache.commons.compress.utils.Lists; | |||
import org.dubhe.biz.base.constant.MagicNumConstant; | |||
import org.dubhe.biz.base.constant.SymbolConstant; | |||
import org.dubhe.biz.base.context.UserContext; | |||
import org.dubhe.biz.base.dto.*; | |||
import org.dubhe.biz.base.dto.PtImageIdDTO; | |||
import org.dubhe.biz.base.dto.PtImageIdsDTO; | |||
import org.dubhe.biz.base.dto.PtModelBranchQueryByIdDTO; | |||
import org.dubhe.biz.base.dto.PtModelBranchQueryByIdsDTO; | |||
import org.dubhe.biz.base.dto.TrainAlgorithmSelectAllBatchIdDTO; | |||
import org.dubhe.biz.base.dto.TrainAlgorithmSelectByIdDTO; | |||
import org.dubhe.biz.base.dto.UserDTO; | |||
import org.dubhe.biz.base.dto.UserSmallDTO; | |||
import org.dubhe.biz.base.enums.BizEnum; | |||
import org.dubhe.biz.base.enums.ResourcesPoolTypeEnum; | |||
import org.dubhe.biz.base.exception.BusinessException; | |||
@@ -49,13 +55,15 @@ import org.dubhe.cloud.authconfig.service.AdminClient; | |||
import org.dubhe.data.client.AlgorithmClient; | |||
import org.dubhe.data.client.ImageClient; | |||
import org.dubhe.data.client.ModelClient; | |||
import org.dubhe.data.config.DataHarborConfig; | |||
import org.dubhe.data.config.DataRedisConfig; | |||
import org.dubhe.data.constant.AutoLabelModelServiceStatusEnum; | |||
import org.dubhe.data.constant.Constant; | |||
import org.dubhe.data.constant.ErrorEnum; | |||
import org.dubhe.data.dao.AutoLabelModelServiceMapper; | |||
import org.dubhe.data.domain.dto.*; | |||
import org.dubhe.data.domain.dto.AutoLabelModelServiceCreateDTO; | |||
import org.dubhe.data.domain.dto.AutoLabelModelServiceQueryDTO; | |||
import org.dubhe.data.domain.dto.AutoLabelModelServiceUpdateDTO; | |||
import org.dubhe.data.domain.dto.DataK8sDeploymentCallbackCreateDTO; | |||
import org.dubhe.data.domain.entity.AutoLabelModelService; | |||
import org.dubhe.data.domain.vo.AutoLabelModelServicePodVO; | |||
import org.dubhe.data.domain.vo.AutoLabelModelServiceVO; | |||
@@ -75,9 +83,16 @@ import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.beans.factory.annotation.Value; | |||
import org.springframework.dao.DuplicateKeyException; | |||
import org.springframework.stereotype.Service; | |||
import javax.annotation.Resource; | |||
import java.sql.Timestamp; | |||
import java.util.*; | |||
import java.util.ArrayList; | |||
import java.util.Arrays; | |||
import java.util.HashMap; | |||
import java.util.List; | |||
import java.util.Map; | |||
import java.util.Objects; | |||
import java.util.Set; | |||
import java.util.concurrent.TimeUnit; | |||
import java.util.function.Function; | |||
import java.util.stream.Collectors; | |||
@@ -116,9 +131,6 @@ public class AutoLabelModelServiceServiceImpl extends ServiceImpl<AutoLabelModel | |||
@Resource | |||
private UserContextService userContextService; | |||
@Resource | |||
private DataHarborConfig dataHarborConfig; | |||
@Resource | |||
private DataRedisConfig dataRedisConfig; | |||
@@ -197,7 +209,7 @@ public class AutoLabelModelServiceServiceImpl extends ServiceImpl<AutoLabelModel | |||
modelServiceBO.setResourceName(k8sNameTool.generateResourceName(BizEnum.DATA, autoLabelModelService.getId().toString())); | |||
modelServiceBO.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.DATA)); | |||
modelServiceBO.setReplicas(autoLabelModelService.getInstanceNum()); | |||
modelServiceBO.setImage(dataHarborConfig.getAddress() + SymbolConstant.SLASH + ptImageVO.getImageUrl()); | |||
modelServiceBO.setImage(ptImageVO.getImageUrl()); | |||
modelServiceBO.setTaskIdentifyLabel(taskIdentify); | |||
String command = String.format(Constant.MODEL_SERVICE_COMMAND, modelServiceId, | |||
@@ -1,142 +0,0 @@ | |||
/** | |||
* Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.image.async; | |||
import cn.hutool.core.util.StrUtil; | |||
import org.dubhe.biz.base.constant.HarborProperties; | |||
import org.dubhe.biz.base.constant.StringConstant; | |||
import org.dubhe.biz.base.enums.ImageStateEnum; | |||
import org.dubhe.biz.base.exception.BusinessException; | |||
import org.dubhe.biz.base.utils.StringUtils; | |||
import org.dubhe.biz.file.utils.IOUtil; | |||
import org.dubhe.biz.log.enums.LogEnum; | |||
import org.dubhe.biz.log.utils.LogUtil; | |||
import org.dubhe.harbor.api.HarborApi; | |||
import org.dubhe.image.dao.PtImageMapper; | |||
import org.dubhe.image.domain.entity.PtImage; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.scheduling.annotation.Async; | |||
import org.springframework.stereotype.Component; | |||
import java.io.BufferedReader; | |||
import java.io.InputStreamReader; | |||
/** | |||
* @description 异步推送镜像到harbor | |||
* @date 2020-07-10 | |||
*/ | |||
@Component | |||
public class HarborImagePushAsync { | |||
@Autowired | |||
private HarborProperties harborProperties; | |||
@Autowired | |||
private HarborApi harborApi; | |||
@Autowired | |||
private PtImageMapper ptImageMapper; | |||
/** | |||
* 组装shell脚本执行命令 | |||
* | |||
* @param imagePath | |||
* @param imageNameandTag | |||
**/ | |||
@Async | |||
public void execShell(String imagePath, String imageNameandTag, PtImage ptImage) { | |||
try { | |||
String imageResource = harborProperties.getAddress() + StrUtil.SLASH + StringConstant.DEFAULT_IMAGE_PROJECT | |||
+ StrUtil.SLASH + imageNameandTag; | |||
String cmdStr = "docker login --username=" + harborProperties.getUsername() + " " + harborProperties.getAddress() + " --password=" + harborProperties.getPassword() + " ; docker " + | |||
"load < " + imagePath + " |awk '{print $3}' |xargs -I str docker tag str " + imageResource + " ; docker push " + imageResource + "; docker rmi " + imageResource; | |||
String[] cmd = {"/bin/bash", "-c", cmdStr}; | |||
LogUtil.info(LogEnum.IMAGE, "镜像上传执行脚本参数:{}", cmd); | |||
Process process = Runtime.getRuntime().exec(cmd); | |||
if (checkImagePushIsOk(ptImage, process)) { | |||
updateImageStatus(ptImage, ImageStateEnum.SUCCESS.getCode()); | |||
} else { | |||
updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); | |||
} | |||
} catch (Exception e) { | |||
LogUtil.error(LogEnum.IMAGE, "upload image failed,exception {}", e); | |||
updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); | |||
throw new BusinessException("上传镜像异常!"); | |||
} | |||
} | |||
/** | |||
* 更新镜像上传状态 | |||
* | |||
* @param ptImage | |||
* @param status | |||
**/ | |||
public void updateImageStatus(PtImage ptImage, Integer status) { | |||
ptImage.setImageStatus(status); | |||
ptImageMapper.updateById(ptImage); | |||
} | |||
/** | |||
* 校验镜像是否上传成功 | |||
* | |||
* @param ptImage 镜像信息 | |||
* @param process process对象 | |||
* @return 是否上传成功 | |||
*/ | |||
public boolean checkImagePushIsOk(PtImage ptImage, Process process) { | |||
//读取标准输出流 | |||
BufferedReader brOut = new BufferedReader(new InputStreamReader(process.getInputStream())); | |||
//读取标准错误流 | |||
BufferedReader brErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); | |||
String line; | |||
StringBuilder outMessage = new StringBuilder(); | |||
StringBuilder errMessage = new StringBuilder(); | |||
boolean isPushOk = true; | |||
try { | |||
while ((line = brOut.readLine()) != null) { | |||
outMessage.append(line); | |||
} | |||
if (StringUtils.isNotEmpty(outMessage)) { | |||
LogUtil.info(LogEnum.IMAGE, "upload image with shell standard output message {}", outMessage.toString()); | |||
} | |||
while ((line = brErr.readLine()) != null) { | |||
errMessage.append(line); | |||
} | |||
if (StringUtils.isNotEmpty(errMessage)) { | |||
LogUtil.error(LogEnum.IMAGE, "upload image with shell standard output message {}", errMessage.toString()); | |||
} | |||
Integer status = process.waitFor(); | |||
LogUtil.info(LogEnum.IMAGE, "上传镜像状态:{}", status); | |||
if (status == null) { | |||
if (!harborApi.isExistImage(ptImage.getImageUrl())) { | |||
isPushOk = false; | |||
} | |||
} else if (status != 0) { | |||
isPushOk = false; | |||
} | |||
} catch (Exception e) { | |||
LogUtil.error(LogEnum.IMAGE, "upload image failed,exception {}", e); | |||
return false; | |||
} finally { | |||
IOUtil.close(brErr, brOut); | |||
} | |||
return isPushOk; | |||
} | |||
} |
@@ -41,9 +41,6 @@ public class PtImageQueryDTO extends PageQueryBase implements Serializable { | |||
@ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") | |||
private Integer imageResource; | |||
@ApiModelProperty(value = "镜像状态,0为制作中,1位制作成功,2位制作失败") | |||
private Integer imageStatus; | |||
@ApiModelProperty(value = "镜像名称或id") | |||
private String imageNameOrId; | |||
@@ -38,7 +38,7 @@ import java.util.List; | |||
*/ | |||
@Data | |||
@Accessors(chain = true) | |||
public class PtImageUploadDTO implements Serializable { | |||
public class PtImageSaveDTO implements Serializable { | |||
private static final long serialVersionUID = 1L; | |||
@@ -50,9 +50,9 @@ public class PtImageUploadDTO implements Serializable { | |||
@FlagValidator(value = {"0", "1"}, message = "无效镜像来源") | |||
private Integer imageResource; | |||
@ApiModelProperty(value = "镜像文件路径", required = true) | |||
@NotBlank(message = "镜像文件路径不能为空") | |||
private String imagePath; | |||
@ApiModelProperty(value = "镜像地址", required = true) | |||
@NotBlank(message = "镜像地址不能为空") | |||
private String imageUrl; | |||
@ApiModelProperty(value = "镜像名称", required = true) | |||
@NotBlank(message = "源镜像名称不能为空") |
@@ -20,10 +20,13 @@ import io.swagger.annotations.ApiModelProperty; | |||
import lombok.Data; | |||
import lombok.experimental.Accessors; | |||
import org.dubhe.biz.base.constant.MagicNumConstant; | |||
import org.dubhe.biz.base.constant.StringConstant; | |||
import org.hibernate.validator.constraints.Length; | |||
import javax.validation.constraints.NotBlank; | |||
import javax.validation.constraints.NotEmpty; | |||
import javax.validation.constraints.NotNull; | |||
import javax.validation.constraints.Pattern; | |||
import java.io.Serializable; | |||
import java.util.List; | |||
@@ -39,14 +42,29 @@ public class PtImageUpdateDTO implements Serializable { | |||
@ApiModelProperty(value = "id", required = true) | |||
@NotNull(message = "镜像id不能为空") | |||
private List<Long> ids; | |||
@ApiModelProperty("镜像描述") | |||
@Length(max = MagicNumConstant.BINARY_TEN_EXP, message = "镜像描述-输入长度不能超过1024个字符") | |||
private String remark; | |||
private Long id; | |||
@ApiModelProperty(value = "镜像用途", required = true) | |||
@NotEmpty(message = "镜像用途(0:notebook , 1:train , 2:serving, 3:terminal, 4:point-cloud)不能为空") | |||
private List<Integer> imageTypes; | |||
@ApiModelProperty(value = "镜像路径", required = true) | |||
@NotBlank(message = "镜像路径不能为空") | |||
private String imageUrl; | |||
@ApiModelProperty(value = "镜像名称", required = true) | |||
@NotBlank(message = "源镜像名称不能为空") | |||
@Length(min = MagicNumConstant.ONE, max = MagicNumConstant.SIXTY_FOUR, message = "镜像名称长度在1-64个字符") | |||
@Pattern(regexp = StringConstant.REGEXP_NAME, message = "镜像名称支持字母、数字、英文横杠和下划线") | |||
private String imageName; | |||
@ApiModelProperty(value = "镜像版本号", required = true) | |||
@NotBlank(message = "镜像版本号不能为空") | |||
@Length(max = MagicNumConstant.SIXTY_FOUR, message = "镜像版本号长度在1-64个字符") | |||
@Pattern(regexp = StringConstant.REGEXP_TAG, message = "镜像版本号支持字母、数字、英文横杠、英文.号和下划线") | |||
private String imageTag; | |||
@ApiModelProperty("镜像描述") | |||
@Length(max = MagicNumConstant.BINARY_TEN_EXP, message = "镜像描述-输入长度不能超过1024个字符") | |||
private String remark; | |||
} |
@@ -80,13 +80,6 @@ public class PtImage extends BaseEntity { | |||
@TableField(value = "image_resource") | |||
private Integer imageResource; | |||
/** | |||
* 镜像状态 | |||
*/ | |||
@TableField(value = "image_status") | |||
private Integer imageStatus; | |||
/** | |||
* 资源拥有者ID | |||
*/ | |||
@@ -41,9 +41,6 @@ public class PtImageQueryVO implements Serializable { | |||
@ApiModelProperty("镜像版本") | |||
private String imageTag; | |||
@ApiModelProperty("镜像状态(0:上传中,1:上传成功,2:上传失败)") | |||
private Integer imageStatus; | |||
@ApiModelProperty("备注") | |||
private String remark; | |||
@@ -58,4 +55,7 @@ public class PtImageQueryVO implements Serializable { | |||
@ApiModelProperty("镜像用途") | |||
private List<Integer> imageTypes; | |||
@ApiModelProperty("镜像地址") | |||
private String imageUrl; | |||
} |
@@ -24,12 +24,25 @@ import org.dubhe.biz.base.dto.PtImageIdDTO; | |||
import org.dubhe.biz.base.dto.PtImageIdsDTO; | |||
import org.dubhe.biz.base.vo.DataResponseBody; | |||
import org.dubhe.biz.base.vo.PtImageVO; | |||
import org.dubhe.image.domain.dto.*; | |||
import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryImageDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryNameDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryUrlDTO; | |||
import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||
import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||
import org.dubhe.image.service.PtImageService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.security.access.prepost.PreAuthorize; | |||
import org.springframework.validation.annotation.Validated; | |||
import org.springframework.web.bind.annotation.*; | |||
import org.springframework.web.bind.annotation.DeleteMapping; | |||
import org.springframework.web.bind.annotation.GetMapping; | |||
import org.springframework.web.bind.annotation.PostMapping; | |||
import org.springframework.web.bind.annotation.PutMapping; | |||
import org.springframework.web.bind.annotation.RequestBody; | |||
import org.springframework.web.bind.annotation.RequestMapping; | |||
import org.springframework.web.bind.annotation.RequestParam; | |||
import org.springframework.web.bind.annotation.RestController; | |||
import java.util.List; | |||
@@ -45,7 +58,7 @@ public class PtImageController { | |||
@Autowired | |||
private PtImageService ptImageService; | |||
@GetMapping("/info") | |||
@GetMapping("/list") | |||
@ApiOperation("查询镜像") | |||
@PreAuthorize(Permissions.IMAGE) | |||
public DataResponseBody getImage(PtImageQueryDTO ptImageQueryDTO) { | |||
@@ -58,11 +71,11 @@ public class PtImageController { | |||
return new DataResponseBody(ptImageService.searchImages(ptImageQueryImageDTO)); | |||
} | |||
@PostMapping("uploadImage") | |||
@ApiOperation("上传镜像包到harbor") | |||
@PreAuthorize(Permissions.IMAGE_UPLOAD) | |||
public DataResponseBody uploadImage(@Validated @RequestBody PtImageUploadDTO ptImageUploadDTO) { | |||
ptImageService.uploadImage(ptImageUploadDTO); | |||
@PostMapping | |||
@ApiOperation("保存镜像信息") | |||
@PreAuthorize(Permissions.IMAGE_SAVE) | |||
public DataResponseBody saveImageInfo(@Validated @RequestBody PtImageSaveDTO ptImageSaveDTO) { | |||
ptImageService.saveImageInfo(ptImageSaveDTO); | |||
return new DataResponseBody(); | |||
} | |||
@@ -18,9 +18,14 @@ | |||
package org.dubhe.image.service; | |||
import org.dubhe.biz.base.vo.PtImageVO; | |||
import org.dubhe.image.domain.dto.*; | |||
import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryImageDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryNameDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryUrlDTO; | |||
import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||
import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||
import org.dubhe.image.domain.entity.PtImage; | |||
import org.dubhe.recycle.domain.dto.RecycleCreateDTO; | |||
import java.util.List; | |||
import java.util.Map; | |||
@@ -42,11 +47,11 @@ public interface PtImageService { | |||
/** | |||
* 上传镜像到harbor | |||
* 保存镜像信息 | |||
* | |||
* @param ptImageUploadDTO 上传条件 | |||
* @param ptImageSaveDTO 镜像信息DTO | |||
*/ | |||
void uploadImage(PtImageUploadDTO ptImageUploadDTO); | |||
void saveImageInfo(PtImageSaveDTO ptImageSaveDTO); | |||
/** | |||
@@ -98,13 +103,6 @@ public interface PtImageService { | |||
*/ | |||
String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO); | |||
/** | |||
* 镜像回收还原 | |||
* | |||
* @param dto 还原DTO对象 | |||
*/ | |||
void recycleRollback(RecycleCreateDTO dto); | |||
/** | |||
* 获取终端镜像列表 | |||
* | |||
@@ -27,12 +27,14 @@ import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; | |||
import com.baomidou.mybatisplus.core.metadata.IPage; | |||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | |||
import org.apache.commons.compress.utils.Lists; | |||
import org.dubhe.biz.base.constant.*; | |||
import org.dubhe.biz.base.constant.MagicNumConstant; | |||
import org.dubhe.biz.base.constant.NumberConstant; | |||
import org.dubhe.biz.base.constant.ResponseCode; | |||
import org.dubhe.biz.base.constant.StringConstant; | |||
import org.dubhe.biz.base.context.DataContext; | |||
import org.dubhe.biz.base.context.UserContext; | |||
import org.dubhe.biz.base.enums.DatasetTypeEnum; | |||
import org.dubhe.biz.base.enums.ImageSourceEnum; | |||
import org.dubhe.biz.base.enums.ImageStateEnum; | |||
import org.dubhe.biz.base.enums.ImageTypeEnum; | |||
import org.dubhe.biz.base.exception.BusinessException; | |||
import org.dubhe.biz.base.service.UserContextService; | |||
@@ -40,31 +42,34 @@ import org.dubhe.biz.base.utils.ReflectionUtils; | |||
import org.dubhe.biz.base.utils.StringUtils; | |||
import org.dubhe.biz.base.vo.PtImageVO; | |||
import org.dubhe.biz.db.utils.PageUtil; | |||
import org.dubhe.biz.file.config.NfsConfig; | |||
import org.dubhe.biz.log.enums.LogEnum; | |||
import org.dubhe.biz.log.utils.LogUtil; | |||
import org.dubhe.biz.permission.annotation.DataPermissionMethod; | |||
import org.dubhe.biz.permission.base.BaseService; | |||
import org.dubhe.image.async.HarborImagePushAsync; | |||
import org.dubhe.image.dao.PtImageMapper; | |||
import org.dubhe.image.domain.dto.*; | |||
import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryImageDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryNameDTO; | |||
import org.dubhe.image.domain.dto.PtImageQueryUrlDTO; | |||
import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||
import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||
import org.dubhe.image.domain.entity.PtImage; | |||
import org.dubhe.image.domain.vo.PtImageQueryVO; | |||
import org.dubhe.image.service.PtImageService; | |||
import org.dubhe.recycle.config.RecycleConfig; | |||
import org.dubhe.recycle.domain.dto.RecycleCreateDTO; | |||
import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO; | |||
import org.dubhe.recycle.enums.RecycleModuleEnum; | |||
import org.dubhe.recycle.enums.RecycleResourceEnum; | |||
import org.dubhe.recycle.enums.RecycleTypeEnum; | |||
import org.dubhe.recycle.service.RecycleService; | |||
import org.dubhe.recycle.utils.RecycleTool; | |||
import org.springframework.beans.BeanUtils; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
import org.springframework.transaction.annotation.Transactional; | |||
import java.util.*; | |||
import java.util.ArrayList; | |||
import java.util.Comparator; | |||
import java.util.HashSet; | |||
import java.util.List; | |||
import java.util.Map; | |||
import java.util.Objects; | |||
import java.util.Set; | |||
import java.util.TreeSet; | |||
import java.util.stream.Collectors; | |||
/** | |||
@@ -77,25 +82,9 @@ public class PtImageServiceImpl implements PtImageService { | |||
@Autowired | |||
private PtImageMapper ptImageMapper; | |||
@Autowired | |||
private NfsConfig nfsConfig; | |||
@Autowired | |||
private HarborImagePushAsync imagePushAsync; | |||
@Autowired | |||
private HarborProperties harborProperties; | |||
@Autowired | |||
private UserContextService userContextService; | |||
@Autowired | |||
private RecycleService recycleService; | |||
@Autowired | |||
private RecycleConfig recycleConfig; | |||
public final static List<String> FIELD_NAMES; | |||
@@ -119,9 +108,7 @@ public class PtImageServiceImpl implements PtImageService { | |||
QueryWrapper<PtImage> query = new QueryWrapper<>(); | |||
query.eq("deleted", NumberConstant.NUMBER_0); | |||
if (ptImageQueryDTO.getImageStatus() != null) { | |||
query.eq("image_status", ptImageQueryDTO.getImageStatus()); | |||
} | |||
if (ptImageQueryDTO.getImageResource() != null) { | |||
query.eq("image_resource", ptImageQueryDTO.getImageResource()); | |||
} | |||
@@ -166,67 +153,55 @@ public class PtImageServiceImpl implements PtImageService { | |||
} | |||
/** | |||
* 上传镜像到harbor | |||
* 保存镜像信息 | |||
* | |||
* @param ptImageUploadDTO 上传条件 | |||
* @param ptImageSaveDTO 镜像信息DTO | |||
*/ | |||
@Override | |||
@Transactional(rollbackFor = Exception.class) | |||
public void uploadImage(PtImageUploadDTO ptImageUploadDTO) { | |||
public void saveImageInfo(PtImageSaveDTO ptImageSaveDTO) { | |||
UserContext user = userContextService.getCurUser(); | |||
//普通用户不支持上传预置镜像 | |||
if (ImageSourceEnum.PRE.getCode().equals(ptImageUploadDTO.getImageResource()) && | |||
validImageInfo(ptImageSaveDTO.getImageUrl(), ptImageSaveDTO.getImageName(), ptImageSaveDTO.getImageTag()); | |||
//普通用户不支持预置镜像信息的保存 | |||
if (ImageSourceEnum.PRE.getCode().equals(ptImageSaveDTO.getImageResource()) && | |||
!BaseService.isAdmin(user)) { | |||
throw new BusinessException(ResponseCode.UNAUTHORIZED, "普通用户不支持上传预置镜像!"); | |||
throw new BusinessException(ResponseCode.UNAUTHORIZED, "普通用户不支持保存预置镜像!"); | |||
} | |||
//校验用户自定义镜像不能和预置镜像重名 | |||
List<PtImage> resList = checkUploadImage(ptImageUploadDTO, null, ImageSourceEnum.PRE.getCode()); | |||
List<PtImage> resList = checkSaveImage(ptImageSaveDTO, null, ImageSourceEnum.PRE.getCode()); | |||
if (CollUtil.isNotEmpty(resList)) { | |||
throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复,请重新上传!"); | |||
throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复!"); | |||
} | |||
//同一用户上传镜像的(userId+imageName+imageTag)存在的情况下是不能重复上传的 | |||
List<PtImage> imageList = checkUploadImage(ptImageUploadDTO, user, ImageSourceEnum.MINE.getCode()); | |||
if (CollUtil.isNotEmpty(imageList) && ImageStateEnum.SUCCESS.getCode().equals(imageList.get(0).getImageStatus())) { | |||
throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在,不允许重复上传!"); | |||
List<PtImage> imageList = checkSaveImage(ptImageSaveDTO, user, ImageSourceEnum.MINE.getCode()); | |||
if (CollUtil.isNotEmpty(imageList)) { | |||
throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在!"); | |||
} | |||
String harborImagePath = StringConstant.DEFAULT_IMAGE_PROJECT + StrUtil.SLASH + ptImageUploadDTO.getImageName() + StrUtil.DASHED + user.getId() + | |||
StrUtil.COLON + ptImageUploadDTO.getImageTag(); | |||
//存储镜像信息 | |||
PtImage ptImage = new PtImage(); | |||
ptImage.setImageName(ptImageUploadDTO.getImageName()) | |||
.setImageUrl(harborImagePath) | |||
.setImageResource(ptImageUploadDTO.getImageResource()) | |||
.setImageStatus(ImageStateEnum.MAKING.getCode()) | |||
.setRemark(ptImageUploadDTO.getRemark()) | |||
.setImageTag(ptImageUploadDTO.getImageTag()) | |||
ptImage.setImageName(ptImageSaveDTO.getImageName()) | |||
.setImageUrl(ptImageSaveDTO.getImageUrl()) | |||
.setImageResource(ptImageSaveDTO.getImageResource()) | |||
.setRemark(ptImageSaveDTO.getRemark()) | |||
.setImageTag(ptImageSaveDTO.getImageTag()) | |||
.setCreateUserId(user.getId()); | |||
if (ImageSourceEnum.PRE.getCode().equals(ptImageUploadDTO.getImageResource())) { | |||
if (ImageSourceEnum.PRE.getCode().equals(ptImageSaveDTO.getImageResource())) { | |||
ptImage.setOriginUserId(MagicNumConstant.ZERO_LONG); | |||
} else { | |||
ptImage.setOriginUserId(user.getId()); | |||
} | |||
int count = ptImageMapper.insert(ptImage); | |||
if (count < 1) { | |||
imagePushAsync.updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); | |||
throw new BusinessException("内部错误!"); | |||
} | |||
for (Integer imageType : ptImageUploadDTO.getImageTypes()) { | |||
ptImageMapper.insert(ptImage); | |||
for (Integer imageType : ptImageSaveDTO.getImageTypes()) { | |||
ptImageMapper.insertImageType(ptImage.getId(), imageType); | |||
} | |||
//shell脚本上传镜像 | |||
try { | |||
String imagePath = nfsConfig.getRootDir() + nfsConfig.getBucket().substring(1) + ptImageUploadDTO.getImagePath(); | |||
String imageNameAndTag = ptImageUploadDTO.getImageName() + StrUtil.DASHED + user.getId() + StrUtil.COLON + ptImageUploadDTO.getImageTag(); | |||
imagePushAsync.execShell(imagePath, imageNameAndTag, ptImage); | |||
} catch (Exception e) { | |||
LogUtil.error(LogEnum.IMAGE, "Image upload exception {}", e); | |||
throw new BusinessException("镜像上传失败!"); | |||
} | |||
} | |||
/** | |||
@@ -240,9 +215,8 @@ public class PtImageServiceImpl implements PtImageService { | |||
public List<PtImage> searchImages(PtImageQueryImageDTO ptImageQueryImageDTO) { | |||
LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | |||
queryWrapper.eq(PtImage::getImageName, ptImageQueryImageDTO.getImageName()) | |||
.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()) | |||
.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | |||
List<PtImage> ptImages = new ArrayList<>(); | |||
List<PtImage> ptImages; | |||
if (ptImageQueryImageDTO.getImageResource() != null) { | |||
queryWrapper.eq(PtImage::getImageResource, ptImageQueryImageDTO.getImageResource()); | |||
} | |||
@@ -252,18 +226,13 @@ public class PtImageServiceImpl implements PtImageService { | |||
ptImages = ptImageMapper.getImagesByTypes(queryWrapper, ptImageQueryImageDTO.getImageTypes()); | |||
} | |||
List<PtImage> list = new ArrayList<>(); | |||
if (CollUtil.isEmpty(ptImages)) { | |||
throw new BusinessException(ResponseCode.BADREQUEST, "未查询到镜像信息!"); | |||
} | |||
ptImages = ptImages.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> | |||
new TreeSet<>(Comparator.comparing(PtImage::getImageTag))), ArrayList::new)); | |||
ptImages.stream().forEach(ptImage -> { | |||
ptImage.setImageUrl(harborProperties.getAddress() + StrUtil.SLASH + ptImage.getImageUrl()); | |||
list.add(ptImage); | |||
}); | |||
return list; | |||
return ptImages; | |||
} | |||
@@ -282,37 +251,9 @@ public class PtImageServiceImpl implements PtImageService { | |||
//删除本地镜像 | |||
imageList.forEach(image -> { | |||
ptImageMapper.deleteById(image.getId()); | |||
//创建镜像回收任务 | |||
createRecycleTask(image); | |||
}); | |||
} | |||
/** | |||
* 文件定时清理 | |||
* | |||
* @param ptImage 镜像实体对象 | |||
*/ | |||
private void createRecycleTask(PtImage ptImage) { | |||
String imageUrl = ImageStateEnum.SUCCESS.getCode().equals(ptImage.getImageStatus()) ? ptImage.getImageUrl() : ""; | |||
RecycleCreateDTO recycleCreateDTO = RecycleCreateDTO.builder() | |||
.recycleModule(RecycleModuleEnum.BIZ_IMAGE.getValue()) | |||
.recycleDelayDate(recycleConfig.getImageValid()) | |||
.recycleNote(RecycleTool.generateRecycleNote("删除镜像", ptImage.getImageName(), ptImage.getId())) | |||
.recycleCustom(RecycleResourceEnum.IMAGE_RECYCLE_FILE.getClassName()) | |||
.restoreCustom(RecycleResourceEnum.IMAGE_RECYCLE_FILE.getClassName()) | |||
.remark(String.valueOf(ptImage.getId())) | |||
.build(); | |||
recycleCreateDTO.addRecycleDetailCreateDTO(RecycleDetailCreateDTO.builder() | |||
.recycleCondition(imageUrl) | |||
.recycleType(RecycleTypeEnum.FILE.getCode()) | |||
.recycleNote(RecycleTool.generateRecycleNote("删除镜像", ptImage.getImageName(), ptImage.getId())) | |||
.remark(String.valueOf(ptImage.getId())) | |||
.build() | |||
); | |||
recycleService.createRecycleTask(recycleCreateDTO); | |||
} | |||
/** | |||
* 修改镜像信息 | |||
* | |||
@@ -324,29 +265,33 @@ public class PtImageServiceImpl implements PtImageService { | |||
public void updateTrainImage(PtImageUpdateDTO imageUpdateDTO) { | |||
UserContext curUser = userContextService.getCurUser(); | |||
List<PtImage> imageList = ptImageMapper.selectList(new LambdaQueryWrapper<PtImage>() | |||
.in(PtImage::getId, imageUpdateDTO.getIds())); | |||
PtImage image = ptImageMapper.selectById(imageUpdateDTO.getId()); | |||
if (CollUtil.isEmpty(imageList)) { | |||
throw new BusinessException("内部错误"); | |||
if (image == null) { | |||
throw new BusinessException("镜像不存在!"); | |||
} | |||
for (PtImage image : imageList) { | |||
//非管理员禁止修改预置镜像 | |||
if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource()) && !BaseService.isAdmin(curUser)) { | |||
throw new BusinessException("非管理员无权限修改预置镜像信息"); | |||
} | |||
image.setRemark(imageUpdateDTO.getRemark()); | |||
ptImageMapper.updateById(image); | |||
List<Integer> imageTypes = ptImageMapper.selectImageType(image.getId()); | |||
for (Integer imageType : imageUpdateDTO.getImageTypes()) { | |||
if (!CollectionUtil.contains(imageTypes, imageType)) { | |||
ptImageMapper.insertImageType(image.getId(), imageType); | |||
} | |||
validImageInfo(imageUpdateDTO.getImageUrl(), imageUpdateDTO.getImageName(), imageUpdateDTO.getImageTag()); | |||
//非管理员禁止修改预置镜像 | |||
if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource()) && !BaseService.isAdmin(curUser)) { | |||
throw new BusinessException("非管理员无权限修改预置镜像信息"); | |||
} | |||
image.setImageTypes(imageUpdateDTO.getImageTypes()) | |||
.setImageName(imageUpdateDTO.getImageName()) | |||
.setImageUrl(imageUpdateDTO.getImageUrl()) | |||
.setImageTag(imageUpdateDTO.getImageTag()) | |||
.setRemark(imageUpdateDTO.getRemark()); | |||
ptImageMapper.updateById(image); | |||
List<Integer> imageTypes = ptImageMapper.selectImageType(image.getId()); | |||
for (Integer imageType : imageUpdateDTO.getImageTypes()) { | |||
if (!CollectionUtil.contains(imageTypes, imageType)) { | |||
ptImageMapper.insertImageType(image.getId(), imageType); | |||
} | |||
for (Integer imageType : imageTypes) { | |||
if (!CollectionUtil.contains(imageUpdateDTO.getImageTypes(), imageType)) { | |||
ptImageMapper.deleteImageType(image.getId(), imageType); | |||
} | |||
} | |||
for (Integer imageType : imageTypes) { | |||
if (!CollectionUtil.contains(imageUpdateDTO.getImageTypes(), imageType)) { | |||
ptImageMapper.deleteImageType(image.getId(), imageType); | |||
} | |||
} | |||
} | |||
@@ -362,10 +307,8 @@ public class PtImageServiceImpl implements PtImageService { | |||
//从会话中获取用户信息 | |||
UserContext user = userContextService.getCurUser(); | |||
QueryWrapper<PtImage> queryWrapper = new QueryWrapper<>(); | |||
queryWrapper.eq("image_status", ImageStateEnum.SUCCESS.getCode()); | |||
queryWrapper.eq("deleted", NumberConstant.NUMBER_0); | |||
if (!BaseService.isAdmin(user)) { | |||
queryWrapper.in("origin_user_id", user.getId(), 0L); | |||
} | |||
if (ptImageQueryNameDTO.getImageResource() != null) { | |||
@@ -403,12 +346,8 @@ public class PtImageServiceImpl implements PtImageService { | |||
throw new BusinessException(ResponseCode.BADREQUEST, "该镜像不存在或镜像用途不支持!"); | |||
} | |||
//仅支持[制作成功]状态镜像设置为默认镜像 | |||
if (!ImageStateEnum.SUCCESS.getCode().equals(image.getImageStatus())) { | |||
throw new BusinessException(ResponseCode.BADREQUEST, "仅支持[制作成功]状态镜像设置为默认镜像!"); | |||
} | |||
if (image.getImageResource() != ImageSourceEnum.PRE.getCode()) { | |||
if (!image.getImageResource().equals(ImageSourceEnum.PRE.getCode())) { | |||
throw new BusinessException(ResponseCode.BADREQUEST, "非预制镜像不能设置为默认镜像!"); | |||
} | |||
@@ -434,7 +373,7 @@ public class PtImageServiceImpl implements PtImageService { | |||
/** | |||
* 获取镜像URL | |||
* | |||
* @param imageQueryUrlDTO 查询镜像路径DTO | |||
* @param imageQueryUrlDTO 查询镜像地址DTO | |||
* @return String 镜像url | |||
*/ | |||
@Override | |||
@@ -453,7 +392,6 @@ public class PtImageServiceImpl implements PtImageService { | |||
if (imageQueryUrlDTO.getIsDefault() != null) { | |||
queryWrapper.eq(PtImage::getIsDefault, imageQueryUrlDTO.getIsDefault()); | |||
} | |||
queryWrapper.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()); | |||
queryWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | |||
List<PtImage> imageList = new ArrayList<>(); | |||
if (CollectionUtil.isNotEmpty(imageQueryUrlDTO.getImageTypes())) { | |||
@@ -470,23 +408,13 @@ public class PtImageServiceImpl implements PtImageService { | |||
return imageUrl; | |||
} | |||
/** | |||
* 自定义镜像回收的还原实现 | |||
* | |||
* @param dto 资源回收DTO对象 | |||
*/ | |||
@Override | |||
public void recycleRollback(RecycleCreateDTO dto) { | |||
String imageId = dto.getRemark(); | |||
ptImageMapper.updateDeletedById(Long.valueOf(imageId), false); | |||
} | |||
@Override | |||
@DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) | |||
public List<PtImage> getTerminalImageList() { | |||
UserContext user = userContextService.getCurUser(); | |||
LambdaQueryWrapper<PtImage> queryTerminalWrapper = new LambdaQueryWrapper<>(); | |||
queryTerminalWrapper.eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()).eq(PtImage::getDeleted, NumberConstant.NUMBER_0);; | |||
queryTerminalWrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0); | |||
; | |||
if (user != null && !BaseService.isAdmin()) { | |||
queryTerminalWrapper.and(wrapper -> wrapper.eq(PtImage::getCreateUserId, user.getId()).or().eq(PtImage::getImageResource, ImageSourceEnum.PRE.getCode())) | |||
.and(wrapper -> wrapper.eq(PtImage::getDeleted, NumberConstant.NUMBER_0)); | |||
@@ -509,17 +437,16 @@ public class PtImageServiceImpl implements PtImageService { | |||
/** | |||
* @param ptImageUploadDTO 镜像上传逻辑校验 | |||
* @param ptImageSaveDTO 镜像保存逻辑校验 | |||
* @param user 用户 | |||
* @return List<PtImage> 镜像列表 | |||
**/ | |||
@DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) | |||
private List<PtImage> checkUploadImage(PtImageUploadDTO ptImageUploadDTO, UserContext user, Integer source) { | |||
private List<PtImage> checkSaveImage(PtImageSaveDTO ptImageSaveDTO, UserContext user, Integer source) { | |||
LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | |||
queryWrapper.eq(PtImage::getImageName, ptImageUploadDTO.getImageName()) | |||
.eq(PtImage::getImageTag, ptImageUploadDTO.getImageTag()) | |||
queryWrapper.eq(PtImage::getImageUrl, ptImageSaveDTO.getImageUrl()) | |||
.eq(PtImage::getImageResource, source); | |||
if (user != null) { | |||
@@ -549,7 +476,7 @@ public class PtImageServiceImpl implements PtImageService { | |||
ptImages.stream().forEach( | |||
ptImage -> { | |||
PtImageVO ptImageVO =new PtImageVO(); | |||
PtImageVO ptImageVO = new PtImageVO(); | |||
ptImageVO.setId(ptImage.getId()); | |||
ptImageVO.setName(ptImage.getImageName()); | |||
ptImageVO.setTag(ptImage.getImageTag()); | |||
@@ -558,4 +485,24 @@ public class PtImageServiceImpl implements PtImageService { | |||
); | |||
return ptImageVOS; | |||
} | |||
} | |||
/** | |||
* 校验镜像地址与镜像名称、tag是否匹配 | |||
* | |||
* @param imageUrl 镜像地址 | |||
* @param imageName 镜像名称 | |||
* @param imageTag 镜像tag | |||
*/ | |||
private void validImageInfo(String imageUrl, String imageName, String imageTag) { | |||
// 标准镜像地址:镜像地址域名/命名空间/镜像名:镜像tag | |||
// 分解镜像地址 | |||
String[] split = imageUrl.substring(imageUrl.lastIndexOf(StrUtil.SLASH) + 1).split(StrUtil.COLON); | |||
if (!split[0].equals(imageName)) { | |||
throw new BusinessException("镜像名称不匹配"); | |||
} | |||
if (!split[1].equals(imageTag)) { | |||
throw new BusinessException("镜像版本号不匹配"); | |||
} | |||
} | |||
} |
@@ -1,77 +0,0 @@ | |||
/** | |||
* Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.image.service.task; | |||
import cn.hutool.core.util.StrUtil; | |||
import org.dubhe.biz.base.constant.HarborProperties; | |||
import org.dubhe.biz.log.enums.LogEnum; | |||
import org.dubhe.biz.log.utils.LogUtil; | |||
import org.dubhe.harbor.api.HarborApi; | |||
import org.dubhe.image.service.PtImageService; | |||
import org.dubhe.recycle.domain.dto.RecycleCreateDTO; | |||
import org.dubhe.recycle.domain.dto.RecycleDetailCreateDTO; | |||
import org.dubhe.recycle.global.AbstractGlobalRecycle; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.cloud.context.config.annotation.RefreshScope; | |||
import org.springframework.stereotype.Component; | |||
/** | |||
* @description 镜像资源回收/还原 | |||
* @date 2021-03-23 | |||
*/ | |||
@RefreshScope | |||
@Component(value = "imageRecycleFile") | |||
public class ImageRecycleFile extends AbstractGlobalRecycle { | |||
@Autowired | |||
private HarborApi harborApi; | |||
@Autowired | |||
private HarborProperties harborProperties; | |||
@Autowired | |||
private PtImageService ptImageService; | |||
/** | |||
* 自定义回收镜像实现 | |||
* | |||
* @param detail 数据清理详情参数 | |||
* @param dto 资源回收创建对象 | |||
* @return true 继续执行,false 中断任务详情回收(本次无法执行完毕,创建新任务到下次执行) | |||
*/ | |||
@Override | |||
protected boolean clearDetail(RecycleDetailCreateDTO detail, RecycleCreateDTO dto) throws Exception { | |||
LogUtil.info(LogEnum.IMAGE, "image custom recycle file,params:{}", detail); | |||
if (StrUtil.isNotBlank(detail.getRecycleCondition())) { | |||
String imageUrl = harborProperties.getAddress() + StrUtil.SLASH + detail.getRecycleCondition(); | |||
LogUtil.info(LogEnum.IMAGE, "delete harbor image url:{}", imageUrl); | |||
//同步删除harbor镜像 | |||
harborApi.deleteImageByTag(imageUrl); | |||
} | |||
return true; | |||
} | |||
/** | |||
* 数据还原 | |||
* | |||
* @param dto 数据清理参数 | |||
*/ | |||
@Override | |||
protected void rollback(RecycleCreateDTO dto) { | |||
ptImageService.recycleRollback(dto); | |||
} | |||
} |
@@ -21,8 +21,8 @@ import com.alibaba.fastjson.JSON; | |||
import org.dubhe.biz.base.constant.AuthConst; | |||
import org.dubhe.cloud.unittest.base.BaseTest; | |||
import org.dubhe.image.domain.dto.PtImageDeleteDTO; | |||
import org.dubhe.image.domain.dto.PtImageSaveDTO; | |||
import org.dubhe.image.domain.dto.PtImageUpdateDTO; | |||
import org.dubhe.image.domain.dto.PtImageUploadDTO; | |||
import org.junit.Test; | |||
import org.junit.runner.RunWith; | |||
import org.springframework.boot.test.context.SpringBootTest; | |||
@@ -78,9 +78,9 @@ public class PtImageTest extends BaseTest { | |||
*/ | |||
@Test | |||
public void uploadImageTest() throws Exception { | |||
PtImageUploadDTO ptImageUploadDTO = new PtImageUploadDTO(); | |||
PtImageSaveDTO ptImageUploadDTO = new PtImageSaveDTO(); | |||
ptImageUploadDTO.setImageName("mysql"); | |||
ptImageUploadDTO.setImagePath("F:/mysql.tar"); | |||
ptImageUploadDTO.setImageUrl("F:/mysql.tar"); | |||
ptImageUploadDTO.setImageTag("5.7"); | |||
ptImageUploadDTO.setRemark("测试上传镜像"); | |||
@@ -94,7 +94,7 @@ public class PtImageTest extends BaseTest { | |||
@Test | |||
public void updateImageTest() throws Exception { | |||
PtImageUpdateDTO imageUpdateDTO = new PtImageUpdateDTO(); | |||
imageUpdateDTO.setIds(Arrays.asList()); | |||
imageUpdateDTO.setId(1L); | |||
imageUpdateDTO.setRemark(""); | |||
mockMvcTest(MockMvcRequestBuilders.put("/ptImage"), JSON.toJSONString(imageUpdateDTO), | |||
MockMvcResultMatchers.status().is2xxSuccessful(), 200); | |||
@@ -75,7 +75,7 @@ public class DeploymentCallback extends Observable { | |||
setChanged(); | |||
notifyObservers(deployment); | |||
if (StringUtils.isNotEmpty(businessLabel)){ | |||
BaseK8sDeploymentCallbackCreateDTO baseK8sDeploymentCallbackCreateDTO = new BaseK8sDeploymentCallbackCreateDTO(deployment.getNamespace(), deployment.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),deployment.getName(), deployment.getReadyReplicas() == null?0:deployment.getReadyReplicas(), deployment.getReplicas() == null?0:deployment.getReplicas()); | |||
BaseK8sDeploymentCallbackCreateDTO baseK8sDeploymentCallbackCreateDTO = new BaseK8sDeploymentCallbackCreateDTO(deployment.getNamespace(), deployment.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),deployment.getName(), deployment.getReadyReplicas() == null?0:deployment.getReadyReplicas(), deployment.getReplicas() == null?0:deployment.getReplicas(),watcherActionEnum.getAction()); | |||
String url = k8sCallBackTool.getDeploymentCallbackUrl(businessLabel); | |||
String token = k8sCallBackTool.generateToken(); | |||
@@ -17,6 +17,7 @@ | |||
package org.dubhe.dubhek8s.observer; | |||
import org.dubhe.biz.base.constant.MagicNumConstant; | |||
import org.dubhe.biz.base.enums.BizEnum; | |||
import org.dubhe.biz.base.utils.SpringContextHolder; | |||
import org.dubhe.biz.log.enums.LogEnum; | |||
@@ -54,8 +55,18 @@ public class TrainJobObserver implements Observer { | |||
BizPod pod = (BizPod)arg; | |||
boolean trainJobFailed = PodPhaseEnum.FAILED.getPhase().equals(pod.getPhase()) && BizEnum.ALGORITHM.getBizCode().equals(pod.getBusinessLabel()) && SpringContextHolder.getActiveProfile().equals(pod.getLabel(K8sLabelConstants.PLATFORM_RUNTIME_ENV)); | |||
if (trainJobFailed){ | |||
LogUtil.warn(LogEnum.BIZ_K8S,"delete failed train job resourceName {};phase {};podName {}",pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),pod.getPhase(),pod.getName()); | |||
trainJobApi.delete(pod.getNamespace(),pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE)); | |||
new Thread(new Runnable(){ | |||
@Override | |||
public void run(){ | |||
try { | |||
Thread.sleep(MagicNumConstant.ONE_MINUTE); | |||
} catch (InterruptedException e) { | |||
LogUtil.error(LogEnum.BIZ_K8S,"TrainJobObserver update error {}",e.getMessage(),e); | |||
} | |||
LogUtil.warn(LogEnum.BIZ_K8S,"delete failed train job resourceName {};phase {};podName {}",pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE),pod.getPhase(),pod.getName()); | |||
trainJobApi.delete(pod.getNamespace(),pod.getLabel(K8sLabelConstants.BASE_TAG_SOURCE)); | |||
} | |||
}).start(); | |||
} | |||
} | |||
} | |||
@@ -79,7 +79,7 @@ public class SystemNodeServiceImpl implements SystemNodeService { | |||
@Resource | |||
private AdminClient adminClient; | |||
private final static String INTERNAL_IP = "InternalIP"; | |||
/** | |||
@@ -231,7 +231,7 @@ public class SystemNodeServiceImpl implements SystemNodeService { | |||
nodeDTO.setUid(node.getUid()); | |||
nodeDTO.setName(node.getName()); | |||
node.getAddresses().stream().forEach(bizNodeAddress -> { | |||
if (INTERNAL_IP.equals(bizNodeAddress.getType())) { | |||
if (K8sParamConstants.INTERNAL_IP.equals(bizNodeAddress.getType())) { | |||
nodeDTO.setIp(bizNodeAddress.getAddress()); | |||
return; | |||
} | |||
@@ -141,7 +141,7 @@ public class PtAtlasCameraServiceImpl implements PtAtlasCameraService { | |||
ResponseEntity<Map> exchange = restTemplate.exchange(atlasUrlProperties.getVms() + "/checkCameraUrl", HttpMethod.POST, requestEntity, Map.class, param); | |||
Map<String, String> statusMap = (Map) exchange.getBody().get("data"); | |||
if (statusMap.containsKey("cameraUrl")) { | |||
ptAtlasCamera.setHlsUrl(statusMap.get("cameraUrl").replace("127.0.0.1", "127.0.0.1")); | |||
ptAtlasCamera.setHlsUrl(statusMap.get("cameraUrl").replace("172.18.26.2", "10.105.10.51")); | |||
ptAtlasCameraMapper.updateById(ptAtlasCamera); | |||
} | |||
} catch (Exception e) { | |||
@@ -21,13 +21,10 @@ import cn.hutool.core.collection.CollUtil; | |||
import cn.hutool.core.date.DateBetween; | |||
import cn.hutool.core.date.DateUnit; | |||
import cn.hutool.core.util.RandomUtil; | |||
import cn.hutool.core.util.StrUtil; | |||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; | |||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; | |||
import com.baomidou.mybatisplus.core.metadata.IPage; | |||
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; | |||
import org.dubhe.biz.base.utils.StringUtils; | |||
import org.dubhe.biz.base.constant.HarborProperties; | |||
import org.dubhe.biz.base.constant.MagicNumConstant; | |||
import org.dubhe.biz.base.constant.NumberConstant; | |||
import org.dubhe.biz.base.constant.StringConstant; | |||
@@ -45,8 +42,10 @@ import org.dubhe.biz.base.service.UserContextService; | |||
import org.dubhe.biz.base.utils.HttpUtils; | |||
import org.dubhe.biz.base.utils.NumberUtil; | |||
import org.dubhe.biz.base.utils.ResultUtil; | |||
import org.dubhe.biz.base.utils.StringUtils; | |||
import org.dubhe.biz.base.vo.DataResponseBody; | |||
import org.dubhe.biz.base.vo.DatasetVO; | |||
import org.dubhe.biz.base.vo.NoteBookVO; | |||
import org.dubhe.biz.db.utils.PageUtil; | |||
import org.dubhe.biz.db.utils.WrapperHelp; | |||
import org.dubhe.biz.file.api.FileStoreApi; | |||
@@ -79,7 +78,6 @@ import org.dubhe.notebook.enums.NoteBookStatusEnum; | |||
import org.dubhe.notebook.service.NoteBookService; | |||
import org.dubhe.notebook.service.ProcessNotebookCommand; | |||
import org.dubhe.notebook.utils.NotebookUtil; | |||
import org.dubhe.biz.base.vo.NoteBookVO; | |||
import org.springframework.beans.BeanUtils; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.beans.factory.annotation.Qualifier; | |||
@@ -89,7 +87,13 @@ import org.springframework.transaction.annotation.Transactional; | |||
import org.springframework.util.CollectionUtils; | |||
import java.io.File; | |||
import java.util.*; | |||
import java.util.ArrayList; | |||
import java.util.Arrays; | |||
import java.util.Collections; | |||
import java.util.Date; | |||
import java.util.List; | |||
import java.util.Map; | |||
import java.util.Set; | |||
import java.util.stream.Collectors; | |||
/** | |||
@@ -127,9 +131,6 @@ public class NoteBookServiceImpl implements NoteBookService { | |||
@Autowired | |||
private ImageClient imageClient; | |||
@Autowired | |||
private HarborProperties harborProperties; | |||
@Autowired | |||
@Qualifier("hostFileStoreApiImpl") | |||
private FileStoreApi fileStoreApi; | |||
@@ -223,7 +224,7 @@ public class NoteBookServiceImpl implements NoteBookService { | |||
throw new BusinessException(ImageTypeEnum.NOTEBOOK.getCode() + "未配置默认镜像!"); | |||
} | |||
return harborProperties.getAddress() + StrUtil.SLASH + imageUrl; | |||
return imageUrl; | |||
} | |||
/** | |||
@@ -1,39 +0,0 @@ | |||
/** | |||
* Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.pointcloud.config; | |||
import lombok.Data; | |||
import org.springframework.boot.context.properties.ConfigurationProperties; | |||
import org.springframework.stereotype.Component; | |||
/** | |||
* @description harbor相关配置 | |||
* @date 2020-07-17 | |||
*/ | |||
@Data | |||
@Component | |||
@ConfigurationProperties(prefix = "harbor") | |||
public class PointCloudHarborConfig { | |||
private String address; | |||
private String username; | |||
private String password; | |||
private String modelName; | |||
} |
@@ -66,7 +66,6 @@ import org.dubhe.pointcloud.client.DataClient; | |||
import org.dubhe.pointcloud.client.ImageClient; | |||
import org.dubhe.pointcloud.client.ModelClient; | |||
import org.dubhe.pointcloud.common.Constant; | |||
import org.dubhe.pointcloud.config.PointCloudHarborConfig; | |||
import org.dubhe.pointcloud.dao.PcAnnotationDetailMapper; | |||
import org.dubhe.pointcloud.dao.PcDatasetFileMapper; | |||
import org.dubhe.pointcloud.dao.PcDatasetMapper; | |||
@@ -141,9 +140,6 @@ public class PcDatasetServiceImpl implements PcDatasetService { | |||
@Resource | |||
private DeployAsyncTask deployAsyncTask; | |||
@Resource | |||
private PointCloudHarborConfig pointCloudHarborConfig; | |||
@Resource | |||
private ImageClient imageClient; | |||
@@ -652,7 +648,7 @@ public class PcDatasetServiceImpl implements PcDatasetService { | |||
if (!dataResponseBody.succeed()) { | |||
throw new BusinessException(ErrorEnum.CALL_IMAGE_SERVER_FAIL); | |||
} | |||
return pointCloudHarborConfig.getAddress() + SymbolConstant.SLASH + dataResponseBody.getData(); | |||
return dataResponseBody.getData(); | |||
} | |||
/** | |||
@@ -18,8 +18,10 @@ | |||
package org.dubhe.servinggateway.config; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.dubhe.biz.base.constant.MagicNumConstant; | |||
import org.dubhe.biz.base.constant.NumberConstant; | |||
import org.dubhe.biz.base.constant.SymbolConstant; | |||
import org.dubhe.biz.base.utils.StringUtils; | |||
import org.dubhe.biz.log.enums.LogEnum; | |||
import org.dubhe.biz.log.utils.LogUtil; | |||
import org.dubhe.servinggateway.constant.GatewayConstant; | |||
@@ -38,8 +40,8 @@ import org.springframework.web.util.UriComponentsBuilder; | |||
import reactor.core.publisher.Mono; | |||
import javax.annotation.Resource; | |||
import java.util.ArrayList; | |||
import java.util.Arrays; | |||
import java.util.Collections; | |||
import java.util.HashMap; | |||
import java.util.List; | |||
import java.util.Map; | |||
@@ -55,21 +57,12 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
private ApplicationEventPublisher publisher; | |||
@Value("${serving.gateway.postfixUrl}") | |||
private String postfixUrl; | |||
@Resource | |||
private RedisRouteDefinitionRepository routeDefinitionRepository; | |||
@Resource | |||
private GatewayRouteService gatewayRouteService; | |||
/** | |||
* 模型服务部署使用的端口 | |||
*/ | |||
@Value("${k8s.port}") | |||
private String httpPort; | |||
@Override | |||
public void run(String... args) { | |||
this.loadRouteConfig(); | |||
@@ -87,6 +80,9 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin load route config"); | |||
List<GatewayRouteQueryVO> activeRoutes = gatewayRouteService.findActiveRoutes(); | |||
for (GatewayRouteQueryVO activeRoute : activeRoutes) { | |||
if (StringUtils.isEmpty(activeRoute.getUri())){ | |||
continue; | |||
} | |||
RouteDefinition definition = this.convert2RouteDefinition(activeRoute); | |||
routeDefinitionRepository.save(Mono.just(definition)).subscribe(); | |||
} | |||
@@ -103,7 +99,19 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
public void saveRouteByRouteId(Long id) { | |||
LogUtil.info(LogEnum.SERVING_GATEWAY, "Deal save route event"); | |||
GatewayRouteQueryVO gatewayRouteQueryVO = gatewayRouteService.findActiveById(id); | |||
if (Objects.nonNull(gatewayRouteQueryVO)) { | |||
//查不到数据进行重试 | |||
int count = 0; | |||
while (gatewayRouteQueryVO == null && count < MagicNumConstant.THREE){ | |||
LogUtil.info(LogEnum.SERVING_GATEWAY, "gatewayRouteQueryVO is null retry:"+count +" id="+id); | |||
gatewayRouteQueryVO = gatewayRouteService.findActiveById(id); | |||
count++; | |||
try { | |||
Thread.sleep(MagicNumConstant.FIVE_HUNDRED); | |||
} catch (InterruptedException e) { | |||
e.printStackTrace(); | |||
} | |||
} | |||
if (Objects.nonNull(gatewayRouteQueryVO) && StringUtils.isNotEmpty(gatewayRouteQueryVO.getUri())) { | |||
routeDefinitionRepository.save(Mono.just(this.convert2RouteDefinition(gatewayRouteQueryVO))).subscribe(); | |||
} | |||
// 发布刷新路由事件 | |||
@@ -131,17 +139,19 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
* @return 网关所需的RouteDefinition | |||
*/ | |||
private RouteDefinition convert2RouteDefinition(GatewayRouteQueryVO gatewayRouteQueryVO) { | |||
if(StringUtils.isEmpty(gatewayRouteQueryVO.getUri())){ | |||
return null; | |||
} | |||
RouteDefinition definition = new RouteDefinition(); | |||
// 设置路由基础信息 | |||
definition.setId(GatewayConstant.ROUTE_PREFIX + gatewayRouteQueryVO.getId()); | |||
definition.setUri(UriComponentsBuilder.fromHttpUrl(SymbolConstant.HTTP_SLASH + gatewayRouteQueryVO.getUri() + | |||
SymbolConstant.COLON + httpPort).build().toUri()); | |||
definition.setUri(UriComponentsBuilder.fromHttpUrl(SymbolConstant.HTTP_SLASH + gatewayRouteQueryVO.getUri()).build().toUri()); | |||
// 定义url匹配规则的断言 | |||
// 举例 {abc}.dubhe.ai 匹配{abc}部分 | |||
PredicateDefinition pathPredicate = new PredicateDefinition(); | |||
pathPredicate.setName("Host"); | |||
pathPredicate.setName("Path"); | |||
Map<String, String> predicateParams = new HashMap<>(NumberConstant.NUMBER_8); | |||
predicateParams.put("pattern", gatewayRouteQueryVO.getPatternPath() + postfixUrl); | |||
predicateParams.put("pattern", "/"+gatewayRouteQueryVO.getPatternPath() + "/**"); | |||
pathPredicate.setArgs(predicateParams); | |||
// 定义权重断言 | |||
// 根据配置的权重信息进行分流 | |||
@@ -152,11 +162,21 @@ public class GatewayServiceHandler implements ApplicationEventPublisherAware, Co | |||
weightParams.put("weight.weight", gatewayRouteQueryVO.getWeight()); | |||
weightPredicate.setArgs(weightParams); | |||
definition.setPredicates(Arrays.asList(pathPredicate, weightPredicate)); | |||
List<FilterDefinition> filters =new ArrayList<>(); | |||
// 传入自定义的监控指标过滤器 | |||
FilterDefinition filterDefinition = new FilterDefinition(); | |||
// name设置为定义的MetricsGatewayFilterFactory中GatewayFilterFactory前部分 | |||
filterDefinition.setName("Metrics"); | |||
definition.setFilters(Collections.singletonList(filterDefinition)); | |||
filters.add(filterDefinition); | |||
//去除一级url转发 | |||
FilterDefinition filterDefinitionUrl = new FilterDefinition(); | |||
filterDefinitionUrl.setName("StripPrefix"); | |||
filterDefinitionUrl.addArg("parts","1"); | |||
filters.add(filterDefinitionUrl); | |||
definition.setFilters(filters); | |||
// 塞入自定义数据,供之后filter使用 | |||
HashMap<String, Object> metadata = new HashMap<>(NumberConstant.NUMBER_4); | |||
metadata.put("servingInfoId", gatewayRouteQueryVO.getServiceInfoId()); | |||
@@ -120,7 +120,7 @@ public class MetricsGatewayFilterFactory extends AbstractGatewayFilterFactory<Me | |||
DataBufferUtils.join(body) | |||
.doOnNext(dataBuffer -> { | |||
// 判断该请求是否为推理请求 | |||
boolean isInference = exchange.getRequest().getPath().toString().startsWith(GatewayConstant.INFERENCE_INTERFACE_NAME); | |||
boolean isInference = exchange.getRequest().getPath().toString().endsWith(GatewayConstant.INFERENCE_INTERFACE_NAME); | |||
if (isInference) { | |||
LogUtil.info(LogEnum.SERVING_GATEWAY, "Begin deal inference request filter"); | |||
// 调用请求是否成功 | |||
@@ -7,7 +7,7 @@ | |||
select smc.id id, si.id serviceInfoId, si.uuid patternPath, smc.release_rate weight, smc.url uri | |||
from serving_model_config smc | |||
left join serving_info si on si.id = smc.serving_id | |||
where si.`status` = 2 and si.deleted = 0 and smc.ready_replicas > 0 and smc.deleted = 0 | |||
where si.deleted = 0 and smc.deleted = 0 | |||
and smc.id = #{id} | |||
</select> | |||
@@ -1,39 +0,0 @@ | |||
/** | |||
* Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.serving.config; | |||
import lombok.Data; | |||
import org.springframework.boot.context.properties.ConfigurationProperties; | |||
import org.springframework.stereotype.Component; | |||
/** | |||
* @description harbor相关配置 | |||
* @date 2020-07-17 | |||
*/ | |||
@Data | |||
@Component | |||
@ConfigurationProperties(prefix = "harbor") | |||
public class TrainHarborConfig { | |||
private String address; | |||
private String username; | |||
private String password; | |||
private String modelName; | |||
} |
@@ -71,7 +71,6 @@ import org.dubhe.serving.client.AlgorithmClient; | |||
import org.dubhe.serving.client.ImageClient; | |||
import org.dubhe.serving.client.ModelBranchClient; | |||
import org.dubhe.serving.client.ModelInfoClient; | |||
import org.dubhe.serving.config.TrainHarborConfig; | |||
import org.dubhe.serving.constant.ServingConstant; | |||
import org.dubhe.serving.dao.BatchServingMapper; | |||
import org.dubhe.serving.domain.dto.*; | |||
@@ -135,8 +134,6 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat | |||
@Resource | |||
private ImageClient imageClient; | |||
@Resource | |||
private TrainHarborConfig trainHarborConfig; | |||
@Resource | |||
private AlgorithmClient algorithmClient; | |||
@Autowired | |||
private RedisUtils redisUtils; | |||
@@ -238,7 +235,7 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat | |||
BeanUtils.copyProperties(batchServingCreateDTO, batchServing); | |||
String imageUrl = getImageUrl(batchServingCreateDTO.getImageName(), batchServingCreateDTO.getImageTag()); | |||
batchServing.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + imageUrl); | |||
batchServing.setImage(imageUrl); | |||
PtModelInfoQueryVO ptModelInfoQueryVO = getPtModelInfo(batchServingCreateDTO.getModelId()); | |||
batchServing.setModelAddress(ptModelInfoQueryVO.getModelAddress()); | |||
if (ptModelInfoQueryVO.getFrameType() > NumberConstant.NUMBER_4) { | |||
@@ -421,7 +418,7 @@ public class BatchServingServiceImpl extends ServiceImpl<BatchServingMapper, Bat | |||
BeanUtils.copyProperties(batchServingUpdateDTO, batchServing); | |||
String imageUrl = getImageUrl(batchServingUpdateDTO.getImageName(), batchServingUpdateDTO.getImageTag()); | |||
batchServing.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + imageUrl); | |||
batchServing.setImage(imageUrl); | |||
checkScriptPath(batchServing); | |||
checkResourceType(batchServing.getFrameType(), batchServing.getResourcesPoolType()); | |||
batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); | |||
@@ -18,6 +18,7 @@ | |||
package org.dubhe.serving.service.impl; | |||
import cn.hutool.core.bean.BeanUtil; | |||
import com.alibaba.fastjson.JSON; | |||
import com.alibaba.fastjson.JSONObject; | |||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; | |||
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; | |||
@@ -56,11 +57,16 @@ import org.dubhe.biz.permission.annotation.DataPermissionMethod; | |||
import org.dubhe.biz.permission.base.BaseService; | |||
import org.dubhe.biz.redis.utils.RedisUtils; | |||
import org.dubhe.k8s.api.MetricsApi; | |||
import org.dubhe.k8s.api.ModelServingApi; | |||
import org.dubhe.k8s.api.NodeApi; | |||
import org.dubhe.k8s.cache.ResourceCache; | |||
import org.dubhe.k8s.domain.dto.PodQueryDTO; | |||
import org.dubhe.k8s.domain.resource.BizServicePort; | |||
import org.dubhe.k8s.domain.vo.ModelServingVO; | |||
import org.dubhe.k8s.domain.vo.PodVO; | |||
import org.dubhe.k8s.domain.vo.PtPodsVO; | |||
import org.dubhe.k8s.enums.PodPhaseEnum; | |||
import org.dubhe.k8s.enums.WatcherActionEnum; | |||
import org.dubhe.k8s.service.PodService; | |||
import org.dubhe.k8s.utils.K8sNameTool; | |||
import org.dubhe.recycle.config.RecycleConfig; | |||
@@ -75,7 +81,6 @@ import org.dubhe.serving.client.AlgorithmClient; | |||
import org.dubhe.serving.client.ImageClient; | |||
import org.dubhe.serving.client.ModelBranchClient; | |||
import org.dubhe.serving.client.ModelInfoClient; | |||
import org.dubhe.serving.config.TrainHarborConfig; | |||
import org.dubhe.serving.constant.ServingConstant; | |||
import org.dubhe.serving.dao.ServingInfoMapper; | |||
import org.dubhe.serving.dao.ServingModelConfigMapper; | |||
@@ -178,8 +183,6 @@ public class ServingServiceImpl implements ServingService { | |||
@Resource | |||
private ImageClient imageClient; | |||
@Resource | |||
private TrainHarborConfig trainHarborConfig; | |||
@Resource | |||
private AlgorithmClient algorithmClient; | |||
@Resource | |||
private RecycleConfig recycleConfig; | |||
@@ -191,6 +194,10 @@ public class ServingServiceImpl implements ServingService { | |||
private ResourceCache resourceCache; | |||
@Value("Task:Serving:" + "${spring.profiles.active}_serving_id_") | |||
private String servingIdPrefix; | |||
@Resource | |||
private ModelServingApi modelServingApi; | |||
@Resource | |||
private NodeApi nodeApi; | |||
private final static List<String> FILE_NAMES; | |||
@@ -265,7 +272,7 @@ public class ServingServiceImpl implements ServingService { | |||
List<ServingInfoQueryVO> queryList = servingInfos.getRecords().stream().map(servingInfo -> { | |||
ServingInfoQueryVO servingInfoQueryVO = new ServingInfoQueryVO(); | |||
BeanUtils.copyProperties(servingInfo, servingInfoQueryVO); | |||
servingInfoQueryVO.setUrl(servingInfo.getUuid() + GATEWAY_URI_POSTFIX); | |||
servingInfoQueryVO.setUrl(GATEWAY_URI_POSTFIX+"/"+servingInfo.getUuid()); | |||
Map<String, String> statistics = servingLuaScriptService.countCallsByServingInfoId(servingInfo.getId()); | |||
servingInfoQueryVO.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO)); | |||
servingInfoQueryVO.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO)); | |||
@@ -419,7 +426,7 @@ public class ServingServiceImpl implements ServingService { | |||
throw new BusinessException(ServingErrorEnum.CALL_IMAGE_SERVER_FAIL); | |||
} | |||
servingModelConfig | |||
.setImage(trainHarborConfig.getAddress() + SymbolConstant.SLASH + dataResponseBody.getData()); | |||
.setImage(dataResponseBody.getData()); | |||
// 校验模型文件是否存在 | |||
String path = k8sNameTool.getAbsolutePath(servingModelConfig.getModelAddress()); | |||
@@ -933,8 +940,8 @@ public class ServingServiceImpl implements ServingService { | |||
predictParamVO.setOutputs(outputs); | |||
predictParamVO.setOther(other); | |||
} else if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | |||
String url = "http://" + servingInfo.getUuid() + GATEWAY_URI_POSTFIX | |||
+ ServingConstant.INFERENCE_INTERFACE_NAME; | |||
String url = "http://" + GATEWAY_URI_POSTFIX | |||
+"/"+ servingInfo.getUuid()+ ServingConstant.INFERENCE_INTERFACE_NAME; | |||
predictParamVO.setUrl(url); | |||
Map<String, String> inputs = new HashMap<>(); | |||
inputs.put("files", "File"); | |||
@@ -1079,6 +1086,7 @@ public class ServingServiceImpl implements ServingService { | |||
*/ | |||
@Override | |||
public boolean servingDeploymentCallback(ServingK8sDeploymentCallbackCreateDTO req) { | |||
LogUtil.info(LogEnum.BIZ_K8S,"servingDeploymentCallback:{}", JSON.toJSONString(req)); | |||
// 根据namespace和podName找到模型配置 | |||
String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.SERVING, req.getResourceName()); | |||
if (StringUtils.isBlank(resourceInfo)) { | |||
@@ -1105,13 +1113,13 @@ public class ServingServiceImpl implements ServingService { | |||
return false; | |||
} | |||
// 增加发送路由信息 | |||
if (req.getReadyReplicas() > NumberConstant.NUMBER_0 | |||
if (WatcherActionEnum.ADDED.getAction().equals(req.getAction()) | |||
&& ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | |||
this.notifyUpdateServingRoute(Collections.singletonList(servingModelConfig.getId()), | |||
Collections.emptyList()); | |||
} | |||
// 增加删除路由信息 | |||
if (req.getReadyReplicas() == NumberConstant.NUMBER_0 | |||
if (WatcherActionEnum.DELETED.getAction().equals(req.getAction()) | |||
&& ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { | |||
this.notifyUpdateServingRoute(Collections.emptyList(), | |||
Collections.singletonList(servingModelConfig.getId())); | |||
@@ -1148,8 +1156,29 @@ public class ServingServiceImpl implements ServingService { | |||
@Transactional(rollbackFor = Exception.class) | |||
public boolean updateByCallback(ServingK8sDeploymentCallbackCreateDTO req, ServingModelConfig servingModelConfig, | |||
ServingInfo servingInfo) { | |||
//更新url | |||
if (StringUtils.isEmpty(servingModelConfig.getUrl())){ | |||
//查询 | |||
ModelServingVO modelServingVO = modelServingApi.get(req.getNamespace(),req.getResourceName()); | |||
if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { | |||
// 获取pod对应的url,并修改模型部署状态 | |||
List<BizServicePort> ports = modelServingVO.getBizService().getPorts(); | |||
if (CollectionUtils.isNotEmpty(ports)) { | |||
//取第一个url | |||
String url = ""; | |||
if (ports.get(NumberConstant.NUMBER_0).getNodePort() != null) { | |||
url = nodeApi.getAvailableNodeIp() + ":" + ports.get(NumberConstant.NUMBER_0).getNodePort(); | |||
} | |||
servingModelConfig.setUrl(url); | |||
} | |||
} | |||
} | |||
// 更新当前模型配置有效节点数 | |||
servingModelConfig.setReadyReplicas(req.getReadyReplicas()); | |||
servingModelConfig.setResourceInfo(null); | |||
int result = servingModelConfigMapper.updateById(servingModelConfig); | |||
if (result < NumberConstant.NUMBER_1) { | |||
return true; | |||
@@ -31,6 +31,7 @@ import org.dubhe.biz.log.enums.LogEnum; | |||
import org.dubhe.biz.log.utils.LogUtil; | |||
import org.dubhe.k8s.api.DistributeTrainApi; | |||
import org.dubhe.k8s.api.ModelServingApi; | |||
import org.dubhe.k8s.api.NodeApi; | |||
import org.dubhe.k8s.api.TrainJobApi; | |||
import org.dubhe.k8s.domain.PtBaseResult; | |||
import org.dubhe.k8s.domain.bo.DistributeTrainBO; | |||
@@ -38,7 +39,7 @@ import org.dubhe.k8s.domain.bo.ModelServingBO; | |||
import org.dubhe.k8s.domain.bo.PtJupyterJobBO; | |||
import org.dubhe.k8s.domain.bo.PtMountDirBO; | |||
import org.dubhe.k8s.domain.resource.BizDistributeTrain; | |||
import org.dubhe.k8s.domain.resource.BizIngressRule; | |||
import org.dubhe.k8s.domain.resource.BizServicePort; | |||
import org.dubhe.k8s.domain.vo.ModelServingVO; | |||
import org.dubhe.k8s.domain.vo.PtJupyterJobVO; | |||
import org.dubhe.k8s.utils.K8sNameTool; | |||
@@ -88,6 +89,9 @@ public class DeployServingAsyncTask { | |||
@Resource | |||
private ModelServingApi modelServingApi; | |||
@Resource | |||
private NodeApi nodeApi; | |||
@Resource | |||
private K8sNameTool k8sNameTool; | |||
@@ -143,11 +147,14 @@ public class DeployServingAsyncTask { | |||
String statusDetailKey = ServingStatusDetailDescUtil.getServingStatusDetailKey(ServingStatusDetailDescUtil.CONTAINER_DEPLOYMENT_EXCEPTION, uniqueName); | |||
if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { | |||
// 获取pod对应的url,并修改模型部署状态 | |||
List<BizIngressRule> rules = modelServingVO.getBizIngress().getRules(); | |||
List<BizServicePort> ports = modelServingVO.getBizService().getPorts(); | |||
if (CollectionUtils.isNotEmpty(rules)) { | |||
if (CollectionUtils.isNotEmpty(ports)) { | |||
//取第一个url | |||
String url = rules.get(NumberConstant.NUMBER_0).getHost(); | |||
String url = ""; | |||
if (ports.get(NumberConstant.NUMBER_0).getNodePort() != null){ | |||
url = nodeApi.getAvailableNodeIp()+":"+ports.get(NumberConstant.NUMBER_0).getNodePort(); | |||
} | |||
servingModelConfig.setUrl(url); | |||
flag = true; | |||
servingInfo.removeStatusDetail(statusDetailKey); | |||
@@ -176,7 +183,6 @@ public class DeployServingAsyncTask { | |||
servingInfo.putStatusDetail(statusDetailKey, e.getMessage()); | |||
LogUtil.error(LogEnum.SERVING, "User {} create serving failed.The name of serving is {}", user.getUsername(), servingInfo.getName(), e); | |||
} | |||
} | |||
//修改服务状态 | |||
@@ -276,7 +282,7 @@ public class DeployServingAsyncTask { | |||
return null; | |||
} | |||
} else { | |||
LogUtil.info(LogEnum.SERVING, "User {} failed to copy the inference script, script {} not exist", user.getUsername(), pyPathList.get(0)); | |||
LogUtil.info(LogEnum.SERVING, "User {} failed to copy the inference script, script {} not exist", user.getUsername(), pyPathList); | |||
return null; | |||
} | |||
servingPath = fileStoreApi.formatPath(targetPath + getSourceName(sourcePath)); | |||
@@ -17,13 +17,9 @@ | |||
package org.dubhe.serving.utils; | |||
import cn.hutool.core.codec.Base64; | |||
import cn.hutool.core.io.FileUtil; | |||
import io.grpc.ManagedChannel; | |||
import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; | |||
import io.grpc.netty.shaded.io.grpc.netty.NegotiationType; | |||
import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; | |||
import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; | |||
import org.dubhe.biz.base.constant.NumberConstant; | |||
import org.dubhe.biz.base.context.UserContext; | |||
import org.dubhe.biz.log.enums.LogEnum; | |||
@@ -31,12 +27,9 @@ import org.dubhe.biz.log.utils.LogUtil; | |||
import org.dubhe.serving.domain.entity.DataInfo; | |||
import org.dubhe.serving.proto.Inference; | |||
import org.dubhe.serving.proto.InferenceServiceGrpc; | |||
import org.springframework.beans.factory.annotation.Value; | |||
import org.springframework.stereotype.Component; | |||
import javax.net.ssl.SSLException; | |||
import java.io.File; | |||
import java.io.InputStream; | |||
import java.util.List; | |||
import java.util.concurrent.ConcurrentHashMap; | |||
import java.util.concurrent.TimeUnit; | |||
@@ -52,51 +45,20 @@ public class GrpcClient { | |||
* <服务id, grpc客户端channel> | |||
*/ | |||
public static ConcurrentHashMap<Long, ManagedChannel> channelMap = new ConcurrentHashMap<>(); | |||
/** | |||
* 项目根路径 | |||
*/ | |||
private final static String USER_DIR_SYSTEM_PROPERTY = "user.dir"; | |||
/** | |||
* k8s crt证书文件名 | |||
*/ | |||
private final static String TLS_CRT = "server.crt"; | |||
/** | |||
* k8s grpc端口 | |||
*/ | |||
@Value("${k8s.https-port:31365}") | |||
private String grpcPort; | |||
/** | |||
* k8s 模型部署tls证书 crt | |||
*/ | |||
@Value("${k8s.serving.tls-crt}") | |||
private String tlsCrt; | |||
/** | |||
* 创建带tls认证的grpc通道 | |||
* 创建grpc通道 | |||
* | |||
* @param url tls证书地址 | |||
* @param url | |||
* @return ManagedChannel 通道 | |||
* @throws SSLException | |||
*/ | |||
public ManagedChannel createTlsChannel(String url) throws Exception { | |||
String path = getTlsCrt(); | |||
SslContext sslContext = GrpcSslContexts.forClient().trustManager(new File(path)).build(); | |||
ManagedChannel channel = NettyChannelBuilder.forAddress(url, Integer.parseInt(grpcPort)).maxInboundMessageSize(NumberConstant.MAX_MESSAGE_LENGTH).negotiationType(NegotiationType.TLS).sslContext(sslContext).build(); | |||
public ManagedChannel createChannel(String url) { | |||
String[] host = url.split(":"); | |||
ManagedChannel channel = NettyChannelBuilder.forAddress(host[0], Integer.parseInt(host[1])).maxInboundMessageSize(NumberConstant.MAX_MESSAGE_LENGTH).negotiationType(NegotiationType.PLAINTEXT).build(); | |||
return channel; | |||
} | |||
/** | |||
* 获取crt证书路径 | |||
*/ | |||
public String getTlsCrt() { | |||
String path = System.getProperty(USER_DIR_SYSTEM_PROPERTY) + File.separator.concat(TLS_CRT); | |||
if(!FileUtil.exist(path)) { | |||
FileUtil.writeUtf8String(Base64.decodeStr(tlsCrt), path); | |||
} | |||
return path; | |||
} | |||
/** | |||
* 关闭通道 | |||
* | |||
@@ -148,7 +110,7 @@ public class GrpcClient { | |||
} | |||
ManagedChannel channel = null; | |||
try { | |||
channel = this.createTlsChannel(url); | |||
channel = this.createChannel(url); | |||
channelMap.put(servingId, channel); | |||
} catch (Exception e) { | |||
LogUtil.error(LogEnum.SERVING, "An Exception occurred when user {} creating the grpc channel, service id:{}", user.getUsername(), servingId, e); | |||
@@ -169,7 +131,7 @@ public class GrpcClient { | |||
} | |||
ManagedChannel channel = null; | |||
try { | |||
channel = this.createTlsChannel(url); | |||
channel = this.createChannel(url); | |||
channelMap.put(servingId, channel); | |||
} catch (Exception e) { | |||
LogUtil.error(LogEnum.SERVING, "An Exception occurred when getting grpc channel, service id:{}", servingId, e); | |||
@@ -71,13 +71,6 @@ public class PtImage extends BaseEntity { | |||
@TableField(value = "image_resource") | |||
private Integer imageResource; | |||
/** | |||
* 镜像状态 | |||
*/ | |||
@TableField(value = "image_status") | |||
private Integer imageStatus; | |||
/** | |||
* 资源拥有者ID | |||
*/ | |||
@@ -182,6 +182,6 @@ public class Terminal extends BaseEntity { | |||
return null; | |||
} | |||
String[] strings = imageUrl.split(SymbolConstant.SLASH); | |||
return strings.length > 0 ? strings[MagicNumConstant.ZERO] : null; | |||
return strings.length > 0 ? strings[MagicNumConstant.ONE] : null; | |||
} | |||
} |
@@ -16,7 +16,6 @@ | |||
*/ | |||
package org.dubhe.terminal.service.impl; | |||
import java.sql.Timestamp; | |||
import cn.hutool.core.util.RandomUtil; | |||
import cn.hutool.core.util.StrUtil; | |||
@@ -38,6 +37,7 @@ import org.dubhe.biz.log.utils.LogUtil; | |||
import org.dubhe.biz.permission.base.BaseService; | |||
import org.dubhe.biz.redis.utils.RedisUtils; | |||
import org.dubhe.docker.api.DockerApi; | |||
import org.dubhe.docker.callback.TerminalPushImageResultCallback; | |||
import org.dubhe.docker.config.DockerClientFactory; | |||
import org.dubhe.docker.enums.DockerOperationEnum; | |||
import org.dubhe.docker.utils.DockerCallbackTool; | |||
@@ -53,7 +53,6 @@ import org.dubhe.k8s.domain.vo.TerminalResourceVO; | |||
import org.dubhe.k8s.enums.BusinessLabelServiceNameEnum; | |||
import org.dubhe.k8s.enums.PodPhaseEnum; | |||
import org.dubhe.k8s.utils.K8sNameTool; | |||
import org.dubhe.docker.callback.TerminalPushImageResultCallback; | |||
import org.dubhe.terminal.config.TerminalConfig; | |||
import org.dubhe.terminal.constant.TerminalConstant; | |||
import org.dubhe.terminal.dao.PtImageMapper; | |||
@@ -77,6 +76,7 @@ import org.springframework.stereotype.Service; | |||
import org.springframework.transaction.annotation.Transactional; | |||
import org.springframework.util.CollectionUtils; | |||
import java.sql.Timestamp; | |||
import java.util.ArrayList; | |||
import java.util.Date; | |||
import java.util.List; | |||
@@ -174,38 +174,43 @@ public class TerminalServiceImpl implements TerminalService { | |||
@Override | |||
public boolean preserve(TerminalPreserveDTO terminalPreserveDTO) { | |||
Terminal terminal = terminalMapper.selectById(terminalPreserveDTO.getId()); | |||
if (terminal == null){ | |||
LogUtil.error(LogEnum.TERMINAL,"preserve terminal 数据不存在 terminalPreserveDTO: {}",terminalPreserveDTO); | |||
if (checkImageExists(terminal.getImageProject(), terminalPreserveDTO.getImageName(), terminalPreserveDTO.getImageTag())) { | |||
throw new BusinessException("请确认镜像版本号是否重复!"); | |||
} | |||
if (terminal == null) { | |||
LogUtil.error(LogEnum.TERMINAL, "preserve terminal 数据不存在 terminalPreserveDTO: {}", terminalPreserveDTO); | |||
throw new BusinessException("数据不存在"); | |||
} | |||
LambdaQueryWrapper<TerminalInfo> wrapper = new LambdaQueryWrapper<>(); | |||
wrapper.eq(TerminalInfo::getTerminalId, terminalPreserveDTO.getId()); | |||
List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectList(wrapper); | |||
if (CollectionUtils.isEmpty(terminalInfoList)){ | |||
LogUtil.error(LogEnum.TERMINAL,"preserve terminalInfoList 数据不存在 terminalPreserveDTO: {}",terminalPreserveDTO); | |||
if (CollectionUtils.isEmpty(terminalInfoList)) { | |||
LogUtil.error(LogEnum.TERMINAL, "preserve terminalInfoList 数据不存在 terminalPreserveDTO: {}", terminalPreserveDTO); | |||
throw new BusinessException("数据不存在"); | |||
} | |||
TerminalInfo masterTerminalInfo = null; | |||
for (TerminalInfo terminalInfo : terminalInfoList) { | |||
if (terminalInfo.isMasterFlag()){ | |||
if (terminalInfo.isMasterFlag()) { | |||
masterTerminalInfo = terminalInfo; | |||
} | |||
} | |||
if (masterTerminalInfo == null){ | |||
LogUtil.error(LogEnum.TERMINAL,"master 节点不存在 terminalPreserveDTO:{}",terminalPreserveDTO); | |||
if (masterTerminalInfo == null) { | |||
LogUtil.error(LogEnum.TERMINAL, "master 节点不存在 terminalPreserveDTO:{}", terminalPreserveDTO); | |||
throw new BusinessException("master 节点不存在"); | |||
} | |||
BizPod pod = podApi.getWithResourceName(k8sNameTool.getNamespace(terminal.getCreateUserId()),masterTerminalInfo.getK8sResourceName()); | |||
if (pod == null){ | |||
LogUtil.error(LogEnum.TERMINAL,"master 容器不存在 terminalPreserveDTO:{}",terminalPreserveDTO); | |||
BizPod pod = podApi.getWithResourceName(k8sNameTool.getNamespace(terminal.getCreateUserId()), masterTerminalInfo.getK8sResourceName()); | |||
if (pod == null) { | |||
LogUtil.error(LogEnum.TERMINAL, "master 容器不存在 terminalPreserveDTO:{}", terminalPreserveDTO); | |||
throw new BusinessException("master 容器不存在"); | |||
} | |||
if (!PodPhaseEnum.RUNNING.getPhase().equals(pod.getPhase()) || pod.getPodIp() == null || StringUtils.isNotEmpty(pod.getContainerStateMessages())){ | |||
LogUtil.error(LogEnum.TERMINAL,"master 容器未运行 terminalPreserveDTO:{}",terminalPreserveDTO); | |||
if (!PodPhaseEnum.RUNNING.getPhase().equals(pod.getPhase()) || pod.getPodIp() == null || StringUtils.isNotEmpty(pod.getContainerStateMessages())) { | |||
LogUtil.error(LogEnum.TERMINAL, "master 容器未运行 terminalPreserveDTO:{}", terminalPreserveDTO); | |||
throw new BusinessException("master 容器未运行"); | |||
} | |||
String containerID = pod.getContainerId(); | |||
@@ -217,14 +222,13 @@ public class TerminalServiceImpl implements TerminalService { | |||
terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"commit 镜像..."); | |||
terminalMapper.updateById(terminal); | |||
DockerClient dockerClient = dockerClientFactory.getDockerClient(pod.getHostIP()); | |||
String newImagePath = terminal.getImageProject()+SymbolConstant.SLASH+terminal.getCreateUserId()+SymbolConstant.SLASH+terminalPreserveDTO.getImageName(); | |||
String newImageRepository = terminalConfig.getHarborAddress()+SymbolConstant.SLASH+newImagePath; | |||
String newImagePath = terminalConfig.getHarborAddress() + StrUtil.SLASH + terminal.getImageProject() + SymbolConstant.SLASH + terminalPreserveDTO.getImageName(); | |||
try { | |||
dockerApi.commit(dockerClient,containerID,newImageRepository,terminalPreserveDTO.getImageTag()); | |||
dockerApi.commit(dockerClient,containerID,newImagePath,terminalPreserveDTO.getImageTag()); | |||
terminal.setStatus(null); | |||
terminal.putStatusDetail(TerminalStatusEnum.SAVING.getDescription(),"push 镜像..."); | |||
terminalMapper.updateById(terminal); | |||
boolean pushResult = dockerApi.push(dockerClient,newImageRepository+SymbolConstant.COLON+terminalPreserveDTO.getImageTag(),new TerminalPushImageResultCallback(dockerCallbackTool.getCallbackUrl(SymbolConstant.LOCAL_HOST,terminalConfig.getServerPort(), DockerOperationEnum.PUSH.getType()),terminal.getId(),dockerClient,terminal.getCreateUserId())); | |||
boolean pushResult = dockerApi.push(dockerClient,newImagePath+SymbolConstant.COLON+terminalPreserveDTO.getImageTag(),new TerminalPushImageResultCallback(dockerCallbackTool.getCallbackUrl(SymbolConstant.LOCAL_HOST,terminalConfig.getServerPort(), DockerOperationEnum.PUSH.getType()),terminal.getId(),dockerClient,terminal.getCreateUserId())); | |||
if (!pushResult){ | |||
LogUtil.error(LogEnum.TERMINAL,"master 推送镜像错误 terminalPreserveDTO:{}",terminalPreserveDTO); | |||
throw new BusinessException("推送镜像错误:"); | |||
@@ -234,7 +238,7 @@ public class TerminalServiceImpl implements TerminalService { | |||
throw new BusinessException("保存容器错误:"+e.getMessage()); | |||
} | |||
terminal.setImageUrl(newImagePath+SymbolConstant.COLON+terminalPreserveDTO.getImageTag()); | |||
terminal.setImageUrl(newImagePath + SymbolConstant.COLON + terminalPreserveDTO.getImageTag()); | |||
terminal.setImageName(terminalPreserveDTO.getImageName()); | |||
terminal.setDescription(terminalPreserveDTO.getImageRemark()); | |||
terminal.setImageTag(terminalPreserveDTO.getImageTag()); | |||
@@ -244,13 +248,24 @@ public class TerminalServiceImpl implements TerminalService { | |||
return terminalMapper.updateById(terminal) > 0; | |||
} | |||
private boolean checkImageExists(String imageProject, String imageName, String imageTag) { | |||
// 拼接镜像的完整路径 | |||
String imageUrl = terminalConfig.getHarborAddress() + StrUtil.SLASH + imageProject + SymbolConstant.SLASH + imageName + StrUtil.COLON + imageTag; | |||
// 校验该待保存镜像是否存在 | |||
LambdaQueryWrapper<PtImage> queryWrapper = new LambdaQueryWrapper<>(); | |||
queryWrapper.eq(PtImage::getImageUrl, imageUrl); | |||
queryWrapper.ne(PtImage::getCreateUserId, userContextService.getCurUserId()); | |||
return ptImageMapper.selectCount(queryWrapper) > 0; | |||
} | |||
@Override | |||
@Transactional(rollbackFor = Exception.class) | |||
public boolean delete(TerminalDTO terminalDTO) { | |||
try{ | |||
try { | |||
Terminal terminal = terminalMapper.selectById(terminalDTO.getId()); | |||
if (terminal == null){ | |||
LogUtil.error(LogEnum.TERMINAL,"delete 数据不存在 terminalDTO: {}",terminalDTO); | |||
if (terminal == null) { | |||
LogUtil.error(LogEnum.TERMINAL, "delete 数据不存在 terminalDTO: {}", terminalDTO); | |||
throw new BusinessException("数据不存在"); | |||
} | |||
terminal.setDeleted(true); | |||
@@ -259,7 +274,7 @@ public class TerminalServiceImpl implements TerminalService { | |||
terminalMapper.deleteById(terminal); | |||
// 删除任务缓存 | |||
String taskIdentify = (String) redisUtils.get(terminalIdPrefix + String.valueOf(terminal.getId())); | |||
if (StringUtils.isNotEmpty(taskIdentify)){ | |||
if (StringUtils.isNotEmpty(taskIdentify)) { | |||
redisUtils.del(taskIdentify, terminalIdPrefix + String.valueOf(terminal.getId())); | |||
} | |||
String namespace = k8sNameTool.getNamespace(terminal.getCreateUserId()); | |||
@@ -477,6 +492,13 @@ public class TerminalServiceImpl implements TerminalService { | |||
return; | |||
} | |||
Terminal terminal = terminalMapper.selectById(terminalId); | |||
List<TerminalInfo> terminalInfoList = terminalInfoMapper.selectByTerminalId(terminalId); | |||
TerminalInfo masterTerminalInfo = null; | |||
for (TerminalInfo info : terminalInfoList){ | |||
if (info.isMasterFlag()){ | |||
masterTerminalInfo = info; | |||
} | |||
} | |||
if (terminal == null){ | |||
LogUtil.error(LogEnum.TERMINAL,"pushImageComplete no terminal found id:{}",terminalId); | |||
return; | |||
@@ -499,10 +521,11 @@ public class TerminalServiceImpl implements TerminalService { | |||
ptImage.setImageTag(terminal.getImageTag()); | |||
ptImage.setRemark(terminal.getDescription()); | |||
ptImage.setImageResource(MagicNumConstant.ZERO); | |||
ptImage.setImageStatus(MagicNumConstant.ONE); | |||
ptImage.setDeleted(false); | |||
ptImage.setUpdateUserId(userId); | |||
ptImage.setUpdateTime(new Timestamp(new java.util.Date().getTime())); | |||
ptImage.setSshUser(masterTerminalInfo.getSshUser()); | |||
ptImage.setSshPwd(masterTerminalInfo.getSshPassword()); | |||
if (ptImage.getId() != null){ | |||
ptImageMapper.updateById(ptImage); | |||
@@ -563,7 +586,7 @@ public class TerminalServiceImpl implements TerminalService { | |||
terminalBO.setGpuNum(terminalInfo.getGpuNum()); | |||
terminalBO.setMemNum(terminalInfo.getMemNum()); | |||
terminalBO.setCpuNum(terminalInfo.getCpuNum()); | |||
terminalBO.setImage(terminalConfig.getHarborAddress()+SymbolConstant.SLASH+terminalCreateDTO.getImageUrl()); | |||
terminalBO.setImage(terminalCreateDTO.getImageUrl()); | |||
terminalBO.setFsMounts(Maps.newHashMap()); | |||
terminalBO.setBusinessLabel(BusinessLabelServiceNameEnum.TERMINAL.getBusinessLabel()); | |||
terminalBO.setTaskIdentifyLabel(taskIdentifyLabel); | |||
@@ -26,7 +26,6 @@ import org.springframework.stereotype.Component; | |||
* @description 镜像远程服务调用熔断类 | |||
* @date 2020-12-15 | |||
*/ | |||
@Component | |||
public class ImageClientFallback implements ImageClient { | |||
@Override | |||
@@ -1,39 +0,0 @@ | |||
/** | |||
* Copyright 2020 Tianshu AI Platform. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.train.config; | |||
import lombok.Data; | |||
import org.springframework.boot.context.properties.ConfigurationProperties; | |||
import org.springframework.stereotype.Component; | |||
/** | |||
* @description harbor相关配置 | |||
* @date 2020-07-17 | |||
*/ | |||
@Data | |||
@Component | |||
@ConfigurationProperties(prefix = "harbor") | |||
public class TrainHarborConfig { | |||
private String address; | |||
private String username; | |||
private String password; | |||
private String modelName; | |||
} |
@@ -96,7 +96,6 @@ import org.dubhe.train.client.ModelBranchClient; | |||
import org.dubhe.train.client.ModelInfoClient; | |||
import org.dubhe.train.client.NoteBookClient; | |||
import org.dubhe.train.client.ResourceSpecsClient; | |||
import org.dubhe.train.config.TrainHarborConfig; | |||
import org.dubhe.train.config.TrainJobConfig; | |||
import org.dubhe.train.constant.ATlasTrainConstant; | |||
import org.dubhe.train.constant.TrainConstant; | |||
@@ -202,9 +201,6 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
@Autowired | |||
private TrainJobConfig trainJobConfig; | |||
@Autowired | |||
private TrainHarborConfig trainHarborConfig; | |||
@Autowired | |||
private K8sNameTool k8sNameTool; | |||
@@ -484,7 +480,12 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
public void buildImageAndTagInfo(PtJobParam ptJobParam, PtTrainJobDetailVO ptTrainJobDetailVO) { | |||
//image信息拼装 | |||
if (StringUtils.isNotBlank(ptJobParam.getImageName())) { | |||
String imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); | |||
String imageNameSuffix; | |||
if (ptJobParam.getImageName().contains(StrUtil.SLASH)) { | |||
imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); | |||
} else { | |||
imageNameSuffix = ptJobParam.getImageName(); | |||
} | |||
String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); | |||
ptTrainJobDetailVO.setImageName(imageNameSuffixArray[0]); | |||
ptTrainJobDetailVO.setImageTag(imageNameSuffixArray[1]); | |||
@@ -563,11 +564,10 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
.setMemNum(ptTrainJobCreateDTO.getMemNum()) | |||
.setWorkspaceRequest(ptTrainJobCreateDTO.getWorkspaceRequest()) | |||
.setTaskIdentify(taskIdentify); | |||
//例如: 将harbor.dubhe.ai/notebook/notebook:v1 去掉 harbor地址 | |||
String userImageName = trimHarborAddress(ptImageAndAlgorithmVO.getImageName()); | |||
String imageNameAndTag = ptTrainJobCreateDTO.getImageName() + StrUtil.COLON + ptTrainJobCreateDTO.getImageTag(); | |||
//结果集处理 | |||
PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, userContextService.getCurUser(), userImageName, trainKey, baseTrainJobDTO); | |||
PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, userContextService.getCurUser(), imageNameAndTag, trainKey, baseTrainJobDTO); | |||
//添加任务缓存 | |||
resourceCache.addTaskCache(taskIdentify, ptTrainJob.getTrainId(), ptTrainJobCreateDTO.getTrainName(), trainIdPrefix); | |||
// 提交job | |||
@@ -575,16 +575,6 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
return Collections.singletonList(ptTrainJob.getTrainId()); | |||
} | |||
/** | |||
* 去掉harbor地址 | |||
* | |||
* @param imageName | |||
* @return | |||
*/ | |||
private String trimHarborAddress(String imageName) { | |||
return StringUtils.isBlank(imageName) ? StringUtils.EMPTY : imageName.replace(trainHarborConfig.getAddress() + StrUtil.SLASH, StringUtils.EMPTY); | |||
} | |||
/** | |||
* 构建镜像和算法目录VO 考虑到无算法创建 | |||
* | |||
@@ -611,9 +601,8 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
String imageUrl = imageUtil.getImageUrl(ptTrainJobBaseDTO, userContextService.getCurUser()); | |||
String userImageName = imageUrl.split(StrUtil.SLASH)[0] + StrUtil.SLASH + ptTrainJobBaseDTO.getImageName() + StrUtil.COLON + ptTrainJobBaseDTO.getImageTag(); | |||
ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobBaseDTO.getAlgorithmId()); | |||
String imageName = trainHarborConfig.getAddress() + StrUtil.SLASH + userImageName; | |||
ptImageAndAlgorithmVO.setImageName(imageName); | |||
ptImageAndAlgorithmVO.setImageUrl(trainHarborConfig.getAddress() + StrUtil.SLASH + imageUrl); | |||
ptImageAndAlgorithmVO.setImageName(ptTrainJobBaseDTO.getImageName()); | |||
ptImageAndAlgorithmVO.setImageUrl(imageUrl); | |||
} | |||
ptImageAndAlgorithmVO.setRunCommand(ptTrainJobBaseDTO.getRunCommand()); | |||
@@ -676,13 +665,13 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
* | |||
* @param ptTrainJobCreateDTO 创建训练任务DTO | |||
* @param currentUser 用户 | |||
* @param imageName 镜像名称 | |||
* @param imageNameAndTag 镜像名称和tag | |||
* @param trainKey 训练key | |||
* @param baseTrainJobDTO 基础训练参数 | |||
* @return PtTrain 训练 | |||
*/ | |||
private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserContext currentUser, | |||
String imageName, String trainKey, BaseTrainJobDTO baseTrainJobDTO) { | |||
String imageNameAndTag, String trainKey, BaseTrainJobDTO baseTrainJobDTO) { | |||
// 添加train表 | |||
PtTrain ptTrain = new PtTrain(); | |||
ptTrain.setTrainName(ptTrainJobCreateDTO.getTrainName()) | |||
@@ -732,7 +721,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
ptJobParam.setTrainJobId(ptTrainJob.getId()) | |||
.setAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId()) | |||
.setRunCommand(ptTrainJobCreateDTO.getRunCommand()) | |||
.setImageName(imageName) | |||
.setImageName(imageNameAndTag) | |||
.setRunParams(ptTrainJobCreateDTO.getRunParams()) | |||
.setRunParamsNameMap(ptTrainJobCreateDTO.getRunParamsNameMap()) | |||
.setCreateUserId(currentUser.getId()); | |||
@@ -1010,9 +999,8 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
.setWorkspaceRequest(ptTrainJobUpdateDTO.getWorkspaceRequest()) | |||
.setTaskIdentify(taskIdentify); | |||
String userImageName = trimHarborAddress(ptImageAndAlgorithmVO.getImageName()); | |||
//结果集处理 | |||
PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, userContextService.getCurUser(), existPtTrainJob, userImageName, ptTrain, baseTrainJobDTO); | |||
PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, userContextService.getCurUser(), existPtTrainJob, ptTrainJobUpdateDTO.getImageName() + StrUtil.COLON + ptTrainJobUpdateDTO.getImageTag(), ptTrain, baseTrainJobDTO); | |||
//提交job | |||
asyncManager.execute(baseTrainJobDTO, ptTrain.getCreateUserId(), ptImageAndAlgorithmVO, ptTrainJob); | |||
@@ -1025,13 +1013,13 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
* @param ptTrainJobUpdateDTO 更新训练任务DTO | |||
* @param currentUser 当前用户 | |||
* @param existPtTrainJob 存在的训练任务 | |||
* @param imageName 镜像名称 | |||
* @param imageNameAndTag 镜像名称和tag | |||
* @param ptTrain 训练 | |||
* @param baseTrainJobDTO 基本训练信息 | |||
* @return PtTrainJob 训练任务 | |||
*/ | |||
private PtTrainJob updateTrainJobTableData(PtTrainJobUpdateDTO ptTrainJobUpdateDTO, UserContext | |||
currentUser, PtTrainJob existPtTrainJob, String imageName, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) { | |||
currentUser, PtTrainJob existPtTrainJob, String imageNameAndTag, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) { | |||
//检查模型是否合法,合法则保存其路径地址 | |||
checkModelAndSavePath(currentUser, baseTrainJobDTO); | |||
@@ -1070,7 +1058,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { | |||
ptJobParam.setTrainJobId(ptTrainJob.getId()) | |||
.setAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId()) | |||
.setRunCommand(ptTrainJobUpdateDTO.getRunCommand()) | |||
.setImageName(imageName) | |||
.setImageName(imageNameAndTag) | |||
.setRunParams(ptTrainJobUpdateDTO.getRunParams()) | |||
.setRunParamsNameMap(ptTrainJobUpdateDTO.getRunParamsNameMap()) | |||
.setCreateUserId(ptTrain.getCreateUserId()); | |||
@@ -252,10 +252,10 @@ ALTER TABLE `data_task` ADD COLUMN `file_type` smallint(3) NULL COMMENT '待处 | |||
ALTER TABLE `data_task` ADD COLUMN `stop` bit(1) NULL COMMENT '是否停止 0-没有 1-已停止' AFTER `file_type`; | |||
ALTER TABLE `data_task` ADD COLUMN `of_record_version` varchar(255) NULL COMMENT '生成ofRecord的版本号' AFTER `stop`; | |||
-- 医学表 | |||
ALTER TABLE `data_medicine` ADD COLUMN `stop` bit(1) NULL AFTER `annotate_type`; | |||
-- 点云数据集表 | |||
create table if not exists `pc_dataset` ( | |||
`id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键id', | |||
@@ -397,3 +397,5 @@ CREATE TABLE `pt_atlas_camera` ( | |||
PRIMARY KEY (`id`), | |||
UNIQUE KEY `camera_index_uniq` (`camera_index_code`) COMMENT '设备标识唯一' | |||
) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8mb4 COMMENT='模型炼知视频流管理'; | |||
ALTER TABLE pt_image DROP image_status; |
@@ -33,11 +33,11 @@ ribbon: | |||
dubhe-proxy: | |||
visual: | |||
keyword: visual | |||
server: 127.0.0.1 | |||
server: 10.5.26.88 | |||
port: 9898 | |||
refine: | |||
keyword: refine | |||
server: 127.0.0.1 | |||
server: localhost | |||
port: 9797 | |||
## 测试标识 | |||
debug: | |||
@@ -17,7 +17,7 @@ spring: | |||
redis: | |||
#数据库索引 | |||
database: 4 | |||
host: 127.0.0.1 | |||
host: 10.5.26.88 | |||
port: 6379 | |||
password: | |||
#连接超时时间 | |||
@@ -26,9 +26,9 @@ spring: | |||
druid: | |||
db-type: com.alibaba.druid.pool.DruidDataSource | |||
driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | |||
url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||
url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||
username: test | |||
password: test | |||
password: zj12345678 | |||
# 初始化配置 | |||
initial-size: 3 | |||
@@ -67,17 +67,17 @@ logging.config: | |||
# 文件存储服务配置 | |||
storage: | |||
# 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 | |||
file-store: 127.0.0.1 | |||
file-store: 10.5.26.234 | |||
# 文件存储服务端 共享目录 | |||
file-store-root-path: /nfs/ | |||
file-store-root-windows-path: "Z:" | |||
# minio配置 | |||
minio: | |||
url: http://127.0.0.1:9000/ | |||
url: http://10.5.26.90:9000/ | |||
accessKey: admin | |||
secretKey: 123@abc.com | |||
bucketName: dubhe-prod | |||
bucketName: dubhe-cloud-test | |||
presignedUrlExpiryTime: 300 | |||
annotation: /annotation/ | |||
@@ -121,13 +121,6 @@ feign: | |||
okhttp: | |||
enabled: false | |||
#配置harbor | |||
harbor: | |||
address: harbor.dubhe.ai | |||
username: admin | |||
password: Harbor12345 | |||
model-name: train | |||
#配置dev环境单元测试用户名与密码 | |||
unittest: | |||
username: admin | |||
@@ -136,10 +129,10 @@ unittest: | |||
model: | |||
measuring: | |||
url: | |||
package: http://127.0.0.1:32760/model_measure/package | |||
json: http://127.0.0.1:32760/model_measure/measure | |||
package: http://10.5.26.90:32760/model_measure/package | |||
json: http://10.5.26.90:32760/model_measure/measure | |||
converter: | |||
url: http://127.0.0.1:32230/model_convert | |||
url: http://10.5.26.90:32230/model_convert | |||
user: | |||
config: | |||
@@ -5,21 +5,17 @@ k8s: | |||
# k8s集群配置文件 | |||
kubeconfig: kubeconfig_test | |||
# nfs服务暴露的IP地址 如需测试需修改为合适的地址 | |||
nfs: 127.0.0.1 | |||
nfs: 10.5.26.234 | |||
#nfs服务端 共享目录 | |||
nfs-root-path: /nfs/ | |||
nfs-root-windows-path: "Z:" | |||
# 文件存储服务暴露的IP地址 如需测试需修改为合适的地址 | |||
file-store: 127.0.0.1 | |||
file-store: 10.5.26.234 | |||
#文件存储服务端 共享目录 | |||
file-store-root-path: /nfs/ | |||
file-store-root-windows-path: "Z:" | |||
# 命名空间关键字 | |||
namespace: namespace | |||
# k8s ingress域名 如需测试需修改为合适的域名,注意:需要为此域名配置泛域名解析到 k8s集群master节点的ip | |||
host: notebooktest.dubhe.club | |||
# k8s ingress-controller 对外port | |||
port: 30865 | |||
# k8s ingress-controller 对外grpc port | |||
https-port: 31287 | |||
# k8s 模型部署配置 | |||
@@ -27,12 +23,12 @@ k8s: | |||
# 在线服务模型部署后容器域名(k8s ingress域名),解析地址为k8s集群地址,如需测试需修改为合适的域名 | |||
host: servingtest.dubhe.club | |||
# tls 证书 crt | |||
tls-crt: | |||
tls-crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURYekNDQWtlZ0F3SUJBZ0lKQUp3Z0VCYlhGdk9HTUEwR0NTcUdTSWIzRFFFQkN3VUFNRVl4SVRBZkJnTlYKQkFNTUdDb3VjMlZ5ZG1sdVozUmxjM1F1WkhWaWFHVXVZMngxWWpFaE1COEdBMVVFQ2d3WUtpNXpaWEoyYVc1bgpkR1Z6ZEM1a2RXSm9aUzVqYkhWaU1CNFhEVEl4TURVeE9UQTNNRGN5TTFvWERUTXhNRFV4TnpBM01EY3lNMW93ClJqRWhNQjhHQTFVRUF3d1lLaTV6WlhKMmFXNW5kR1Z6ZEM1a2RXSm9aUzVqYkhWaU1TRXdId1lEVlFRS0RCZ3EKTG5ObGNuWnBibWQwWlhOMExtUjFZbWhsTG1Oc2RXSXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBdwpnZ0VLQW9JQkFRREwxNDBMUXk2dXFHUi9WckVXeVNUdFppVGwvOEo1eGlHenB5WUJtTldUTzVQSTczTHRKbXdkCmFPMGNCNXhBcy96SXlpYjM3b0loR0FQejZNWlBBZjJEeUMxNXJLb3Via3h2Vkswa3hJY0k2L1dsWFUxMjdZZzQKdUllNzJEbFNIak9tbStVQ2JDWkxtc0VlQVI2S3RwNW9lVUFUQVgwOWEyL012ZkdtNU1URWZFSVplTENoL21rVAppZGFKcnl0NThwYVFLZmpJcVdKQnlnUGpJYlFMUlZnSG5mckIzdlB5b3RmMFBZZWVKVHlEa1lNQnNzblMxS044ClY2Qldvb2dzWUxQcCs2VE1wOXBHS3pBdksxdXJMWkFsa1N0SUw1TkppQWJOZ1lMOXN1UVE3OU9aano2c1ZrbWQKait6aHZHSGUrYk9zdlBrYmtqNS9lbzQyQUdQcW9zdTlBZ01CQUFHalVEQk9NQjBHQTFVZERnUVdCQlFzWDlCegpxVEhNN21xMUJrOFRNWXgzY1ZRaFB6QWZCZ05WSFNNRUdEQVdnQlFzWDlCenFUSE03bXExQms4VE1ZeDNjVlFoClB6QU1CZ05WSFJNRUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCc1BJSUpPMm1qUFNJV1lTaloKQmdjbFJoUTRudEkvdjFWczVjb0xOVzlrZUhqUGlCYlhvVzgzZExqSWh5ZmVYSk93Y3lpRnJkaVlZeVV5SEprTAprTmlxajRxVmZuaGkwcUoweDQyN3MzUG40TkZZRnNJYVM2OHA1WlZpUlJSZGNWY0dIMkNiRUxzYW41ZUhkNExYCnlTUUdDZmVkUVdtMEVyWWxDMGJWSER1d3RCa0NzWElPRmJEb0tSMW5wMDc3YlBrbEpIRS9rUi82QVpJR3hFM28KbDd1SDVRWWRsWldoanY5WFZRM1plVnZLUzVkM1pHYjcxZ0dwSEU2b2hmL1IweVhoS1RadUREYkhjK2lFc1dWbQowa24xenlvTUVHSW5nM0N6cVk4aE1tdHhRVStLekJrSEliVzh0UnkrS0MxZ2ZvNktFa01JVGpjK2pwQU4wZ2VDCm1LWWYKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= | |||
# tls 证书 key | |||
tls-key: | |||
tls-key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRREwxNDBMUXk2dXFHUi8KVnJFV3lTVHRaaVRsLzhKNXhpR3pweVlCbU5XVE81UEk3M0x0Sm13ZGFPMGNCNXhBcy96SXlpYjM3b0loR0FQego2TVpQQWYyRHlDMTVyS291Ymt4dlZLMGt4SWNJNi9XbFhVMTI3WWc0dUllNzJEbFNIak9tbStVQ2JDWkxtc0VlCkFSNkt0cDVvZVVBVEFYMDlhMi9NdmZHbTVNVEVmRUlaZUxDaC9ta1RpZGFKcnl0NThwYVFLZmpJcVdKQnlnUGoKSWJRTFJWZ0huZnJCM3ZQeW90ZjBQWWVlSlR5RGtZTUJzc25TMUtOOFY2Qldvb2dzWUxQcCs2VE1wOXBHS3pBdgpLMXVyTFpBbGtTdElMNU5KaUFiTmdZTDlzdVFRNzlPWmp6NnNWa21kait6aHZHSGUrYk9zdlBrYmtqNS9lbzQyCkFHUHFvc3U5QWdNQkFBRUNnZ0VCQUtJRG83WDA0dEN0UkpzMVV6RnlFK2lnUyttR201TGdHSDlLOWRsRkdWT1oKZ2IrdUdhRkY3WjUrZkI3UUthLy9ub1lVcmw1VzhwVE5HcTh0THhreTBzV0FRQ3k2UU5VQWs3M1ZCbnozdXdBcgpVeVhvLzUzcjgxNXB4SEJYems5bmM5UVRpalNmc3R4YWx1MTdKRVJLRzZPYjQ0SjNwNHcrclRDRk0rRmJhTTFsCkxuWUh2bGR4ZnRoMmhBWHdaU2dFYXdCNXRBbTY2YnIxbUc4ZW5saEIvbisrY09iYkZWYnk4OUcyUmVMMjJxdHIKa0R0RHVQOGY1NVdxNVB0a25TRE4zaGoweEYydG9aSC80RHpKOWVibVNMVHpUK1pQSjVjK1dlZ3BNdCtVd3YzbwpNNFdGQnFUUUZSR0MyQVJTaXVFTEdtRUJnVk50VDRmRVhRSmFveGJKeWYwQ2dZRUE2MDRJOXlDZUttY2hjSEJvCmRFeFJ2bnNhcGhwbkJqc3pHazZIWVhhTFI0NGpvalRCWm5ZWVhHYklreDdZaytla1RQYjMwRUpzQjlRRmxpVUUKQ25oZXhCS0pjTk5VcTdWdHFGTXBhT0JKWG9Md0cySmVwVmNhN1MzcFo3THZMWGRQR2wvOXFTTW5pMXROV1kxdwpmTGl1Zzlqb3JCeEJaL250NU43MDJFTWpaL2NDZ1lFQTNjVWV2U3U1VmlHaGNPSmFVSlBoUjdnclViMW9EWDF4CnROdDdXNS9pNkFyNDBsY1FZK2RCcll2VEljRllEL3piZnRWNGFnMDhsLzVHamxQK2grWW01QWNRb1c1Z3BaaHgKd1ZhTFFzRTg4ZUdPY2pLakVqd2krTEZRNE5HRmdXdU9pc1hZWk5GTlBvT1JDbHRUbkRMRnFjS0FNY2tBTEgwVwpRT2poTkJqeGhPc0NnWUFRaHRPVVgrTWNBVkJVOEdBMXd1Sis2WENPdUE0Q1h3Z1EwZkVxUkVRMkMyS3ZVdHMrCmtnN1Y5cFloMXluSkFaMEZsdGNDOXBkVjJXdG5CMFNJWTduc05ZMFhzcEFnMjBaUGF6L2VVTnAyVytYM1ZtcGwKWEgvVXBzUGM5N0ZhMVNWbUtkWE1HbDc5cDdVQUZESVJZSHRKWVdPK0t1SGhKcW14eUlNVDZXdEVNd0tCZ0R0ZAo4WVNpbDlLKzNnRGlGMXRLdXh2LzZWalFZM0o1Q2w5b0FmWGRMMWorMXUwMzhXTk5IUC9nVm56S3pWQTZXR2Z6CnJYQjJhcW9sbjYycVBwRVN3NFozZmJRNVlCWDBZVDlvYzQ5RE81VmsxRVV0MlFtZ241d0RtNnNUYTdIaG9SNzEKSjZDVmh5QWRDRTdGYy9SMGd3V1cwOHFBREZQY2lJQ0gxd0dqUzhSUkFvR0JBTklUNW5yTzNwZmNKYTROZWFZeApKMzd5RzFodUJGcWdRRWpVU3pmbHQ3UWwvNjI3LzNWT2lIRjVqSkhES0JqODBEdzh1UjhoQ2VhWDY2WkJqUmxKCk5vOUEvUm43Q2F4YUl2dkNjWG4rUW5oLy9BVlNHRzU4WWI2UUlQNFVqT1RKOW1GUlFFajN0cm9KNnVpL3FCZTQKUkhXSE44WC91Z242dzZUSGRWNktJbSs4Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K | |||
# elasticsearch暴露的服务地址 | |||
elasticsearch: | |||
hostlist: ${eshostlist:127.0.0.1:30498} | |||
hostlist: ${eshostlist:10.5.26.90:30498} | |||
# 日志采集配置信息 | |||
log: | |||
type: _doc | |||
@@ -50,9 +46,9 @@ k8s: | |||
# 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址 | |||
pod: | |||
metrics: | |||
grafanaUrl: http://127.0.0.1:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod= | |||
grafanaUrl: http://10.5.26.90:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod= | |||
prometheus: | |||
url: http://127.0.0.1:30003/ | |||
url: http://10.5.26.90:30003/ | |||
query: api/v1/query | |||
query-range: api/v1/query_range | |||
gpu-query-param: sum(DCGM_FI_DEV_GPU_UTIL{pod="pod-name-placeholder"})by(pod,UUID) | |||
@@ -75,21 +71,40 @@ k8s: | |||
cpu: 10 | |||
memory: 32 | |||
gpu: 2 | |||
#配置harbor | |||
harbor: | |||
address: harbor.dubhe.ai | |||
username: admin | |||
password: Harbor12345 | |||
model-name: train | |||
# minio配置 | |||
minio: | |||
url: http://127.0.0.1:9000/ | |||
url: http://10.5.26.90:9000/ | |||
accessKey: admin | |||
secretKey: 123@abc.com | |||
bucketName: dubhe-prod | |||
bucketName: dubhe-cloud-test | |||
presignedUrlExpiryTime: 300 | |||
annotation: /annotation/ | |||
docker: | |||
remote-api-port: 2375 | |||
clusters: | |||
- cluster: | |||
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1USXdPREE0TWpnd00xb1hEVE14TVRJd05qQTRNamd3TTFvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWVKCkRjTThQUHFlV0tHazE3S2JtN3kwbzkzTGtxb2w5dG1QbVN0YTVyUXZSWU9rOFJYK2QvVEpVaXE2S2c4NXFyMGUKZFZ3SG5vTjg3Q0VFbXhOK2o4OUhlcnY1dDFaRFBHMVRHS3U5N1l1WnV4RCt2R1hGQkhDZ2ViT0s0Zmc0am1iSgpwUmlJeCtUZmlyUjM4TlVhZTBkR1g5c1o1TWF2NkhiSVVEK2hNTm9VU2pyWUZ2bTcxbGtYdlhVWW9wd1FSR3FvCkZiWG5KTFFVakd3UFFRclRCQ0wycFE2c1dGbkpkQ0VBSHhXU0grTUN4cXV0L25SUHhORlFOWGFMZmU3clB3a3MKd3NrVlFlTzgyL205aFZkWnNnNUFnRDJtTS9jS2VybE9acVBGcWFEaFBoYnpkRzF4QjRtcVZmTGJVUUppa3I1SQpYQ1FhTWR2c2FRTnhPcmNPOHQwQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZFek92OGhVUGtRYS90bGNhQTE0OERmcmF5TXRNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFBMTFqcDR4V1hyZkRPNDJlblBpdm9HNXZxY3hqbysxaktZV1YxR1NIdHNVdlBNU1UzVAptdVFCQWU2VVRzM0JtSTgzMUQ4aFd4cDlka3BkMlhWbCt6OTZabFIvQ2Qrb3NSWWE1MUpZd2RvMUp4Z29hbUk2CmszOS9sNTRBQVpWMkFPWWNQU0pKNkNzdVcxZ2NTMDlaTlA0djlhK0NIKzY5OVpQMGlhUStEc0FPRFJzUnhRcVMKK08zNUU4K0RzRjBSOWZCYlhvbkxiK05VSHlTQ2pQSnVVT21lS0RYMFpQQm81blNscmh5cUIxbHVNZnhEaUltdgpIU3BscXB4M1duMzJnNEZYOCs3clZBNUZjWkIvNDg3bUZmdG1CTXc3aENidlFJeWY2Ymd6VTUycUdaeFZGTTVjCnUxeEZyRXlXaFA1eElJY09lY1M1RCtJa29ZOHUyT3ozQldEeQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== | |||
server: https://10.105.0.6:6443 | |||
name: kubernetes | |||
contexts: | |||
- context: | |||
cluster: kubernetes | |||
user: kubernetes-admin | |||
name: kubernetes-admin@kubernetes | |||
current-context: kubernetes-admin@kubernetes | |||
kind: Config | |||
preferences: {} | |||
users: | |||
- name: kubernetes-admin | |||
user: | |||
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFekNDQWZ1Z0F3SUJBZ0lJQjUxN3hkT3V2cnd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TVRFeU1EZ3dPREk0TUROYUZ3MHlNakV5TURnd09ESTRNRFZhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXlFaURrYVJ1WmVUbVJ4WmsKRkxUbkRPMUxIM1k5dE80WXpXREt4QVA2bFphTUJwZXJWRWpiU1VMOGIzdG9WWFppVnVnaHhvMzRHdjF4dlJKLwpDN3NhWGVaU3EzcTVMcEFhQjBrelRDNFRtbGtrdE03YkJTZFNhMG9SeGJXTUdvVEM0WUVBckJCNS9FRmdUYjRCCnE4aG1USUlmdUcwa05sNmd5a2M4UnhSL3N5Q1lSVjZCTTZTYWFzODR1Tmx3ZU9vV2RaQkQ0UVZQbzJueUI4NGQKUnc0d0w2VEQ5MlUwMG5KcjY3bUtpTjhBNGtBampFbWNrNm1JbHdsUTB2MHR5Uk8zTFBXZFdNazZ5QVIyVEZ6RgpBZCs4dUd5L3VDQlB6UDlVcDFERkI5TDF0Y25sd1lTeTYycGdFWC9wMzl6ZjZ0RFpZRVJwK0Jpd21Eb3RmVmFaClNLam9md0lEQVFBQm8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0h3WURWUjBqQkJnd0ZvQVVUTTYveUZRK1JCcisyVnhvRFhqd04rdHJJeTB3RFFZSktvWklodmNOQVFFTApCUUFEZ2dFQkFCQ01US2VEV1dHa0VFZWp6dlN3TTFKZit0QUJaYklrd0FoZEt1MjFnUkxyQktzbHQ4Y0NHMG1KCmc3eDhndFJYcW0xRGlEU01DMnlWMW9OWTFSdGlLZVhZT29RbVY2SFh2bEQybWZyY2VzS3VSNHlIbCtlWHcyL1AKb2FLZVFEM21yVm9ZZ1ZMMVNqOWpzcDRBa1hEM1ErUzdKYXN4Y3MvVkR3SmlRQzhGamkzaWZMVFFKQkxCdm1WVwpCZ3QvM0xPZ1lCdWxYRWg3cFlQUzduTVFqNFpxQ0QwWTNJOENicUM0VGlZQjB1aTZlbVRDd3cvRnBNSzNIRFZ4CnZOd3FPRHBaWW5DK0IxSzR5RUk4NGIvM1o3ZXA0MUE1NGk3QkpxTklORUVtRjdGcmpQQy91V1RuYlpzVUNFQjAKWTFUR1Y3czNBUTZMZXNCRFF6UVRzVnBENWFRRk9VYz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= | |||
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBeUVpRGthUnVaZVRtUnhaa0ZMVG5ETzFMSDNZOXRPNFl6V0RLeEFQNmxaYU1CcGVyClZFamJTVUw4YjN0b1ZYWmlWdWdoeG8zNEd2MXh2UkovQzdzYVhlWlNxM3E1THBBYUIwa3pUQzRUbWxra3RNN2IKQlNkU2Ewb1J4YldNR29UQzRZRUFyQkI1L0VGZ1RiNEJxOGhtVElJZnVHMGtObDZneWtjOFJ4Ui9zeUNZUlY2QgpNNlNhYXM4NHVObHdlT29XZFpCRDRRVlBvMm55Qjg0ZFJ3NHdMNlREOTJVMDBuSnI2N21LaU44QTRrQWpqRW1jCms2bUlsd2xRMHYwdHlSTzNMUFdkV01rNnlBUjJURnpGQWQrOHVHeS91Q0JQelA5VXAxREZCOUwxdGNubHdZU3kKNjJwZ0VYL3AzOXpmNnREWllFUnArQml3bURvdGZWYVpTS2pvZndJREFRQUJBb0lCQUgrbHY3ME9TSkpHZmdHbQpvcWlUMTRKa3BuRnA0ZEF6dzdqNXpLRjdTN1VWR3krRWNOeXFCcUM5d2JlbnRvcHBoaW1Qang5R0VtL1pRaWxYCjVZTHJmOVdDMndPUmx2NjNOdStYMXNyaHZ2cXJmL3FBc0JTcnlCcTdQWEo5ejhxQy9OWE9hMGcreEJCaTltYjIKQjRpZGs1MkZmWVFFZzRUbmNLRWJINjdKd00yL2dmZ3BXMGNEd0RiNVc2aHIxclUzNkNGc1AyVTlEYytMVkV5cQpvK3FZZ0RFZ0MxSWxucGVIdzROejJmK3RkbFpLVHN5dzV1ZG5xVm1kWGNkLyszR3NvQVM3Q25DTmQ3a1pKQ1BPClNpRktZSDZZUS93ZmJLZjFibC84eERXU1UvKytQUTZyODJGV21mMFBjd3ExbWV5OXIvUW1kaHc5YndiamkxVDQKdURjSzUva0NnWUVBOURIZi8xZnN4czlqVldoaE94UXFzUFN3UzdDbVZYcHV2S2Q3MjlMRGFIV1VQT3E5Y0l4QQpaV01Ibi9EbEN3U2dxaitPL0FNSDdaQVpPbDV5V2RiZFVEM2I0WWVKLzlubXdvN25ReG9ndTlUdTlNZEVnM0pvCm1wTzVqQTYzYTE2Z0sxRWlmTDJkdllRQkJGQzg2aDJzYlV0WHNGRDV6R3QzTU96bVF6SFFQM3NDZ1lFQTBmY3gKMzlUR0YrZFdVb3MzOXRLaE9MZVpFaWtUSmtqckdrTUdldHpYNEVCRE5oRUUvdWNkYUZCWmJoU3ltZHAwMmtKNgpKQkpvaEJRaEN4Zk9nUXhRdFp2dUt3WUMvTEYrMVF1TG1MaERGV2NEa05Ma1E0RWU5WVRFV1VuNE9NNkJJTmVOCldseklzcncyeHRvaUVoMXZFWW92ODdobEZCQ0E4N0xYSElpa1NjMENnWUFnZ1pNajFueDZhcGo2Z1k2UDRydGMKR3ZaczNQTUhaZWpmekJ1OHcyMm50aDhwak1YeStYaUpCb201VE1Qd2w0a2JvS0pVQWNOSWFHb3pUdWRCOWt3Mwpwa1JpM1R3Tmh1QWsvZ2ppOGROeHJZS3hxdVQvNm5icVBNZnlVUE14bzhNR2dTSnFJSU9pK3JOMzg1ZHlRc2oxCk01WlVyakxTZDJ2Q1k0YXpZeW9waVFLQmdBS0RBb1c2OWVPelVNVU1CVTllbGE4b0F4ekVnUVlrT1N2SFFYeVcKeDc1WEJuYlBIVkF2VTVxNzIxWUZ5VjB0ejlnTWs0bDY3dVVsbUgrWmVVN3g1c1ZGYUVQN2ZtMm5jZXo2aG1EOQpVMUFlTzF5d0tTcmxrSWsvWFZuMEdKUVZaRllRUGhDbXEzLzM0Ry9nakNmTFVsalRYbDk4QlRtSU9RS3hVUUYyCllRREJBb0dBRTB5Q0NzcFE3Y1lYaVFyQnF4VVkxU2N5SjViVm1iaFN5cmgyb3FiUmRMRmxzWHFjUnBIK2tORjYKdXdjUzhZalJodmdQazgyZm1QcXlieE5ZWGxTYWoydGdzSFdpcGNETXZSM1dtSlQzTGpjeXZObDh6cUlkZDNBdQpJT1pNeG0vWWVMemNFOUdteDRsQkhmMnU5MkhiRjRiSCtqeVp3d3RWcHpRdmphaFF6dGs9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== | |||
#配置harbor | |||
harbor: | |||
address: harbor.dubhe.ai | |||
username: admin | |||
password: Harbor12345 |
@@ -6,9 +6,9 @@ spring: | |||
master: | |||
type: com.alibaba.druid.pool.DruidDataSource | |||
driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | |||
url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||
url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true | |||
username: test | |||
password: test | |||
password: zj12345678 | |||
props: | |||
sql: | |||
show: true | |||
@@ -18,7 +18,7 @@ data: | |||
# 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 | |||
dcm: | |||
host: 127.0.0.1 | |||
host: 10.5.29.100 | |||
port: 11112 | |||
# 资源回收 | |||
@@ -27,13 +27,13 @@ recycle: | |||
# 数据处理医学影像数据集dcm服务器配置,查看影像功能需要使用该服务 | |||
dcm: | |||
host: 127.0.0.1 | |||
host: 10.5.29.100 | |||
port: 11112 | |||
# ES服务地址及端口 | |||
es: | |||
host: 127.0.0.1 | |||
serverPort: 31321 | |||
transportPort: 31322 | |||
clusterName: kubenets-logging | |||
host: 10.5.29.100 | |||
serverPort: 9200 | |||
transportPort: 9300 | |||
clusterName: docker-cluster | |||
index: dataset_text_test |
@@ -32,11 +32,11 @@ spring: | |||
precise-algorithm-class-name: org.dubhe.data.util.MyPreciseShardingAlgorithm | |||
k8s: | |||
# nfs服务暴露的IP地址 如需测试需修改为合适的地址 | |||
nfs: 127.0.0.1 | |||
nfs: 10.5.26.234 | |||
#nfs服务端 共享目录 | |||
nfs-root-path: /nfs/ | |||
nfs-root-windows-path: "Z:" | |||
file-store: 127.0.0.1 | |||
file-store: 10.5.26.234 | |||
#文件存储服务端 共享目录 | |||
file-store-root-path: /nfs/ | |||
file-store-root-windows-path: "Z:" | |||
@@ -80,8 +80,8 @@ recycle: | |||
# ES服务地址及端口 | |||
es: | |||
host: 127.0.0.1 | |||
serverPort: 32321 | |||
transportPort: 32322 | |||
clusterName: kubenets-logging | |||
host: 10.5.29.100 | |||
serverPort: 9200 | |||
transportPort: 9300 | |||
clusterName: docker-cluster | |||
index: dataset_text_test |
@@ -3,7 +3,7 @@ notebook-specs: | |||
# cpu数量(核) | |||
cpu-num: 1 | |||
# gpu数量(核) | |||
gpu-num: 1 | |||
gpu-num: 0 | |||
# 内存大小(M) | |||
mem-num: 1024 | |||
# 工作空间配额(m) | |||
@@ -22,8 +22,8 @@ management: | |||
serving: | |||
gateway: | |||
# 在线服务http接口域名,解析地址为serving网关部署的服务器地址 | |||
postfixUrl: .openapi.dubhe.club | |||
# 在线服务http接口 host:port,解析地址为serving网关部署的服务地址 | |||
postfixUrl: 10.101.12.41:8081 | |||
token: | |||
secret-key: 1n3512x | |||
expire-seconds: 108000 | |||
@@ -31,7 +31,6 @@ serving: | |||
httpclient: | |||
connect-timeout: 1000 | |||
response-timeout: 3000 | |||
#关闭熔断机制 | |||
feign: | |||
hystrix: |
@@ -1,52 +0,0 @@ | |||
# serving预测上传文件大小限制 | |||
spring: | |||
servlet: | |||
multipart: | |||
# 设置单个文件的大小 | |||
max-file-size: 10MB | |||
# 设置单次请求的文件的总大小 | |||
max-request-size: 100MB | |||
k8s: | |||
# nfs服务暴露的IP地址 如需测试需修改为合适的地址 | |||
nfs: 127.0.0.1 | |||
#nfs服务端 共享目录 | |||
nfs-root-path: /nfs/ | |||
nfs-root-windows-path: "Z:" | |||
# data模块配置 | |||
data: | |||
annotation: | |||
task: | |||
# 自动标注任务分割的文件split size | |||
splitSize: 16 | |||
# 单位ms | |||
retryInterval: 5000 | |||
# 任务不更新置为失败的时间单位秒 | |||
failTime: 43200 | |||
server: | |||
# 文件存储服务器用户名 | |||
userName: root | |||
#数据集训练配置 | |||
ptversion: http://127.0.0.1:8000/ | |||
minioweb: | |||
GetToken: | |||
url: minio/webrpc | |||
param: | |||
id: 2 | |||
jsonrpc: 2.0 | |||
method: Web.Login | |||
zip: | |||
url: minio/zip?token= | |||
#logback | |||
logging.config: | |||
classpath:logback-spring-dev.xml | |||
# ES服务地址及端口 | |||
es: | |||
host: 127.0.0.1 | |||
serverPort: 32321 | |||
transportPort: 32322 | |||
index: dataset_text_test |
@@ -5,7 +5,7 @@ terminal: | |||
# 用户workspace目录 | |||
workspace-dir: "/workspace" | |||
# ssh主机 | |||
ssh-host: 127.0.0.1 | |||
ssh-host: 10.5.26.91 | |||
# 可匿名访问路径 | |||
security: | |||
@@ -4,149 +4,139 @@ spring: | |||
gateway: | |||
enabled: true | |||
routes: | |||
# id唯一表示 路由至admin服务 | |||
- id: admin-route | |||
# uri=> lb://服务名称 | |||
uri: lb://admin | |||
# 路由规则 | |||
predicates: | |||
- Path=/api/v1/admin/** | |||
filters: | |||
- StripPrefix=3 | |||
- id: visual-route | |||
# uri=> lb://服务名称 | |||
uri: lb://admin | |||
# 路由规则 | |||
predicates: | |||
- Path=/visual/api/** | |||
# 路由至auth服务 | |||
- id: auth-route | |||
uri: lb://auth | |||
predicates: | |||
- Path=/api/v1/auth/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至notebook服务 | |||
- id: notebook-route | |||
uri: lb://dubhe-notebook | |||
predicates: | |||
- Path=/api/v1/notebook/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至镜像管理服务 | |||
- id: image-route | |||
uri: lb://dubhe-image | |||
predicates: | |||
- Path=/api/v1/image/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至度量管理服务 | |||
- id: measure-route | |||
uri: lb://dubhe-measure | |||
predicates: | |||
- Path=/api/v1/measure/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-k8s服务 | |||
- id: dubhe-k8s | |||
uri: lb://dubhe-k8s | |||
predicates: | |||
- Path=/api/v1/k8s/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-model(模型管理)服务 | |||
- id: dubhe-model | |||
uri: lb://dubhe-model | |||
predicates: | |||
- Path=/api/v1/model/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-algorithm(算法管理)服务 | |||
- id: dubhe-algorithm | |||
uri: lb://dubhe-algorithm | |||
predicates: | |||
- Path=/api/v1/algorithm/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-train(训练管理)服务 | |||
- id: dubhe-train | |||
uri: lb://dubhe-train | |||
predicates: | |||
- Path=/api/v1/train/** | |||
filters: | |||
- StripPrefix=3 | |||
# id唯一表示 路由至data(数据集)服务 | |||
- id: dubhe-data | |||
# uri=> lb://服务名称 | |||
uri: lb://dubhe-data | |||
# 路由规则 | |||
predicates: | |||
- Path=/api/v1/data/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-optimize(模型优化)服务 | |||
- id: dubhe-optimize | |||
uri: lb://dubhe-optimize | |||
predicates: | |||
- Path=/api/v1/optimize/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-serving(云端Serving)服务 | |||
- id: dubhe-serving | |||
uri: lb://dubhe-serving | |||
predicates: | |||
- Path=/api/v1/serving/**,/api/v1/batchServing/** | |||
filters: | |||
- StripPrefix=3 | |||
# 负载均衡样例 | |||
- id: weight-route1 | |||
uri: lb://demo-client | |||
# 路由规则 | |||
predicates: | |||
# 接口匹配前缀 | |||
- Path=/api/v1/w/** | |||
# 权重组weight-group1,权重比例 1/4 | |||
- Weight=weight-group1, 1 | |||
filters: | |||
# 根据接口匹配前缀截取 | |||
- StripPrefix=3 | |||
- id: weight-route2 | |||
uri: lb://demo-provider | |||
# 路由规则 | |||
predicates: | |||
- Path=/api/v1/w/** | |||
# 权重组weight-group1,权重比例 3/4 | |||
- Weight=weight-group1, 3 | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-data-dcm(医学数据集)服务 | |||
- id: dubhe-data-dcm | |||
uri: lb://dubhe-data-dcm | |||
predicates: | |||
- Path=/api/v1/dcm/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-terminal 天枢专业版终端 | |||
- id: dubhe-terminal | |||
uri: lb://dubhe-terminal | |||
predicates: | |||
- Path=/api/v1/terminal/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-tadl服务 | |||
- id: dubhe-tadl | |||
uri: lb://dubhe-tadl | |||
predicates: | |||
- Path=/api/v1/tadl/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-point-cloud服务 | |||
- id: dubhe-point-cloud | |||
uri: lb://dubhe-point-cloud | |||
predicates: | |||
- Path=/api/v1/pointCloud/** | |||
filters: | |||
- StripPrefix=3 | |||
server: | |||
maxPostSize: -1 | |||
maxHttpHeaderSize: 10240000 | |||
# id唯一表示 路由至admin服务 | |||
- id: admin-route | |||
# uri=> lb://服务名称 | |||
uri: lb://admin | |||
# 路由规则 | |||
predicates: | |||
- Path=/api/v1/admin/** | |||
filters: | |||
- StripPrefix=3 | |||
- id: visual-route | |||
# uri=> lb://服务名称 | |||
uri: lb://admin | |||
# 路由规则 | |||
predicates: | |||
- Path=/visual/api/** | |||
# 路由至demo-provider服务 | |||
- id: nacos-server-route | |||
uri: lb://demo-provider | |||
predicates: | |||
- Path=/api/v1/server/**,/api/v1/service/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至auth服务 | |||
- id: auth-route | |||
uri: lb://auth | |||
predicates: | |||
- Path=/api/v1/auth/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至notebook服务 | |||
- id: notebook-route | |||
uri: lb://dubhe-notebook | |||
predicates: | |||
- Path=/api/v1/notebook/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至镜像管理服务 | |||
- id: image-route | |||
uri: lb://dubhe-image | |||
predicates: | |||
- Path=/api/v1/image/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至度量管理服务 | |||
- id: measure-route | |||
uri: lb://dubhe-measure | |||
predicates: | |||
- Path=/api/v1/measure/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-k8s服务 | |||
- id: dubhe-k8s | |||
uri: lb://dubhe-k8s | |||
predicates: | |||
- Path=/api/v1/k8s/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-model(模型管理)服务 | |||
- id: dubhe-model | |||
uri: lb://dubhe-model | |||
predicates: | |||
- Path=/api/v1/model/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-algorithm(算法管理)服务 | |||
- id: dubhe-algorithm | |||
uri: lb://dubhe-algorithm | |||
predicates: | |||
- Path=/api/v1/algorithm/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-train(训练管理)服务 | |||
- id: dubhe-train | |||
uri: lb://dubhe-train | |||
predicates: | |||
- Path=/api/v1/train/** | |||
filters: | |||
- StripPrefix=3 | |||
# id唯一表示 路由至data(数据集)服务 | |||
- id: dubhe-data | |||
# uri=> lb://服务名称 | |||
uri: lb://dubhe-data | |||
# 路由规则 | |||
predicates: | |||
- Path=/api/v1/data/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-optimize(模型优化)服务 | |||
- id: dubhe-optimize | |||
uri: lb://dubhe-optimize | |||
predicates: | |||
- Path=/api/v1/optimize/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-serving(云端Serving)服务 | |||
- id: dubhe-serving | |||
uri: lb://dubhe-serving | |||
predicates: | |||
- Path=/api/v1/serving/**,/api/v1/batchServing/** | |||
filters: | |||
- StripPrefix=3 | |||
# 负载均衡样例 | |||
- id: weight-route1 | |||
uri: lb://demo-client | |||
# 路由规则 | |||
predicates: | |||
# 接口匹配前缀 | |||
- Path=/api/v1/w/** | |||
# 权重组weight-group1,权重比例 1/4 | |||
- Weight=weight-group1, 1 | |||
filters: | |||
# 根据接口匹配前缀截取 | |||
- StripPrefix=3 | |||
- id: weight-route2 | |||
uri: lb://demo-provider | |||
# 路由规则 | |||
predicates: | |||
- Path=/api/v1/w/** | |||
# 权重组weight-group1,权重比例 3/4 | |||
- Weight=weight-group1, 3 | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-data-dcm(医学数据集)服务 | |||
- id: dubhe-data-dcm | |||
uri: lb://dubhe-data-dcm | |||
predicates: | |||
- Path=/api/v1/dcm/** | |||
filters: | |||
- StripPrefix=3 | |||
# 路由至dubhe-tadl服务 | |||
- id: dubhe-tadl | |||
uri: lb://dubhe-tadl | |||
predicates: | |||
- Path=/api/v1/tadl/** | |||
filters: | |||
- StripPrefix=3 |
@@ -3,7 +3,7 @@ spring: | |||
redis: | |||
#数据库索引 | |||
database: 1 | |||
host: 127.0.0.1 | |||
host: 10.5.26.88 | |||
port: 6379 | |||
password: | |||
#连接超时时间 | |||
@@ -12,9 +12,9 @@ spring: | |||
druid: | |||
db-type: com.alibaba.druid.pool.DruidDataSource | |||
driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy | |||
url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-cloud-prod?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true | |||
url: jdbc:log4jdbc:mysql://127.0.0.1/dubhe-cloud-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true | |||
username: test | |||
password: test | |||
password: zj12345678 | |||
# 初始化配置 | |||
initial-size: 3 | |||
@@ -43,11 +43,5 @@ spring: | |||
url-pattern: /* | |||
exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" | |||
#配置harbor | |||
harbor: | |||
address: harbor.dubhe.ai | |||
username: admin | |||
password: Harbor12345 | |||
@@ -1,19 +0,0 @@ | |||
apiVersion: v1 | |||
clusters: | |||
- cluster: | |||
certificate-authority-data: | |||
server: https://127.0.0.1:6443 | |||
name: kubernetes | |||
contexts: | |||
- context: | |||
cluster: kubernetes | |||
user: kubernetes-admin | |||
name: kubernetes-admin@kubernetes | |||
current-context: kubernetes-admin@kubernetes | |||
kind: Config | |||
preferences: {} | |||
users: | |||
- name: kubernetes-admin | |||
user: | |||
client-certificate-data: | |||
client-key-data: |
@@ -3,7 +3,7 @@ spring: | |||
redis: | |||
#数据库索引 | |||
database: 1 | |||
host: 127.0.0.1 | |||
host: 10.5.26.88 | |||
port: 6379 | |||
password: | |||
#连接超时时间 | |||
@@ -16,10 +16,10 @@ model: | |||
atlas: | |||
url: | |||
# 流服务地址 | |||
rtmp: rtmp://127.0.0.1:1935/live/ | |||
rtmp: rtmp://10.5.30.42:1935/live/ | |||
# vms接口地址 | |||
vms: http://127.0.0.1:8000/api/vms | |||
vms: http://10.105.10.51:8000/api/vms | |||
# 推理服务地址 | |||
video-serve: http://127.0.0.1:32761/video | |||
video-serve: http://10.105.0.6:32761/video | |||
# 拉流地址,需要拼接streamId | |||
stream: http://127.0.0.1:8055/live?port=1935&app=live&stream= | |||
stream: http://10.5.30.42:8055/live?port=1935&app=live&stream= |
@@ -14,16 +14,16 @@ VUE_APP_DATA_API = '/' | |||
VUE_APP_VISUAL_API = '/' | |||
# minio | |||
VUE_APP_MINIO_API = 'http://127.0.0.1:9000/minio' | |||
VUE_APP_MINIO_API = 'http://10.105.1.132:9000/minio' | |||
# atlas | |||
VUE_APP_ATLAS_HOST = 'http://127.0.0.1' | |||
VUE_APP_ATLAS_HOST = 'http://183.129.174.186' | |||
# DCM4CHEE | |||
VUE_APP_DCM_API = 'http://pre.dubhe.club/dcm4chee/dcm4chee-arc/aets/DCM4CHEE_ADMIN' | |||
# minIO 服务 IP | |||
VUE_APP_MINIO_ENDPOINT = '127.0.0.1' | |||
VUE_APP_MINIO_ENDPOINT = '10.105.1.132' | |||
# minIO 服务 端口 | |||
VUE_APP_MINIO_PORT = '9000' | |||
# 是否开启 SSL | |||
@@ -5,4 +5,5 @@ dist | |||
src/components/Crud | |||
mock | |||
src/views/visual | |||
src/store/modules/Visual | |||
src/store/modules/Visual | |||
src/utils/VisualUtils |
@@ -1,12 +1,9 @@ | |||
const plugins = [ | |||
"@vue/babel-plugin-transform-vue-jsx", | |||
"@babel/plugin-proposal-optional-chaining", | |||
]; | |||
const plugins = ['@vue/babel-plugin-transform-vue-jsx', '@babel/plugin-proposal-optional-chaining']; | |||
// 生产环境移除console | |||
if (process.env.NODE_ENV === "production") { | |||
plugins.push("transform-remove-console"); | |||
if (process.env.NODE_ENV === 'production') { | |||
plugins.push('transform-remove-console'); | |||
} | |||
module.exports = { | |||
plugins, | |||
presets: [["@vue/app",{ useBuiltIns: "entry" }]], | |||
presets: [['@vue/app', { useBuiltIns: 'entry' }]], | |||
}; |
@@ -24,6 +24,18 @@ | |||
"lint:style": "stylelint src/**/*.{html,vue,css,sass,scss}", | |||
"fix:style": "stylelint src/**/*.{html,vue,css,sass,scss} --fix" | |||
}, | |||
"husky": { | |||
"hooks": { | |||
"pre-commit": "lint-staged", | |||
"commit-msg": "commitlint -E HUSKY_GIT_PARAMS" | |||
} | |||
}, | |||
"lint-staged": { | |||
"src/**/*.{js,vue}": [ | |||
"eslint --fix", | |||
"git add" | |||
] | |||
}, | |||
"repository": { | |||
"type": "git", | |||
"url": "git@codeup.teambition.com:zhejianglab/dubhe-web.git" | |||
@@ -19,7 +19,7 @@ import { API_MODULE_NAME } from '@/config'; | |||
export function list(params) { | |||
return request({ | |||
url: `/${API_MODULE_NAME.IMAGE}/ptImage/info`, | |||
url: `/${API_MODULE_NAME.IMAGE}/ptImage/list`, | |||
method: 'get', | |||
params, | |||
}); | |||
@@ -27,7 +27,7 @@ export function list(params) { | |||
export function add(data) { | |||
return request({ | |||
url: `/${API_MODULE_NAME.IMAGE}/ptImage/uploadImage`, | |||
url: `/${API_MODULE_NAME.IMAGE}/ptImage`, | |||
method: 'post', | |||
data, | |||
}); | |||
@@ -65,14 +65,6 @@ export function getImageTagList(params) { | |||
}); | |||
} | |||
export function setPrecast(params) { | |||
return request({ | |||
url: `/${API_MODULE_NAME.IMAGE}/ptImage/imageResource`, | |||
method: 'put', | |||
params, | |||
}); | |||
} | |||
// 设置notebook默认镜像-获取默认镜像 | |||
export function getDefaultImage(params) { | |||
return request({ | |||
@@ -332,6 +332,10 @@ | |||
width: 320px; | |||
} | |||
.w-400 { | |||
width: 400px; | |||
} | |||
.w-500 { | |||
width: 500px; | |||
} | |||
@@ -9,6 +9,9 @@ the License. * ============================================================= */ | |||
<template> | |||
<el-tooltip v-bind="mergedAttrs"> | |||
<i class="primary f18 vm" :class="[icon]" /> | |||
<template #content> | |||
<slot name="content" /> | |||
</template> | |||
</el-tooltip> | |||
</template> | |||
@@ -541,6 +541,7 @@ import { | |||
RESOURCES_MODULE_ENUM, | |||
RESOURCES_POOL_TYPE_ENUM, | |||
RESOURCES_POOL_TYPE_MAP, | |||
IMAGE_TYPE_ENUM, | |||
} from '@/utils'; | |||
import { list as getAlgorithmList, getAlgorithmInfo } from '@/api/algorithm/algorithm'; | |||
import { getModelByResource } from '@/api/model/model'; | |||
@@ -551,7 +552,7 @@ import { list as getSpecsNames } from '@/api/system/resources'; | |||
import { list as getNotebooks } from '@/api/development/notebook'; | |||
import { trainConfig } from '@/config'; | |||
import { NOTEBOOK_STATUS_ENUM } from '@/views/development/utils'; | |||
import { IMAGE_TYPE, TRAINING_TYPE_ENUM } from '@/views/trainingJob/utils'; | |||
import { TRAINING_TYPE_ENUM } from '@/views/trainingJob/utils'; | |||
import BaseTooltip from '@/components/BaseTooltip'; | |||
import DataSourceSelector from './dataSourceSelector'; | |||
@@ -929,7 +930,7 @@ export default { | |||
}, | |||
async getHarborProjects() { | |||
this.harborProjectList = await getImageNameList({ | |||
imageTypes: [IMAGE_TYPE.TRAIN], | |||
imageTypes: [IMAGE_TYPE_ENUM.TRAIN], | |||
}); | |||
if ( | |||
this.form.imageName && | |||
@@ -39,7 +39,7 @@ const covertSVG2Image = (node, name, width, height, type = 'png') => { | |||
const context = canvas.getContext('2d'); | |||
context.fillStyle = '#fff'; | |||
context.fillRect(0, 0, 10000, 10000); | |||
image.onload = function() { | |||
image.onload = () => { | |||
context.drawImage(image, 0, 0); | |||
const a = document.createElement('a'); | |||
a.download = `${name}.${type}`; | |||
@@ -133,6 +133,24 @@ export const K8S_BUSINESS_LABEL_MAP = { | |||
[K8S_BUSINESS_LABEL_ENUM.POINT_CLOUD]: '点云数据集', | |||
}; | |||
// 镜像分类枚举 | |||
export const IMAGE_TYPE_ENUM = { | |||
NOTEBOOK: 0, // notebook镜像类型 | |||
TRAIN: 1, // 训练镜像和预置镜像类型 | |||
SERVING: 2, // serving镜像 | |||
TERMINAL: 3, // 终端镜像 | |||
POINTCLOUD: 4, // 点云镜像 | |||
DATASETMARKED: 5, // 数据标注镜像 | |||
}; | |||
export const IMAGE_TYPE_MAP = { | |||
[IMAGE_TYPE_ENUM.NOTEBOOK]: 'Notebook镜像', | |||
[IMAGE_TYPE_ENUM.TRAIN]: '训练镜像', | |||
[IMAGE_TYPE_ENUM.SERVING]: 'Serving镜像', | |||
[IMAGE_TYPE_ENUM.TERMINAL]: '终端镜像', | |||
[IMAGE_TYPE_ENUM.POINTCLOUD]: '点云镜像', | |||
[IMAGE_TYPE_ENUM.DATASETMARKED]: '数据标注镜像', | |||
}; | |||
// 默认进度条颜色 | |||
export const defaultProcessColors = [ | |||
{ color: '#909399', percentage: 40 }, | |||
@@ -20,7 +20,7 @@ import ZIP from './zip'; | |||
const pMap = require('p-map'); | |||
//streamSaver.mitm = 'https://static.zhejianglab.com/mitm.html'; | |||
streamSaver.mitm = 'https://static.zhejianglab.com/mitm.html'; | |||
// 默认名字解析 | |||
const defaultName = (file) => file.name; | |||
@@ -171,13 +171,13 @@ import { | |||
invalidFileNameChar, | |||
defaultProcessColors, | |||
ALGORITHM_RESOURCE_ENUM, | |||
IMAGE_TYPE_ENUM, | |||
} from '@/utils'; | |||
import UploadInline from '@/components/UploadForm/inline'; | |||
import UploadProgress from '@/components/UploadProgress'; | |||
import { useMapGetters } from '@/hooks'; | |||
import { algorithmConfig } from '@/config'; | |||
import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
const defaultForm = { | |||
id: null, | |||
@@ -289,7 +289,7 @@ export default { | |||
}; | |||
// 获取镜像名列表 | |||
const getImageNames = async (keepValue = false) => { | |||
state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.TRAIN] }); | |||
state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.TRAIN] }); | |||
if (!keepValue || !form.imageName) { | |||
form.imageTag = null; | |||
} else if (!state.imageNameList.includes(form.imageName)) { | |||
@@ -199,14 +199,13 @@ | |||
<script> | |||
import { mapGetters } from 'vuex'; | |||
import { getUniqueId, invalidFileNameChar, RESOURCES_MODULE_ENUM } from '@/utils'; | |||
import { getUniqueId, invalidFileNameChar, RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils'; | |||
import { getServingModel } from '@/api/model/model'; | |||
import { list as getModelVersions } from '@/api/model/modelVersion'; | |||
import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; | |||
import { list as getSpecsNames } from '@/api/system/resources'; | |||
import { validateNameWithHyphen } from '@/utils/validate'; | |||
import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
import RunParamForm from '@/components/Training/runParamForm'; | |||
import BaseModal from '@/components/BaseModal'; | |||
import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; | |||
@@ -541,7 +540,7 @@ export default { | |||
// 镜像选择 | |||
async getImageNames(keepValue = false) { | |||
this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.SERVING] }); | |||
this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.SERVING] }); | |||
if (!keepValue || !this.form.imageName) { | |||
this.form.imageTag = null; | |||
} else if (!this.imageNameList.includes(this.form.imageName)) { | |||
@@ -204,8 +204,7 @@ import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
import { getInferenceAlgorithm, add as addAlgorithm } from '@/api/algorithm/algorithm'; | |||
import { list as getSpecsNames } from '@/api/system/resources'; | |||
import { servingConfig } from '@/config'; | |||
import { RESOURCES_MODULE_ENUM } from '@/utils'; | |||
import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
import { RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils'; | |||
import RunParamForm from '@/components/Training/runParamForm'; | |||
import BaseModal from '@/components/BaseModal'; | |||
import AlgorithmForm from '@/views/algorithm/components/algorithmForm'; | |||
@@ -540,7 +539,7 @@ export default { | |||
// 镜像选择 | |||
// 获取镜像名称列表 | |||
async getImageNames(keepValue = false) { | |||
this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.SERVING] }); | |||
this.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.SERVING] }); | |||
if (!keepValue || !this.form.imageName) { | |||
this.form.imageTag = null; | |||
} else if (!this.imageNameList.includes(this.form.imageName)) { | |||
@@ -554,7 +553,7 @@ export default { | |||
async getImageTags(imageName, keepValue = false) { | |||
this.imageTagList = await getImageTagList({ | |||
imageName, | |||
imageTypes: [IMAGE_TYPE.SERVING], | |||
imageTypes: [IMAGE_TYPE_ENUM.SERVING], | |||
}); | |||
if (keepValue && this.form.imageTag) { | |||
if (!this.imageTagList.some((image) => image.imageTag === this.form.imageTag)) { | |||
@@ -134,8 +134,12 @@ import { list as listBranchModel } from '@/api/model/modelVersion'; | |||
import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
import { list as listResourceSpec } from '@/api/system/resources'; | |||
import { types } from '@/utils/validate'; | |||
import { RESOURCES_POOL_TYPE_ENUM, RESOURCES_MODULE_ENUM, modelTypeSymbol } from '@/utils/constant'; | |||
import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
import { | |||
RESOURCES_POOL_TYPE_ENUM, | |||
RESOURCES_MODULE_ENUM, | |||
modelTypeSymbol, | |||
IMAGE_TYPE_ENUM, | |||
} from '@/utils/constant'; | |||
const initialForm = { | |||
name: undefined, | |||
@@ -246,7 +250,7 @@ export default { | |||
}; | |||
const queryImages = () => { | |||
return getImageNameList({ imageTypes: [IMAGE_TYPE.DATASETMARKED] }); | |||
return getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.DATASETMARKED] }); | |||
}; | |||
const handleModelParentChange = () => { | |||
@@ -143,8 +143,12 @@ import { list as listBranchModel } from '@/api/model/modelVersion'; | |||
import { getImageNameList, getImageTagList } from '@/api/trainingImage'; | |||
import { list as listResourceSpec } from '@/api/system/resources'; | |||
import { types } from '@/utils/validate'; | |||
import { RESOURCES_POOL_TYPE_ENUM, RESOURCES_MODULE_ENUM, modelTypeSymbol } from '@/utils/constant'; | |||
import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
import { | |||
RESOURCES_POOL_TYPE_ENUM, | |||
RESOURCES_MODULE_ENUM, | |||
modelTypeSymbol, | |||
IMAGE_TYPE_ENUM, | |||
} from '@/utils/constant'; | |||
const initialForm = { | |||
name: undefined, | |||
@@ -283,7 +287,7 @@ export default { | |||
// 查询镜像列表 | |||
const queryImages = () => { | |||
return getImageNameList({ imageTypes: [IMAGE_TYPE.DATASETMARKED] }); | |||
return getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.DATASETMARKED] }); | |||
}; | |||
// 查询镜像版本列表 | |||
@@ -163,8 +163,8 @@ import { | |||
ALGORITHM_RESOURCE_ENUM, | |||
MODEL_RESOURCE_ENUM, | |||
RESOURCES_POOL_TYPE_ENUM, | |||
IMAGE_TYPE_ENUM, | |||
} from '@/utils'; | |||
import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
const initFormState = { | |||
modelId: null, | |||
@@ -285,7 +285,7 @@ export default { | |||
return Promise.reject(); | |||
} | |||
return getImageTagList({ | |||
imageTypes: IMAGE_TYPE.POINTCLOUD, | |||
imageTypes: IMAGE_TYPE_ENUM.POINTCLOUD, | |||
imageName: modelState.imageName, | |||
}).then((res) => { | |||
state.imageTagList = res; | |||
@@ -294,7 +294,7 @@ export default { | |||
// 镜像选择 | |||
const getImageNames = async () => { | |||
state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE.POINTCLOUD] }); | |||
state.imageNameList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.POINTCLOUD] }); | |||
if ( | |||
modelState.imageName && | |||
!state.imageNameList.some((image) => image === modelState.imageName) | |||
@@ -122,14 +122,13 @@ | |||
</template> | |||
<script> | |||
import { validateNameWithHyphen, RESOURCES_MODULE_ENUM } from '@/utils'; | |||
import { validateNameWithHyphen, RESOURCES_MODULE_ENUM, IMAGE_TYPE_ENUM } from '@/utils'; | |||
import BaseModal from '@/components/BaseModal'; | |||
import InfoSelect from '@/components/InfoSelect'; | |||
import { getPublishedDatasets, getDatasetVersions } from '@/api/preparation/dataset'; | |||
import { add as addNotebook } from '@/api/development/notebook'; | |||
import { list as getSpecsNames } from '@/api/system/resources'; | |||
import { getImageNameList, getImageTagList } from '@/api/trainingImage/index'; | |||
import { IMAGE_TYPE } from '@/views/trainingJob/utils'; | |||
const defaultForm = { | |||
noteBookName: null, | |||
@@ -208,7 +207,7 @@ export default { | |||
} | |||
}, | |||
async getHarborProjects() { | |||
this.harborProjectList = await getImageNameList({ imageTypes: [IMAGE_TYPE.NOTEBOOK] }); | |||
this.harborProjectList = await getImageNameList({ imageTypes: [IMAGE_TYPE_ENUM.NOTEBOOK] }); | |||
if ( | |||
this.form.imageName && | |||
!this.harborProjectList.some((project) => project === this.form.imageName) | |||
@@ -237,7 +236,7 @@ export default { | |||
} | |||
return getImageTagList({ | |||
imageName: this.form.imageName, | |||
imageTypes: [IMAGE_TYPE.NOTEBOOK], | |||
imageTypes: [IMAGE_TYPE_ENUM.NOTEBOOK], | |||
}).then((res) => { | |||
this.harborImageList = res; | |||
}); | |||
@@ -48,7 +48,7 @@ | |||
<el-table | |||
v-if="prefabricate" | |||
ref="table" | |||
v-loading="crud.loading || disableEdit" | |||
v-loading="crud.loading" | |||
:data="crud.data" | |||
highlight-current-row | |||
@selection-change="crud.selectionChangeHandler" | |||
@@ -57,21 +57,6 @@ | |||
<el-table-column v-if="!isPreset" prop="id" label="ID" sortable="custom" width="80px" /> | |||
<el-table-column prop="imageName" label="镜像名称" sortable="custom" /> | |||
<el-table-column prop="imageTag" label="镜像版本号" sortable="custom" /> | |||
<el-table-column prop="imageStatus" label="状态" width="130px"> | |||
<template #header> | |||
<dropdown-header | |||
title="状态" | |||
:list="imageStatusList" | |||
:filtered="Boolean(localQuery.imageStatus)" | |||
@command="filterStatus" | |||
/> | |||
</template> | |||
<template slot-scope="scope"> | |||
<el-tag effect="plain" :type="map[scope.row.imageStatus]"> | |||
{{ statusMap[scope.row.imageStatus] }} | |||
</el-tag> | |||
</template> | |||
</el-table-column> | |||
<el-table-column prop="imageTypes" label="镜像用途" width="180px"> | |||
<template #header> | |||
<dropdown-header | |||
@@ -91,31 +76,8 @@ | |||
<span>{{ parseTime(scope.row.createTime) }}</span> | |||
</template> | |||
</el-table-column> | |||
<el-table-column v-if="isNotebook" prop="imageResource" label="是否为默认" align="center"> | |||
<template slot-scope="scope"> | |||
<i | |||
:class="resourceObj(scope.row.imageResource).icon" | |||
:style="{ color: resourceObj(scope.row.imageResource).color, fontSize: '20px' }" | |||
></i> | |||
</template> | |||
</el-table-column> | |||
<el-table-column v-if="isAdmin || isCustom" label="操作" width="200px" fixed="right"> | |||
<template slot-scope="scope"> | |||
<el-tooltip | |||
v-if="isAdmin && isNotebook" | |||
effect="dark" | |||
content="设为在线编辑算法时创建nootbook的默认镜像" | |||
placement="top" | |||
> | |||
<el-button | |||
:id="`doPrecast_` + scope.$index" | |||
:disabled="Boolean(scope.row.imageResource)" | |||
type="text" | |||
@click.stop="doPrecast(scope.row.id)" | |||
> | |||
{{ resourceObj(scope.row.imageResource).butText }} | |||
</el-button> | |||
</el-tooltip> | |||
<el-button | |||
v-if="(hasPermission('training:image:edit') && isCustom) || (isPreset && isAdmin)" | |||
:id="`doEdit_` + scope.$index" | |||
@@ -143,15 +105,13 @@ | |||
:visible="crud.status.cu > 0" | |||
:title="crud.status.title" | |||
:loading="crud.status.cu === 2" | |||
:disabled="loading" | |||
width="600px" | |||
@open="onDialogOpen" | |||
@close="onDialogClose" | |||
@cancel="crud.cancelCU" | |||
@ok="crud.submitCU" | |||
> | |||
<el-form ref="form" :model="form" :rules="rules" label-width="120px"> | |||
<el-form-item v-if="isEdit && isAdmin" label="镜像类别" prop="imageResource"> | |||
<el-form-item v-if="isFormAdd && isAdmin" label="镜像类别" prop="imageResource"> | |||
<el-radio-group v-model="form.imageResource" @change="imageResourceChange"> | |||
<el-radio :label="Number(IMAGE_RESOURCE_ENUM.CUSTOM)" border class="mr-0" | |||
>我的镜像</el-radio | |||
@@ -159,21 +119,20 @@ | |||
<el-radio :label="Number(IMAGE_RESOURCE_ENUM.PRESET)" border>预置镜像</el-radio> | |||
</el-radio-group> | |||
</el-form-item> | |||
<el-form-item v-if="isEdit" label="镜像名称" prop="imageName"> | |||
<el-form-item label="镜像名称" prop="imageName"> | |||
<el-autocomplete | |||
ref="imageName" | |||
v-model="form.imageName" | |||
class="inline-input" | |||
class="inline-input w-400" | |||
:fetch-suggestions="querySearchAsync" | |||
placeholder="请选择或输入镜像名称" | |||
style="width: 400px;" | |||
></el-autocomplete> | |||
</el-form-item> | |||
<el-form-item label="镜像用途"> | |||
<el-select | |||
v-model="form.imageTypes" | |||
placeholder="请选择或输入镜像用途" | |||
style="width: 400px;" | |||
class="w-400" | |||
multiple | |||
filterable | |||
allow-create | |||
@@ -187,36 +146,18 @@ | |||
/> | |||
</el-select> | |||
</el-form-item> | |||
<el-form-item v-if="isEdit" ref="imagePath" label="镜像文件路径" prop="imagePath"> | |||
<upload-inline | |||
v-if="crud.status.cu > 0" | |||
ref="upload" | |||
action="fakeApi" | |||
accept=".zip,.tar,.rar,.gz" | |||
list-type="text" | |||
:acceptSize="imageConfig.uploadFileAcceptSize" | |||
:acceptSizeFormat="uploadSizeFomatter" | |||
:params="uploadParams" | |||
:show-file-count="false" | |||
:auto-upload="true" | |||
:filters="uploadFilters" | |||
:limit="1" | |||
:on-remove="onFileRemove" | |||
@uploadStart="uploadStart" | |||
@uploadSuccess="uploadSuccess" | |||
@uploadError="uploadError" | |||
/> | |||
<upload-progress | |||
v-if="loading" | |||
:progress="progress" | |||
:color="customColors" | |||
:status="status" | |||
:size="size" | |||
@onSetProgress="onSetProgress" | |||
/> | |||
<el-form-item ref="imageUrl" label="镜像地址" prop="imageUrl"> | |||
<el-input v-model="form.imageUrl" placeholder="请输入镜像地址" class="w-400" /> | |||
<BaseTooltip icon="el-icon-warning" class="c-info"> | |||
<template #content> | |||
<div> | |||
镜像地址标准格式:镜像仓库域名/命名空间/镜像名:镜像版本号<br />示例:registry.cn-hangzhou.aliyuncs.com/enlin/notebook:v1 | |||
</div> | |||
</template> | |||
</BaseTooltip> | |||
</el-form-item> | |||
<el-form-item v-if="isEdit" label="镜像版本号" prop="imageTag"> | |||
<el-input id="imageTag" v-model="form.imageTag" style="width: 400px;" /> | |||
<el-form-item label="镜像版本号" prop="imageTag"> | |||
<el-input id="imageTag" v-model="form.imageTag" class="w-400" /> | |||
</el-form-item> | |||
<el-form-item label="描述" prop="remark"> | |||
<el-input | |||
@@ -227,19 +168,19 @@ | |||
maxlength="1024" | |||
show-word-limit | |||
placeholder | |||
style="width: 400px;" | |||
class="w-400" | |||
/> | |||
</el-form-item> | |||
</el-form> | |||
</BaseModal> | |||
<!--Notebook默认镜像设置表单--> | |||
<BaseModal | |||
:visible.sync="formVisible" | |||
:visible.sync="notebookFormVisible" | |||
title="Notebook默认镜像设置" | |||
:loading="formSubmitting" | |||
:loading="notebookFormSubmitting" | |||
width="800px" | |||
@cancel="formVisible = false" | |||
@ok="onSubmitForm" | |||
@cancel="notebookFormVisible = false" | |||
@ok="onSubmitNotebookForm" | |||
> | |||
<el-form | |||
ref="noteBookFormRef" | |||
@@ -253,7 +194,7 @@ | |||
id="defaultImage" | |||
v-model="noteBookForm.defaultImage" | |||
placeholder="请选择镜像" | |||
style="width: 400px;" | |||
class="w-400" | |||
filterable | |||
allow-create | |||
default-first-option | |||
@@ -283,8 +224,6 @@ | |||
<script> | |||
import { mapGetters } from 'vuex'; | |||
// eslint-disable-next-line import/no-extraneous-dependencies | |||
import { debounce } from 'throttle-debounce'; | |||
import cdOperation from '@crud/CD.operation'; | |||
import rrOperation from '@crud/RR.operation'; | |||
@@ -292,40 +231,36 @@ import pagination from '@crud/Pagination'; | |||
import CRUD, { presenter, header, form, crud } from '@crud/crud'; | |||
import trainingImageApi, { | |||
del, | |||
setPrecast, | |||
setDefaultImage, | |||
getDefaultImage, | |||
getImageNameList, | |||
getImageTagList, | |||
} from '@/api/trainingImage/index'; | |||
import { | |||
getUniqueId, | |||
uploadSizeFomatter, | |||
invalidFileNameChar, | |||
ADMIN_ROLE_ID, | |||
hasPermission, | |||
validateImageName, | |||
validateImageTag, | |||
IMAGE_TYPE_ENUM, | |||
IMAGE_TYPE_MAP, | |||
} from '@/utils'; | |||
import BaseModal from '@/components/BaseModal'; | |||
import UploadInline from '@/components/UploadForm/inline'; | |||
import BaseTooltip from '@/components/BaseTooltip'; | |||
import DropdownHeader from '@/components/DropdownHeader'; | |||
import UploadProgress from '@/components/UploadProgress'; | |||
import { imageConfig } from '@/config'; | |||
import { IMAGE_RESOURCE_ENUM, IMAGE_TYPE } from '../trainingJob/utils'; | |||
import { IMAGE_RESOURCE_ENUM } from '../trainingJob/utils'; | |||
const defaultForm = { | |||
imagePath: null, | |||
imageUrl: null, | |||
imageTag: null, | |||
remark: null, | |||
imageTypes: Number(IMAGE_TYPE), | |||
imageTypes: Number(IMAGE_TYPE_ENUM), | |||
imageResource: Number(IMAGE_RESOURCE_ENUM.CUSTOM), | |||
imageName: null, | |||
}; | |||
const defaultQuery = { | |||
imageStatus: null, | |||
imageNameOrId: null, | |||
imageTypes: null, | |||
}; | |||
@@ -334,28 +269,27 @@ export default { | |||
name: 'TrainingImage', | |||
components: { | |||
BaseModal, | |||
BaseTooltip, | |||
pagination, | |||
cdOperation, | |||
rrOperation, | |||
UploadInline, | |||
DropdownHeader, | |||
UploadProgress, | |||
}, | |||
cruds() { | |||
return CRUD({ | |||
title: '镜像', | |||
crudMethod: { ...trainingImageApi }, | |||
optShow: { | |||
add: imageConfig.allowUploadImage && hasPermission('training:image:upload'), | |||
add: imageConfig.allowUploadImage && hasPermission('training:image:save'), | |||
del: false, | |||
}, | |||
queryOnPresenterCreated: false, | |||
props: { | |||
optText: { | |||
add: '上传镜像', | |||
add: '创建镜像', | |||
}, | |||
optTitle: { | |||
add: '上传', | |||
add: '创建', | |||
}, | |||
}, | |||
}); | |||
@@ -365,24 +299,6 @@ export default { | |||
return { | |||
active: IMAGE_RESOURCE_ENUM.CUSTOM, | |||
localQuery: { ...defaultQuery }, | |||
map: { | |||
0: 'info', | |||
1: 'success', | |||
2: 'danger', | |||
}, | |||
statusMap: { | |||
0: '制作中', | |||
1: '制作成功', | |||
2: '制作失败', | |||
}, | |||
imageTypesMap: { | |||
0: 'Notebook镜像', | |||
1: '训练镜像', | |||
2: 'Serving镜像', | |||
3: '终端镜像', | |||
4: '点云镜像', | |||
5: '数据标注镜像', | |||
}, | |||
rules: { | |||
imageTypes: [{ required: true, message: '请选择镜像类型', trigger: 'change' }], | |||
imageResource: [{ required: true, message: '请选择镜像来源', trigger: 'change' }], | |||
@@ -390,7 +306,7 @@ export default { | |||
{ required: true, message: '请选择项目名称', trigger: 'change' }, | |||
{ validator: validateImageName, trigger: ['blur', 'change'] }, | |||
], | |||
imagePath: [{ required: true, message: '请输入镜像路径', trigger: ['blur', 'manual'] }], | |||
imageUrl: [{ required: true, message: '请输入镜像地址', trigger: ['blur', 'manual'] }], | |||
imageTag: [ | |||
{ required: true, message: '请输入镜像版本号', trigger: 'blur' }, | |||
{ validator: validateImageTag, trigger: ['blur', 'change'] }, | |||
@@ -401,32 +317,17 @@ export default { | |||
id: [{ required: true, message: '请选择默认镜像版本', trigger: 'blur' }], | |||
}, | |||
harborProjectList: [], | |||
drawer: false, | |||
uploadParams: { | |||
objectPath: null, // 对象存储路径 | |||
}, | |||
progress: 0, | |||
size: 0, | |||
customColors: [ | |||
{ color: '#909399', percentage: 40 }, | |||
{ color: '#e6a23c', percentage: 80 }, | |||
{ color: '#67c23a', percentage: 100 }, | |||
], | |||
disableEdit: false, | |||
loading: false, | |||
isEdit: false, | |||
prefabricate: true, | |||
// 以下为配置参数及常量参数 | |||
imageConfig, | |||
IMAGE_RESOURCE_ENUM, | |||
IMAGE_TYPE, | |||
uploadFilters: [invalidFileNameChar], | |||
// 设置notebook默认镜像相关参数 | |||
noteBookImages: [], | |||
noteBookTags: [], | |||
noteBookForm: { defaultImage: '', defaultTag: '', id: '' }, | |||
formVisible: false, | |||
formSubmitting: false, | |||
formType: 'add', | |||
notebookFormVisible: false, | |||
notebookFormSubmitting: false, | |||
}; | |||
}, | |||
computed: { | |||
@@ -441,14 +342,8 @@ export default { | |||
isPreset() { | |||
return this.active === IMAGE_RESOURCE_ENUM.PRESET; | |||
}, | |||
isNotebook() { | |||
return this.active === IMAGE_RESOURCE_ENUM.NOTEBOOK; | |||
}, | |||
isTerminal() { | |||
return this.active === IMAGE_RESOURCE_ENUM.TERMINAL; | |||
}, | |||
disableAdd() { | |||
if (this.isAdmin) return this.isTerminal; // 管理员只有在终端镜像处无法点击上传 | |||
if (this.isAdmin) return false; // 管理员可以创建我的镜像和预置镜像 | |||
return !this.isCustom; // 其他角色只有在我的镜像处可以点击上传 | |||
}, | |||
operationProps() { | |||
@@ -456,37 +351,24 @@ export default { | |||
disabled: this.disableAdd, | |||
}; | |||
}, | |||
imageStatusList() { | |||
const arr = [{ label: '全部', value: null }]; | |||
for (const key in this.statusMap) { | |||
arr.push({ label: this.statusMap[key], value: key }); | |||
} | |||
return arr; | |||
}, | |||
imageTypesList() { | |||
const arr = [{ label: '全部', value: null }]; | |||
for (const key in this.imageTypesMap) { | |||
arr.push({ label: this.imageTypesMap[key], value: +key }); | |||
for (const key in IMAGE_TYPE_MAP) { | |||
arr.push({ label: IMAGE_TYPE_MAP[key], value: +key }); | |||
} | |||
return arr; | |||
}, | |||
status() { | |||
return this.progress === 100 ? 'success' : null; | |||
isFormAdd() { | |||
return this.formType === 'add'; | |||
}, | |||
}, | |||
mounted() { | |||
this.crud.refresh(); | |||
this.refetch = debounce(3000, this.crud.refresh); | |||
this.updateImagePath(); | |||
}, | |||
methods: { | |||
hasPermission, | |||
getImageTypes(imageTypes) { | |||
const usageStr = []; | |||
imageTypes.forEach((item) => { | |||
usageStr.push(this.imageTypesMap[item]); | |||
}); | |||
return usageStr.join(','); | |||
return imageTypes.map((type) => IMAGE_TYPE_MAP[type]).join(','); | |||
}, | |||
getNoteBookTags() { | |||
this.noteBookForm.defaultTag = null; | |||
@@ -497,7 +379,7 @@ export default { | |||
} | |||
return getImageTagList({ | |||
imageName: this.noteBookForm.defaultImage, | |||
imageTypes: IMAGE_TYPE.NOTEBOOK, | |||
imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK, | |||
imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | |||
}).then((res) => { | |||
this.noteBookTags = res; | |||
@@ -513,46 +395,11 @@ export default { | |||
this.prefabricate = true; | |||
}); | |||
}, | |||
onFileRemove() { | |||
this.form.imagePath = null; | |||
this.loading = false; | |||
this.$refs.imagePath.validate('manual'); | |||
}, | |||
uploadStart(files) { | |||
this.updateImagePath(); | |||
[this.loading, this.size, this.progress] = [true, files.size, 0]; | |||
}, | |||
onSetProgress(val) { | |||
this.progress += val; | |||
}, | |||
uploadSuccess(res) { | |||
this.progress = 100; | |||
setTimeout(() => { | |||
this.loading = false; | |||
}, 1000); | |||
if (this.loading) { | |||
this.form.imagePath = res[0].data.objectName; | |||
this.$refs.imagePath.validate('manual'); | |||
} | |||
}, | |||
uploadError() { | |||
this.$message({ | |||
message: '上传文件失败', | |||
type: 'error', | |||
}); | |||
this.loading = false; | |||
}, | |||
// hook | |||
[CRUD.HOOK.afterRefresh]() { | |||
this.checkStatus(); | |||
}, | |||
[CRUD.HOOK.beforeToAdd]() { | |||
this.isEdit = true; | |||
this.formType = 'add'; | |||
if (this.isPreset) { | |||
this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.PRESET); | |||
} else if (this.isNotebook) { | |||
this.form.imageTypes = IMAGE_TYPE.NOTEBOOK; | |||
} | |||
}, | |||
[CRUD.HOOK.beforeRefresh]() { | |||
@@ -567,7 +414,7 @@ export default { | |||
} | |||
}, | |||
[CRUD.HOOK.beforeToEdit]() { | |||
this.isEdit = false; | |||
this.formType = 'edit'; | |||
}, | |||
async querySearchAsync(queryString, cb) { | |||
let { harborProjectList } = this; | |||
@@ -597,24 +444,9 @@ export default { | |||
this.form.imageResource = Number(IMAGE_RESOURCE_ENUM.CUSTOM); | |||
this.getImageNameList(); | |||
}, | |||
onDialogClose() { | |||
if (this.isEdit) { | |||
this.$refs.upload.formRef.reset(); | |||
} | |||
this.loading = false; | |||
}, | |||
async onDialogOpen() { | |||
this.getImageNameList(); | |||
}, | |||
checkStatus() { | |||
if (this.crud.data.some((item) => [0].includes(item.imageStatus))) { | |||
this.refetch(); | |||
} | |||
}, | |||
filterStatus(status) { | |||
this.localQuery.imageStatus = status; | |||
this.crud.toQuery(); | |||
}, | |||
filterImageTypes(imageTypes) { | |||
this.localQuery.imageTypes = imageTypes; | |||
this.crud.toQuery(); | |||
@@ -622,9 +454,6 @@ export default { | |||
resetQuery() { | |||
this.localQuery = { ...defaultQuery }; | |||
}, | |||
updateImagePath() { | |||
this.uploadParams.objectPath = `upload-temp/${this.user.id}/${getUniqueId()}`; | |||
}, | |||
async doEdit(imageObj) { | |||
const dataObj = { | |||
ids: [imageObj.id], | |||
@@ -642,21 +471,6 @@ export default { | |||
this.crud.refresh(); | |||
}); | |||
}, | |||
doPrecast(i) { | |||
setPrecast({ id: i }).then(() => { | |||
this.$message({ | |||
message: '设置成功', | |||
type: 'success', | |||
}); | |||
this.crud.refresh(); | |||
}); | |||
}, | |||
resourceObj(resource) { | |||
return resource | |||
? { icon: 'el-icon-circle-check', color: '#67C23A', butText: '当前为默认' } | |||
: { icon: 'el-icon-circle-close', color: '#F56C6C', butText: '设为默认' }; | |||
}, | |||
uploadSizeFomatter, | |||
async doSetDefaultImage() { | |||
// 获取默认镜像 | |||
@@ -666,20 +480,20 @@ export default { | |||
this.noteBookForm.id = defaultImage.length ? defaultImage[0].id : ''; | |||
// 获取镜像列表 | |||
this.noteBookImages = await getImageNameList({ | |||
imageTypes: IMAGE_TYPE.NOTEBOOK, | |||
imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK, | |||
imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | |||
}); | |||
if (this.noteBookForm.defaultImage) { | |||
this.noteBookTags = await getImageTagList({ | |||
imageName: this.noteBookForm.defaultImage, | |||
imageTypes: IMAGE_TYPE.NOTEBOOK, | |||
imageTypes: IMAGE_TYPE_ENUM.NOTEBOOK, | |||
imageResource: Number(IMAGE_RESOURCE_ENUM.PRESET), | |||
}); | |||
} else { | |||
this.noteBookTags = []; | |||
} | |||
this.formVisible = true; | |||
this.notebookFormVisible = true; | |||
this.$nextTick(() => { | |||
this.clearValidate(); | |||
}); | |||
@@ -690,17 +504,17 @@ export default { | |||
validateField(field) { | |||
this.$refs.noteBookFormRef.validateField(field); | |||
}, | |||
onSubmitForm() { | |||
onSubmitNotebookForm() { | |||
this.$refs.noteBookFormRef.validate((valid) => { | |||
if (valid) { | |||
this.formSubmitting = true; | |||
this.notebookFormSubmitting = true; | |||
setDefaultImage({ id: this.noteBookForm.id }) | |||
.then(() => { | |||
this.formVisible = false; | |||
this.notebookFormVisible = false; | |||
this.crud.toQuery(); | |||
}) | |||
.finally(() => { | |||
this.formSubmitting = false; | |||
this.notebookFormSubmitting = false; | |||
}); | |||
} | |||
}); | |||
@@ -96,15 +96,6 @@ export const ATLAS_ALGORITHM_TYPE_ENUM = { | |||
DDRL: 4, // ddrl | |||
}; | |||
export const IMAGE_TYPE = { | |||
NOTEBOOK: 0, // notebook镜像类型 | |||
TRAIN: 1, // 训练镜像和预置镜像类型 | |||
SERVING: 2, // serving镜像 | |||
TERMINAL: 3, // 终端镜像 | |||
POINTCLOUD: 4, // 点云镜像 | |||
DATASETMARKED: 5, // 数据标注镜像 | |||
}; | |||
// 训练可视化列表页查询项 | |||
export const trainVisualQueryFormItems = [ | |||
{ | |||