From b07e94fa2a90768bae28e4af9b98b2edf7a72cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=8B=E6=B1=9F=E5=AE=9E=E9=AA=8C=E5=AE=A4?= Date: Wed, 20 Jan 2021 14:33:44 +0800 Subject: [PATCH] update config file --- dubhe-server/.gitignore | 1 + dubhe-server/CHANGELOG.md | 28 --- dubhe-server/Dockerfile | 62 +++++ dubhe-server/LICENSE | 211 ++++++++++++++++++ .../org/dubhe/constant/ModelOptConstant.java | 2 +- .../org/dubhe/constant/ModelOptErrorEnum.java | 2 +- .../constant/ModelOptInstanceStatusEnum.java | 2 +- .../ModelOptK8sPodCallbackCreateDTO.java | 2 +- dubhe-server/deploy-docker.sh | 71 ++++++ .../org/dubhe/dao/ModelOptBuiltInMapper.java | 2 +- .../org/dubhe/dao/ModelOptDatasetMapper.java | 2 +- .../dubhe/dao/ModelOptTaskInstanceMapper.java | 2 +- .../org/dubhe/dao/ModelOptTaskMapper.java | 2 +- .../dubhe/domain/dto/ModelOptLogQueryDTO.java | 2 +- .../domain/dto/ModelOptTaskCreateDTO.java | 2 +- .../domain/dto/ModelOptTaskDeleteDTO.java | 2 +- .../dto/ModelOptTaskInstanceCancelDTO.java | 2 +- .../dto/ModelOptTaskInstanceDeleteDTO.java | 2 +- .../dto/ModelOptTaskInstanceDetailDTO.java | 2 +- .../dto/ModelOptTaskInstanceQueryDTO.java | 2 +- .../dto/ModelOptTaskInstanceResubmitDTO.java | 2 +- .../domain/dto/ModelOptTaskQueryDTO.java | 2 +- .../domain/dto/ModelOptTaskRunParamDTO.java | 2 +- .../domain/dto/ModelOptTaskSubmitDTO.java | 2 +- .../domain/dto/ModelOptTaskUpdateDTO.java | 2 +- .../domain/dto/PtModelAlgorithmCreateDTO.java | 2 +- .../dubhe/domain/entity/ModelOptBuiltIn.java | 2 +- .../dubhe/domain/entity/ModelOptDataset.java | 2 +- .../org/dubhe/domain/entity/ModelOptTask.java | 2 +- .../domain/entity/ModelOptTaskInstance.java | 2 +- .../domain/vo/ModelOptAlgorithmQueryVO.java | 2 +- .../org/dubhe/domain/vo/ModelOptCreateVO.java | 2 +- .../domain/vo/ModelOptDatasetQueryVO.java | 2 +- .../dubhe/domain/vo/ModelOptDatasetVO.java | 2 +- .../dubhe/domain/vo/ModelOptModelQueryVO.java | 2 +- .../domain/vo/ModelOptResultQueryVO.java | 2 +- .../vo/ModelOptTaskInstanceQueryVO.java | 2 +- .../dubhe/domain/vo/ModelOptTaskQueryVO.java | 2 +- .../org/dubhe/domain/vo/ModelOptUpdateVO.java | 2 +- .../org/dubhe/enums/DistillCommandEnum.java | 2 +- .../org/dubhe/enums/OptimizeTypeEnum.java | 2 +- .../dubhe/rest/ModelOptTaskController.java | 2 +- .../rest/ModelOptTaskInstanceController.java | 2 +- .../service/ModelOptTaskInstanceService.java | 2 +- .../dubhe/service/ModelOptTaskService.java | 2 +- .../impl/ModelOptAsyncServiceImpl.java | 2 +- .../impl/ModelOptTaskInstanceServiceImpl.java | 2 +- .../service/impl/ModelOptTaskServiceImpl.java | 2 +- .../resources/config/application-prod.yml | 8 +- .../statemachine/GlobalStateMachine.java | 2 +- .../cr/DistributeTrainInitContainer.java | 90 -------- .../k8s/observer/BatchservingObserver.java | 2 +- .../dubhe/k8s/observer/ServingObserver.java | 2 +- .../dto/PtModelOptimizationCreateDTO.java | 2 +- .../opt/ModelOptTaskInstanceSyncTask.java | 2 +- .../resources/config/application-prod.yml | 8 +- .../src/main/resources/config/application.yml | 17 +- 57 files changed, 421 insertions(+), 171 deletions(-) delete mode 100644 dubhe-server/CHANGELOG.md create mode 100644 dubhe-server/Dockerfile create mode 100644 dubhe-server/LICENSE create mode 100644 dubhe-server/deploy-docker.sh delete mode 100644 dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/domain/cr/DistributeTrainInitContainer.java diff --git a/dubhe-server/.gitignore b/dubhe-server/.gitignore index 3b38ee9..a103ef3 100644 --- a/dubhe-server/.gitignore +++ b/dubhe-server/.gitignore @@ -2,6 +2,7 @@ .mvn* mvnw mvnw.cmd +kubeconfig ### STS ### .apt_generated diff --git a/dubhe-server/CHANGELOG.md b/dubhe-server/CHANGELOG.md deleted file mode 100644 index 5995bd7..0000000 --- a/dubhe-server/CHANGELOG.md +++ /dev/null @@ -1,28 +0,0 @@ -## 0.2.0 (2020-10-26) -### Breaking Changes - -1. 支持 OneFlow、TensorFlow、PyTorch 等主流框架的多机多卡模式分布式训练 -2. 基于状态机的数据集状态重构，将业务代码和状态变更完全解耦，实现零延时的状态变更响应能力 -3. 添加分布式算法调度。引入任务调度，将算法和应用解耦，支持多算法程序节点并行处理任务，并基于 k8s 实现算法节点自动伸缩 -4. 将标签和数据集拆分，引入「标签组」统一管理标签，实现标签可复用、可重组 -5. 训练时支持将已有模型作为训练入参 -6. 训练时支持区分训练数据集与验证数据集 -7. 平台后端日志分流，区分用户请求日志、平台日志、定时任务日志；精简日志信息 -8. 增加定时任务，异步清理垃圾数据（用户上传文件、训练/模型删除文件、大批量数据集文件） - -### Features - -1. 数据集图片手动标注优化。支持对标注像素级位置、大小调整，支持常见缩放、拖拽、平移等操作 -2. 通用数据权限方案改造。对业务代码零侵入性的权限方案，实现基于用户，角色，资源的权限数据管理 -3. 超大数据集操作流程优化。实现超大数据集（ 10w+ 文件）的前端、后端、存储全流程平滑操作 -4. 支持本地已有数据集兼容。系统提供标准数据集模板，用户按照规范导入数据集文件，实现数据集全功能兼容 -5. 模型开发Notebook 超时（默认 4h ）自动关闭并回收资源 -6. 断点续训功能、模型下载功能、模型保存功能支持通过目录树选择模型文件/文件夹 -7. 文件上传增加进度条展示 -8. 训练创建页，增加运行命令预览功能；训练详情页，增加算法在线编辑跳转功能 -9. 训练支持延时启动、定时停止功能 -10. 训练日志、运行日志下载功能优化，避免大文件导致的浏览器卡死 -11. 镜像管理功能，镜像名称支持自定义；支持镜像的删除、修改等操作；镜像上传后自动清除 docker load 镜像 -12. 上传算法功能、训练创建功能、训练终止功能性能优化 -13. CPU/GPU 规格类配置支持管理员控制台管理 -14. 增加训练失败异常信息反馈 diff --git a/dubhe-server/Dockerfile b/dubhe-server/Dockerfile new file mode 100644 index 0000000..09b4f13 --- /dev/null +++ b/dubhe-server/Dockerfile @@ -0,0 +1,62 @@ +FROM docker +ENV LANG=C.UTF-8 + +#更换为aliyun源 +RUN echo "http://mirrors.aliyun.com/alpine/latest-stable/community" > /etc/apk/repositories +RUN echo "http://mirrors.aliyun.com/alpine/latest-stable/main" >> /etc/apk/repositories +#update apk bash vim wget curl net-tools sshpass +RUN apk update \ + && apk upgrade \ + && apk --no-cache add ca-certificates \ + && apk add bash \ + bash-doc \ + bash-completion \ + vim \ + wget \ + curl \ + net-tools \ + sshpass \ + && rm -rf /var/cache/apk/* \ + && /bin/bash + +#setup glibc +RUN wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.32-r0/glibc-2.32-r0.apk +RUN wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.32-r0/glibc-bin-2.32-r0.apk +RUN wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.32-r0/glibc-i18n-2.32-r0.apk +RUN apk add --no-cache --virtual=.build-dependencies wget ca-certificates && \ + echo \ + "-----BEGIN PUBLIC KEY-----\ + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApZ2u1KJKUu/fW4A25y9m\ + y70AGEa/J3Wi5ibNVGNn1gT1r0VfgeWd0pUybS4UmcHdiNzxJPgoWQhV2SSW1JYu\ + tOqKZF5QSN6X937PTUpNBjUvLtTQ1ve1fp39uf/lEXPpFpOPL88LKnDBgbh7wkCp\ + m2KzLVGChf83MS0ShL6G9EQIAUxLm99VpgRjwqTQ/KfzGtpke1wqws4au0Ab4qPY\ + KXvMLSPLUp7cfulWvhmZSegr5AdhNw5KNizPqCJT8ZrGvgHypXyiFvvAH5YRtSsc\ + Zvo9GI2e2MaZyo9/lvb+LbLEJZKEQckqRj4P26gmASrZEPStwc+yqy1ShHLA0j6m\ + 1QIDAQAB\ + -----END PUBLIC KEY-----" | sed 's/ */\n/g' > "/etc/apk/keys/sgerrand.rsa.pub" +RUN apk add glibc-2.32-r0.apk +RUN apk add glibc-bin-2.32-r0.apk +RUN apk add glibc-i18n-2.32-r0.apk +RUN rm -rf *.apk + +#setup date +RUN apk add tzdata +RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + +#安装jdk +RUN wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz" +RUN tar xzf jdk-8u141-linux-x64.tar.gz -C /usr/local/ +ENV JAVA_HOME /usr/local/jdk1.8.0_141 +ENV PATH ${PATH}:${JAVA_HOME}/bin +ENV CLASSPATH $JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar + +#设置到nfs-host-ip的免密登录 +RUN ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa >/dev/null 2>&1 && \ + #让连接新主机时，不进行公钥确认 + echo "StrictHostKeyChecking no">>/etc/ssh/ssh_config && \ + #发送公钥到目标服务器 + sshpass -p nfs-host-pwd ssh-copy-id -p 22 -i ~/.ssh/id_rsa.pub nfs-host-user@nfs-host-ip + +COPY dubhe-admin/target/dubhe-admin-1.0-exec.jar /dubhe/ +COPY dubhe-task/target/dubhe-task-1.0.jar /dubhe/ +COPY dubhe-serving-gateway/target/dubhe-serving-gateway.jar /dubhe/ \ No newline at end of file diff --git a/dubhe-server/LICENSE b/dubhe-server/LICENSE new file mode 100644 index 0000000..87562c3 --- /dev/null +++ b/dubhe-server/LICENSE @@ -0,0 +1,211 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Other dependencies and licenses: +---------------------------------------------------------------------------------------- + +Open Source Software Licensed Under the Apache License, Version 2.0: +The below software in this distribution may have been modified. +---------------------------------------------------------------------------------------- +1. EL-ADMIN +Copyright 2019-2020 Zheng Jie \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java index 8fc6f9d..b191317 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java index 7bfba57..d16073b 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java index 99aa735..235d74b 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java index 12aa34f..770f963 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/deploy-docker.sh b/dubhe-server/deploy-docker.sh new file mode 100644 index 0000000..6ae01a9 --- /dev/null +++ b/dubhe-server/deploy-docker.sh @@ -0,0 +1,71 @@ +#!/bin/bash +#环境，用以区分部署的命名空间，日志路径 +ENV=$1 + +SOURCE_CODE_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd ) +#harbor 地址 +HARBOR_URL=harbor.dubhe.ai +#harbor 用户名 +HARBOR_USERNAME=admin +#harbor 密码 +HARBOR_PWD=123456 +#nfs 地址 +NFS_ADDRESS=127.0.0.1 +#nfs 服务器 root权限用户名 +NFS_HOST_USER=root +#nfs 服务器 root权限用户密码 +NFS_HOST_PWD=123456 +#nfs 共享目录 +NFS_PATH=/nfs +#容器日志路径 +CONTAINER_LOG_PATH=/logs +#宿主机日志路径 +HOST_LOG_PATH=/logs/dubhe-${ENV} +#admin模块端口 +ADMIN_PORT=8000 +#serving-gateway模块端口 +GATEWAY_PORT=8081 + +#挂载nfs +mount_nfs() { + mkdir ${NFS_PATH} + mount -t nfs ${NFS_ADDRESS}:${NFS_PATH} ${NFS_PATH} +} + +update_dockerfile() { + cd ${SOURCE_CODE_PATH} && sed -i "s#nfs-host-pwd#${NFS_HOST_PWD}#g;s#nfs-host-user#${NFS_HOST_USER}#g;s#nfs-host-ip#${NFS_ADDRESS}#g" Dockerfile +} +mvn_build() { + cd ${SOURCE_CODE_PATH} && mvn clean compile package + cd ${SOURCE_CODE_PATH}/dubhe-serving-gateway && mvn clean compile package +} +#删除镜像 +delete_old_image() { + docker rmi -f ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} +} +#构建镜像 +build_image() { + cd ${SOURCE_CODE_PATH} && docker build -t ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} . +} + +#运行admin模块 +run_admin() { + docker run -d -v ${HOST_LOG_PATH}:${CONTAINER_LOG_PATH} -v ${NFS_PATH}:${NFS_PATH} -v /var/run/docker.sock:/var/run/docker.sock -p ${ADMIN_PORT}:${ADMIN_PORT} ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} java -jar /dubhe/dubhe-admin-1.0-exec.jar --spring.profiles.active=${ENV} 2>&1 > /dev/null +} +#运行task模块 +run_task() { + docker run -d -v ${HOST_LOG_PATH}:${CONTAINER_LOG_PATH} -v ${NFS_PATH}:${NFS_PATH} -v /var/run/docker.sock:/var/run/docker.sock ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} java -jar /dubhe/dubhe-task-1.0.jar --spring.profiles.active=${ENV} 2>&1 > /dev/null +} +#运行serving-gateway模块 +run_gateway() { + docker run -d -v ${HOST_LOG_PATH}:${CONTAINER_LOG_PATH} -v ${NFS_PATH}:${NFS_PATH} -v /var/run/docker.sock:/var/run/docker.sock -p ${GATEWAY_PORT}:${GATEWAY_PORT} ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} java -jar /dubhe/dubhe-serving-gateway.jar --spring.profiles.active=${ENV} 2>&1 > /dev/null +} + +mount_nfs +update_dockerfile +mvn_build +delete_old_image +build_image +run_admin +run_task +run_gateway diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java index 13c2997..3afe75b 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java index 31d7799..9537712 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java index c49c1c7..2ec4961 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java index 9ed43a4..66c5648 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java index 17532f4..53caa33 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java index d4e8751..8e7e9cd 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java index f25cdc0..18de210 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java index 34599be..4145e2d 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java index 528c288..3ff1574 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java index 4e7b570..9f6bee3 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java index ce556ee..e1abeed 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java index 1bbe465..ebac902 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java index 85fe54a..82cda7a 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java index e0bce93..0417029 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java index 2d20c22..2ff55da 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java index 84058d8..cffc132 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java index 040ab1a..faf6f78 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java index d581346..0aa61d6 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java index 3ad3024..d42d737 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java index 6a95bf4..e426469 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java index fa66abb..239a9d9 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java index 71815ae..4ab46a2 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java index 83aa2d6..9ec1a69 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java index 1e90ea5..8f5fb53 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java index fb0af79..50b358e 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java index 79dcc4e..e9db090 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java index e1b31c2..b4b085b 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java index 47bc2e5..d4c22fa 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java index b40783c..c3ba79d 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java index c7144ee..7ab2e56 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java index d56b6be..3e085e5 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java index 6f0d749..52aac0b 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java index 25203da..23f5dba 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java index 0b0df29..82f487d 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java index 56e61d5..0b6de2f 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java index 8f1d4f9..8257dbf 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java index 4374c15..531b0f9 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java index a99cf16..1d38dba 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java index 1900bd0..3cc0f7f 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/resources/config/application-prod.yml b/dubhe-server/dubhe-admin/src/main/resources/config/application-prod.yml index 739a15a..7e70595 100644 --- a/dubhe-server/dubhe-admin/src/main/resources/config/application-prod.yml +++ b/dubhe-server/dubhe-admin/src/main/resources/config/application-prod.yml @@ -47,6 +47,8 @@ spring: web-stat-filter: url-pattern: /* exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" + + # 配置允许后面的Bean覆盖前面名称重复的Bean main: allow-bean-definition-overriding: true @@ -113,7 +115,7 @@ k8s: log: type: _doc # 过滤源字段 - source_field: log,@timestamp,kubernetes.pod_name + source_field: message,@timestamp,kubernetes.pod_name # 异步回调 callback: # boot 单机部署即回调本机实例 @@ -138,6 +140,7 @@ harbor: username: admin password: Harbor12345 model-name: train + # data模块配置 data: annotation: @@ -169,6 +172,7 @@ dcm: #镜像脚本路径 image: shell-path: /data/prod/dubhe-prod/ + #logback logging.config: classpath:logback-spring-dev.xml @@ -197,4 +201,4 @@ serving: gateway-uri-postfix: .api.dubhe.ai rootPath: "/serving/deployment/" batchRootPath: "/serving/batch/" - group: serving_prod_group \ No newline at end of file + group: serving_prod_group diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/statemachine/GlobalStateMachine.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/statemachine/GlobalStateMachine.java index 874b8eb..1962030 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/statemachine/GlobalStateMachine.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/statemachine/GlobalStateMachine.java @@ -37,4 +37,4 @@ public class GlobalStateMachine { */ private FileStateMachine fileStateMachine; -} \ No newline at end of file +} diff --git a/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/domain/cr/DistributeTrainInitContainer.java b/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/domain/cr/DistributeTrainInitContainer.java deleted file mode 100644 index ae1967d..0000000 --- a/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/domain/cr/DistributeTrainInitContainer.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.k8s.domain.cr; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import io.fabric8.kubernetes.api.model.KubernetesResource; -import io.fabric8.kubernetes.api.model.VolumeMount; -import java.util.ArrayList; -import java.util.List; - -/** - * @description 自定义资源初始化类 DistributeTrainInitContainer - * @date 2020-07-07 - */ -@JsonDeserialize( - using = JsonDeserializer.None.class -) -@JsonIgnoreProperties(ignoreUnknown = true) -public class DistributeTrainInitContainer implements KubernetesResource { - private String name; - private String image; - private String imagePullPolicy; - private List volumeMounts = new ArrayList(); - private List command; - private List args = new ArrayList(); - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getImage() { - return image; - } - - public void setImage(String image) { - this.image = image; - } - - public String getImagePullPolicy() { - return imagePullPolicy; - } - - public void setImagePullPolicy(String imagePullPolicy) { - this.imagePullPolicy = imagePullPolicy; - } - - public List getVolumeMounts() { - return volumeMounts; - } - - public void setVolumeMounts(List volumeMounts) { - this.volumeMounts = volumeMounts; - } - - public List getCommand() { - return command; - } - - public void setCommand(List command) { - this.command = command; - } - - public List getArgs() { - return args; - } - - public void setArgs(List args) { - this.args = args; - } -} diff --git a/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/BatchservingObserver.java b/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/BatchservingObserver.java index 9b9ee39..2134a37 100644 --- a/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/BatchservingObserver.java +++ b/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/BatchservingObserver.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/ServingObserver.java b/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/ServingObserver.java index b2f4982..82ee890 100644 --- a/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/ServingObserver.java +++ b/dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/ServingObserver.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-model/src/main/java/org/dubhe/domain/dto/PtModelOptimizationCreateDTO.java b/dubhe-server/dubhe-model/src/main/java/org/dubhe/domain/dto/PtModelOptimizationCreateDTO.java index d0e6295..3b0ff12 100644 --- a/dubhe-server/dubhe-model/src/main/java/org/dubhe/domain/dto/PtModelOptimizationCreateDTO.java +++ b/dubhe-server/dubhe-model/src/main/java/org/dubhe/domain/dto/PtModelOptimizationCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-task/src/main/java/org/dubhe/task/opt/ModelOptTaskInstanceSyncTask.java b/dubhe-server/dubhe-task/src/main/java/org/dubhe/task/opt/ModelOptTaskInstanceSyncTask.java index ec4a55c..2d419b0 100644 --- a/dubhe-server/dubhe-task/src/main/java/org/dubhe/task/opt/ModelOptTaskInstanceSyncTask.java +++ b/dubhe-server/dubhe-task/src/main/java/org/dubhe/task/opt/ModelOptTaskInstanceSyncTask.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-task/src/main/resources/config/application-prod.yml b/dubhe-server/dubhe-task/src/main/resources/config/application-prod.yml index 739a15a..7e70595 100644 --- a/dubhe-server/dubhe-task/src/main/resources/config/application-prod.yml +++ b/dubhe-server/dubhe-task/src/main/resources/config/application-prod.yml @@ -47,6 +47,8 @@ spring: web-stat-filter: url-pattern: /* exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" + + # 配置允许后面的Bean覆盖前面名称重复的Bean main: allow-bean-definition-overriding: true @@ -113,7 +115,7 @@ k8s: log: type: _doc # 过滤源字段 - source_field: log,@timestamp,kubernetes.pod_name + source_field: message,@timestamp,kubernetes.pod_name # 异步回调 callback: # boot 单机部署即回调本机实例 @@ -138,6 +140,7 @@ harbor: username: admin password: Harbor12345 model-name: train + # data模块配置 data: annotation: @@ -169,6 +172,7 @@ dcm: #镜像脚本路径 image: shell-path: /data/prod/dubhe-prod/ + #logback logging.config: classpath:logback-spring-dev.xml @@ -197,4 +201,4 @@ serving: gateway-uri-postfix: .api.dubhe.ai rootPath: "/serving/deployment/" batchRootPath: "/serving/batch/" - group: serving_prod_group \ No newline at end of file + group: serving_prod_group diff --git a/dubhe-server/dubhe-task/src/main/resources/config/application.yml b/dubhe-server/dubhe-task/src/main/resources/config/application.yml index 94630c0..b69f95a 100644 --- a/dubhe-server/dubhe-task/src/main/resources/config/application.yml +++ b/dubhe-server/dubhe-task/src/main/resources/config/application.yml @@ -79,6 +79,12 @@ train-job: docker-log-path: "train_log=${train-job.docker-train-path}/${train-job.log-path}" docker-visualized-log-path: "train_visualized_log=${train-job.docker-train-path}/${train-job.visualized-log-path}" docker-dataset: "data_url=${train-job.docker-dataset-path}" + docker-teacher-model-key: "teacher_path_list" + docker-teacher-model-path: "/model/teacher" + docker-student-model-key: "student_path_list" + docker-student-model-path: "/model/student" + atlas-anaconda: "cd /root/anaconda3/bin && source activate pytorch_1_3" + atlas-pythonioencoding: "PYTHONIOENCODING=utf-8 " eight: "8" plus-eight: "+8" node-ips: "node_ips" @@ -111,7 +117,7 @@ train-algorithm: fork: false -# 配置slq打印日志 +# 配置sql打印日志 logging: level: com: @@ -138,6 +144,15 @@ delay: #模型开发延时关闭时间 delete: 240 +# 模型优化容器镜像配置 +optimize: + image: harbor.dubhe.ai/optimize/oneflow-gpu:base + +# # serving模块容器镜像配置 +serving: + gpu-image: harbor.dubhe.ai/serving/serving-gpu:base + cpu-image: harbor.dubhe.ai/serving/serving-cpu:base + #垃圾回收默认配置 recycle: #过期时间设置