update config file

dubhe-server/.gitignore

@@ -2,6 +2,7 @@
 .mvn*
 mvnw
 mvnw.cmd
+kubeconfig
 
 ### STS ###
 .apt_generated

dubhe-server/CHANGELOG.md

@@ -1,28 +0,0 @@
-## 0.2.0 (2020-10-26)
-### Breaking Changes
-
-1. 支持 OneFlow、TensorFlow、PyTorch 等主流框架的多机多卡模式分布式训练
-2. 基于状态机的数据集状态重构，将业务代码和状态变更完全解耦，实现零延时的状态变更响应能力
-3. 添加分布式算法调度。引入任务调度，将算法和应用解耦，支持多算法程序节点并行处理任务，并基于 k8s 实现算法节点自动伸缩
-4. 将标签和数据集拆分，引入「标签组」统一管理标签，实现标签可复用、可重组
-5. 训练时支持将已有模型作为训练入参
-6. 训练时支持区分训练数据集与验证数据集
-7. 平台后端日志分流，区分用户请求日志、平台日志、定时任务日志；精简日志信息
-8. 增加定时任务，异步清理垃圾数据（用户上传文件、训练/模型删除文件、大批量数据集文件）
-
-### Features
-
-1. 数据集图片手动标注优化。支持对标注像素级位置、大小调整，支持常见缩放、拖拽、平移等操作
-2. 通用数据权限方案改造。对业务代码零侵入性的权限方案，实现基于用户，角色，资源的权限数据管理
-3. 超大数据集操作流程优化。实现超大数据集（ 10w+ 文件）的前端、后端、存储全流程平滑操作
-4. 支持本地已有数据集兼容。系统提供标准数据集模板，用户按照规范导入数据集文件，实现数据集全功能兼容
-5. 模型开发Notebook 超时（默认 4h ）自动关闭并回收资源
-6. 断点续训功能、模型下载功能、模型保存功能支持通过目录树选择模型文件/文件夹
-7. 文件上传增加进度条展示
-8. 训练创建页，增加运行命令预览功能；训练详情页，增加算法在线编辑跳转功能
-9. 训练支持延时启动、定时停止功能
-10. 训练日志、运行日志下载功能优化，避免大文件导致的浏览器卡死
-11. 镜像管理功能，镜像名称支持自定义；支持镜像的删除、修改等操作；镜像上传后自动清除 docker load 镜像
-12. 上传算法功能、训练创建功能、训练终止功能性能优化
-13. CPU/GPU 规格类配置支持管理员控制台管理
-14. 增加训练失败异常信息反馈

dubhe-server/Dockerfile

@@ -0,0 +1,62 @@
+FROM docker
+ENV LANG=C.UTF-8
+
+#更换为aliyun源
+RUN echo "http://mirrors.aliyun.com/alpine/latest-stable/community" > /etc/apk/repositories
+RUN echo "http://mirrors.aliyun.com/alpine/latest-stable/main" >> /etc/apk/repositories
+#update apk bash vim wget curl net-tools sshpass
+RUN apk update \
+	&& apk upgrade \
+	&& apk --no-cache add ca-certificates \
+	&& apk add bash \
+	bash-doc \
+	bash-completion \
+	vim \
+	wget \
+	curl \
+	net-tools \
+	sshpass \
+    && rm -rf /var/cache/apk/* \
+    && /bin/bash
+
+#setup glibc
+RUN wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.32-r0/glibc-2.32-r0.apk
+RUN wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.32-r0/glibc-bin-2.32-r0.apk
+RUN wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.32-r0/glibc-i18n-2.32-r0.apk
+RUN apk add --no-cache --virtual=.build-dependencies wget ca-certificates && \
+    echo \
+        "-----BEGIN PUBLIC KEY-----\
+        MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApZ2u1KJKUu/fW4A25y9m\
+        y70AGEa/J3Wi5ibNVGNn1gT1r0VfgeWd0pUybS4UmcHdiNzxJPgoWQhV2SSW1JYu\
+        tOqKZF5QSN6X937PTUpNBjUvLtTQ1ve1fp39uf/lEXPpFpOPL88LKnDBgbh7wkCp\
+        m2KzLVGChf83MS0ShL6G9EQIAUxLm99VpgRjwqTQ/KfzGtpke1wqws4au0Ab4qPY\
+        KXvMLSPLUp7cfulWvhmZSegr5AdhNw5KNizPqCJT8ZrGvgHypXyiFvvAH5YRtSsc\
+        Zvo9GI2e2MaZyo9/lvb+LbLEJZKEQckqRj4P26gmASrZEPStwc+yqy1ShHLA0j6m\
+        1QIDAQAB\
+        -----END PUBLIC KEY-----" | sed 's/   */\n/g' > "/etc/apk/keys/sgerrand.rsa.pub"
+RUN apk add glibc-2.32-r0.apk
+RUN apk add glibc-bin-2.32-r0.apk
+RUN apk add glibc-i18n-2.32-r0.apk
+RUN rm -rf *.apk
+
+#setup date
+RUN apk add tzdata
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+
+#安装jdk
+RUN wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz"
+RUN tar xzf jdk-8u141-linux-x64.tar.gz -C /usr/local/
+ENV JAVA_HOME /usr/local/jdk1.8.0_141
+ENV PATH ${PATH}:${JAVA_HOME}/bin
+ENV CLASSPATH $JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
+
+#设置到nfs-host-ip的免密登录
+RUN ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa >/dev/null 2>&1 && \
+	#让连接新主机时，不进行公钥确认
+    echo "StrictHostKeyChecking no">>/etc/ssh/ssh_config && \
+	#发送公钥到目标服务器
+    sshpass -p nfs-host-pwd ssh-copy-id -p 22 -i ~/.ssh/id_rsa.pub nfs-host-user@nfs-host-ip
+	
+COPY dubhe-admin/target/dubhe-admin-1.0-exec.jar /dubhe/
+COPY dubhe-task/target/dubhe-task-1.0.jar /dubhe/
+COPY dubhe-serving-gateway/target/dubhe-serving-gateway.jar /dubhe/

dubhe-server/LICENSE

@@ -0,0 +1,211 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+Other dependencies and licenses:
+----------------------------------------------------------------------------------------
+
+Open Source Software Licensed Under the Apache License, Version 2.0:
+The below software in this distribution may have been modified.
+----------------------------------------------------------------------------------------
+1. EL-ADMIN
+Copyright 2019-2020 Zheng Jie

dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/deploy-docker.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+#环境，用以区分部署的命名空间，日志路径
+ENV=$1
+
+SOURCE_CODE_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd )
+#harbor 地址
+HARBOR_URL=harbor.dubhe.ai
+#harbor 用户名
+HARBOR_USERNAME=admin
+#harbor 密码
+HARBOR_PWD=123456
+#nfs 地址
+NFS_ADDRESS=127.0.0.1
+#nfs 服务器 root权限用户名
+NFS_HOST_USER=root
+#nfs 服务器 root权限用户密码
+NFS_HOST_PWD=123456
+#nfs 共享目录
+NFS_PATH=/nfs
+#容器日志路径
+CONTAINER_LOG_PATH=/logs
+#宿主机日志路径
+HOST_LOG_PATH=/logs/dubhe-${ENV}
+#admin模块端口
+ADMIN_PORT=8000
+#serving-gateway模块端口
+GATEWAY_PORT=8081
+
+#挂载nfs
+mount_nfs() {
+	mkdir ${NFS_PATH}
+	mount -t nfs ${NFS_ADDRESS}:${NFS_PATH} ${NFS_PATH}
+}
+
+update_dockerfile() {
+	cd ${SOURCE_CODE_PATH} && sed -i "s#nfs-host-pwd#${NFS_HOST_PWD}#g;s#nfs-host-user#${NFS_HOST_USER}#g;s#nfs-host-ip#${NFS_ADDRESS}#g" Dockerfile
+}
+mvn_build() {
+	cd ${SOURCE_CODE_PATH} && mvn clean compile package
+	cd ${SOURCE_CODE_PATH}/dubhe-serving-gateway && mvn clean compile package
+}
+#删除镜像
+delete_old_image() {
+	docker rmi -f ${HARBOR_URL}/dubhe/dubhe-spring:${ENV}
+}
+#构建镜像
+build_image() {
+	cd ${SOURCE_CODE_PATH} && docker build -t ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} .
+}
+
+#运行admin模块
+run_admin() {
+	docker run -d -v ${HOST_LOG_PATH}:${CONTAINER_LOG_PATH} -v  ${NFS_PATH}:${NFS_PATH} -v /var/run/docker.sock:/var/run/docker.sock -p ${ADMIN_PORT}:${ADMIN_PORT}  ${HARBOR_URL}/dubhe/dubhe-spring:${ENV}  java -jar /dubhe/dubhe-admin-1.0-exec.jar --spring.profiles.active=${ENV} 2>&1 > /dev/null
+}
+#运行task模块
+run_task() {
+	docker run -d -v ${HOST_LOG_PATH}:${CONTAINER_LOG_PATH} -v ${NFS_PATH}:${NFS_PATH} -v /var/run/docker.sock:/var/run/docker.sock ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} java -jar /dubhe/dubhe-task-1.0.jar --spring.profiles.active=${ENV} 2>&1 > /dev/null
+}
+#运行serving-gateway模块
+run_gateway() {
+	docker run -d -v ${HOST_LOG_PATH}:${CONTAINER_LOG_PATH} -v ${NFS_PATH}:${NFS_PATH} -v /var/run/docker.sock:/var/run/docker.sock -p ${GATEWAY_PORT}:${GATEWAY_PORT} ${HARBOR_URL}/dubhe/dubhe-spring:${ENV}  java -jar /dubhe/dubhe-serving-gateway.jar --spring.profiles.active=${ENV} 2>&1 > /dev/null
+}
+
+mount_nfs
+update_dockerfile
+mvn_build
+delete_old_image
+build_image
+run_admin
+run_task
+run_gateway

dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-admin/src/main/resources/config/application-prod.yml

@@ -47,6 +47,8 @@ spring:
       web-stat-filter:
         url-pattern: /*
         exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*"
+
+
   # 配置允许后面的Bean覆盖前面名称重复的Bean
   main:
     allow-bean-definition-overriding: true
@@ -113,7 +115,7 @@ k8s:
     log:
       type: _doc
       # 过滤源字段
-      source_field: log,@timestamp,kubernetes.pod_name
+      source_field: message,@timestamp,kubernetes.pod_name
   # 异步回调
   callback:
     # boot 单机部署即回调本机实例
@@ -138,6 +140,7 @@ harbor:
   username: admin
   password: Harbor12345
   model-name: train
+
 # data模块配置
 data:
   annotation:
@@ -169,6 +172,7 @@ dcm:
 #镜像脚本路径
 image:
   shell-path: /data/prod/dubhe-prod/
+
 #logback
 logging.config:
   classpath:logback-spring-dev.xml
@@ -197,4 +201,4 @@ serving:
   gateway-uri-postfix: .api.dubhe.ai
   rootPath: "/serving/deployment/"
   batchRootPath: "/serving/batch/"
-  group: serving_prod_group
+  group: serving_prod_group

dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/statemachine/GlobalStateMachine.java

@@ -37,4 +37,4 @@ public class GlobalStateMachine {
      */
     private FileStateMachine fileStateMachine;
 
-}
+}

dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/domain/cr/DistributeTrainInitContainer.java

@@ -1,90 +0,0 @@
-/**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =============================================================
- */
-package org.dubhe.k8s.domain.cr;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.databind.JsonDeserializer;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import io.fabric8.kubernetes.api.model.KubernetesResource;
-import io.fabric8.kubernetes.api.model.VolumeMount;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @description 自定义资源初始化类 DistributeTrainInitContainer
- * @date 2020-07-07
- */
-@JsonDeserialize(
-        using = JsonDeserializer.None.class
-)
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class DistributeTrainInitContainer implements KubernetesResource {
-    private String name;
-    private String image;
-    private String imagePullPolicy;
-    private List<VolumeMount> volumeMounts = new ArrayList();
-    private List<String> command;
-    private List<String> args = new ArrayList();
-
-    public String getName() {
-        return name;
-    }
-
-    public void setName(String name) {
-        this.name = name;
-    }
-
-    public String getImage() {
-        return image;
-    }
-
-    public void setImage(String image) {
-        this.image = image;
-    }
-
-    public String getImagePullPolicy() {
-        return imagePullPolicy;
-    }
-
-    public void setImagePullPolicy(String imagePullPolicy) {
-        this.imagePullPolicy = imagePullPolicy;
-    }
-
-    public List<VolumeMount> getVolumeMounts() {
-        return volumeMounts;
-    }
-
-    public void setVolumeMounts(List<VolumeMount> volumeMounts) {
-        this.volumeMounts = volumeMounts;
-    }
-
-    public List<String> getCommand() {
-        return command;
-    }
-
-    public void setCommand(List<String> command) {
-        this.command = command;
-    }
-
-    public List<String> getArgs() {
-        return args;
-    }
-
-    public void setArgs(List<String> args) {
-        this.args = args;
-    }
-}

dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/BatchservingObserver.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-k8s/src/main/java/org/dubhe/k8s/observer/ServingObserver.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-model/src/main/java/org/dubhe/domain/dto/PtModelOptimizationCreateDTO.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-task/src/main/java/org/dubhe/task/opt/ModelOptTaskInstanceSyncTask.java

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * Copyright 2020 Tianshu AI Platform. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

dubhe-server/dubhe-task/src/main/resources/config/application-prod.yml

@@ -47,6 +47,8 @@ spring:
       web-stat-filter:
         url-pattern: /*
         exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*"
+
+
   # 配置允许后面的Bean覆盖前面名称重复的Bean
   main:
     allow-bean-definition-overriding: true
@@ -113,7 +115,7 @@ k8s:
     log:
       type: _doc
       # 过滤源字段
-      source_field: log,@timestamp,kubernetes.pod_name
+      source_field: message,@timestamp,kubernetes.pod_name
   # 异步回调
   callback:
     # boot 单机部署即回调本机实例
@@ -138,6 +140,7 @@ harbor:
   username: admin
   password: Harbor12345
   model-name: train
+
 # data模块配置
 data:
   annotation:
@@ -169,6 +172,7 @@ dcm:
 #镜像脚本路径
 image:
   shell-path: /data/prod/dubhe-prod/
+
 #logback
 logging.config:
   classpath:logback-spring-dev.xml
@@ -197,4 +201,4 @@ serving:
   gateway-uri-postfix: .api.dubhe.ai
   rootPath: "/serving/deployment/"
   batchRootPath: "/serving/batch/"
-  group: serving_prod_group
+  group: serving_prod_group

dubhe-server/dubhe-task/src/main/resources/config/application.yml

@@ -79,6 +79,12 @@ train-job:
   docker-log-path: "train_log=${train-job.docker-train-path}/${train-job.log-path}"
   docker-visualized-log-path: "train_visualized_log=${train-job.docker-train-path}/${train-job.visualized-log-path}"
   docker-dataset: "data_url=${train-job.docker-dataset-path}"
+  docker-teacher-model-key: "teacher_path_list"
+  docker-teacher-model-path: "/model/teacher"
+  docker-student-model-key: "student_path_list"
+  docker-student-model-path: "/model/student"
+  atlas-anaconda: "cd /root/anaconda3/bin && source activate pytorch_1_3"
+  atlas-pythonioencoding: "PYTHONIOENCODING=utf-8 "
   eight: "8"
   plus-eight: "+8"
   node-ips: "node_ips"
@@ -111,7 +117,7 @@ train-algorithm:
   fork: false
 
 
-# 配置slq打印日志
+# 配置sql打印日志
 logging:
   level:
     com:
@@ -138,6 +144,15 @@ delay:
     #模型开发延时关闭时间
     delete: 240
 
+# 模型优化容器镜像配置
+optimize:
+  image: harbor.dubhe.ai/optimize/oneflow-gpu:base
+
+# # serving模块容器镜像配置
+serving:
+  gpu-image: harbor.dubhe.ai/serving/serving-gpu:base
+  cpu-image: harbor.dubhe.ai/serving/serving-cpu:base
+
 #垃圾回收默认配置
 recycle:
   #过期时间设置