commit e2743c1873c1c1724bd0e0d1eee9b35694a8edc5 Author: 之江实验室 Date: Fri Jul 31 17:23:02 2020 +0800 init diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ef4ea8b --- /dev/null +++ b/LICENSE @@ -0,0 +1,211 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Other dependencies and licenses: +---------------------------------------------------------------------------------------- + +Open Source Software Licensed Under the Apache License, Version 2.0: +The below software in this distribution may have been modified. +---------------------------------------------------------------------------------------- +1. EL-ADMIN +Copyright 2019-2020 Zheng Jie \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c755c4d --- /dev/null +++ b/README.md @@ -0,0 +1,103 @@ +

+ + + +

+ +

天枢一站式AI模型开发平台

+ +**天枢人工智能开源开放平台**（简称：**天枢平台**）是天枢平台由之江实验室牵头，联合北京一流科技、中国信通院和浙江大学共同自研的人工智能开源平台。整个平台由一站式AI模型开发平台、高性能深度学习框架和模型炼知框架三大子系统组成。 + +其中， **一站式AI模型开发平台面**（简称：**一站式开发平台**）面向AI模型生产的生命周期，提供了包括数据处理、模型开发、模型训练和模型管理等功能，方便用户一站式构建AI算法。 + +## 平台优势 + +* **一站式开发**：为用户提供一站式深度学习开发功能，通过智能数据处理、便利的模型开发和模型训练，打通深度学习全链路； + +* **集成先进算法**：除了囊括常规AI算法外，之江天枢还集成了多领域的独家算法，提供业界领先性能。 + +* **灵活易用**：除了一站式深度学习开发平台，亦提供可视化和动静结合编码方式，调试灵活，小白亦可快速上手。 + +* **性能优越**：集成自主研发的分布式训练平台，提供高性能的分布式计算体验，节省训练成本和训练时间。 + +## 页面预览 +![概览](/docs/images/dubhe_dashboard.png "概览") + + +## 功能列表 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
一站式开发平台
数据管理数据集管理
模型开发Notebook
算法管理
训练管理镜像管理
训练任务
模型管理模型列表
控制台用户管理
角色管理
菜单管理
字典管理
集群状态
+ +## 技术架构 +![技术架构](./docs/images/tech-arc.jpg "技术架构") + +## 技术栈 +- 后端: [Spring Boot](https://spring.io/projects/spring-boot) +- 前端: [Vue.js](https://vuejs.org/), [Element](https://element.eleme.cn/) +- 数据处理 [Yolo](https://pjreddie.com/darknet/yolo/) ... +- 可视化: [Django](https://www.djangoproject.com/) ... +- 中间件: [MySQL](https://www.mysql.com/), [MyBatis-Plus](https://mp.baomidou.com/), [Redis](https://redis.io/) +- 基础设施: [Docker](https://www.docker.com/), [Kubernetes](https://kubernetes.io/) + +## 目录结构 + +``` +├── dubhe_data_process 数据处理服务 +├── dubhe-server 后端服务 +├── dubhe-visual-server 可视化服务 +├── webapp 前端服务 +``` + +## 反馈问题 + +- [在线社区](http://www.aiiaos.cn/index.php?s=/forum/index/forum/id/45.html) +- 钉钉交流群 + + + + +## 许可证书 +本项目的发布受[Apache 2.0 license](./LICENSE)许可认证。 diff --git a/docs/images/dingtalk.jpg b/docs/images/dingtalk.jpg new file mode 100644 index 0000000..d047e63 Binary files /dev/null and b/docs/images/dingtalk.jpg differ diff --git a/docs/images/dubhe_dashboard.png b/docs/images/dubhe_dashboard.png new file mode 100644 index 0000000..f417819 Binary files /dev/null and b/docs/images/dubhe_dashboard.png differ diff --git a/docs/images/logo.svg b/docs/images/logo.svg new file mode 100644 index 0000000..317b636 --- /dev/null +++ b/docs/images/logo.svg @@ -0,0 +1,22 @@ + + + 天枢logo-横板-浅色底用 + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/images/tech-arc.jpg b/docs/images/tech-arc.jpg new file mode 100644 index 0000000..150291c Binary files /dev/null and b/docs/images/tech-arc.jpg differ diff --git a/dubhe-server/.gitignore b/dubhe-server/.gitignore new file mode 100644 index 0000000..ab86f52 --- /dev/null +++ b/dubhe-server/.gitignore @@ -0,0 +1,53 @@ +/target/ +.mvn* +mvnw +mvnw.cmd +kubeconfig + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/build/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +# system ignore +.DS_Store +Thumbs.db +*.orig + +# maven ignore +target/ +output/ +*.jar +!.mvn/wrapper/* +*.war +*.zip +*.tar +#*.tar.gz + +# eclipse ignore +.settings/ +.project +.classpath +logs/ +/dubhe-k8s/src/main/resources/kubeconfig +/dubhe-k8s/src/main/resources +*.log +/dubhe-admin/kubeconfig \ No newline at end of file diff --git a/dubhe-server/.gitmessage b/dubhe-server/.gitmessage new file mode 100644 index 0000000..f893509 --- /dev/null +++ b/dubhe-server/.gitmessage @@ -0,0 +1,10 @@ + +# type: subject +# type:feat增加新功能；fix修复错误；docs修改文档；style修改样式；refactor代码重构；test增加测试模块，不涉及生产环境的代码；chore更新核心模块，包配置文件，不涉及生产环境的代码 +# subject: 一两句话简述提交原因，与下方详细描述间隔一行 + + +# body：提交详细原因，与下方脚注间隔一行 + + +# footer（可选）：1.提供链接关联issue、2.关闭issue diff --git a/dubhe-server/README.md b/dubhe-server/README.md new file mode 100644 index 0000000..609205f --- /dev/null +++ b/dubhe-server/README.md @@ -0,0 +1,98 @@ +# 之江天枢-服务端 + +**之江天枢一站式人工智能开源平台**（简称：**之江天枢**），包括海量数据处理、交互式模型构建（包含Notebook和模型可视化）、AI模型高效训练。多维度产品形态满足从开发者到大型企业的不同需求，将提升人工智能技术的研发效率、扩大算法模型的应用范围，进一步构建人工智能生态“朋友圈”。 + +## 源码部署 + +### 准备环境 +安装如下软件环境。 +- OpenJDK：1.8+ +- Redis: 3.0+ +- Maven: 3.0+ +- MYSQL: 5.5.0+ + +### 下载源码 +``` bash +git clone https://codeup.teambition.com/zhejianglab/dubhe-server.git +# 进入项目根目录 +cd dubhe-server +``` + +### 创建DB +在MySQL中依次执行如下sql文件 +``` +sql/v1/00-Dubhe-DB.sql +sql/v1/01-Dubhe-DDL.sql +sql/v1/02-Dubhe-DML.sql +``` + +### 配置 +根据实际情况修改如下配置文件。 +``` +dubhe-admin/src/main/resources/config/application-prod.yml +``` + +### 构建 +``` bash +# 构建，生成的 jar 包位于 ./dubhe-admin/target/dubhe-admin-1.0.jar +mvn clean compile package +``` + +### 启动 +``` bash +# 指定启动环境为 prod +java -jar ./dubhe-admin/target/dubhe-admin-1.0.jar --spring.profiles.active=prod +``` + +## 本地开发 + +### 必要条件： + 导入maven项目，下载所需的依赖包 + mysql下创建数据库dubhe，初始化数据脚本 + 安装redis + +### 启动： + mvn spring-boot:run + +## 代码结构： +``` +├── common 公共模块 +├── dubhe-admin 开发与训练模块 +│   ├── src +│   │   └── main +│   │   ├── java +│   │   │   └── org +│   │   │   └── dubhe +│   │   │   ├── AppRun.java +│   │   │   ├── domain 实体对象 +│   │   │   ├── repository 数据库层 +│   │   │   ├── rest 控制层 +│   │   │   └── service 服务层 +│   │   │   ├── dto 数据传输对象 +│   │   │   ├── impl 服务实现 +│   │   │   └── mapper 对象转化 +│   │   └── resources 配置文件 +├── dubhe-data 数据处理模块 +├── dubhe-model 模型管理模块 +├── dubhe-system 系统管理 +``` + +## docker服务器 + 上传镜像功能依赖docker服务，harbor与dokcer的信任配置如下： +### 1、对外开放端口 + vi /lib/systemd/system/docker.service + ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock -H tcp://0.0.0.0:2375 -H unix://var/run/docker.sock +### 2、信任harbor地址 + vi /etc/docker/daemon.json + { + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver": "json-file", + "insecure-registries":[harbor地址], + "log-opts": { + "max-size": "100m" + } + } +### 3、重新启动 + systemctl daemon-reload + service docker restart + systemctl status docker diff --git a/dubhe-server/common/pom.xml b/dubhe-server/common/pom.xml new file mode 100644 index 0000000..21b9ea8 --- /dev/null +++ b/dubhe-server/common/pom.xml @@ -0,0 +1,121 @@ + + + + dubheplatform + zhejianglab + 1.0 + + 4.0.0 + + 5.0.6 + 7.0.2 + 1.8.9 + + + common + 公共模块 + + + + + cn.hutool + hutool-all + ${hutool.version} + + + com.google.guava + guava + 21.0 + + + com.github.penggle + kaptcha + ${kaptcha.version} + + + + org.apache.shiro + shiro-spring + ${shiro.version} + + + com.auth0 + java-jwt + ${jwt.version} + + + * + * + + + + + io.minio + minio + ${minio.version} + + + + + org.aspectj + aspectjweaver + ${aspectjweaver.version} + + + + javax.xml.bind + jaxb-api + 2.3.0 + + + com.sun.xml.bind + jaxb-impl + 2.3.0 + + + com.sun.xml.bind + jaxb-core + 2.3.0 + + + commons-io + commons-io + 1.3.2 + compile + + + net.lingala.zip4j + zip4j + 1.3.2 + + + org.apache.commons + commons-compress + 1.20 + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + + true + src/main/resources + + + + diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java new file mode 100644 index 0000000..6cc1bb5 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.annotation; + +import java.lang.annotation.*; + +/** + * @description API版本控制注解 + * @date 2020-04-06 + */ +@Target({ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface ApiVersion { + //标识版本号 + int value() default 1; +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java new file mode 100644 index 0000000..96361bb --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.annotation; + +import java.lang.annotation.*; + +/** + * 数据权限过滤Mapper拦截 + * + * @date 2020-06-22 + */ +@Target({ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface DataPermission { + + /** + * 不需要数据权限的方法名 + */ + String[] ignores() default {}; + + /** + * 只在方法的注解上使用，代表方法的数据权限类型，如果不加注解，只会识别带"select"方法名的方法 + * + * @return + */ + String[] permission() default {}; + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java new file mode 100644 index 0000000..1fb7c93 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java @@ -0,0 +1,87 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import javax.validation.Constraint; +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import javax.validation.Payload; + +/** + * @date: 2020-05-21 + */ +@Target({ ElementType.METHOD, ElementType.FIELD, ElementType.ANNOTATION_TYPE }) +@Retention(RetentionPolicy.RUNTIME) +@Constraint(validatedBy = EnumValue.Validator.class) +public @interface EnumValue { + + String message() default "custom.value.invalid"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + Class> enumClass(); + + String enumMethod(); + + class Validator implements ConstraintValidator { + + private Class> enumClass; + private String enumMethod; + + @Override + public void initialize(EnumValue enumValue) { + enumMethod = enumValue.enumMethod(); + enumClass = enumValue.enumClass(); + } + + @Override + public boolean isValid(Object value, ConstraintValidatorContext constraintValidatorContext) { + if (value == null) { + return Boolean.TRUE; + } + + if (enumClass == null || enumMethod == null) { + return Boolean.TRUE; + } + Class valueClass = value.getClass(); + + try { + Method method = enumClass.getMethod(enumMethod, valueClass); + if (!Boolean.TYPE.equals(method.getReturnType()) && !Boolean.class.equals(method.getReturnType())) { + throw new RuntimeException(String.format("%s method return is not boolean type in the %s class", enumMethod, enumClass)); + } + + Boolean result = (Boolean)method.invoke(null, value); + return result == null ? false : result; + } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + throw new RuntimeException(e); + } catch (NoSuchMethodException | SecurityException e) { + throw new RuntimeException(String.format("This %s(%s) method does not exist in the %s", enumMethod, valueClass, enumClass), e); + } + } + + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/Log.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/Log.java new file mode 100644 index 0000000..15714ae --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/Log.java @@ -0,0 +1,23 @@ + /** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.dubhe.annotation; + +/** + * @description 日志 + * @date 2020-03-15 + */ +public @interface Log { +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/Query.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/Query.java new file mode 100644 index 0000000..40e1b77 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/Query.java @@ -0,0 +1,68 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @description 构建Wrapper的注解 + * @date 2020-03-26 + */ +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Query { + + + String propName() default ""; + + Type type() default Type.EQ; + + + String blurry() default ""; + + enum Type { + // 相等 + EQ + // 不等于 + , NE + // 大于 + , GT + // 大于等于 + , GE + // 小于 + , LT + // 小于等于 + , LE, + BETWEEN, + NOT_BETWEEN, + LIKE, + NOT_LIKE, + LIkE_LEFT, + LIKE_RIGHT, + IS_NULL, + IS_NOT_NULL, + IN, + NOT_IN, + INSQL, + NOT_INSQL, + ORDER_BY + } + +} + diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java new file mode 100644 index 0000000..f7bc427 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @description 限流 + * @date 2020-03-15 + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface RateLimit { + double limitNum() default 20; +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/aspect/LogAspect.java b/dubhe-server/common/src/main/java/org/dubhe/aspect/LogAspect.java new file mode 100644 index 0000000..ce7485f --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/aspect/LogAspect.java @@ -0,0 +1,87 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.aspect; + +import java.util.UUID; + +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.dubhe.enums.LogEnum; +import org.dubhe.utils.LogUtil; +import org.slf4j.MDC; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import lombok.extern.slf4j.Slf4j; + +/** + * @date 2020/04/10 + */ +@Component +@Aspect +@Slf4j +public class LogAspect { + + public static final String TRACE_ID = "traceId"; + + @Pointcut("execution(* org.dubhe..service..*.*(..))) " + + "and execution(* org.dubhe..schedule..*.*(..))) " + + "and execution(* org.dubhe..util*..*.*(..)))") + public void serviceAspect() { + } + + @Pointcut("execution(* org.dubhe..rest..*.*(..))) ") + public void restAspect() { + } + + @Pointcut(" execution(* org.dubhe..task..*.*(..))) ") + public void taskAspect() { + } + + @Pointcut(" serviceAspect() || taskAspect() ") + public void aroundAspect() { + } + + @Around("aroundAspect()") + public Object around(JoinPoint joinPoint) throws Throwable { + if (StringUtils.isEmpty(MDC.get(TRACE_ID))) { + MDC.put(TRACE_ID, UUID.randomUUID().toString()); + } + return combineLogInfo(joinPoint); + } + + @Around("restAspect()") + public Object aroundRest(JoinPoint joinPoint) throws Throwable { + MDC.clear(); + MDC.put(TRACE_ID, UUID.randomUUID().toString()); + return combineLogInfo(joinPoint); + } + + private Object combineLogInfo(JoinPoint joinPoint) throws Throwable { + Object[] param = joinPoint.getArgs(); + LogUtil.info(LogEnum.REST_REQ, "uri:{},input:{},==>begin", joinPoint.getSignature(), param); + long start = System.currentTimeMillis(); + Object result = ((ProceedingJoinPoint) joinPoint).proceed(); + long end = System.currentTimeMillis(); + LogUtil.info(LogEnum.REST_REQ, "uri:{},output:{},proc_time:{},<==end", joinPoint.getSignature().toString(), + result, end - start); + return result; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseConvert.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseConvert.java new file mode 100644 index 0000000..4b75c40 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseConvert.java @@ -0,0 +1,58 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.base; + +import java.util.List; + +/** + * @description DTO Entity 转换 + * @date 2020-03-15 + */ +public interface BaseConvert { + + /** + * DTO转Entity + * + * @param dto / + * @return / + */ + E toEntity(D dto); + + /** + * Entity转DTO + * + * @param entity / + * @return / + */ + D toDto(E entity); + + /** + * DTO集合转Entity集合 + * + * @param dtoList / + * @return / + */ + List toEntity(List dtoList); + + /** + * Entity集合转DTO集合 + * + * @param entityList / + * @return / + */ + List toDto(List entityList); + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java new file mode 100644 index 0000000..c868f28 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java @@ -0,0 +1,48 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.base; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description DTO基础类 + * @date 2020-03-15 + */ +@Getter +@Setter +public class BaseDTO implements Serializable { + + private Boolean deleted; + + private Timestamp createTime; + + private Timestamp updateTime; + + @Override + public String toString() { + return "BaseDTO{" + + "deleted=" + deleted + + ", createTime=" + createTime + + ", updateTime=" + updateTime + + '}'; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java new file mode 100644 index 0000000..95d527c --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java @@ -0,0 +1,76 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.base; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableLogic; +import lombok.Data; +import org.apache.commons.lang3.builder.ToStringBuilder; + +import java.io.Serializable; +import java.lang.reflect.Field; +import java.sql.Timestamp; + +/** + * @description Entity基础类 + * @date 2020-03-15 + */ +@Data +public class BaseEntity implements Serializable { + + private static final long serialVersionUID = 4936056317364745513L; + + /** + * 删除标识 + **/ + @TableField(value = "deleted",fill = FieldFill.INSERT) + @TableLogic + private Boolean deleted = false; + + @TableField(value = "create_user_id",fill = FieldFill.INSERT) + private Long createUserId; + + @TableField(value = "update_user_id",fill = FieldFill.INSERT_UPDATE) + private Long updateUserId; + + @TableField(value = "create_time",fill = FieldFill.INSERT) + private Timestamp createTime; + + + @TableField(value = "update_time",fill = FieldFill.INSERT_UPDATE) + private Timestamp updateTime; + + @Override + public String toString() { + ToStringBuilder builder = new ToStringBuilder(this); + Field[] fields = this.getClass().getDeclaredFields(); + try { + for (Field f : fields) { + f.setAccessible(true); + builder.append(f.getName(), f.get(this)).append("\n"); + } + } catch (Exception e) { + builder.append("toString builder encounter an error"); + } + return builder.toString(); + } + + public @interface Update { + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java new file mode 100644 index 0000000..950e453 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.base; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotBlank; +import java.io.Serializable; + +/** + * @description 镜像基础类DTO + * @date: 2020-07-14 + */ +@Data +@Accessors(chain = true) +public class BaseImageDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "镜像版本", required = true) + @NotBlank(message = "镜像版本不能为空") + private String imageTag; + + @ApiModelProperty(value = "镜像名称", required = true) + @NotBlank(message = "镜像名称不能为空") + private String imageName; + +} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java new file mode 100644 index 0000000..7a6fff0 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.base; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description: VO基础类 + * @date: 2020-05-22 + */ +@Data +public class BaseVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("创建人") + private Long createUserId; + + @ApiModelProperty("创建时间") + private Timestamp createTime; + + @ApiModelProperty("修改人") + private Long updateUserId; + + @ApiModelProperty("修改时间") + private Timestamp updateTime; +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java b/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java new file mode 100644 index 0000000..97ff300 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java @@ -0,0 +1,76 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.base; + + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.slf4j.MDC; + +import java.io.Serializable; + +/** + * @description 统一的公共响应体 + * @date 2020-03-16 + */ +@Data +@ApiModel(description = "统一的公共响应体") +public class DataResponseBody implements Serializable { + + /** + * 返回状态码 + */ + @ApiModelProperty("返回状态码") + private Integer code; + /** + * 返回信息 + */ + @ApiModelProperty("返回信息") + private String msg; + /** + * 泛型数据 + */ + @ApiModelProperty("泛型数据") + private T data; + /** + * 链路追踪ID + */ + @ApiModelProperty("链路追踪ID") + private String traceId; + + public DataResponseBody() { + this(ResponseCode.SUCCESS, null); + } + + public DataResponseBody(T data) { + this(ResponseCode.SUCCESS, null, data); + } + + public DataResponseBody(Integer code, String msg) { + this(code, msg, null); + } + + public DataResponseBody(Integer code, String msg, T data) { + this.code = code; + this.msg = msg; + this.data = data; + this.traceId = MDC.get("traceId"); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java b/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java new file mode 100644 index 0000000..a4376e7 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java @@ -0,0 +1,99 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.base; + +/** + * @date: 2020-05-14 + */ +public final class MagicNumConstant { + + public static final int NEGATIVE_ONE = -1; + public static final int ZERO = 0; + public static final int ONE = 1; + public static final int TWO = 2; + public static final int THREE = 3; + public static final int FOUR = 4; + public static final int FIVE = 5; + public static final int SIX = 6; + public static final int SEVEN = 7; + public static final int EIGHT = 8; + public static final int NINE = 9; + public static final int TEN = 10; + + public static final int ELEVEN = 11; + public static final int SIXTEEN = 16; + public static final int TWENTY = 20; + public static final int FIFTY = 50; + public static final int SIXTY = 60; + public static final int SIXTY_TWO = 62; + public static final int SIXTY_FOUR = 64; + public static final int INTEGER_TWO_HUNDRED_AND_FIFTY_FIVE = 255; + public static final int ONE_HUNDRED = 100; + public static final int ONE_HUNDRED_TWENTY_EIGHT = 128; + public static final int TWO_HUNDRED = 200; + public static final int FIVE_HUNDRED = 500; + public static final int FIVE_HUNDRED_AND_SIXTEEN = 516; + public static final int ONE_THOUSAND = 1000; + public static final int BINARY_TEN_EXP = 1024; + public static final int ONE_THOUSAND_ONE_HUNDRED = 1100; + public static final int ONE_THOUSAND_ONE_HUNDRED_ONE = 1101; + public static final int ONE_THOUSAND_TWO_HUNDRED = 1200; + public static final int ONE_THOUSAND_TWO_HUNDRED_ONE = 1201; + public static final int ONE_THOUSAND_TWENTY_FOUR = 1024; + public static final int ONE_THOUSAND_THREE_HUNDRED = 1300; + public static final int ONE_THOUSAND_THREE_HUNDRED_ONE = 1301; + public static final int ONE_THOUSAND_THREE_HUNDRED_NINE = 1309; + public static final int ONE_THOUSAND_FIVE_HUNDRED = 1500; + public static final int TWO_THOUSAND = 2000; + public static final int TWO_THOUSAND_TWENTY_EIGHT = 2048; + public static final int THREE_THOUSAND = 3000; + public static final int FOUR_THOUSAND = 4000; + public static final int NINE_THOUSAND = 9000; + public static final int NINE_THOUSAND_NINE_HUNDRED_NINTY_NINE = 9999; + public static final int TEN_THOUSAND = 10000; + public static final int FIFTEEN_THOUSAND = 15000; + public static final int HUNDRED_THOUSAND = 100000; + public static final int MILLION = 1000000; + public static final int ONE_MINUTE = 60000; + + public static final long NEGATIVE_ONE__LONG = -1L; + public static final long ZERO_LONG = 0L; + public static final long ONE_LONG = 1L; + public static final long TWO_LONG = 2L; + public static final long THREE_LONG = 3L; + public static final long FOUR_LONG = 4L; + public static final long FIVE_LONG = 5L; + public static final long SIX_LONG = 6L; + public static final long SEVEN_LONG = 7L; + public static final long EIGHT_LONG = 8L; + public static final long NINE_LONG = 9L; + public static final long TEN_LONG = 10L; + + public static final long TWELVE_LONG = 12L; + public static final long SIXTY_LONG = 60L; + public static final long TEN_THOUSAND_LONG = 10000L; + public static final long ONE_ZERO_ONE_ZERO_ONE_ZERO_LONG = 101010L; + public static final long NINE_ZERO_NINE_ZERO_NINE_ZERO_LONG = 909090L; + public static final long ONE_YEAR_BEFORE_LONG = 1552579200000L; + + public static final int SIXITY_0XFF = 0xFF; + + + private MagicNumConstant() { + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java b/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java new file mode 100644 index 0000000..e39aaa2 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java @@ -0,0 +1,62 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.base; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.constant.NumberConstant; + +import javax.validation.constraints.Min; + +/** + * @description: 分页基类 + * @date: 2020-05-8 + */ +@Data +@Accessors(chain = true) +public class PageQueryBase { + + @ApiModelProperty(value = "分页-当前页数") + @Min(value = 1, message = "current不能小于1") + private Integer current; + + @ApiModelProperty(value = "分页-每页展示数") + @Min(value = 1, message = "size不能小于1") + private Integer size; + + + @ApiModelProperty(value = "排序字段") + private String sort; + + @ApiModelProperty(value = "排序方式,asc | desc") + private String order; + + public Page toPage() { + Page page = new Page(); + if (this.current != null) { + page.setCurrent(this.current); + } + if (this.size != null && this.size < NumberConstant.MAX_PAGE_SIZE) { + page.setSize(this.size); + } + return page; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java b/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java new file mode 100644 index 0000000..5ee446f --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.base; + +/** + * @description 返回状态码 + * @date 2020-02-23 + */ +public class ResponseCode { + public static Integer SUCCESS = 200; + public static Integer UNAUTHORIZED = 401; + public static Integer ERROR = 10000; + public static Integer ENTITY_NOT_EXIST = 10001; + public static Integer BADREQUEST = 10002; + public static Integer SERVICE_ERROR = 10003; + public static Integer DOCKER_ERROR = 10004; + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/GlobalCorsConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/GlobalCorsConfig.java new file mode 100644 index 0000000..45a57c8 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/GlobalCorsConfig.java @@ -0,0 +1,56 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.UrlBasedCorsConfigurationSource; +import org.springframework.web.filter.CorsFilter; + +/** + * @description 允许跨域 + * @date 2020-02-23 + */ +@Configuration +public class GlobalCorsConfig { + + @Bean + public CorsFilter corsFilter() { + //1.添加CORS配置信息 + CorsConfiguration config = new CorsConfiguration(); + //放行哪些原始域 + config.addAllowedOrigin("*"); + //是否发送Cookie信息 + config.setAllowCredentials(true); + //放行哪些原始域(请求方式) + config.addAllowedMethod("OPTIONS"); + config.addAllowedMethod("HEAD"); + config.addAllowedMethod("GET"); + config.addAllowedMethod("PUT"); + config.addAllowedMethod("POST"); + config.addAllowedMethod("DELETE"); + config.addAllowedMethod("PATCH"); + config.addAllowedHeader("*"); + + //2.添加映射路径 + UrlBasedCorsConfigurationSource configSource = new UrlBasedCorsConfigurationSource(); + configSource.registerCorsConfiguration("/**", config); + + //3.返回新的CorsFilter. + return new CorsFilter(configSource); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java new file mode 100644 index 0000000..9c03757 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +/** + * @description k8s命名相关配置 + * + * @date 2020-05-13 + */ +@Getter +@Configuration +public class K8sNameConfig { + + @Value("${k8s.nfs-root-path}") + private String nfsRootPath; + + @Value("${k8s.namespace}") + private String namespace; + + @Value("${minio.bucketName}") + private String fileBucket; + + @Value("${train-job.docker-dataset-path}") + private String datasetPath; +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/KaptchaConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/KaptchaConfig.java new file mode 100644 index 0000000..86bf416 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/KaptchaConfig.java @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import com.google.code.kaptcha.impl.DefaultKaptcha; +import com.google.code.kaptcha.util.Config; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +import java.util.Properties; + +/** + * @description 验证码配置 + * @date 2020-02-23 + */ +@Component +public class KaptchaConfig { + private final static String CODE_LENGTH = "4"; + private final static String SESSION_KEY = "verification_session_key"; + + @Bean + public DefaultKaptcha defaultKaptcha() { + DefaultKaptcha defaultKaptcha = new DefaultKaptcha(); + Properties properties = new Properties(); + + // 设置边框 + properties.setProperty("kaptcha.border", "yes"); + // 设置边框颜色 + properties.setProperty("kaptcha.border.color", "105,179,90"); + // 设置字体颜色 + properties.setProperty("kaptcha.textproducer.font.color", "blue"); + // 设置图片宽度 + properties.setProperty("kaptcha.image.width", "108"); + // 设置图片高度 + properties.setProperty("kaptcha.image.height", "28"); + // 设置字体尺寸 + properties.setProperty("kaptcha.textproducer.font.size", "26"); + // 设置session key + properties.setProperty("kaptcha.session.key", SESSION_KEY); + // 设置验证码长度 + properties.setProperty("kaptcha.textproducer.char.length", CODE_LENGTH); + // 设置字体 + properties.setProperty("kaptcha.textproducer.font.names", "宋体,楷体,黑体"); + //去噪点 + properties.setProperty("kaptcha.noise.impl", "com.google.code.kaptcha.impl.NoNoise"); + Config config = new Config(properties); + defaultKaptcha.setConfig(config); + return defaultKaptcha; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java new file mode 100644 index 0000000..ffb7788 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +@Configuration +public class LocalDateTimeSerializerConfig { + @Value("${spring.jackson.date-format}") + private String pattern; + + @Bean + public LocalDateTimeSerializer localDateTimeDeserializer() { + return new LocalDateTimeSerializer(DateTimeFormatter.ofPattern(pattern)); + } + + @Bean + public Jackson2ObjectMapperBuilderCustomizer jackson2ObjectMapperBuilderCustomizer() { + return builder -> builder.serializerByType(LocalDateTime.class, localDateTimeDeserializer()); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java new file mode 100644 index 0000000..b7e9860 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java @@ -0,0 +1,95 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler; +import org.apache.ibatis.reflection.MetaObject; +import org.dubhe.constant.StringConstant; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.enums.SwitchEnum; +import org.dubhe.utils.DateUtil; +import org.dubhe.utils.JwtUtils; +import org.springframework.stereotype.Component; + +import java.util.Objects; + +/** + * @description 处理新增和更新的基础数据填充，配合BaseEntity和MyBatisPlusConfig使用 + * @date 2020-6-10 + */ +@Component +public class MetaHandlerConfig implements MetaObjectHandler { + + + private final String LOCK_USER_ID = "LOCK_USER_ID"; + + + /** + * 新增数据执行 + * + * @param metaObject + */ + + @Override + public void insertFill(MetaObject metaObject) { + if (Objects.isNull(getFieldValByName(StringConstant.CREATE_TIME, metaObject))) { + this.setFieldValByName(StringConstant.CREATE_TIME, DateUtil.getCurrentTimestamp(), metaObject); + } + if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_TIME, metaObject))) { + this.setFieldValByName(StringConstant.UPDATE_TIME, DateUtil.getCurrentTimestamp(), metaObject); + } + synchronized (LOCK_USER_ID){ + if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_USER_ID, metaObject))) { + this.setFieldValByName(StringConstant.UPDATE_USER_ID, getUserId(), metaObject); + } + if (Objects.isNull(getFieldValByName(StringConstant.CREATE_USER_ID, metaObject))) { + this.setFieldValByName(StringConstant.CREATE_USER_ID, getUserId(), metaObject); + } + } + if (Objects.isNull(getFieldValByName(StringConstant.DELETED, metaObject))) { + this.setFieldValByName(StringConstant.DELETED, SwitchEnum.getBooleanValue(SwitchEnum.OFF.getValue()), metaObject); + } + } + + /** + * 更新数据执行 + * + * @param metaObject + */ + @Override + public void updateFill(MetaObject metaObject) { + if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_TIME, metaObject))) { + this.setFieldValByName(StringConstant.UPDATE_TIME, DateUtil.getCurrentTimestamp(), metaObject); + } + if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_USER_ID, metaObject))) { + this.setFieldValByName(StringConstant.UPDATE_USER_ID, getUserId(), metaObject); + } + + } + + + /** + * 获取用户ID + * + * @return + */ + private Long getUserId() { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + return Objects.isNull(userDTO) ? null : userDTO.getId(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java new file mode 100644 index 0000000..18deab6 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java @@ -0,0 +1,310 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import com.baomidou.mybatisplus.core.override.MybatisMapperProxy; +import com.baomidou.mybatisplus.core.parser.ISqlParser; +import com.baomidou.mybatisplus.core.parser.SqlParserHelper; +import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; +import com.baomidou.mybatisplus.extension.plugins.tenant.TenantHandler; +import com.baomidou.mybatisplus.extension.plugins.tenant.TenantSqlParser; +import com.google.common.collect.Sets; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.expression.operators.relational.InExpression; +import net.sf.jsqlparser.schema.Column; +import org.apache.ibatis.mapping.MappedStatement; +import org.apache.shiro.UnavailableSecurityManagerException; +import org.dubhe.annotation.DataPermission; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.constant.PermissionConstant; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.Role; +import org.dubhe.enums.LogEnum; +import org.dubhe.utils.JwtUtils; +import org.dubhe.utils.LogUtil; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.ApplicationListener; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.core.annotation.AnnotationUtils; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.util.CollectionUtils; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @description MybatisPlus配置类 + * @date 2020-06-24 + */ +@EnableTransactionManagement +@Configuration +public class MybatisPlusConfig implements ApplicationListener, ApplicationContextAware { + + /** + * 以此字段作为租户实现数据隔离 + */ + private static final String TENANT_ID_COLUMN = "create_user_id"; + /** + * 以0作为公共数据的标识 + */ + private static final long PUBLIC_TENANT_ID = MagicNumConstant.ZERO; + private static final Set PUBLIC_TENANT_ID_SET = new HashSet() {{ + add(PUBLIC_TENANT_ID); + }}; + private static final String PACKAGE_SEPARATOR = "."; + private static final Set SELECT_PERMISSION = new HashSet() {{ + add(PermissionConstant.SELECT); + }}; + private static final Set UPDATE_DELETE_PERMISSION = new HashSet() {{ + add(PermissionConstant.UPDATE); + add(PermissionConstant.DELETE); + }}; + + private static final String SELECT_STR = "select"; + /** + * 优先级高于dataFilters，如果ignore，则不进行sql注入 + */ + private Map> dataFilters = new HashMap<>(); + + private ApplicationContext applicationContext; + public Set tenantId; + + /** + * mybatis plus 分页插件 + * 其中增加了通过多租户实现了数据权限功能 + * + * @return + */ + @Bean + public PaginationInterceptor paginationInterceptor() { + PaginationInterceptor paginationInterceptor = new PaginationInterceptor(); + List sqlParserList = new ArrayList<>(); + TenantSqlParser tenantSqlParser = new TenantSqlParser(); + tenantSqlParser.setTenantHandler(new TenantHandler() { + @Override + public Expression getTenantId(boolean where) { + Set tenants = tenantId; + + final boolean multipleTenantIds = tenants.size() > MagicNumConstant.ONE; + if (multipleTenantIds) { + return multipleTenantIdCondition(tenants); + } else { + return singleTenantIdCondition(tenants); + } + } + + private Expression singleTenantIdCondition(Set tenants) { + return new LongValue((Long) tenants.toArray()[0]); + } + + private Expression multipleTenantIdCondition(Set tenants) { + final InExpression inExpression = new InExpression(); + inExpression.setLeftExpression(new Column(getTenantIdColumn())); + final ExpressionList itemsList = new ExpressionList(); + final List inValues = new ArrayList<>(tenants.size()); + tenants.forEach(i -> + inValues.add(new LongValue(i)) + ); + itemsList.setExpressions(inValues); + inExpression.setRightItemsList(itemsList); + return inExpression; + } + + @Override + public String getTenantIdColumn() { + return TENANT_ID_COLUMN; + } + + @Override + public boolean doTableFilter(String tableName) { + return false; + } + }); + sqlParserList.add(tenantSqlParser); + paginationInterceptor.setSqlParserList(sqlParserList); + paginationInterceptor.setSqlParserFilter(metaObject -> { + MappedStatement ms = SqlParserHelper.getMappedStatement(metaObject); + String method = ms.getId(); + if (!dataFilters.containsKey(method) || isAdmin()) { + return true; + } + Set permission = dataFilters.get(method); + tenantId = getTenantId(permission); + return false; + }); + return paginationInterceptor; + } + + /** + * 判断用户是否是管理员 + * 如果未登录，无法请求任何接口，所以不会到该层，因此匿名认为是定时任务，给予admin权限。 + * + * @return 判断用户是否是管理员 + */ + private boolean isAdmin() { + UserDTO user; + try { + user = JwtUtils.getCurrentUserDto(); + } catch (UnavailableSecurityManagerException e) { + return true; + } + if (Objects.isNull(user)) { + return true; + } + List roles; + if ((roles = user.getRoles()) == null) { + return false; + } + Set permissions = roles.stream().map(Role::getPermission).collect(Collectors.toSet()); + if (CollectionUtils.isEmpty(permissions)) { + return false; + } + return user.getId() == PermissionConstant.ANONYMOUS_USER || user.getId() == PermissionConstant.ADMIN_USER_ID; + } + + /** + * 如果是管理员，在前一步isAdmin已过滤； + * 如果是匿名用户，在shiro层被过滤； + * 因此只会是无角色、权限用户或普通用户 + * + * @return Set 租户ID集合 + */ + private Set getTenantId(Set permission) { + UserDTO user = JwtUtils.getCurrentUserDto(); + List roles; + if (Objects.isNull(user) || (roles = user.getRoles()) == null) { + if (permission.contains(PermissionConstant.SELECT)) { + return PUBLIC_TENANT_ID_SET; + } + return Collections.EMPTY_SET; + } + Set permissions = roles.stream().map(Role::getPermission).collect(Collectors.toSet()); + if (CollectionUtils.isEmpty(permissions)) { + if (permission.contains(PermissionConstant.SELECT)) { + return PUBLIC_TENANT_ID_SET; + } + return Collections.EMPTY_SET; + } + if (permission.contains(PermissionConstant.SELECT)) { + return new HashSet() {{ + add(PUBLIC_TENANT_ID); + add(user.getId()); + }}; + } + return new HashSet() {{ + add(user.getId()); + }}; + } + + /** + * 设置上下文 + * #需要通过上下文 获取SpringBean + * + * @param applicationContext spring上下文 + * @throws BeansException 找不到bean异常 + */ + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + + @Override + public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) { + Class annotationClass = DataPermission.class; + Map beanWithAnnotation = applicationContext.getBeansWithAnnotation(annotationClass); + Set> entitySet = beanWithAnnotation.entrySet(); + for (Map.Entry entry : entitySet) { + Proxy proxy = (Proxy) entry.getValue(); + Class clazz = getMapperClass(proxy); + populateDataFilters(clazz); + } + } + + /** + * 根据mapper对应代理对象获取Class + * + * @param proxy mapper对应代理对象 + * @return + */ + private Class getMapperClass(Proxy proxy) { + try { + Field field = proxy.getClass().getSuperclass().getDeclaredField("h"); + field.setAccessible(true); + MybatisMapperProxy mapperProxy = (MybatisMapperProxy) field.get(proxy); + field = mapperProxy.getClass().getDeclaredField("mapperInterface"); + field.setAccessible(true); + return (Class) field.get(mapperProxy); + } catch (NoSuchFieldException | IllegalAccessException e) { + LogUtil.error(LogEnum.BIZ_DATASET, "reflect error", e); + } + return null; + } + + /** + * 填充数据权限过滤，处理那些需要排除的方法 + * + * @param clazz 需要处理的类(mapper) + */ + private void populateDataFilters(Class clazz) { + if (clazz == null) { + return; + } + Method[] methods = clazz.getMethods(); + DataPermission dataPermission = AnnotationUtils.findAnnotation((Class) clazz, DataPermission.class); + Set ignores = Sets.newHashSet(dataPermission.ignores()); + for (Method method : methods) { + if (ignores.contains(method.getName())) { + continue; + } + Set permission = getDataPermission(method); + dataFilters.put(clazz.getName() + PACKAGE_SEPARATOR + method.getName(), permission); + } + } + + /** + * 获取方法上权限注解 + * 权限注解包含 + * 1.用户拥有指定权限才可以执行该方法：比如 PermissionConstant.SELECT 表示用户必须拥有select权限，才可以使用该方法 + * 2.方法权限校验排除：比如 ignores = {"insert"} 表示insert方法不做权限处理 + * + * @param method 方法对象 + * @return + */ + private Set getDataPermission(Method method) { + DataPermission dataPermission = AnnotationUtils.findAnnotation(method, DataPermission.class); + // 无注解时以方法名判断 + if (dataPermission == null) { + if (method.getName().contains(SELECT_STR)) { + return SELECT_PERMISSION; + } + return UPDATE_DELETE_PERMISSION; + } + return Sets.newHashSet(dataPermission.permission()); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java new file mode 100644 index 0000000..c399147 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * @description NFS config + * @create 2020/5/13 + */ +@Data +@Component +public class NfsConfig { + + @Value("${k8s.nfs}") + private String nfsIp; + + @Value("${k8s.nfs-root-path}") + private String rootDir; + + @Value("/${minio.bucketName}/") + private String bucket; + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java new file mode 100644 index 0000000..dad8e7c --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java @@ -0,0 +1,214 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import cn.hutool.core.lang.Assert; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.parser.ParserConfig; +import com.alibaba.fastjson.serializer.SerializerFeature; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.dubhe.utils.StringUtils; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cache.Cache; +import org.springframework.cache.annotation.CachingConfigurerSupport; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.interceptor.CacheErrorHandler; +import org.springframework.cache.interceptor.KeyGenerator; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.cache.RedisCacheConfiguration; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisOperations; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.serializer.RedisSerializationContext; +import org.springframework.data.redis.serializer.RedisSerializer; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +/** + * @description redis配置类 + * @date 2020-03-25 + */ +@Slf4j +@Configuration +@EnableCaching +@ConditionalOnClass(RedisOperations.class) +@EnableConfigurationProperties(RedisProperties.class) +public class RedisConfig extends CachingConfigurerSupport { + + /** + * 设置 redis 数据默认过期时间，默认2小时 + * 设置@cacheable 序列化方式 + */ + @Bean + public RedisCacheConfiguration redisCacheConfiguration() { + FastJsonRedisSerializer fastJsonRedisSerializer = new FastJsonRedisSerializer<>(Object.class); + RedisCacheConfiguration configuration = RedisCacheConfiguration.defaultCacheConfig(); + configuration = configuration.serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(fastJsonRedisSerializer)).entryTtl(Duration.ofHours(2)); + return configuration; + } + + @SuppressWarnings("all") + @Bean(name = "redisTemplate") + @ConditionalOnMissingBean(name = "redisTemplate") + public RedisTemplate redisTemplate(RedisConnectionFactory redisConnectionFactory) { + RedisTemplate template = new RedisTemplate<>(); + //序列化 + FastJsonRedisSerializer fastJsonRedisSerializer = new FastJsonRedisSerializer<>(Object.class); + // value值的序列化采用fastJsonRedisSerializer + template.setValueSerializer(fastJsonRedisSerializer); + template.setHashValueSerializer(fastJsonRedisSerializer); + // 全局开启AutoType，这里方便开发，使用全局的方式 + ParserConfig.getGlobalInstance().setAutoTypeSupport(true); + // 建议使用这种方式，小范围指定白名单 + // ParserConfig.getGlobalInstance().addAccept("org.dubhe.domain"); + // key的序列化采用StringRedisSerializer + template.setKeySerializer(new StringRedisSerializer()); + template.setHashKeySerializer(new StringRedisSerializer()); + template.setConnectionFactory(redisConnectionFactory); + return template; + } + + /** + * 自定义缓存key生成策略，默认将使用该策略 + */ + @Bean + @Override + public KeyGenerator keyGenerator() { + return (target, method, params) -> { + Map container = new HashMap<>(3); + Class targetClassClass = target.getClass(); + // 类地址 + container.put("class", targetClassClass.toGenericString()); + // 方法名称 + container.put("methodName", method.getName()); + // 包名称 + container.put("package", targetClassClass.getPackage()); + // 参数列表 + for (int i = 0; i < params.length; i++) { + container.put(String.valueOf(i), params[i]); + } + // 转为JSON字符串 + String jsonString = JSON.toJSONString(container); + // 做SHA256 Hash计算，得到一个SHA256摘要作为Key + return DigestUtils.sha256Hex(jsonString); + }; + } + + @Bean + @Override + public CacheErrorHandler errorHandler() { + // 异常处理，当Redis发生异常时，打印日志，但是程序正常走 + log.info("初始化 -> [{}]", "Redis CacheErrorHandler"); + return new CacheErrorHandler() { + @Override + public void handleCacheGetError(RuntimeException e, Cache cache, Object key) { + log.error("Redis occur handleCacheGetError：key -> [{}]", key, e); + } + + @Override + public void handleCachePutError(RuntimeException e, Cache cache, Object key, Object value) { + log.error("Redis occur handleCachePutError：key -> [{}]；value -> [{}]", key, value, e); + } + + @Override + public void handleCacheEvictError(RuntimeException e, Cache cache, Object key) { + log.error("Redis occur handleCacheEvictError：key -> [{}]", key, e); + } + + @Override + public void handleCacheClearError(RuntimeException e, Cache cache) { + log.error("Redis occur handleCacheClearError：", e); + } + }; + } + +} + +/** + * Value 序列化 + * + * @param + */ +class FastJsonRedisSerializer implements RedisSerializer { + + private Class clazz; + + FastJsonRedisSerializer(Class clazz) { + super(); + this.clazz = clazz; + } + + @Override + public byte[] serialize(T t) { + if (t == null) { + return new byte[0]; + } + return JSON.toJSONString(t, SerializerFeature.WriteClassName).getBytes(StandardCharsets.UTF_8); + } + + @Override + public T deserialize(byte[] bytes) { + if (bytes == null || bytes.length <= 0) { + return null; + } + String str = new String(bytes, StandardCharsets.UTF_8); + return JSON.parseObject(str, clazz); + } + +} + +/** + * 重写序列化器 + * + */ +class StringRedisSerializer implements RedisSerializer { + + private final Charset charset; + + StringRedisSerializer() { + this(StandardCharsets.UTF_8); + } + + private StringRedisSerializer(Charset charset) { + Assert.notNull(charset, "Charset must not be null!"); + this.charset = charset; + } + + @Override + public String deserialize(byte[] bytes) { + return (bytes == null ? null : new String(bytes, charset)); + } + + @Override + public byte[] serialize(Object object) { + String string = JSON.toJSONString(object); + if (StringUtils.isBlank(string)) { + return null; + } + string = string.replace("\"", ""); + return string.getBytes(charset); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java b/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java new file mode 100644 index 0000000..143b7f2 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java @@ -0,0 +1,70 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import org.springframework.context.annotation.PropertySource; +import org.springframework.context.i18n.LocaleContextHolder; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.ResourceBundle; + +/** + * @description 加载配置 + * @date 2020-03-15 + */ +@PropertySource(value = {"classpath:config/thirdParty.properties", "classpath:i18n/messages*.properties"}) +public final class Resources { + /** + * 第三方登录配置 + */ + public static final ResourceBundle THIRDPARTY = ResourceBundle.getBundle("config/thirdParty"); + /** + * 国际化信息 + */ + private static final Map MESSAGES = new HashMap(); + + /** + * 国际化信息 + */ + public static String getMessage(String key, Object... params) { + Locale locale = LocaleContextHolder.getLocale(); + ResourceBundle message = MESSAGES.get(locale.getLanguage()); + if (message == null) { + synchronized (MESSAGES) { + message = MESSAGES.get(locale.getLanguage()); + if (message == null) { + message = ResourceBundle.getBundle("i18n/messages", locale); + MESSAGES.put(locale.getLanguage(), message); + } + } + } + if (params != null && params.length > 0) { + return String.format(message.getString(key), params); + } + return message.getString(key); + } + + /** + * 清除国际化信息 + */ + public static void flushMessage() { + MESSAGES.clear(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java new file mode 100644 index 0000000..8880a09 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; + +/** + * @description 关闭Security + * @date 2020-03-25 + */ +@Configuration +public class SecurityConfig extends WebSecurityConfigurerAdapter { + + @Override + protected void configure(HttpSecurity http) throws Exception { + http.authorizeRequests().antMatchers("/**").permitAll().and().csrf().disable(); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java new file mode 100644 index 0000000..14a4bd7 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java @@ -0,0 +1,138 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import com.fasterxml.classmate.TypeResolver; +import com.google.common.base.Predicates; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.Ordered; +import org.springframework.data.domain.Pageable; +import springfox.documentation.builders.ApiInfoBuilder; +import springfox.documentation.builders.ParameterBuilder; +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.schema.AlternateTypeRule; +import springfox.documentation.schema.AlternateTypeRuleConvention; +import springfox.documentation.schema.ModelRef; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.service.Parameter; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static com.google.common.collect.Lists.newArrayList; +import static springfox.documentation.schema.AlternateTypeRules.newRule; + +/** + * @description api页面 /swagger-ui.html + * @date 2020-03-25 + */ + +@Configuration +@EnableSwagger2 +public class SwaggerConfig { + + @Value("${jwt.header}") + private String tokenHeader; + + @Value("${jwt.token-start-with}") + private String tokenStartWith; + + @Value("${swagger.enabled}") + private Boolean enabled; + + @Value("${server.rest-version}") + private String restVersion; + + @Bean + @SuppressWarnings("all") + public Docket createRestApi() { + List pars = new ArrayList<>(); + pars.add(new ParameterBuilder().name(tokenHeader).description("token") + .modelRef(new ModelRef("string")) + .parameterType("header") + .defaultValue(tokenStartWith + " ") + .required(true) + .build()); + pars.add(new ParameterBuilder().name("version").description("版本") + .modelRef(new ModelRef("string")) + .parameterType("path") + .defaultValue(restVersion) + .required(false) + .build()); + return new Docket(DocumentationType.SWAGGER_2) + .enable(enabled) + .apiInfo(apiInfo()) + .select() + .paths(Predicates.not(PathSelectors.regex("/error.*"))) + .build().directModelSubstitute(Timestamp.class, Date.class) + .globalOperationParameters(pars); + } + + private ApiInfo apiInfo() { + return new ApiInfoBuilder() + .title("之江天枢一站式人工智能开源平台") + .version("1.0") + .build(); + } + +} + +/** + * 将Pageable转换展示在swagger中 + */ +@Configuration +class SwaggerDataConfig { + + @Bean + public AlternateTypeRuleConvention pageableConvention(final TypeResolver resolver) { + return new AlternateTypeRuleConvention() { + @Override + public int getOrder() { + return Ordered.HIGHEST_PRECEDENCE; + } + + @Override + public List rules() { + return newArrayList(newRule(resolver.resolve(Pageable.class), resolver.resolve(Page.class))); + } + }; + } + + @ApiModel + @Data + private static class Page { + @ApiModelProperty("页码 (0..N)") + private Integer page; + + @ApiModelProperty("每页显示的数目") + private Integer size; + + @ApiModelProperty("以下列格式排序标准：property[,asc | desc]。 默认排序顺序为升序。 支持多种排序条件：如：id,asc") + private List sort; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java new file mode 100644 index 0000000..bc4341c --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * @description harbor相关配置 + * @date 2020-07-17 + */ +@Data +@Component +@ConfigurationProperties(prefix = "harbor") +public class TrainHarborConfig { + + private String address; + + private String username; + + private String password; + + private String modelName; +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java new file mode 100644 index 0000000..85a83be --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import org.dubhe.interceptor.K8sCallBackPodInterceptor; +import org.dubhe.utils.K8sCallBackTool; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import javax.annotation.Resource; + +/** + * @description Web Mvc Config + * + * @date 2020-05-28 + */ +@Configuration +public class WebMvcConfig implements WebMvcConfigurer { + + @Resource + private K8sCallBackPodInterceptor k8sCallBackPodInterceptor; + + @Override + public void addInterceptors(InterceptorRegistry registry) { + InterceptorRegistration registration = registry.addInterceptor(k8sCallBackPodInterceptor); + // 拦截配置 + registration.addPathPatterns(K8sCallBackTool.getK8sCallbackPaths()); + + } + + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java new file mode 100644 index 0000000..60c2962 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import org.dubhe.handle.CustomRequestMappingHandlerMapping; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.web.servlet.WebMvcRegistrations; +import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping; + +/** + * @description API版本控制 + * @date 2020-04-06 + */ +@SpringBootConfiguration +public class WebMvcRegistrationsConfig implements WebMvcRegistrations { + @Override + public RequestMappingHandlerMapping getRequestMappingHandlerMapping() { + return new CustomRequestMappingHandlerMapping(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java b/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java new file mode 100644 index 0000000..1f8e234 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +import lombok.Getter; + +/** + * @description 算法枚举类 + * @date 2020-05-12 + */ +@Getter +public enum AlgorithmSourceEnum { + + /** + * MINE 算法来源 我的算法 + */ + MINE(1, "MINE"), + /** + * PRE 算法来源 预置算法 + */ + PRE(2,"PRE"); + + private Integer status; + + private String message; + + AlgorithmSourceEnum(Integer status, String message) { + this.status = status; + this.message = message; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java new file mode 100644 index 0000000..08e124f --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +/** + * @Description 数字常量 + * @Date 2020-6-9 + */ +public class NumberConstant { + + public final static int NUMBER_0 = 0; + public final static int NUMBER_1 = 1; + public final static int NUMBER_2 = 2; + public final static int NUMBER_3 = 3; + public final static int NUMBER_5 = 5; + public final static int NUMBER_10 = 10; + public final static int NUMBER_30 = 30; + public final static int NUMBER_50 = 50; + public final static int NUMBER_60 = 60; + public final static int HOUR_SECOND = 60 * 60; + public final static int DAY_SECOND = 60 * 60 * 24; + public final static int WEEK_SECOND = 60 * 60 * 24 * 7; + public final static int MAX_PAGE_SIZE = 2000; +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java new file mode 100644 index 0000000..8989218 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +import lombok.Data; +import org.springframework.stereotype.Component; + +/** + * @description: 权限常量 + * @since: 2020-05-25 14:39 + */ +@Component +@Data +public class PermissionConstant { + + /** + * 超级用户 + */ + public static final long ADMIN_USER_ID = 1L; + public static final long ANONYMOUS_USER = -1L; + public static final String SELECT = "select"; + public static final String UPDATE = "update"; + public static final String DELETE = "delete"; + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java b/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java new file mode 100644 index 0000000..d9d4f7a --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +/** + * @description 权限标识，对应 menu 表中的 permission 字段 + * @date 2020-05-14 + */ +public final class Permissions { + + /** + * 数据管理 + */ + public static final String DATA = "data"; + public static final String DATA_DATASET = "data:dataset"; + + /** + * 模型开发 + */ + public static final String DEVELOPMENT = "development"; + public static final String DEVELOPMENT_NOTEBOOK = "development:notebook"; + public static final String DEVELOPMENT_ALGORITHM = "development:algorithm"; + + /** + * 训练管理 + */ + public static final String TRAINING = "training"; + public static final String TRAINING_IMAGE = "training:image"; + public static final String TRAINING_JOB = "training:job"; + + /** + * 模型管理 + */ + public static final String MODEL = "model"; + public static final String MODEL_MODEL = "model:model"; + + /** + * 控制台 + */ + public static final String SYSTEM = "system"; + public static final String SYSTEM_USER = "system:user"; + public static final String SYSTEM_ROLE = "system:role"; + public static final String SYSTEM_MENU = "system:menu"; + public static final String SYSTEM_DICT = "system:dict"; + public static final String SYSTEM_NODE = "system:node"; + public static final String SYSTEM_LOG = "system:log"; + public static final String SYSTEM_TEAM = "system:team"; + + private Permissions() { + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java new file mode 100644 index 0000000..bd97aad --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +/** + * @description 字符串constant + * @date 2020-05-14 + */ +public final class StringConstant { + + public static final String MSIE = "MSIE"; + public static final String MOZILLA = "Mozilla"; + + /** + * 公共字段 + */ + public static final String CREATE_TIME = "createTime"; + public static final String UPDATE_TIME = "updateTime"; + public static final String UPDATE_USER_ID = "updateUserId"; + public static final String CREATE_USER_ID = "createUserId"; + public static final String DELETED = "deleted"; + public static final String UTF8 = "utf-8"; + + /** + * 测试环境 + */ + public static final String PROFILE_ACTIVE_TEST = "test"; + + private StringConstant() { + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java new file mode 100644 index 0000000..1e0476a --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +/** + * @description 符号常量 + * @Date 2020-5-29 + */ +public class SymbolConstant { + public static final String SLASH = "/"; + public static final String COMMA = ","; + public static final String COLON = ":"; + public static final String LINEBREAK = "\n"; + public static final String BLANK = ""; + public static final String QUESTION = "?"; + public static final String ZERO = "0"; + public static final String DOT = "."; + public static final String TOKEN = "token"; + public static final String GET = "get"; + public static final String SET = "set"; + + public static final String BRACKETS = "{}"; + public static final String BACKSLASH = "\\"; + public static final String BACKSLASH_MARK= "\\\""; + public static final String DOUBLE_MARK= "\"\""; + public static final String MARK= "\""; + + private SymbolConstant() { + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java new file mode 100644 index 0000000..71477d0 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java @@ -0,0 +1,47 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * @description 算法常量 + * @date 2020-06-02 + */ +@Data +@Component +public class TrainAlgorithmConstant { + + @Value("${train-algorithm.is-train-out}") + private Boolean isTrainOut; + + @Value("${train-algorithm.is-train-log}") + private Boolean isTrainLog; + + @Value("${train-algorithm.is-visualized-log}") + private Boolean isVisualizedLog; + + @Value("${train-algorithm.algorithm-source}") + private Integer algorithmSource; + + @Value("${train-algorithm.fork}") + private Boolean fork; + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/TrainJobConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/TrainJobConstant.java new file mode 100644 index 0000000..196d6c5 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/TrainJobConstant.java @@ -0,0 +1,100 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * @description 训练常量 + * @create: 2020-05-12 + */ +@Component +@Data +public class TrainJobConstant { + + @Value("${train-job.namespace}") + private String namespace; + + @Value("${train-job.version-label}") + private String versionLabel; + + @Value("${train-job.separator}") + private String separator; + + @Value("${train-job.pod-name}") + private String podName; + + @Value("${train-job.python-format}") + private String pythonFormat; + + @Value("${train-job.manage}") + private String manage; + + @Value("${train-job.out-path}") + private String outPath; + + @Value("${train-job.log-path}") + private String logPath; + + @Value("${train-job.visualized-log-path}") + private String visualizedLogPath; + + @Value("${train-job.docker-dataset-path}") + private String dockerDatasetPath; + + @Value("${train-job.docker-train-path}") + private String dockerTrainPath; + + @Value("${train-job.docker-out-path}") + private String dockerOutPath; + + @Value("${train-job.docker-log-path}") + private String dockerLogPath; + + @Value("${train-job.docker-dataset}") + private String dockerDataset; + + @Value("${train-job.docker-visualized-log-path}") + private String dockerVisualizedLogPath; + + @Value("${train-job.load-path}") + private String loadPath; + + @Value("${train-job.load-key}") + private String loadKey; + + @Value("${train-job.eight}") + private String eight; + + @Value("${train-job.plus-eight}") + private String plusEight; + + public static final String TRAIN_ID = "trainId"; + + public static final String TRAIN_VERSION = "trainVersion"; + + public static final String RUN_TIME = "runtime"; + + public static final String TRAIN_STATUS = "trainStatus"; + + public static final String CREATE_TIME = "createTime"; + + public static final String ALGORITHM_NAME = "algorithmName"; +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java new file mode 100644 index 0000000..27b19d8 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +import org.springframework.stereotype.Component; + +import lombok.Data; + +/** + * @description 算法用途 + * @date: 2020-06-23 + */ +@Component +@Data +public class UserAuxiliaryInfoConstant { + + + + public static final String ALGORITHM_USAGE ="algorithem_usage"; + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/dto/UserDTO.java b/dubhe-server/common/src/main/java/org/dubhe/domain/dto/UserDTO.java new file mode 100644 index 0000000..cf506f5 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/dto/UserDTO.java @@ -0,0 +1,65 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.domain.dto; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.dubhe.domain.entity.Role; +import org.dubhe.domain.entity.UserAvatar; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.Date; +import java.util.List; + +/** + * @description 用户信息 + * @date 2020-06-29 + */ +@Data +public class UserDTO implements Serializable { + + @ApiModelProperty(hidden = true) + private Long id; + + private String username; + + private String nickName; + + private String sex; + + private String email; + + private String phone; + + private Boolean enabled; + + private String remark; + + @JsonIgnore + private String password; + + private Date lastPasswordResetTime; + + private Timestamp createTime; + + private List roles; + + private UserAvatar userAvatar; + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java new file mode 100644 index 0000000..d37791d --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java @@ -0,0 +1,64 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; +import java.io.Serializable; + +import org.dubhe.base.MagicNumConstant; + +import com.alibaba.fastjson.annotation.JSONField; + +import cn.hutool.core.date.DateUtil; +import lombok.Data; +import lombok.experimental.Accessors; + +/** + * @description 日志对象封装类 + * @date 2020-06-29 + */ +@Data +@Accessors(chain = true) +public class LogInfo implements Serializable { + + @JSONField(ordinal = MagicNumConstant.ONE) + private String traceId; + + @JSONField(ordinal = MagicNumConstant.TWO) + private String type; + + @JSONField(ordinal = MagicNumConstant.THREE) + private String level; + + @JSONField(ordinal = MagicNumConstant.FOUR) + private String cName; + + @JSONField(ordinal = MagicNumConstant.FIVE) + private String mName; + + @JSONField(ordinal = MagicNumConstant.SIX) + private String line; + + @JSONField(ordinal = MagicNumConstant.SEVEN) + private String time = DateUtil.now(); + + @JSONField(ordinal = MagicNumConstant.EIGHT) + private Object info; + + public void setInfo(Object info) { + this.info = info; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Menu.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Menu.java new file mode 100644 index 0000000..32e3162 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Menu.java @@ -0,0 +1,120 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.dubhe.base.BaseEntity; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.Objects; + +/** + * @description 菜单实体 + * @date 2020-06-29 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +@TableName("menu") +public class Menu extends BaseEntity implements Serializable { + + private static final long serialVersionUID = 3100515433018008777L; + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @NotBlank + private String name; + + @TableField(value = "sort") + private Long sort = 999L; + + @TableField(value = "path") + private String path; + + @TableField(value = "component") + private String component; + + /** + * 类型，目录、菜单、按钮 + */ + @TableField(value = "type") + private Integer type; + + /** + * 权限 + */ + @TableField(value = "permission") + private String permission; + + @TableField(value = "component_name") + private String componentName; + + @TableField(value = "icon") + private String icon; + + /** + * 布局类型 + */ + @TableField(value = "layout") + private String layout; + + @TableField(value = "cache") + private Boolean cache; + + @TableField(value = "hidden") + private Boolean hidden; + + /** + * 上级菜单ID + */ + @TableField(value = "pid") + private Long pid; + + @TableField(value = "deleted",fill = FieldFill.INSERT) + private Boolean deleted; + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Menu menu = (Menu) o; + return Objects.equals(id, menu.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + public @interface Update { + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Role.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Role.java new file mode 100644 index 0000000..77227ec --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Role.java @@ -0,0 +1,87 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.dubhe.base.BaseEntity; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.Objects; +import java.util.Set; + +/** + * @description 角色实体 + * @date 2020-06-29 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +@TableName("role") +public class Role extends BaseEntity implements Serializable { + + private static final long serialVersionUID = -812009584744832371L; + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @TableField(value = "name") + private String name; + + /** + * 权限 + */ + @TableField(value = "permission") + private String permission; + + @TableField(value = "remark") + private String remark; + + @TableField(exist = false) + private Set menus; + + @TableField(value = "deleted",fill = FieldFill.INSERT) + private Boolean deleted = false; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Role role = (Role) o; + return Objects.equals(id, role.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + public @interface Update { + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/RoleMenu.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/RoleMenu.java new file mode 100644 index 0000000..18715c5 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/RoleMenu.java @@ -0,0 +1,49 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + + +/** + * @description 角色菜单关系实体 + * @date 2020-06-29 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +@TableName("roles_menus") +public class RoleMenu implements Serializable { + + private static final long serialVersionUID = -6296866205797727963L; + + @TableField(value = "menu_id") + private Long menuId; + + @TableField(value = "role_id") + private Long roleId; + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java new file mode 100644 index 0000000..d0eb560 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java @@ -0,0 +1,85 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import java.sql.Timestamp; +import java.util.List; +import java.util.Objects; + +/** + * @description 团队实体 + * @date 2020-06-29 + */ +@TableName("team") +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Team { + @TableId(value = "id", type = IdType.AUTO) + @NotNull(groups = Update.class) + private Long id; + + @TableField(value = "name") + @NotBlank + private String name; + + @TableField(value = "enabled") + @NotNull + private Boolean enabled; + + /** + * 团队成员 + */ + @TableField(exist = false) + private List teamUserList; + + @TableField(value = "create_time") + private Timestamp createTime; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Team dept = (Team) o; + return Objects.equals(id, dept.id) && + Objects.equals(name, dept.name); + } + + @Override + public int hashCode() { + return Objects.hash(id, name); + } + + public @interface Update { + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java new file mode 100644 index 0000000..c6ec6c3 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 团队用户关系实体 + * @date 2020-06-29 + */ +@Data +@TableName("teams_users_roles") +public class TeamUserRole implements Serializable { + + @TableId(value = "id", type = IdType.AUTO) + @NotNull() + private Long id; + + /** + * 团队 + */ + @TableField(exist = false) + private Team team; + + /** + * 用户 + */ + @TableField(exist = false) + private User user; + + /** + * 角色 + */ + @TableField(exist = false) + private Role role; + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/User.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/User.java new file mode 100644 index 0000000..fdbe3f7 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/User.java @@ -0,0 +1,126 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.dubhe.base.BaseEntity; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.io.Serializable; +import java.util.Date; +import java.util.List; +import java.util.Objects; + + +/** + * @description 用户实体 + * @date 2020-06-29 + */ +@Data +@TableName("user") +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class User extends BaseEntity implements Serializable { + + private static final long serialVersionUID = -3836401769559845765L; + + @TableId(value = "id", type = IdType.AUTO) + @NotNull(groups = Update.class) + private Long id; + + @NotBlank + @TableField(value = "username") + private String username; + + /** + * 用户昵称 + */ + @TableField(value = "nick_name") + @NotBlank + private String nickName; + + /** + * 性别 + */ + @TableField(value = "sex") + private String sex; + + @NotBlank + @TableField(value = "email") + private String email; + + @NotBlank + @TableField(value = "phone") + private String phone; + + @NotNull + @TableField(value = "enabled") + private Boolean enabled; + + @TableField(value = "password") + private String password; + + @TableField(value = "last_password_reset_time") + private Date lastPasswordResetTime; + + @TableField(value = "remark") + private String remark; + + @TableField(value = "avatar_id") + private Long avatarId; + + + @TableField(exist = false) + private UserAvatar userAvatar; + + + @NotEmpty + @TableField(exist = false) + private List roles; + + @TableField(value = "deleted",fill = FieldFill.INSERT) + private Boolean deleted = false; + + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + User user = (User) o; + return Objects.equals(id, user.id) && + Objects.equals(username, user.username); + } + + @Override + public int hashCode() { + return Objects.hash(id, username); + } + + public @interface Update { + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserAvatar.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserAvatar.java new file mode 100644 index 0000000..b46c67b --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserAvatar.java @@ -0,0 +1,58 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.domain.entity; + +import cn.hutool.core.util.ObjectUtil; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.dubhe.base.BaseEntity; + +import java.io.Serializable; + + +/** + * @description 用户头像实体 + * @date 2020-06-29 + */ +@Data +@NoArgsConstructor +@TableName("user_avatar") +public class UserAvatar extends BaseEntity implements Serializable { + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @TableField(value = "real_name") + private String realName; + + @TableField(value = "path") + private String path; + + @TableField(value = "size") + private String size; + + + public UserAvatar(UserAvatar userAvatar, String realName, String path, String size) { + this.id = ObjectUtil.isNotEmpty(userAvatar) ? userAvatar.getId() : null; + this.realName = realName; + this.path = path; + this.size = size; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserRole.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserRole.java new file mode 100644 index 0000000..756326b --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserRole.java @@ -0,0 +1,47 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +/** + * @description 用户角色关系实体 + * @date 2020-06-29 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +@TableName("users_roles") +public class UserRole implements Serializable { + + private static final long serialVersionUID = -6296866205797727963L; + + @TableField(value = "user_id") + private Long userId; + + @TableField(value = "role_id") + private Long roleId; + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java new file mode 100644 index 0000000..e291579 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dto.callback; + +import io.swagger.annotations.ApiModel; +import lombok.Data; + +/** + * @descripton 个性化参数实现 与 个性化参数个性化校验 + * + * @date 2020-05-28 + */ +@ApiModel(description = "k8s pod异步回调Algorithm") +@Data +public class AlgorithmK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO { + @Override + public String toString() { + return super.toString(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java new file mode 100644 index 0000000..5fd0740 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dto.callback; + +import io.swagger.annotations.ApiModel; +import lombok.Data; + +/** + * @descripton k8s方异步回调统一汇总类，即不管什么业务所有请求参数都放在这个类中 + * + * @date 2020-05-28 + */ +@ApiModel(description = "k8s方 pod异步回调汇总类") +@Data +public class AllK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO{ + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java new file mode 100644 index 0000000..bc5d4f8 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java @@ -0,0 +1,76 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dto.callback; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotEmpty; + +/** + * @descripton 统一通用参数实现与校验 + * + * @date 2020-05-28 + */ +@ApiModel(description = "k8s pod异步回调基类") +@Data +public class BaseK8sPodCallbackCreateDTO { + + @ApiModelProperty(required = true,value = "k8s namespace") + @NotEmpty(message = "namespace 不能为空！") + private String namespace; + + @ApiModelProperty(required = true,value = "k8s resource name") + @NotEmpty(message = "resourceName 不能为空！") + private String resourceName; + + @ApiModelProperty(required = true,value = "k8s pod name") + @NotEmpty(message = "podName 不能为空！") + private String podName; + + @ApiModelProperty(value = "k8s pod phase",notes = "对应PodPhaseEnum") + @NotEmpty(message = "phase 不能为空！") + private String phase; + + @ApiModelProperty(value = "k8s pod containerStatuses state") + private String messages; + + public BaseK8sPodCallbackCreateDTO(){ + + } + + public BaseK8sPodCallbackCreateDTO(String namespace,String resourceName,String podName,String phase,String messages){ + this.namespace = namespace; + this.resourceName = resourceName; + this.podName = podName; + this.phase = phase; + this.messages = messages; + } + + @Override + public String toString() { + return "BaseK8sPodCallbackReq{" + + "namespace='" + namespace + '\'' + + ", resourceName='" + resourceName + '\'' + + ", podName='" + podName + '\'' + + ", phase='" + phase + '\'' + + ", messages=" + messages + + '}'; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java new file mode 100644 index 0000000..57b41eb --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dto.callback; + +import io.swagger.annotations.ApiModel; +import lombok.Data; + +/** + * @descripton k8s pod异步回调Notebook + * + * @date 2020-05-28 + */ +@ApiModel(description = "k8s pod异步回调Notebook") +@Data +public class NotebookK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO { + + @Override + public String toString() { + return super.toString(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java new file mode 100644 index 0000000..8a87acb --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java @@ -0,0 +1,80 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + +import lombok.Getter; + +import java.util.HashMap; +import java.util.Map; + +/** + * @desc: 业务模块 + * + * @date 2020.05.25 + */ +@Getter +public enum BizEnum { + + /** + * 模型开发 + */ + NOTEBOOK("模型开发","notebook",0), + /** + * 算法管理 + */ + ALGORITHM("算法管理","algorithm",1), + ; + + /** + * 业务模块名称 + */ + private String bizName; + /** + * 业务模块名称 + */ + private String bizCode; + /** + * 业务源代号 + */ + private Integer createResource; + + BizEnum(String bizName,String bizCode, Integer createResource) { + this.createResource = createResource; + this.bizName = bizName; + this.bizCode = bizCode; + } + + private static final Map RESOURCE_ENUM_MAP = new HashMap(){ + { + for (BizEnum enums:BizEnum.values()){ + put(enums.getCreateResource(),enums); + } + } + }; + + /** + * 根据createResource获取BizEnum + * @param createResource + * @return + */ + public static BizEnum getByCreateResource(int createResource){ + return RESOURCE_ENUM_MAP.get(createResource); + } + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java new file mode 100644 index 0000000..2b06087 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java @@ -0,0 +1,91 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + +import java.util.HashMap; +import java.util.Map; + +/** + * @desc: 业务NFS路径枚举 + * @date 2020.05.13 + */ +public enum BizNfsEnum { + /** + * 模型开发 NFS 路径命名 + */ + NOTEBOOK(BizEnum.NOTEBOOK, "notebook"), + /** + * 算法管理 NFS 路径命名 + */ + ALGORITHM(BizEnum.ALGORITHM, "algorithm-manage"), + ; + + BizNfsEnum(BizEnum bizEnum, String bizNfsPath) { + this.bizEnum = bizEnum; + this.bizNfsPath = bizNfsPath; + } + + /** + * 业务模块 + */ + private BizEnum bizEnum; + /** + * 业务模块nfs路径 + */ + private String bizNfsPath; + + + private static final Map RESOURCE_ENUM_MAP = new HashMap() { + { + for (BizNfsEnum enums : BizNfsEnum.values()) { + put(enums.getCreateResource(), enums); + } + } + }; + + /** + * 根据createResource获取BizNfsEnum + * + * @param createResource + * @return + */ + public static BizNfsEnum getByCreateResource(int createResource) { + return RESOURCE_ENUM_MAP.get(createResource); + } + + + public String getBizName() { + return bizEnum == null ? null : bizEnum.getBizName(); + } + + public Integer getCreateResource() { + return bizEnum == null ? null : bizEnum.getCreateResource(); + } + + public String getBizNfsPath() { + return bizNfsPath; + } + + public BizEnum getBizEnum(){ + return bizEnum; + } + + public String getBizCode() { + return bizEnum == null ? null :bizEnum.getBizCode(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java new file mode 100644 index 0000000..2b985bc --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + +import lombok.Getter; + +/** + * @description 日志类型枚举类 + * @date 2020-06-23 + */ +@Getter +public enum LogEnum { + + // 系统报错日志 + SYS_ERR, + // 用户请求日志 + REST_REQ, + // 训练模块 + BIZ_TRAIN, + // 系统模块 + BIZ_SYS, + // 模型模块 + BIZ_MODEL, + // 数据集模块 + BIZ_DATASET, + // k8s模块 + BIZ_K8S, + //note book + NOTE_BOOK, + //NFS UTILS + NFS_UTIL; + + /** + * 判断日志类型不能为空 + * + * @param logType 日志类型 + * @return boolean 返回类型 + */ + public static boolean isLogType(LogEnum logType) { + + if (logType != null) { + return true; + } + return false; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java new file mode 100644 index 0000000..2c4fcaa --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java @@ -0,0 +1,95 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + +/** + * @Description 是否开关枚举 + * @Date 2020-06-01 + */ +public enum SwitchEnum { + /** + * OFF 否 + */ + OFF(0, "否"), + + /** + * ON 否 + */ + ON(1, "是"), + + ; + + private Integer value; + + private String desc; + + SwitchEnum(Integer value, String desc) { + this.value = value; + this.desc = desc; + } + + public Integer getValue() { + return this.value; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public static SwitchEnum getEnumValue(Integer value) { + switch (value) { + case 0: + return OFF; + case 1: + return ON; + default: + return OFF; + } + } + + public static Boolean getBooleanValue(Integer value) { + switch (value) { + case 1: + return true; + case 0: + return false; + default: + return false; + } + } + + public static boolean isExist(Integer value) { + for (SwitchEnum itm : SwitchEnum.values()) { + if (value.compareTo(itm.getValue()) == 0) { + return true; + } + } + return false; + } + + + @Override + public String toString() { + return "[" + this.value + "]" + this.desc; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java new file mode 100644 index 0000000..560d4ce --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java @@ -0,0 +1,101 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + +import lombok.Getter; + +import java.util.Arrays; +import java.util.List; + +/** + * @description 训练任务枚举类 + * @date 2020-04-27 + */ +@Getter +public enum TrainJobStatusEnum { + + /** + * PENDING + */ + PENDING(0, "PENDING"), + /** + * RUNNING + */ + RUNNING(1, "RUNNING"), + /** + * SUCCEEDED + */ + SUCCEEDED(2, "SUCCEEDED"), + /** + * FAILED + */ + FAILED(3, "FAILED"), + /** + * STOP + */ + STOP(4, "STOP"), + /** + * UNKNOWN + */ + UNKNOWN(5, "UNKNOWN"), + /** + * DELETED + */ + DELETED(6, "DELETED"), + + /** + * CREATE_FAILED + */ + CREATE_FAILED(7, "CREATE_FAILED"); + + private Integer status; + + private String message; + + TrainJobStatusEnum(Integer status, String message) { + this.status = status; + this.message = message; + } + + public static TrainJobStatusEnum get(String msg) { + for (TrainJobStatusEnum statusEnum : values()) { + if (statusEnum.message.equalsIgnoreCase(msg)) { + return statusEnum; + } + } + return UNKNOWN; + } + + public static boolean isEnd(String msg) { + List endList = Arrays.asList("SUCCEEDED", "FAILED", "STOP", "CREATE_FAILED"); + return endList.stream().anyMatch(s -> s.equalsIgnoreCase(msg)); + } + + public static boolean isEnd(Integer num) { + List endList = Arrays.asList(2, 3, 4, 7); + return endList.stream().anyMatch(s -> s.equals(num)); + } + + public static boolean checkStopStatus(Integer num) { + return SUCCEEDED.getStatus().equals(num) || + FAILED.getStatus().equals(num) || + STOP.getStatus().equals(num) || + CREATE_FAILED.getStatus().equals(num) || + DELETED.getStatus().equals(num); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java b/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java new file mode 100644 index 0000000..dcd18b2 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java @@ -0,0 +1,69 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.exception; + +import lombok.Getter; + +/** + * @description 通用异常code + * @date 2020-03-26 + */ +@Getter +public enum BaseErrorCode implements ErrorCode { + + /** + * undefined error + */ + UNDEFINED(10000, "操作成功!"), + ERROR(10001, "操作失败!"), + ERROR_SYSTEM(10002, "系统繁忙!"), + UNAUTHORIZED(401, "无权访问!"), + /** + * system 模块异常码 + */ + SYSTEM_USERNAME_ALREADY_EXISTS(20000, "账号已存在!"), + SYSTEM_CODE_ALREADY_EXISTS(20001, "Code already exists!"), + SYSTEM_USER_IS_NOT_EXISTS(20002, "用户不存在!"), + SYSTEM_USER_ALREADY_REGISTER(20003, "账号已注册!"), + SYSTEM_USER_REGISTER_EMAIL_INFO_EXPIRED(20004, "邮箱验证码已过期!"), + SYSTEM_USER_EMAIL_ALREADY_EXISTS(20004, "该邮箱已被注册!"), + SYSTEM_USER_EMAIL_PASSWORD_ERROR(20005, "邮件密码错误!"), + SYSTEM_USER_EMAIL_CODE_CANNOT_EXCEED_TIMES(20006, "邮件发送不能超过三次!"), + SYSTEM_USER_EMAIL_OR_CODE_ERROR(20007, "邮箱地址或验证码错误、请重新输入!"), + SYSTEM_USER_IS_LOCKED(20008, "用户已锁定!"), + SYSTEM_USER_USERNAME_OR_PASSWORD_ERROR(20009, "账号或密码不正确!"), + SYSTEM_USER_USERNAME_IS_LOCKED(20010, "账号已锁定，6小时后解锁!"), + SYSTEM_USER_CANNOT_UPDATE_ADMIN(20011, "仅超级管理员可操作!"), + SYSTEM_USER_TOKEN_INFO_IS_NULL(20012, "登录信息不存在!"), + SYSTEM_USER_EMAIL_NOT_EXISTS(20013, "该邮箱未注册!"), + SYSTEM_USER_CANNOT_DELETE(20014, "系统默认用户不可删除!"), + SYSTEM_ROLE_CANNOT_DELETE(20015, "系统默认角色不可删除!"), + + + ; + + + Integer code; + String msg; + + BaseErrorCode(Integer code, String msg) { + this.code = code; + this.msg = msg; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java new file mode 100644 index 0000000..9cb5a8b --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java @@ -0,0 +1,72 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.exception; + +import lombok.Getter; +import org.dubhe.base.DataResponseBody; +import org.dubhe.base.ResponseCode; + +/** + * @description 业务异常 + * @date 2020-03-13 + */ +@Getter +public class BusinessException extends RuntimeException { + + private DataResponseBody responseBody; + + public BusinessException(String msg) { + super(msg); + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); + } + + public BusinessException(String msg, Throwable cause) { + super(msg,cause); + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); + } + + public BusinessException(Throwable cause) { + super(cause); + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST); + } + + public BusinessException(Integer code, String msg, String info, Throwable cause) { + super(msg,cause); + if (info == null) { + this.responseBody = new DataResponseBody(code, msg); + } else { + this.responseBody = new DataResponseBody(code, msg + ":" + info); + } + } + + public BusinessException(ErrorCode errorCode, Throwable cause) { + this(errorCode.getCode(), errorCode.getMsg(), null, cause); + } + + public BusinessException(ErrorCode errorCode, String info, Throwable cause) { + this(errorCode.getCode(), errorCode.getMsg(), info, cause); + } + + public BusinessException(ErrorCode errorCode) { + this(errorCode, null); + } + + public BusinessException(Integer code,String msg) { + this.responseBody = new DataResponseBody(code, msg); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java new file mode 100644 index 0000000..fc8e9aa --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java @@ -0,0 +1,48 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.exception; + +import lombok.Getter; +import org.dubhe.base.DataResponseBody; +import org.dubhe.base.ResponseCode; + +/** + * @description 验证码异常 + * @date 2020-02-23 + */ +@Getter +public class CaptchaException extends RuntimeException { + private static final long serialVersionUID = 1L; + + private DataResponseBody responseBody; + private Throwable cause; + + public CaptchaException(String msg) { + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); + } + + public CaptchaException(String msg, Throwable cause) { + this.cause = cause; + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); + } + + public CaptchaException(Throwable cause) { + this.cause = cause; + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java new file mode 100644 index 0000000..8f80555 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java @@ -0,0 +1,48 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.exception; + +import lombok.Getter; +import org.dubhe.base.DataResponseBody; +import org.dubhe.base.ResponseCode; + +/** + * @Description docker 异常 + * @Date 2020-6-6 + **/ +@Getter +public class DockerBizException extends RuntimeException { + private static final long serialVersionUID = 1L; + + private DataResponseBody responseBody; + private Throwable cause; + + public DockerBizException(String msg) { + this.responseBody = new DataResponseBody(ResponseCode.DOCKER_ERROR, msg); + } + + public DockerBizException(String msg, Throwable cause) { + this.cause = cause; + this.responseBody = new DataResponseBody(ResponseCode.DOCKER_ERROR, msg); + } + + public DockerBizException(Throwable cause) { + this.cause = cause; + this.responseBody = new DataResponseBody(ResponseCode.DOCKER_ERROR); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java b/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java new file mode 100644 index 0000000..2bbc60e --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.exception; + + +/** + * @description 异常code + * @date 2020-03-26 + */ +public interface ErrorCode { + + /** + * 错误码 + * @return code + */ + Integer getCode(); + + /** + * error info + * @return + */ + String getMsg(); + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/LoginException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/LoginException.java new file mode 100644 index 0000000..23668d9 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/LoginException.java @@ -0,0 +1,44 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.exception; + +import lombok.Getter; +import org.dubhe.base.DataResponseBody; +import org.dubhe.base.ResponseCode; + +/** + * @description 登录异常 + * @date 2020-02-23 + */ +@Getter +public class LoginException extends RuntimeException { + private DataResponseBody responseBody; + private Throwable cause; + + public LoginException(String msg) { + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); + } + + public LoginException(String msg, Throwable cause) { + this.cause = cause; + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); + } + + public LoginException(Throwable cause) { + this.cause = cause; + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java new file mode 100644 index 0000000..e1124f3 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java @@ -0,0 +1,42 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.exception; + +import lombok.Getter; + +/** + * Description : NFS utils 工具异常 + * + * @date 15.06.2020 + */ +@Getter +public class NfsBizException extends BusinessException { + + private static final long serialVersionUID = 1L; + + + public NfsBizException(Throwable cause){ + super(cause); + } + + public NfsBizException(String msg){ + super(msg); + } + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java new file mode 100644 index 0000000..9f48beb --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.exception; + +import lombok.Getter; + +/** + * @description: Notebook 业务处理异常 + * + * @date 2020.04.27 + */ +@Getter +public class NotebookBizException extends BusinessException{ + + private static final long serialVersionUID = 1L; + + public NotebookBizException(String msg) { + super(msg); + } + + public NotebookBizException(String msg, Throwable cause) { + super(msg,cause); + } + + public NotebookBizException(Throwable cause) { + super(cause); + } + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java new file mode 100644 index 0000000..fed0939 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java @@ -0,0 +1,47 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.exception; + +import lombok.Getter; +import org.dubhe.base.DataResponseBody; +import org.dubhe.base.ResponseCode; + +/** + * @description 权限异常 + * @date 2020-02-23 + */ +@Getter +public class UnauthorizedException extends RuntimeException { + + private DataResponseBody responseBody; + private Throwable cause; + + public UnauthorizedException(String msg) { + this.responseBody = new DataResponseBody(ResponseCode.UNAUTHORIZED, msg); + } + + public UnauthorizedException(String msg, Throwable cause) { + this.cause = cause; + this.responseBody = new DataResponseBody(ResponseCode.UNAUTHORIZED, msg); + } + + public UnauthorizedException(Throwable cause) { + this.cause = cause; + this.responseBody = new DataResponseBody(ResponseCode.UNAUTHORIZED); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java b/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java new file mode 100644 index 0000000..b042abc --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java @@ -0,0 +1,190 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.exception.handler; + +import java.util.Objects; + +import org.apache.shiro.ShiroException; +import org.apache.shiro.authc.AuthenticationException; +import org.apache.shiro.authc.IncorrectCredentialsException; +import org.apache.shiro.authc.LockedAccountException; +import org.apache.shiro.authc.UnknownAccountException; +import org.dubhe.base.DataResponseBody; +import org.dubhe.base.ResponseCode; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.exception.CaptchaException; +import org.dubhe.exception.LoginException; +import org.dubhe.exception.NotebookBizException; +import org.dubhe.exception.UnauthorizedException; +import org.dubhe.utils.LogUtil; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.validation.BindException; +import org.springframework.validation.ObjectError; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +import lombok.extern.slf4j.Slf4j; + +/** + * @description 处理异常 + * @date 2020-02-23 + */ +@Slf4j +@RestControllerAdvice +public class GlobalExceptionHandler { + + /** + * 处理所有不可知的异常 + */ + @ExceptionHandler(Throwable.class) + public ResponseEntity handleException(Throwable e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + return buildResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR, + new DataResponseBody(ResponseCode.ERROR, e.getMessage())); + } + + /** + * UnauthorizedException + */ + @ExceptionHandler(UnauthorizedException.class) + public ResponseEntity badCredentialsException(UnauthorizedException e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + String message = "坏的凭证".equals(e.getMessage()) ? "用户名或密码不正确" : e.getMessage(); + return buildResponseEntity(HttpStatus.UNAUTHORIZED, new DataResponseBody(ResponseCode.ERROR, message)); + } + + /** + * 处理自定义异常 + */ + @ExceptionHandler(value = BusinessException.class) + public ResponseEntity badRequestException(BusinessException e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + return buildResponseEntity(HttpStatus.OK, e.getResponseBody()); + } + + /** + * 处理自定义异常 + */ + @ExceptionHandler(value = AuthenticationException.class) + public ResponseEntity badRequestException(AuthenticationException e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + return buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.UNAUTHORIZED, "无权访问")); + } + + /** + * shiro 异常捕捉 + */ + @ExceptionHandler(value = ShiroException.class) + public ResponseEntity accountException(ShiroException e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + ResponseEntity responseEntity; + if (e instanceof IncorrectCredentialsException) { + responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "密码不正确")); + } else if (e instanceof UnknownAccountException) { + responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "此账户不存在")); + } + + else if (e instanceof LockedAccountException) { + responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "未知的账号")); + } + + else if (e instanceof UnknownAccountException) { + responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "账户已被禁用")); + } + + else { + responseEntity = buildResponseEntity(HttpStatus.OK, + new DataResponseBody(ResponseCode.UNAUTHORIZED, "无权访问")); + } + return responseEntity; + } + + /** + * 处理自定义异常 + */ + @ExceptionHandler(value = LoginException.class) + public ResponseEntity loginException(LoginException e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + return buildResponseEntity(HttpStatus.UNAUTHORIZED, e.getResponseBody()); + } + + /** + * 处理自定义异常 + */ + @ExceptionHandler(value = CaptchaException.class) + public ResponseEntity captchaException(CaptchaException e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + return buildResponseEntity(HttpStatus.OK, e.getResponseBody()); + } + + /** + * 处理自定义异常 + */ + @ExceptionHandler(value = NotebookBizException.class) + public ResponseEntity captchaException(NotebookBizException e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + return buildResponseEntity(HttpStatus.OK, e.getResponseBody()); + } + + /** + * 处理所有接口数据验证异常 + */ + @ExceptionHandler(MethodArgumentNotValidException.class) + public ResponseEntity handleMethodArgumentNotValidException(MethodArgumentNotValidException e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + String[] str = Objects.requireNonNull(e.getBindingResult().getAllErrors().get(0).getCodes())[1].split("\\."); + String message = e.getBindingResult().getAllErrors().get(0).getDefaultMessage(); + String msg = "不能为空"; + if (msg.equals(message)) { + message = str[1] + ":" + message; + } + return buildResponseEntity(HttpStatus.BAD_REQUEST, new DataResponseBody(ResponseCode.ERROR, message)); + } + + @ExceptionHandler(BindException.class) + public ResponseEntity bindException(BindException e) { + // 打印堆栈信息 + LogUtil.error(LogEnum.SYS_ERR, e); + ObjectError error = e.getAllErrors().get(0); + return buildResponseEntity(HttpStatus.BAD_REQUEST, + new DataResponseBody(ResponseCode.ERROR, error.getDefaultMessage())); + } + + /** + * 统一返回 + * + * @param httpStatus + * @param responseBody + * @return + */ + private ResponseEntity buildResponseEntity(HttpStatus httpStatus, DataResponseBody responseBody) { + return new ResponseEntity<>(responseBody, httpStatus); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java b/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java new file mode 100644 index 0000000..86a20a4 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java @@ -0,0 +1,121 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.factory; + +import org.dubhe.base.DataResponseBody; +import org.dubhe.base.ResponseCode; + +/** + * @desc DataResponseBody 工厂类 + * + * @date 2020.05.28 + */ +public class DataResponseFactory { + + private DataResponseFactory(){ + + } + + /** + * 成功响应 + * + * @param + * @return + */ + public static DataResponseBody success(){ + return success(null,null); + } + + /** + * 成功响应 + * + * @param data + * @param + * @return + */ + public static DataResponseBody success(T data){ + return success(null,data); + } + + /** + * 成功响应 + * + * @param msg + * @return + */ + public static DataResponseBody successWithMsg(String msg){ + return success(msg,null); + } + + /** + * 成功响应 + * + * @param msg + * @param data + * @param + * @return + */ + public static DataResponseBody success(String msg, T data){ + return new DataResponseBody(ResponseCode.SUCCESS,msg,data); + } + + /** + * 失败响应 msg + * + * @param msg + * @return + */ + public static DataResponseBody failed(String msg){ + return failed(ResponseCode.ERROR,msg,null); + } + + /** + * 失败响应 + * + * @param failedCode + * @param msg + * @return + */ + public static DataResponseBody failed(Integer failedCode,String msg){ + return failed(failedCode,msg,null); + } + + /** + * 失败响应 + * + * @param failedCode + * @return + */ + public static DataResponseBody failed(Integer failedCode){ + return failed(failedCode,null,null); + } + + /** + * 失败响应 + * + * @param failedCode + * @param msg + * @param data + * @param + * @return + */ + public static DataResponseBody failed(Integer failedCode, String msg, T data){ + return new DataResponseBody(failedCode,msg,data); + } + + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/filter/FileLogFilter.java b/dubhe-server/common/src/main/java/org/dubhe/filter/FileLogFilter.java new file mode 100644 index 0000000..9fe2b47 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/filter/FileLogFilter.java @@ -0,0 +1,60 @@ +/** + * Copyright 2019-2020 Zheng Jie + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.filter; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.filter.AbstractMatcherFilter; +import ch.qos.logback.core.spi.FilterReply; + +/** + * @description 自定义日志过滤器 + * @date 2020-07-21 + */ +public class FileLogFilter extends AbstractMatcherFilter { + + Level level; + + /** + * 重写decide方法 + * + * @param iLoggingEvent event to decide upon. + * @return FilterReply + */ + @Override + public FilterReply decide(ILoggingEvent iLoggingEvent) { + if (!isStarted()) { + return FilterReply.NEUTRAL; + } + if (iLoggingEvent.getLevel().equals(level) && iLoggingEvent.getMessage() != null + && iLoggingEvent.getMessage().startsWith("{") && iLoggingEvent.getMessage().endsWith("}")) { + return onMatch; + } + return onMismatch; + + } + + public void setLevel(Level level) { + this.level = level; + } + + @Override + public void start() { + if (this.level != null) { + super.start(); + } + } +} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java b/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java new file mode 100644 index 0000000..029457f --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.handle; + +import org.springframework.web.servlet.mvc.condition.RequestCondition; + +import javax.servlet.http.HttpServletRequest; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @description API版本控制 + * @date 2020-04-06 + */ +public class ApiVersionCondition implements RequestCondition { + /** + * 路径中版本的前缀， 这里用 /v[1-9]/的形式 + */ + private final static Pattern VERSION_PREFIX_PATTERN = Pattern.compile("v(\\d+)/"); + private int apiVersion; + + public ApiVersionCondition(int apiVersion) { + this.apiVersion = apiVersion; + } + + @Override + public ApiVersionCondition combine(ApiVersionCondition other) { + // 采用最后定义优先原则，则方法上的定义覆盖类上面的定义 + return new ApiVersionCondition(other.getApiVersion()); + } + + @Override + public ApiVersionCondition getMatchingCondition(HttpServletRequest request) { + Matcher m = VERSION_PREFIX_PATTERN.matcher(request.getRequestURI()); + if (m.find()) { + Integer version = Integer.valueOf(m.group(1)); + if (version >= this.apiVersion) { + return this; + } + } + return null; + } + + + @Override + public int compareTo(ApiVersionCondition other, HttpServletRequest request) { + // 优先匹配最新的版本号 + return other.getApiVersion() - this.apiVersion; + } + public int getApiVersion() { + return apiVersion; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java b/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java new file mode 100644 index 0000000..de9d09f --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java @@ -0,0 +1,64 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.handle; + +import lombok.Data; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.annotation.ApiVersion; +import org.springframework.core.annotation.AnnotationUtils; +import org.springframework.stereotype.Component; +import org.springframework.web.servlet.mvc.condition.RequestCondition; +import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping; + +import java.lang.reflect.Method; +import java.util.HashSet; +import java.util.Set; +/** + * @description 权限注解收集 + * @date 2020-04-06 + */ +@Component +@Data +public class CustomRequestMappingHandlerMapping extends RequestMappingHandlerMapping { + Set permissionsSet =new HashSet(); + @Override + protected RequestCondition getCustomTypeCondition(Class handlerType) { + ApiVersion apiVersion = AnnotationUtils.findAnnotation(handlerType, ApiVersion.class); + return createCondition(apiVersion); + } + + @Override + protected RequestCondition getCustomMethodCondition(Method method) { + RequiresPermissions requiresPermissions = AnnotationUtils.findAnnotation(method, RequiresPermissions.class); + if(requiresPermissions != null){ + if(requiresPermissions.value()!=null){ + for(int i=0;i createCondition(ApiVersion apiVersion) { + return apiVersion == null ? null : new ApiVersionCondition(apiVersion.value()); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java b/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java new file mode 100644 index 0000000..90beef9 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.interceptor; + +import org.dubhe.enums.LogEnum; +import org.dubhe.utils.K8sCallBackTool; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * @description k8s pod 异步回调拦截器 + * + * @date 2020-05-28 + */ +@Component +public class K8sCallBackPodInterceptor extends HandlerInterceptorAdapter { + + @Autowired + private K8sCallBackTool k8sCallBackTool; + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { + String uri = request.getRequestURI(); + LogUtil.debug(LogEnum.BIZ_K8S,"接收到k8s异步请求,URI:{}",uri); + String k8sCallbackToken = request.getHeader(K8sCallBackTool.K8S_CALLBACK_TOKEN); + if (StringUtils.isBlank(k8sCallbackToken)){ + LogUtil.warn(LogEnum.BIZ_K8S,"k8s异步回调没有配置【{}】,URI:{}",K8sCallBackTool.K8S_CALLBACK_TOKEN,uri); + return false; + } + boolean pass = k8sCallBackTool.validateToken(k8sCallbackToken); + if (!pass){ + LogUtil.warn(LogEnum.BIZ_K8S,"k8s异步回调token:【{}】 验证不通过,URI:{}",k8sCallbackToken,uri); + } + return pass; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java new file mode 100644 index 0000000..e4ecd90 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java @@ -0,0 +1,97 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import cn.hutool.core.util.HexUtil; +import org.dubhe.enums.LogEnum; + +import javax.crypto.Cipher; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * @description AES加解密工具 + * @date 2020-06-01 + */ +public class AesUtil { + + private static final String AES = "AES"; + + + private AesUtil(){ + + } + + + /** + * + * @param mode Cipher mode + * @param key 秘钥 + * @return Cipher + * @throws NoSuchAlgorithmException + * @throws NoSuchPaddingException + * @throws InvalidKeyException + */ + private static Cipher getCipher(int mode,String key) throws NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException { + MessageDigest md5Digest = MessageDigest.getInstance("MD5"); + SecretKeySpec secretKeySpec = new SecretKeySpec(md5Digest.digest(key.getBytes(StandardCharsets.UTF_8)), AES); + Cipher cipher = Cipher.getInstance(AES); + cipher.init(mode, secretKeySpec); + return cipher; + } + + /** + * 加密 + * + * @param data 原文 + * @param key 秘钥 + * @return String 密文 + */ + public static String encrypt(String data, String key) { + try { + Cipher cipher = getCipher(Cipher.ENCRYPT_MODE,key); + byte[] content = data.getBytes(StandardCharsets.UTF_8); + return new String(HexUtil.encodeHex(cipher.doFinal(content), false)); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_SYS,"Md5Util 加密失败 ",e); + return null; + } + } + + /** + * 解密 + * @param hexData 十六进制密文 + * @param key 秘钥 + * @return String 密文 + */ + public static String decrypt(String hexData, String key) { + try { + Cipher cipher = getCipher(Cipher.DECRYPT_MODE,key); + byte[] content = HexUtil.decodeHex(hexData); + return new String(cipher.doFinal(content), StandardCharsets.UTF_8); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_SYS,"Md5Util 解密失败 ",e); + return null; + } + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java new file mode 100644 index 0000000..5db3ee9 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java @@ -0,0 +1,80 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; + +/** + * @description 日期工具类 + * @date 2020-6-10 + */ +public class DateUtil { + /** + * 获取当前时间戳 + * + * @return + */ + public static Timestamp getCurrentTimestamp() { + return Timestamp.valueOf(LocalDateTime.now()); + } + + + + /** + * 获取六小时后时间 + * @return + */ + public static long getAfterSixHourTime(){ + long l1 = getTimestampOfDateTime(LocalDateTime.now()); + long milli = getTimestampOfDateTime(LocalDateTime.now().plusHours(6)); + return (milli-l1); + } + + + /** + * LocalDateTime -> long + * @param localDateTime + * @return + */ + public static long getTimestampOfDateTime(LocalDateTime localDateTime) { + ZoneId zone = ZoneId.systemDefault(); + Instant instant = localDateTime.atZone(zone).toInstant(); + return instant.toEpochMilli(); + } + + /** + * 获取第二天凌晨时间 + * @return + */ + public static long getSecondTime(){ + LocalDateTime localDateTime = LocalDateTime.now(); + long l1 = localDateTime.atZone(ZoneId.of("Asia/Shanghai")).toInstant().toEpochMilli(); + + LocalDate localDate = LocalDate.now(); + LocalDate localDate1 = localDate.plusDays(1); + LocalDateTime localDateTime1 = localDate1.atStartOfDay(); + long milli = localDateTime1.atZone(ZoneId.of("Asia/Shanghai")).toInstant().toEpochMilli(); + System.out.println("当前时间到第二天凌晨的毫秒数"+(milli-l1)); + return (milli-l1); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java new file mode 100644 index 0000000..768f527 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java @@ -0,0 +1,47 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +/** + * @description 日期工具类 + * @date 2020-5-29 + **/ +public class DubheDateUtil { + + private static final int SIXTY = 60; + + /** + * 将时间 秒 转为 小时 + * + * @param second 秒数 + * @return + */ + public static String convert2Str(Long second) { + if (null == second || second < 1) { + return ""; + } + Long hh = second / (SIXTY * SIXTY); + second = second % (SIXTY * SIXTY); + Long mm = second / SIXTY; + Long ss = second % SIXTY; + + StringBuilder sb = new StringBuilder(); + sb.append(String.format("%02d", hh)).append(":").append(String.format("%02d", mm)).append(":").append(String.format("%02d", ss)); + return sb.toString(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java new file mode 100644 index 0000000..33ad123 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java @@ -0,0 +1,100 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.DESKeySpec; +import javax.crypto.spec.IvParameterSpec; +import java.nio.charset.StandardCharsets; + +/** + * @description 加密 + * @date 2020-03-13 + */ +public class EncryptUtils { + + private static String strParam = "Passw0rd"; + + private static Cipher cipher; + + private static IvParameterSpec iv = new IvParameterSpec(strParam.getBytes(StandardCharsets.UTF_8)); + + private static DESKeySpec getDesKeySpec(String source) throws Exception { + if (source == null || source.length() == 0) { + return null; + } + cipher = Cipher.getInstance("DES/CBC/PKCS5Padding"); + String strKey = "Passw0rd"; + return new DESKeySpec(strKey.getBytes(StandardCharsets.UTF_8)); + } + + /** + * 对称加密 + */ + public static String desEncrypt(String source) throws Exception { + DESKeySpec desKeySpec = getDesKeySpec(source); + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES"); + SecretKey secretKey = keyFactory.generateSecret(desKeySpec); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, iv); + return byte2hex( + cipher.doFinal(source.getBytes(StandardCharsets.UTF_8))).toUpperCase(); + } + + /** + * 对称解密 + */ + public static String desDecrypt(String source) throws Exception { + byte[] src = hex2byte(source.getBytes()); + DESKeySpec desKeySpec = getDesKeySpec(source); + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES"); + SecretKey secretKey = keyFactory.generateSecret(desKeySpec); + cipher.init(Cipher.DECRYPT_MODE, secretKey, iv); + byte[] retByte = cipher.doFinal(src); + return new String(retByte); + } + + private static String byte2hex(byte[] inStr) { + String stmp; + StringBuilder out = new StringBuilder(inStr.length * 2); + for (byte b : inStr) { + stmp = Integer.toHexString(b & 0xFF); + if (stmp.length() == 1) { + // 如果是0至F的单位字符串，则添加0 + out.append("0").append(stmp); + } else { + out.append(stmp); + } + } + return out.toString(); + } + + private static byte[] hex2byte(byte[] b) { + int size = 2; + if ((b.length % size) != 0) { + throw new IllegalArgumentException("长度不是偶数"); + } + byte[] b2 = new byte[b.length / 2]; + for (int n = 0; n < b.length; n += size) { + String item = new String(b, n, 2); + b2[n / 2] = (byte) Integer.parseInt(item, 16); + } + return b2; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java new file mode 100644 index 0000000..430f2c6 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java @@ -0,0 +1,355 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import cn.hutool.core.codec.Base64; +import cn.hutool.core.io.IoUtil; +import cn.hutool.core.util.IdUtil; +import cn.hutool.poi.excel.BigExcelWriter; +import cn.hutool.poi.excel.ExcelUtil; +import org.apache.poi.util.IOUtils; +import org.dubhe.exception.BusinessException; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.*; +import java.security.MessageDigest; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * @description File工具类，扩展 hutool 工具包 + * @date 2020-03-14 + */ +public class FileUtil extends cn.hutool.core.io.FileUtil { + + /** + * 定义GB的计算常量 + */ + private static final int GB = 1024 * 1024 * 1024; + /** + * 定义MB的计算常量 + */ + private static final int MB = 1024 * 1024; + /** + * 定义KB的计算常量 + */ + private static final int KB = 1024; + + /** + * 格式化小数 + */ + private static final DecimalFormat DF = new DecimalFormat("0.00"); + + /** + * MultipartFile转File + */ + public static File toFile(MultipartFile multipartFile) { + // 获取文件名 + String fileName = multipartFile.getOriginalFilename(); + // 获取文件后缀 + String prefix = "." + getExtensionName(fileName); + File file = null; + try { + // 用uuid作为文件名，防止生成的临时文件重复 + file = File.createTempFile(IdUtil.simpleUUID(), prefix); + // MultipartFile to File + multipartFile.transferTo(file); + } catch (IOException e) { + e.printStackTrace(); + } + return file; + } + + /** + * 获取文件扩展名，不带 . + */ + public static String getExtensionName(String filename) { + if ((filename != null) && (filename.length() > 0)) { + int dot = filename.lastIndexOf('.'); + if ((dot > -1) && (dot < (filename.length() - 1))) { + return filename.substring(dot + 1); + } + } + return filename; + } + + /** + * Java文件操作 获取不带扩展名的文件名 + */ + public static String getFileNameNoEx(String filename) { + if ((filename != null) && (filename.length() > 0)) { + int dot = filename.lastIndexOf('.'); + if ((dot > -1) && (dot < (filename.length()))) { + return filename.substring(0, dot); + } + } + return filename; + } + + /** + * 文件大小转换 + */ + public static String getSize(long size) { + String resultSize; + if (size / GB >= 1) { + //如果当前Byte的值大于等于1GB + resultSize = DF.format(size / (float) GB) + "GB "; + } else if (size / MB >= 1) { + //如果当前Byte的值大于等于1MB + resultSize = DF.format(size / (float) MB) + "MB "; + } else if (size / KB >= 1) { + //如果当前Byte的值大于等于1KB + resultSize = DF.format(size / (float) KB) + "KB "; + } else { + resultSize = size + "B "; + } + return resultSize; + } + + /** + * inputStream 转 File + */ + static File inputStreamToFile(InputStream ins, String name) throws Exception { + File file = new File(System.getProperty("java.io.tmpdir") + File.separator + name); + if (file.exists()) { + return file; + } + OutputStream os = new FileOutputStream(file); + int bytesRead; + int len = 8192; + byte[] buffer = new byte[len]; + while ((bytesRead = ins.read(buffer, 0, len)) != -1) { + os.write(buffer, 0, bytesRead); + } + os.close(); + ins.close(); + return file; + } + + /** + * 将文件名解析成文件的上传路径 + */ + public static File upload(MultipartFile file, String filePath) { + Date date = new Date(); + SimpleDateFormat format = new SimpleDateFormat("yyyyMMddhhmmssS"); + String name = getFileNameNoEx(file.getOriginalFilename()); + String suffix = getExtensionName(file.getOriginalFilename()); + String nowStr = "-" + format.format(date); + try { + String fileName = name + nowStr + "." + suffix; + String path = filePath + fileName; + // getCanonicalFile 可解析正确各种路径 + File dest = new File(path).getCanonicalFile(); + // 检测是否存在目录 + if (!dest.getParentFile().exists()) { + dest.getParentFile().mkdirs(); + } + // 文件写入 + file.transferTo(dest); + return dest; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public static String fileToBase64(File file) throws Exception { + FileInputStream inputFile = new FileInputStream(file); + String base64; + byte[] buffer = new byte[(int) file.length()]; + inputFile.read(buffer); + inputFile.close(); + base64 = Base64.encode(buffer); + return base64.replaceAll("[\\s*\t\n\r]", ""); + } + + /** + * 导出excel + */ + public static void downloadExcel(List> list, HttpServletResponse response) throws IOException { + String tempPath = System.getProperty("java.io.tmpdir") + IdUtil.fastSimpleUUID() + ".xlsx"; + File file = new File(tempPath); + BigExcelWriter writer = ExcelUtil.getBigWriter(file); + // 一次性写出内容，使用默认样式，强制输出标题 + writer.write(list, true); + //response为HttpServletResponse对象 + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;charset=utf-8"); + //test.xls是弹出下载对话框的文件名，不能为中文，中文请自行编码 + response.setHeader("Content-Disposition", "attachment;filename=file.xlsx"); + ServletOutputStream out = response.getOutputStream(); + // 终止后删除临时文件 + file.deleteOnExit(); + writer.flush(out, true); + //此处记得关闭输出Servlet流 + IoUtil.close(out); + } + + /** + * 下载文件 + */ + public static void download(String path, HttpServletResponse response) { + if (path == null) { + return; + } + FileInputStream fis = null; + ServletOutputStream out = null; + try { + File file = new File(path); + fis = new FileInputStream(file); + out = response.getOutputStream(); + IOUtils.copy(fis, out); + response.flushBuffer(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (fis != null) { + try { + fis.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + //此处记得关闭输出Servlet流 + IoUtil.close(out); + } + } + + public static String getFileType(String type) { + String documents = "txt doc pdf ppt pps xlsx xls docx"; + String music = "mp3 wav wma mpa ram ra aac aif m4a"; + String video = "avi mpg mpe mpeg asf wmv mov qt rm mp4 flv m4v webm ogv ogg"; + String image = "bmp dib pcp dif wmf gif jpg tif eps psd cdr iff tga pcd mpt png jpeg"; + if (image.contains(type)) { + return "图片"; + } else if (documents.contains(type)) { + return "文档"; + } else if (music.contains(type)) { + return "音乐"; + } else if (video.contains(type)) { + return "视频"; + } else { + return "其他"; + } + } + + public static void checkSize(long maxSize, long size) { + // 1M + int len = 1024 * 1024; + if (size > (maxSize * len)) { + throw new BusinessException("文件超出规定大小"); + } + } + + /** + * 判断两个文件是否相同 + */ + public static boolean check(File file1, File file2) { + String img1Md5 = getMd5(file1); + String img2Md5 = getMd5(file2); + return img1Md5.equals(img2Md5); + } + + /** + * 判断两个文件是否相同 + */ + public static boolean check(String file1Md5, String file2Md5) { + return file1Md5.equals(file2Md5); + } + + private static byte[] getByte(File file) { + // 得到文件长度 + byte[] b = new byte[(int) file.length()]; + try { + InputStream in = new FileInputStream(file); + try { + in.read(b); + } catch (IOException e) { + e.printStackTrace(); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + return null; + } + return b; + } + + private static String getMd5(byte[] bytes) { + // 16进制字符 + char[] hexDigits = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + try { + MessageDigest mdTemp = MessageDigest.getInstance("MD5"); + mdTemp.update(bytes); + byte[] md = mdTemp.digest(); + int j = md.length; + char[] str = new char[j * 2]; + int k = 0; + // 移位 输出字符串 + for (byte byte0 : md) { + str[k++] = hexDigits[byte0 >>> 4 & 0xf]; + str[k++] = hexDigits[byte0 & 0xf]; + } + return new String(str); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + /** + * 下载文件 + * + * @param request / + * @param response / + * @param file / + */ + public static void downloadFile(HttpServletRequest request, HttpServletResponse response, File file, boolean deleteOnExit) { + response.setCharacterEncoding(request.getCharacterEncoding()); + response.setContentType("application/octet-stream"); + FileInputStream fis = null; + try { + fis = new FileInputStream(file); + response.setHeader("Content-Disposition", "attachment; filename=" + file.getName()); + IOUtils.copy(fis, response.getOutputStream()); + response.flushBuffer(); + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (fis != null) { + try { + fis.close(); + if (deleteOnExit) { + file.deleteOnExit(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + public static String getMd5(File file) { + return getMd5(getByte(file)); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java new file mode 100644 index 0000000..a2250b5 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java @@ -0,0 +1,131 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; +import org.apache.commons.io.IOUtils; +import org.dubhe.enums.LogEnum; + + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +import static org.dubhe.constant.StringConstant.UTF8; +import static org.dubhe.constant.SymbolConstant.BLANK; + +/** + * @description: httpClient工具类，不校验SSL证书 + * @date: 2020-5-21 + */ +public class HttpClientUtils { + + public static String sendHttps(String path) { + InputStream inputStream = null; + BufferedReader bufferedReader = null; + InputStreamReader inputStreamReader = null; + StringBuilder stringBuider = new StringBuilder(); + String result = BLANK; + HttpsURLConnection con = null; + try { + con = getConnection(path); + con.connect(); + + /**将返回的输入流转换成字符串**/ + inputStream = con.getInputStream(); + inputStreamReader = new InputStreamReader(inputStream, UTF8); + bufferedReader = new BufferedReader(inputStreamReader); + + String str = null; + while ((str = bufferedReader.readLine()) != null) { + stringBuider.append(str); + } + + result = stringBuider.toString(); + LogUtil.info(LogEnum.BIZ_SYS,"Request path:{}, SUCCESS, result:{}", path, result); + + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_SYS,"Request path:{}, ERROR, exception:{}", path, e); + return result; + } finally { + closeResource(bufferedReader,inputStreamReader,inputStream,con); + } + + return result; + } + + private static void closeResource(BufferedReader bufferedReader,InputStreamReader inputStreamReader,InputStream inputStream,HttpsURLConnection con) { + + IOUtils.closeQuietly(bufferedReader); + + if (inputStreamReader != null) { + IOUtils.closeQuietly(inputStreamReader); + } + if (inputStream != null) { + IOUtils.closeQuietly(inputStream); + } + if (con != null) { + con.disconnect(); + } + } + + + private static HttpsURLConnection getConnection(String path){ + HttpsURLConnection con = null; + try { + /**创建并初始化SSLContext对象**/ + TrustManager[] trustManagers = {new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { + } + + @Override + public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }}; + SSLContext sslContext = SSLContext.getInstance("SSL", "SunJSSE"); + sslContext.init(null, trustManagers, new java.security.SecureRandom()); + + /**得到SSLSocketFactory对象**/ + SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); + + URL url = new URL(path); + con = (HttpsURLConnection) url.openConnection(); + con.setSSLSocketFactory(sslSocketFactory); + con.setUseCaches(false); + }catch (Exception e){ + LogUtil.error(LogEnum.BIZ_SYS,"Request path:{}, error, exception:{}", path, e); + } + return con; + + } + +} + + diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java new file mode 100644 index 0000000..30e8309 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import lombok.extern.slf4j.Slf4j; + +/** + * @description: HttpUtil + * @date 2020.04.30 + */ +@Slf4j +public class HttpUtils { + + private HttpUtils() { + + } + + /** + * 判断http请求是否成功 + * + * @param httpCode + * 1XX Informational(信息性状态码) + * 2XX Success(成功状态码) + * 3XX Redirection(重定向状态码) + * 4XX Client Error(客户端错误状态码) + * 5XX Server Error(服务器错误状态码) + * @return + */ + public static boolean isSuccess(String httpCode){ + if (StringUtils.isBlank(httpCode)){ + return false; + } + return httpCode.length() == 3 && httpCode.startsWith("2"); + } + + /** + * 判断http请求是否成功 + * + * @param httpCode + * 1XX Informational(信息性状态码) + * 2XX Success(成功状态码) + * 3XX Redirection(重定向状态码) + * 4XX Client Error(客户端错误状态码) + * 5XX Server Error(服务器错误状态码) + * @return + */ + public static boolean isSuccess(int httpCode) { + return isSuccess(String.valueOf(httpCode)); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java b/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java new file mode 100644 index 0000000..d0d433b --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +/** + * @description JWT配置类 + * @date 2020-04-15 + */ +@Configuration +public class JwcConfig { + @Value("${jwt.online-key}") + public String onlineKey; + @Autowired + public RedisUtils redisUtils; + @Value("${jwt.token-validity-in-seconds}") + public Long jwtExpiration; + @Value("${jwt.base64-secret}") + public String secret; +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java new file mode 100644 index 0000000..573880d --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java @@ -0,0 +1,203 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.JWTCreator.Builder; +import com.auth0.jwt.JWTVerifier; +import com.auth0.jwt.algorithms.Algorithm; +import com.auth0.jwt.exceptions.JWTDecodeException; +import com.auth0.jwt.interfaces.Claim; +import com.auth0.jwt.interfaces.DecodedJWT; +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.crypto.SecureRandomNumberGenerator; +import org.apache.shiro.mgt.SecurityManager; +import org.apache.shiro.util.ThreadContext; +import org.dubhe.domain.dto.UserDTO; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.UnsupportedEncodingException; +import java.util.Calendar; +import java.util.Date; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +/** + * @description JWT + * @date 2020-03-14 + */ +@Component +public class JwtUtils { + + + /** + * 请求头 + */ + public static final String AUTH_HEADER = "Authorization"; + /** + * 过期时间20分钟 + */ + private static final long EXPIRE_TIME = 20 * 60 * 1000; + private static final String FIELD_NAME = "USER_NAME"; + private static JwcConfig jwcConfig; + private static RedisUtils redisUtils; + + + @Autowired + public void setJwcConfig(JwcConfig mtJwcConfig) { + jwcConfig = mtJwcConfig; + } + + @Autowired + public void setRedisUtils(RedisUtils myredisUtils) { + redisUtils = myredisUtils; + } + + /** + * 验证token是否正确 + */ + public static boolean verify(String token, String username) { + try { + Algorithm algorithm = null; + algorithm = Algorithm.HMAC256(jwcConfig.secret); + JWTVerifier verifier = JWT.require(algorithm).withClaim(FIELD_NAME, username).build(); + verifier.verify(token); + return true; + } catch (UnsupportedEncodingException exception) { + return false; + } + } + + /** + * 获得token中的自定义信息，无需secret解密也能获得 + */ + public static String getClaimFiled(String token, String filed) { + try { + DecodedJWT jwt = JWT.decode(token); + return jwt.getClaim(filed).asString(); + } catch (JWTDecodeException e) { + return null; + } + } + + /** + * 生成签名 + */ + public static String sign(String userName) { + try { + Date date = new Date(System.currentTimeMillis() + EXPIRE_TIME); + Algorithm algorithm = Algorithm.HMAC256(jwcConfig.secret); + // 附带username， + String token = JWT.create().withClaim(FIELD_NAME, userName).withExpiresAt(date).sign(algorithm); + saveToken(userName, token); + return token; + } catch (UnsupportedEncodingException e) { + return null; + } + } + + /** + * 获取用户名 + */ + public static String getUserName(String token) { + return getClaimFiled(token, FIELD_NAME); + } + + /** + * 获取 token的签发时间 + */ + public static Date getIssuedAt(String token) { + try { + DecodedJWT jwt = JWT.decode(token); + return jwt.getIssuedAt(); + } catch (JWTDecodeException e) { + return null; + } + } + + /** + * 验证 token是否过期 + */ + public static boolean isTokenExpired(String token) { + Date now = Calendar.getInstance().getTime(); + DecodedJWT jwt = JWT.decode(token); + return jwt.getExpiresAt().before(now); + } + + /** + * 刷新 token的过期时间 + */ + public static String refreshTokenExpired(String token) { + DecodedJWT jwt = JWT.decode(token); + Map claims = jwt.getClaims(); + try { + Date date = new Date(System.currentTimeMillis() + EXPIRE_TIME); + Algorithm algorithm = Algorithm.HMAC256(jwcConfig.secret); + Builder builer = JWT.create().withExpiresAt(date); + for (Entry entry : claims.entrySet()) { + builer.withClaim(entry.getKey(), entry.getValue().asString()); + } + String newToken = builer.sign(algorithm); + String userName = claims.get(FIELD_NAME).asString(); + String tokenKey = getUserTokenKey(userName); + if (redisUtils.get(tokenKey) == null) { + return null; + } + saveToken(userName, newToken); + return newToken; + } catch (UnsupportedEncodingException e) { + return null; + } + } + + /** + * 生成16位随机盐 + */ + public static String generateSalt() { + SecureRandomNumberGenerator secureRandom = new SecureRandomNumberGenerator(); + String hex = secureRandom.nextBytes(16).toHex(); + return hex; + } + + /** + * 获取当前用户 + */ + public static UserDTO getCurrentUserDto() { + Object principal = null; + SecurityManager manager = ThreadContext.getSecurityManager(); + if (!Objects.isNull(manager)) { + principal = SecurityUtils.getSubject().getPrincipal(); + } + return Objects.isNull(principal) ? null : (UserDTO) principal; + } + + /** + * 获取用户token key + */ + private static String getUserTokenKey(String userName) { + return jwcConfig.onlineKey + userName; + } + + private static void saveToken(String userName, String token) { + String tokenKey = getUserTokenKey(userName); + redisUtils.set(tokenKey, token, jwcConfig.jwtExpiration, TimeUnit.MINUTES); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java new file mode 100644 index 0000000..7e2cfbd --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java @@ -0,0 +1,168 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import cn.hutool.core.date.DateField; +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * @description k8s命名相关工具类 token使用MD5加密，并设置超时时间 + * @date 2020-05-28 + */ +@Component +public class K8sCallBackTool { + + /** + * k8s 回调 token秘钥 + */ + @Value("${k8s.callback.token.secret-key}") + private String secretKey; + /** + * k8s 回调token超时时间 + */ + @Value("${k8s.callback.token.expire-seconds}") + private Integer expireSeconds; + /** + * k8s 回调域名或IP:Port + */ + @Value("${k8s.callback.url}") + private String url; + + /** + * k8s 回调token key + */ + public static final String K8S_CALLBACK_TOKEN = "k8sCallbackToken"; + /** + * 失败重试次数 + */ + private static final int RETRY_COUNT = 3; + + /** + * k8s 回调匹配地址 + */ + private static final List K8S_CALLBACK_PATH; + /** + * k8s 回调路径 + */ + private static final String K8S_CALLBACK_PATH_POD = "/api/k8s/callback/pod/"; + + static { + K8S_CALLBACK_PATH = new ArrayList<>(); + // 添加需要token权限校验的地址（Shiro匿名访问的地址） + K8S_CALLBACK_PATH.add(K8S_CALLBACK_PATH_POD + "**"); + } + + /** + * 获取 k8s 回调匹配地址 + * + * @return List + */ + public static List getK8sCallbackPaths() { + return new ArrayList<>(K8S_CALLBACK_PATH); + } + + + /** + * 生成k8s回调 + * + * @return String + */ + public String generateToken() { + String expireTime = DateUtil.format( + DateUtil.offset(new Date(), DateField.SECOND, expireSeconds), + DatePattern.PURE_DATETIME_PATTERN + ); + return AesUtil.encrypt(expireTime, secretKey); + } + + /** + * 验证token + * + * @param token + * @return boolean + */ + public boolean validateToken(String token) { + String expireTime = AesUtil.decrypt(token, secretKey); + String nowTime = DateUtil.format( + new Date(), + DatePattern.PURE_DATETIME_PATTERN + ); + return expireTime.compareTo(nowTime) > 0; + } + + + /** + * 判断当前是否可以再次重试 + * + * @param retryTimes 第n次试图重试 + * @return boolean + */ + public boolean continueRetry(int retryTimes) { + return retryTimes <= RETRY_COUNT; + } + + /** + * 获取回调地址 + * + * @param podLabel + * @return String + */ + public String getPodCallbackUrl(String podLabel) { + return url + K8S_CALLBACK_PATH_POD + podLabel; + } + + + /** + * 获取超时时间秒 + * + * @param timeoutSecond 超时秒数 + * @return Long + */ + public static Long getTimeoutSecondLong(int timeoutSecond) { + return Long.valueOf( + DateUtil.format( + DateUtil.offset( + new Date(), DateField.SECOND, timeoutSecond + ), + DatePattern.PURE_DATETIME_PATTERN + ) + ); + } + + /** + * 获取当前秒数 + * + * @return Long + */ + public static Long getCurrentSecondLong() { + return Long.valueOf( + DateUtil.format( + new Date(), + DatePattern.PURE_DATETIME_PATTERN + ) + ); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java new file mode 100644 index 0000000..af25452 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java @@ -0,0 +1,255 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.RandomUtil; +import org.dubhe.config.K8sNameConfig; +import org.dubhe.constant.SymbolConstant; +import org.dubhe.enums.BizEnum; +import org.dubhe.enums.BizNfsEnum; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.apache.commons.lang3.StringUtils; + +import java.util.Date; + +/** + * @description k8s命名相关工具类 + * @date 2020-05-13 + */ +@Component +public class K8sNameTool { + @Autowired + private K8sNameConfig k8sNameConfig; + /** + * 命名分隔符 + */ + private static final char SEPARATOR = '-'; + /** + * 文件分隔符 + */ + private static final String K8S_FILE_SEPARATOR = "/"; + /** + * 资源名称前缀 + */ + private static final String RESOURCE_NAME = "rn"; + /** + * 随机长度值 + */ + private static final int RANDOM_LENGTH = 4; + + /** + * 生成 ResourceName + * + * @param bizEnum 业务枚举 + * @param resourceInfo 资源备注信息（保证同业务下唯一并且命名规范） + * @return String + */ + public String generateResourceName(BizEnum bizEnum, String resourceInfo) { + return bizEnum.getBizCode() + SEPARATOR + RESOURCE_NAME + SEPARATOR + resourceInfo; + } + + /** + * 生成 Notebook的NameSpace + * + * @param userId + * @return namespace + */ + public String generateNameSpace(long userId) { + return this.k8sNameConfig.getNamespace() + SEPARATOR + userId; + } + + /** + * 从resourceName中获取资源信息 + * + * @param bizEnum 业务枚举 + * @param resourceName + * @return resourceInfo + */ + public String getResourceInfoFromResourceName(BizEnum bizEnum, String resourceName) { + if (StringUtils.isEmpty(resourceName) || !resourceName.contains(bizEnum.getBizCode() + SEPARATOR)) { + return null; + } + return resourceName.replace(bizEnum.getBizCode() + SEPARATOR + RESOURCE_NAME + SEPARATOR, SymbolConstant.BLANK); + } + + /** + * 从namespace 获取使用者ID + * + * @param namespace + * @return Long + */ + public Long getUserIdFromNameSpace(String namespace) { + if (StringUtils.isEmpty(namespace) || !namespace.contains(this.k8sNameConfig.getNamespace() + SEPARATOR)) { + return null; + } + return Long.valueOf(namespace.replace(this.k8sNameConfig.getNamespace() + SEPARATOR, "")); + } + + + /** + * 生成业务模块相对路径 + * + * @param bizNfsEnum 业务NFS路径枚举 + * @return String 例如： /{biz}/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/ + */ + public String getNfsPath(BizNfsEnum bizNfsEnum, long userId) { + if (bizNfsEnum == null) { + return null; + } + return optimizationPath(K8S_FILE_SEPARATOR + + bizNfsEnum.getBizNfsPath() + + K8S_FILE_SEPARATOR + + userId + + K8S_FILE_SEPARATOR + + DateUtil.format(new Date(), DatePattern.PURE_DATETIME_MS_FORMAT) + RandomUtil.randomString(RANDOM_LENGTH) + + K8S_FILE_SEPARATOR); + } + + /** + * 去除NFS根路径 + * + * @param nfsPath + * @return String + */ + public String removeNfsRootPath(String nfsPath) { + if (StringUtils.isBlank(nfsPath) || !nfsPath.startsWith(k8sNameConfig.getNfsRootPath())) { + return nfsPath; + } + return optimizationPath(K8S_FILE_SEPARATOR + nfsPath.replace(k8sNameConfig.getNfsRootPath(), "")); + } + + /** + * 路径添加bucket + * + * @param nfsPath + * @return String + */ + public String appendBucket(String nfsPath) { + return optimizationPath(K8S_FILE_SEPARATOR + + k8sNameConfig.getFileBucket() + + K8S_FILE_SEPARATOR + + nfsPath); + } + + /** + * 路径添加时间戳随机数 + * + * @param nfsPath + * @return String + */ + public String appendTimeStampAndRandomNum(String nfsPath) { + return optimizationPath(nfsPath + + DateUtil.format(new Date(), DatePattern.PURE_DATETIME_MS_FORMAT) + RandomUtil.randomString(RANDOM_LENGTH) + + K8S_FILE_SEPARATOR); + } + + /** + * nfs路径根据业务转换 + * + * @param nfsPath + * @param sourceBizNfsEnum 源业务 NFS Path + * @param targetBizNfsEnum 目标业务 NFS Path + * @return String + */ + public String convertNfsPath(String nfsPath, BizNfsEnum sourceBizNfsEnum, BizNfsEnum targetBizNfsEnum) { + if (!validateBizNfsPath(nfsPath, sourceBizNfsEnum) || targetBizNfsEnum == null) { + return nfsPath; + } + return optimizationPath(nfsPath.replace(K8S_FILE_SEPARATOR + sourceBizNfsEnum.getBizNfsPath() + K8S_FILE_SEPARATOR + , K8S_FILE_SEPARATOR + targetBizNfsEnum.getBizNfsPath() + K8S_FILE_SEPARATOR)); + } + + /** + * 获取NFS绝对路径 + * + * @param nfsPath + * @return String + */ + public String getAbsoluteNfsPath(String nfsPath) { + if (StringUtils.isBlank(nfsPath)) { + return nfsPath; + } + return optimizationPath(k8sNameConfig.getNfsRootPath() + + K8S_FILE_SEPARATOR + + k8sNameConfig.getFileBucket() + + K8S_FILE_SEPARATOR + + nfsPath); + } + + /** + * 验证 nfsPath 是否是所属业务路径 + * + * @param nfsPath + * @param bizNfsEnum + * @return boolean + */ + public boolean validateBizNfsPath(String nfsPath, BizNfsEnum bizNfsEnum) { + return org.apache.commons.lang3.StringUtils.isNotBlank(nfsPath) + && bizNfsEnum != null + && nfsPath.contains(bizNfsEnum.getBizNfsPath()) + && !nfsPath.contains(K8S_FILE_SEPARATOR + k8sNameConfig.getFileBucket() + K8S_FILE_SEPARATOR) + && !nfsPath.contains(K8S_FILE_SEPARATOR + k8sNameConfig.getNfsRootPath() + K8S_FILE_SEPARATOR); + } + + /** + * 路径优化 + * + * @param path + * @return String + */ + private String optimizationPath(String path) { + if (StringUtils.isBlank(path)) { + return path; + } + return path.replaceAll("///*", K8S_FILE_SEPARATOR); + } + + /** + * 获取k8s pod标签 + * + * @param bizEnum + * @return String + */ + public String getPodLabel(BizEnum bizEnum) { + return bizEnum == null ? null : bizEnum.getBizCode(); + } + + /** + * 获取数据集在镜像中路径 + * + * @return String + */ + public String getDatasetPath() { + return k8sNameConfig.getDatasetPath(); + } + + + /** + * 自送生成K8S名称，供K8S使用 + * + * @return String + */ + public String getK8sName() { + return DateUtil.format(new Date(), DatePattern.PURE_DATETIME_MS_FORMAT) + + RandomUtil.randomString(RANDOM_LENGTH); + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java new file mode 100644 index 0000000..13470d4 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import java.text.SimpleDateFormat; +import java.util.Calendar; + +/** + * @description Key生成工具类 + * @date 2020-06-08 + */ +public class KeyUtil { + + /** + * 生成训练任务的key，目前规则为：train-{user_id}-{time} + * @param userId 用户id + * @return 训练任务的key + */ + public static String generateTrainKey(long userId) { + Calendar calendar = Calendar.getInstance(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); + return "train-" + userId + '-' + dateFormat.format(calendar.getTime()); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java new file mode 100644 index 0000000..36b9dc0 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java @@ -0,0 +1,255 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import ch.qos.logback.classic.Level; +import com.alibaba.fastjson.JSON; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.dubhe.aspect.LogAspect; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.constant.SymbolConstant; +import org.dubhe.domain.entity.LogInfo; +import org.dubhe.enums.LogEnum; +import org.slf4j.MDC; +import org.slf4j.helpers.MessageFormatter; + +import java.util.Arrays; +import java.util.UUID; + +/** + * @description 日志工具类 + * @date 2020-06-29 + */ +@Slf4j +public class LogUtil { + /** + * info级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + public static void info(LogEnum logType, Object... object) { + + logHandle(logType, Level.INFO, object); + } + + /** + * debug级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + public static void debug(LogEnum logType, Object... object) { + logHandle(logType, Level.DEBUG, object); + } + + /** + * error级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + public static void error(LogEnum logType, Object... object) { + errorObjectHandle(object); + logHandle(logType, Level.ERROR, object); + } + + /** + * warn级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + public static void warn(LogEnum logType, Object... object) { + logHandle(logType, Level.WARN, object); + } + + /** + * trace级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + public static void trace(LogEnum logType, Object... object) { + logHandle(logType, Level.TRACE, object); + } + + /** + * 日志处理 + * + * @param logType 日志类型 + * @param level 日志级别 + * @param object 打印的日志参数 + */ + private static void logHandle(LogEnum logType, Level level, Object[] object) { + + LogInfo logInfo = generateLogInfo(logType, level, object); + String logInfoJsonStr = logJsonStringLengthLimit(logInfo); + switch (Level.toLevel(logInfo.getLevel()).levelInt) { + case Level.TRACE_INT: + log.trace(logInfoJsonStr); + break; + case Level.DEBUG_INT: + log.debug(logInfoJsonStr); + break; + case Level.INFO_INT: + log.info(logInfoJsonStr); + break; + case Level.WARN_INT: + log.warn(logInfoJsonStr); + break; + case Level.ERROR_INT: + log.error(logInfoJsonStr); + break; + default: + } + + } + + /** + * 日志信息组装的内部方法 + * + * @param logType 日志类型 + * @param level 日志级别 + * @param object 打印的日志参数 + * @return LogInfo日志对象信息 + */ + private static LogInfo generateLogInfo(LogEnum logType, Level level, Object[] object) { + LogInfo logInfo = new LogInfo(); + // 日志类型检测 + if (!LogEnum.isLogType(logType)) { + level = Level.ERROR; + object = new Object[MagicNumConstant.ONE]; + object[MagicNumConstant.ZERO] = String.valueOf("logType【").concat(String.valueOf(logType)) + .concat("】is error !"); + logType = LogEnum.SYS_ERR; + } + + // 获取trace_id + if (StringUtils.isEmpty(MDC.get(LogAspect.TRACE_ID))) { + MDC.put(LogAspect.TRACE_ID, UUID.randomUUID().toString()); + } + // 设置logInfo的level,type,traceId属性 + logInfo.setLevel(level.levelStr).setType(logType.toString()).setTraceId(MDC.get(LogAspect.TRACE_ID)); + // 设置logInfo的堆栈信息 + setLogStackInfo(logInfo); + // 设置logInfo的info信息 + setLogInfo(logInfo, object); + // 截取loginfo的长度并转换成json字符串 + return logInfo; + + } + + /** + * 设置loginfo的堆栈信息 + * + * @param logInfo 日志对象 + */ + private static void setLogStackInfo(LogInfo logInfo) { + StackTraceElement[] elements = Thread.currentThread().getStackTrace(); + if (elements.length >= MagicNumConstant.SIX) { + logInfo.setCName(elements[MagicNumConstant.FIVE].getClassName()) + .setMName(elements[MagicNumConstant.FIVE].getMethodName()) + .setLine(String.valueOf(elements[MagicNumConstant.FIVE].getLineNumber())); + } + } + + /** + * 限制log日志的长度并转换成json + * + * @param logInfo 日志对象 + * @return String 日志对象Json字符串 + */ + private static String logJsonStringLengthLimit(LogInfo logInfo) { + try { + String jsonString = JSON.toJSONString(logInfo.getInfo()); + if (jsonString.length() > MagicNumConstant.TEN_THOUSAND) { + jsonString = jsonString.substring(MagicNumConstant.ZERO, MagicNumConstant.TEN_THOUSAND); + } + + logInfo.setInfo(jsonString); + jsonString = JSON.toJSONString(logInfo); + jsonString = jsonString.replace(SymbolConstant.BACKSLASH_MARK, SymbolConstant.MARK) + .replace(SymbolConstant.DOUBLE_MARK, SymbolConstant.MARK) + .replace(SymbolConstant.BRACKETS, SymbolConstant.BLANK); + + return jsonString; + + } catch (Exception e) { + logInfo.setLevel(Level.ERROR.levelStr).setType(LogEnum.SYS_ERR.toString()) + .setInfo("cannot serialize exception: " + ExceptionUtils.getStackTrace(e)); + return JSON.toJSONString(logInfo); + } + } + + /** + * 设置日志对象的info信息 + * + * @param logInfo 日志对象 + * @param object 打印的日志参数 + */ + private static void setLogInfo(LogInfo logInfo, Object[] object) { + for (Object obj : object) { + if (obj instanceof Exception) { + log.error((ExceptionUtils.getStackTrace((Throwable) obj))); + } + } + + if (object.length > MagicNumConstant.ONE) { + logInfo.setInfo(MessageFormatter.arrayFormat(object[MagicNumConstant.ZERO].toString(), + Arrays.copyOfRange(object, MagicNumConstant.ONE, object.length)).getMessage()); + + } else if (object.length == MagicNumConstant.ONE && object[MagicNumConstant.ZERO] instanceof Exception) { + logInfo.setInfo((ExceptionUtils.getStackTrace((Exception) object[MagicNumConstant.ZERO]))); + } else if (object.length == MagicNumConstant.ONE) { + logInfo.setInfo( + object[MagicNumConstant.ZERO] == null ? SymbolConstant.BLANK : object[MagicNumConstant.ZERO]); + } else { + logInfo.setInfo(SymbolConstant.BLANK); + } + + } + + /** + * 处理Exception的情况 + * + * @param object 打印的日志参数 + */ + private static void errorObjectHandle(Object[] object) { + if (object.length >= MagicNumConstant.TWO) { + object[MagicNumConstant.ZERO] = String.valueOf(object[MagicNumConstant.ZERO]) + .concat(SymbolConstant.BRACKETS); + } + + if (object.length == MagicNumConstant.TWO && object[MagicNumConstant.ONE] instanceof Exception) { + log.error((ExceptionUtils.getStackTrace((Throwable) object[MagicNumConstant.ONE]))); + object[MagicNumConstant.ONE] = ExceptionUtils.getStackTrace((Exception) object[MagicNumConstant.ONE]); + + } else if (object.length >= MagicNumConstant.THREE) { + for (int i = 0; i < object.length; i++) { + if (object[i] instanceof Exception) { + log.error((ExceptionUtils.getStackTrace((Throwable) object[i]))); + object[i] = ExceptionUtils.getStackTrace((Exception) object[i]); + } + + } + } + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java new file mode 100644 index 0000000..250cf84 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import org.dubhe.base.MagicNumConstant; + +/** + * @description: 计算工具类 + * @create: 2020/6/4 14:53 + */ +public class MathUtils { + + + /** + * 字符串整数加法 + * num1+num2 + * @param num1 + * @param num2 + * @return + */ + public static String add(String num1,String num2){ + return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO:Integer.valueOf(num1)) + (!RegexUtil.isDigits(num2)?MagicNumConstant.ZERO:Integer.valueOf(num2))); + } + /** + * 字符串整数减法 + * num1 - num2 + * @param num1 + * @param num2 + * @return + */ + public static String reduce(String num1,String num2){ + return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO:Integer.valueOf(num1)) - (!RegexUtil.isDigits(num2)?MagicNumConstant.ZERO:Integer.valueOf(num2))); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java new file mode 100644 index 0000000..e526ef2 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java @@ -0,0 +1,286 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import cn.hutool.core.io.IoUtil; +import cn.hutool.http.HttpRequest; +import com.alibaba.fastjson.JSONObject; +import io.minio.MinioClient; +import io.minio.PutObjectOptions; +import io.minio.Result; +import io.minio.errors.*; +import io.minio.messages.DeleteError; +import io.minio.messages.Item; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.apache.commons.lang.StringUtils; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.BusinessException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @description Minio工具类 + * @date 2020-05-09 + */ +@Service +public class MinioUtil { + + @Value("${minio.url}") + private String url; + @Value("${minio.accessKey}") + private String accessKey; + @Value("${minio.secretKey}") + private String secretKey; + + private MinioClient client; + + @PostConstruct + public void init() { + try { + client = new MinioClient(url, accessKey, secretKey); + } catch (InvalidEndpointException e) { + LogUtil.warn(LogEnum.BIZ_DATASET, "MinIO endpoint invalid. e:", e); + } catch (InvalidPortException e) { + LogUtil.warn(LogEnum.BIZ_DATASET, "MinIO endpoint port invalid. e:", e); + } + } + + /** + * 写文件 + * + * @param bucket 桶名称 + * @param fullFilePath 文件存储的全路径，包括文件名，非'/'开头. e.g. dataset/12/annotation/test.txt + * @param content file content. can not be null + */ + public void writeString(String bucket, String fullFilePath, String content) throws Exception { + boolean isExist = client.bucketExists(bucket); + if (!isExist) { + client.makeBucket(bucket); + } + InputStream inputStream = IoUtil.toUtf8Stream(content); + PutObjectOptions options = new PutObjectOptions(inputStream.available(), MagicNumConstant.NEGATIVE_ONE); + client.putObject(bucket, fullFilePath, inputStream, options); + } + + /** + * 读取文件 + * + * @param bucket + * @param fullFilePath 文件存储的全路径，包括文件名，非'/'开头. e.g. dataset/12/annotation/test.txt + * @return String + */ + public String readString(String bucket, String fullFilePath) throws Exception { + try (InputStream is = client.getObject(bucket, fullFilePath)) { + return IoUtil.read(is, Charset.defaultCharset()); + } + } + + /** + * 文件删除 + * + * @param bucket + * @param fullFilePath 文件存储的全路径，包括文件名，非'/'开头. e.g. dataset/12/annotation/test.txt + */ + public void del(String bucket, String fullFilePath) throws Exception { + Iterable> items = client.listObjects(bucket, fullFilePath); + Set files = new HashSet<>(); + for (Result item : items) { + files.add(item.get().objectName()); + } + Iterable> results = client.removeObjects(bucket, files); + for (Result result : results) { + result.get(); + } + } + + /** + * 批量删除文件 + * + * @param bucket + * @param objectNames + */ + public void delFiles(String bucket,List objectNames) throws Exception{ + Iterable> results = client.removeObjects(bucket, objectNames); + for (Result result : results) { + result.get(); + } + } + + /** + * 获取对象名称 + * + */ + public List getObjects(String bucketName, String prefix)throws Exception{ + List fileNames = new ArrayList<>(); + Iterable> results = client.listObjects(bucketName, prefix); + for(Result result:results){ + Item item = result.get(); + fileNames.add(item.objectName()); + } + return fileNames; + } + + /** + * 获取文件流 + * + */ + public InputStream getObjectInputStream(String bucket,String objectName)throws Exception{ + return client.getObject(bucket, objectName); + } + + /** + * 文件夹复制 + * + * @param bucket + * @param sourceFiles 源文件 + * @param targetDir 目标文件夹 + */ + public void copyDir(String bucket, List sourceFiles, String targetDir) { + sourceFiles.forEach(sourceFile -> { + InputStream inputStream = null; + try { + String sourceObjectName = sourceFile; + String targetObjectName = targetDir + "/" + StringUtils.substringAfterLast(sourceObjectName, "/"); + inputStream = client.getObject(bucket, sourceObjectName); + byte[] buf = new byte[512]; + int bytesRead; + int count = MagicNumConstant.ZERO; + while ((bytesRead = inputStream.read(buf, MagicNumConstant.ZERO, buf.length)) >= MagicNumConstant.ZERO) { + count += bytesRead; + } + PutObjectOptions options = new PutObjectOptions(count, MagicNumConstant.ZERO); + client.putObject(bucket, targetObjectName, client.getObject(bucket, sourceObjectName), options); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_DATASET, "MinIO file copy exception, {}", e); + } finally { + try { + if (inputStream != null) { + inputStream.close(); + } + } catch (IOException e) { + LogUtil.error(LogEnum.BIZ_DATASET, "MinIO file read stream closed failed, {}", e); + } + } + }); + } + + @Data + @Service + public class MinioWebTokenBody { + + @Value("${minioweb.GetToken.url}") + private String tokenUrl; + @Value("${minioweb.GetToken.param.id}") + private int id; + @Value("${minioweb.GetToken.param.jsonrpc}") + private String jsonrpc; + @Value("${minioweb.GetToken.param.method}") + private String method; + @Value("${minioweb.zip.url}") + private String zipUrl; + + /** + * 生成文件下载请求参数方法 + * + * @param bucketName + * @param prefix + * @param objects + * @return MinioDownloadDto + */ + public MinioDownloadDto getDownloadParam(String bucketName, String prefix, List objects, String zipName) { + String paramTemplate = "{\"id\":%d,\"jsonrpc\":\"%s\",\"params\":{\"username\":\"%s\",\"password\":\"%s\"},\"method\":\"%s\"}"; + String downloadBodyTemplate = "{\"bucketName\":\"%s\",\"prefix\":\"%s\",\"objects\":[%s]}"; + String param = String.format(paramTemplate, id, jsonrpc, accessKey, secretKey, method); + String result = HttpRequest.post(url + tokenUrl).contentType("application/json").body(param).execute().body(); + String token = JSONObject.parseObject(result).getJSONObject("result").getString("token"); + return new MinioDownloadDto(token, String.format(downloadBodyTemplate, bucketName, prefix, getStrFromList(objects)), zipName); + } + + public String getStrFromList(List objects) { + List result = new ArrayList<>(); + objects.stream().forEach(s -> { + result.add("\"" + s + "\""); + }); + return StringUtils.join(result, ","); + } + + } + + @ApiModel + @Data + public class MinioDownloadDto { + @ApiModelProperty("下载压缩包请求token") + private String token; + @ApiModelProperty("下载压缩包请求参数") + private String body; + @ApiModelProperty("下载压缩包请求需要的header") + private Map headers; + @ApiModelProperty("下载压缩包文件名称") + private String zipName; + + public MinioDownloadDto() { + } + + public MinioDownloadDto(String token, String body, String zipName) { + this.token = token; + this.body = body; + this.zipName = zipName; + Map headers = new HashMap<>(); + headers.put("Content-Type", "text/plain;charset=UTF-8"); + this.headers = headers; + } + + } + + + /** + * 生成一个给HTTP PUT请求用的presigned URL。浏览器/移动端的客户端可以用这个URL进行上传， + * 即使其所在的存储桶是私有的。这个presigned URL可以设置一个失效时间，默认值是7天 + * + * @param bucketName 存储桶名称 + * @param objectName 存储桶里的对象名称 + * @param expires 失效时间（以秒为单位），默认是7天，不得大于七天 + * @return String + */ + public String getEncryptedPutUrl(String bucketName, String objectName, Integer expires) { + if (StringUtils.isEmpty(objectName)) { + throw new BusinessException("object name cannot be empty"); + } + try { + return client.presignedPutObject(bucketName, objectName, expires); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_DATASET, e.getMessage()); + throw new BusinessException("MinIO an error occurred, please contact the administrator"); + } + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java new file mode 100644 index 0000000..197d15c --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java @@ -0,0 +1,76 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import com.emc.ecs.nfsclient.nfs.nfs3.Nfs3; +import com.emc.ecs.nfsclient.rpc.CredentialUnix; +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.DefaultPooledObject; +import org.dubhe.config.NfsConfig; +import org.dubhe.enums.LogEnum; +import org.springframework.stereotype.Component; + +import java.io.IOException; + +/** + * @description NFS工厂类 + * @date 2020-05-13 + */ +@Component +public class NfsFactory implements PooledObjectFactory { + + private final NfsConfig nfsConfig; + + public NfsFactory(NfsConfig nfsConfig) { + this.nfsConfig = nfsConfig; + } + + @Override + public PooledObject makeObject() { + Nfs3 nfs3 = null; + try { + nfs3 = new Nfs3(nfsConfig.getNfsIp(), nfsConfig.getRootDir(), new CredentialUnix(0, 0, null), 3); + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "创建NFS对象失败: ", e); + } + return new DefaultPooledObject<>(nfs3); + } + + @Override + public void destroyObject(PooledObject pooledObject) { + LogUtil.info(LogEnum.NFS_UTIL, "销毁NFS对象: ", pooledObject.getObject()); + } + + @Override + public boolean validateObject(PooledObject pooledObject) { + LogUtil.info(LogEnum.NFS_UTIL, "验证NFS对象: ", pooledObject.getObject()); + return true; + } + + @Override + public void activateObject(PooledObject pooledObject) { + LogUtil.info(LogEnum.NFS_UTIL, "激活NFS对象: ", pooledObject.getObject()); + + } + + @Override + public void passivateObject(PooledObject pooledObject) { + LogUtil.info(LogEnum.NFS_UTIL, "钝化NFS对象: ", pooledObject.getObject()); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java new file mode 100644 index 0000000..862a97a --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java @@ -0,0 +1,114 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import com.emc.ecs.nfsclient.nfs.nfs3.Nfs3; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.dubhe.enums.LogEnum; +import org.springframework.stereotype.Component; + +/** + * @description NFS3 连接池 + * @date 2020-05-18 + */ +@Component +public class NfsPool { + /** + * NFS工厂对象 + */ + private NfsFactory nfsFactory; + /** + * GenericObjectPool对象 + */ + private final GenericObjectPool genericObjectPool; + /** + * 最大总共连接数 + */ + public static final int MAX_TOTAL = 300; + /** + * 最小连接数 + */ + public static final int MIN = 20; + /** + * 最大连接数 + */ + public static final int MAX = 300; + /** + * 最大等待时间 单位毫秒 + */ + public static final int MAX_WAIT_TIME = 3000; + + /** + * 初始化连接池 + * + * @param nfsFactory + */ + public NfsPool(NfsFactory nfsFactory) { + this.nfsFactory = nfsFactory; + GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig(); + poolConfig.setMaxTotal(MAX_TOTAL); + poolConfig.setMinIdle(MIN); + poolConfig.setMaxIdle(MAX); + poolConfig.setMaxWaitMillis(MAX_WAIT_TIME); + this.genericObjectPool = new GenericObjectPool<>(nfsFactory, poolConfig); + } + + /** + * 从连接池中取连接 + * + * @return nfs3 + */ + public Nfs3 getNfs() { + try { + LogUtil.info(LogEnum.NFS_UTIL,"NFS线程 活跃数量：{} ,空闲数量: {} , 等待队列数量 ： {}",genericObjectPool.getNumActive(),genericObjectPool.getNumIdle(),genericObjectPool.getNumWaiters()); + return genericObjectPool.borrowObject(); + } catch (Exception e) { + LogUtil.error(LogEnum.NFS_UTIL, "获取NFS连接失败: {} ", e); + return null; + } + } + + /** + * 释放连接到连接池 + * + * @param nfs3 + */ + public void revertNfs(Nfs3 nfs3) { + try { + if(nfs3 != null){ + LogUtil.info(LogEnum.NFS_UTIL,"成功释放对象 : {} ", nfs3); + genericObjectPool.returnObject(nfs3); + } + } catch (Exception e) { + LogUtil.error(LogEnum.NFS_UTIL, " 释放NFS连接失败: ", e); + } + } + + /** + * 销毁公共池 + */ + public void destroyPool() { + try { + genericObjectPool.close(); + } catch (Exception e) { + LogUtil.error(LogEnum.NFS_UTIL, "销毁NFS连接失败: ", e); + } + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java new file mode 100644 index 0000000..f7c7b41 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java @@ -0,0 +1,904 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import com.emc.ecs.nfsclient.nfs.io.Nfs3File; +import com.emc.ecs.nfsclient.nfs.io.NfsFileInputStream; +import com.emc.ecs.nfsclient.nfs.io.NfsFileOutputStream; +import lombok.Getter; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.io.IOUtils; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.config.NfsConfig; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.NfsBizException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import org.springframework.util.FileCopyUtils; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +/** + * @description NFS Util + * ！！！注意： 使用NFSFile 对象 业务上自行需要关闭 NFS 对象 + * @date 2020-05-12 + */ +@Component +@Getter +public class NfsUtil { + + private final NfsConfig nfsConfig; + + private static final String FILE_SEPARATOR = "/"; + + private static final String ZIP = ".zip"; + + private static final String CHARACTER_GBK = "GBK"; + + private static final String CHARACTER_UTF_8 = "UTF-8"; + + private static final String UNDER_LINE = "_"; + + @Autowired + private NfsPool nfsPool; + + + public NfsUtil(NfsConfig nfsConfig) { + this.nfsConfig = nfsConfig; + } + + private String getReplaceRootPathRegex() { + String rootPath = nfsConfig.getRootDir(); + return "^" + rootPath.substring(MagicNumConstant.ZERO, rootPath.length() - MagicNumConstant.ONE); + } + + /** + * 获取NFS3File 对象 + * + * @param path + * @return Nfs3File + */ + public Nfs3File getNfs3File(String path) { + if (StringUtils.isEmpty(formatPath(path))) { + LogUtil.error(LogEnum.NFS_UTIL, "传入的NFS3初始化路径 {} ,无法初始化NFS3 ", path); + throw new NfsBizException("初始化路径：" + path + " 不合法"); + } + Nfs3File nfs3File; + try { + nfs3File = new Nfs3File(nfsPool.getNfs(), path); + LogUtil.info(LogEnum.NFS_UTIL, "成功获取NFS3File对象 ： {} ", nfs3File.getName()); + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "获取NFS3File对象失败 ：", e); + throw new NfsBizException("未获取到NFS连接或者NFS连接池已满" + e.getMessage()); + } + return nfs3File; + } + + /** + * 获取NFS3File 对象文件的输入流 + * + * @param nfs3File + * @return BufferedInputStream + */ + public BufferedInputStream getInputStream(Nfs3File nfs3File) { + BufferedInputStream stream = null; + try { + if (!nfs3File.isFile()) { + throw new NfsBizException("此路径下查找到的对象不是文件类型，请检查文件类型是否正确！"); + } + stream = new BufferedInputStream(new NfsFileInputStream(nfs3File)); + } catch (IOException e) { + throw new NfsBizException("nfs获取对象输出流失败！"); + } + return stream; + } + + /** + * 获取NFS3File 对象文件的输入流 + * + * @param path + * @return BufferedInputStream + */ + public BufferedInputStream getInputStream(String path) { + Nfs3File nfs3File = getNfs3File(formatPath(path)); + if (nfs3File == null) { + throw new NfsBizException("此路径" + path + "下没有文件可以加载！"); + } + return getInputStream(nfs3File); + } + + /** + * 校验文件或文件夹是否存在 + * + * @param path 文件路径 + * @return boolean + */ + public boolean fileOrDirIsEmpty(String path) { + if (!StringUtils.isEmpty(path)) { + path = formatPath(path); + Nfs3File nfs3File = getNfs3File(path); + try { + if (nfs3File.exists()) { + return false; + } + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "判断NFS File异常: ", e); + return true; + } finally { + nfsPool.revertNfs(nfs3File.getNfs()); + } + } + return true; + + } + + + /** + * 创建指定NFS目录 + * + * @param dir 需要创建的目录 例如：/abc/def + * @return boolean + */ + public boolean createDir(String dir) { + if (!StringUtils.isEmpty(dir)) { + dir = formatPath(dir); + String[] paths = dir.substring(MagicNumConstant.ONE).split(FILE_SEPARATOR); + StringBuilder sbPath = new StringBuilder(); + for (String path : paths) { + sbPath.append(FILE_SEPARATOR).append(path); + Nfs3File nfs3File = getNfs3File(sbPath.toString()); + try { + if (!nfs3File.exists()) { + nfs3File.mkdirs(); + } + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "创建NFS目录失败:", e); + return false; + } finally { + nfsPool.revertNfs(nfs3File.getNfs()); + } + } + return true; + } + return false; + } + + /** + * 创建多个指定目录 + * + * @param removeNfsRootPath + * @param paths + * @return boolean + */ + private boolean createDir(boolean removeNfsRootPath, String... paths) { + for (String path : paths) { + if (path == null) { + continue; + } + String formatPath = path; + if (removeNfsRootPath) { + formatPath = formatPath.replaceAll(this.getReplaceRootPathRegex(), ""); + } + boolean res = createDir(formatPath); + if (!res) { + return false; + } + } + return true; + } + + + /** + * 创建指定NFS目录 + * + * @param paths 路径 例如：/nfs/abc/def/ /abc/def/ + * @param nfsRootPath 是否包含nfs根目录 true:包含 false:不包含 + * @return boolean + */ + public boolean createDirs(boolean nfsRootPath, String... paths) { + if (null == paths || paths.length < MagicNumConstant.ONE) { + return true; + } + for (String path : paths) { + if (path == null) { + continue; + } + String formatPath = path; + if (nfsRootPath) { + formatPath = formatPath.replaceAll(this.getReplaceRootPathRegex(), ""); + } + if (!createDir(formatPath)) { + return false; + } + } + return true; + } + + /** + * 指定目录NFS中创建文件 + * + * @param dir 需要创建的目录 例如：/abc/def + * @param fileName 需要创建的文件 例如：dd.txt + * @return boolean + */ + public boolean createFile(String dir, String fileName) { + if (!StringUtils.isEmpty(dir) && !StringUtils.isEmpty(fileName)) { + dir = formatPath(dir); + Nfs3File nfs3File = getNfs3File(dir + FILE_SEPARATOR + fileName); + try { + if (!nfs3File.exists()) { + nfs3File.mkdirs(); + } + nfs3File.createNewFile(); + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "创建NFS文件失败: ", e); + } finally { + nfsPool.revertNfs(nfs3File.getNfs()); + } + } + return false; + } + + /** + * 删除NFS目录 或者文件 + * + * @param dirOrFile 需要删除的目录 或者文件 例如：/abc/def 或者 /abc/def/dd.txt + * @return boolean + */ + public boolean deleteDirOrFile(String dirOrFile) { + if (!StringUtils.isEmpty(dirOrFile)) { + dirOrFile = formatPath(dirOrFile); + try { + List nfs3FileList = getNfs3File(dirOrFile).listFiles(); + //删除目录下的子文件 + if (!CollectionUtils.isEmpty(nfs3FileList)) { + for (Nfs3File nfs3File : nfs3FileList) { + if (nfs3File.isDirectory()) { + deleteDirOrFile(nfs3File.getPath()); + } else if (nfs3File.isFile()) { + try { + nfs3File.delete(); + } finally { + nfsPool.revertNfs(nfs3File.getNfs()); + } + } + } + } + Nfs3File sourceNfsFile = getNfs3File(dirOrFile); + try { + sourceNfsFile.delete(); + } finally { + nfsPool.revertNfs(sourceNfsFile.getNfs()); + } + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.NFS_UTIL, "删除NFS目录失败:", e); + } + } + return false; + } + + + /** + * 上传文件到 NFS 指定目录 + * + * @param sourceFile 本地文件 包含路径 例如：/abc/def/gg.txt + * @param targetDir 指定目录 例如：/abc/def + * @return boolean + */ + public boolean uploadFileToNfs(String sourceFile, String targetDir) { + if (StringUtils.isEmpty(sourceFile) || StringUtils.isEmpty(targetDir)) { + return false; + } + sourceFile = formatPath(sourceFile); + targetDir = formatPath(targetDir); + //本地文件对象 + File localFile = new File(sourceFile); + Nfs3File nfs3File = getNfs3File(targetDir + FILE_SEPARATOR + localFile.getName()); + try (BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(localFile)); + BufferedOutputStream outputStream = new BufferedOutputStream(new NfsFileOutputStream(nfs3File))) { + IOUtils.copyLarge(inputStream, outputStream); + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "上传失败: ", e); + } finally { + nfsPool.revertNfs(nfs3File.getNfs()); + } + return false; + } + + /** + * 下载NFS文件到本地目录 + * + * @param sourceFile 指定文件 例如：/abc/def/dd.txt + * @param targetPath 目标目录 例如: /abc/dd + * @return boolean + */ + public boolean downFileFormNfs(String sourceFile, String targetPath) { + if (StringUtils.isEmpty(sourceFile) || StringUtils.isEmpty(targetPath)) { + return false; + } + sourceFile = formatPath(sourceFile); + targetPath = formatPath(targetPath); + Nfs3File nfsFile = getNfs3File(sourceFile); + if (nfsFile != null) { + try (InputStream inputStream = new BufferedInputStream(new NfsFileInputStream(nfsFile)); + OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(new File(targetPath + FILE_SEPARATOR + nfsFile.getName())))) { + IOUtils.copyLarge(inputStream, outputStream); + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "下载失败:", e); + } finally { + nfsPool.revertNfs(nfsFile.getNfs()); + } + } + return false; + } + + + /** + * NFS 复制文件 到指定目录下 单个文件 + * + * @param sourceFile 需要复制的文件 例如：/abc/def/dd.txt + * @param targetPath 需要放置的目标目录 例如：/abc/dd + * @return boolean + */ + public boolean copyFile(String sourceFile, String targetPath) { + if (StringUtils.isEmpty(sourceFile) || StringUtils.isEmpty(targetPath)) { + return false; + } + sourceFile = formatPath(sourceFile); + targetPath = formatPath(targetPath); + Nfs3File sourceNfsFile = null; + Nfs3File targetNfsFileNew = null; + try { + sourceNfsFile = getNfs3File(sourceFile); + targetNfsFileNew = getNfs3File(targetPath + FILE_SEPARATOR + sourceNfsFile.getName()); + if (!targetNfsFileNew.exists()) { + createDir(targetPath); + } + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "创建目标文件失败： ", e); + } + try (InputStream inputStream = new BufferedInputStream(new NfsFileInputStream(sourceNfsFile)); + OutputStream outputStream = new BufferedOutputStream(new NfsFileOutputStream(targetNfsFileNew))) { + targetNfsFileNew.createNewFile(); + IOUtils.copyLarge(inputStream, outputStream); + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "复制失败:", e); + return false; + } finally { + nfsPool.revertNfs(sourceNfsFile.getNfs()); + nfsPool.revertNfs(targetNfsFileNew.getNfs()); + } + } + + + /** + * NFS 复制目录到指定目录下 多个文件 包含目录与文件并存情况 + * + * 通过本地文件复制方式 + * + * @param sourcePath 需要复制的文件目录 例如：/abc/def + * @param targetPath 需要放置的目标目录 例如：/abc/dd + * @return boolean + */ + public boolean copyPath(String sourcePath, String targetPath) { + if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { + return false; + } + sourcePath = formatPath(sourcePath); + targetPath = formatPath(targetPath); + try { + return copyLocalPath(nfsConfig.getRootDir() + sourcePath, nfsConfig.getRootDir() + targetPath); + } catch (Exception e) { + LogUtil.error(LogEnum.NFS_UTIL, " copyPath 复制失败: ", e); + return false; + } + } + + + /** + * NFS 复制目录到指定目录下 多个文件 包含目录与文件并存情况 + * + * 通过NFS文件复制方式 可能存在NFS RPC协议超时情况 + * + * @param sourcePath 需要复制的文件目录 例如：/abc/def + * @param targetPath 需要放置的目标目录 例如：/abc/dd + * @return boolean + */ + public boolean copyNfsPath(String sourcePath, String targetPath) { + if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { + return false; + } + sourcePath = formatPath(sourcePath); + targetPath = formatPath(targetPath); + try { + Nfs3File sourceNfsFile = getNfs3File(sourcePath); + List nfs3FileList = sourceNfsFile.listFiles(); + if (CollectionUtils.isEmpty(nfs3FileList)) { + createDir(targetPath + sourcePath.substring(sourcePath.lastIndexOf(FILE_SEPARATOR))); + } else { + for (Nfs3File nfs3File : nfs3FileList) { + if (nfs3File.isDirectory()) { + String newTargetPath = nfs3File.getPath().substring(nfs3File.getPath().lastIndexOf(FILE_SEPARATOR)); + Nfs3File newNfs3File = getNfs3File(newTargetPath); + try { + if (!newNfs3File.exists()) { + createDir(targetPath + newTargetPath); + } + copyNfsPath(nfs3File.getPath(), targetPath + newTargetPath); + } finally { + nfsPool.revertNfs(newNfs3File.getNfs()); + } + } + if (nfs3File.isFile()) { + copyFile(sourcePath + FILE_SEPARATOR + nfs3File.getName(), targetPath); + } + } + } + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "复制失败: ", e); + return false; + } + } + + + /** + * 复制文件到指定目录下 单个文件 + * + * @param sourcePath 需要复制的文件 例如：/abc/def/cc.txt + * @param targetPath 需要放置的目标目录 例如：/abc/dd + * @return boolean + */ + public boolean copyLocalFile(String sourcePath, String targetPath) { + LogUtil.info(LogEnum.NFS_UTIL, "复制文件原路径: {} ,目标路径： {}", sourcePath, targetPath); + if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { + return false; + } + sourcePath = formatPath(sourcePath); + targetPath = formatPath(targetPath); + LogUtil.info(LogEnum.NFS_UTIL, "过滤后文件原路径: {} ,目标路径：{}", sourcePath, targetPath); + try (InputStream input = new FileInputStream(sourcePath); + FileOutputStream output = new FileOutputStream(targetPath)) { + FileCopyUtils.copy(input, output); + LogUtil.info(LogEnum.NFS_UTIL, "复制文件成功"); + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, " copyLocalFile 复制失败: ", e); + return false; + } + } + + + /** + * 复制文件 到指定目录下 多个文件 包含目录与文件并存情况 + * + * @param sourcePath 需要复制的文件目录 例如：/abc/def + * @param targetPath 需要放置的目标目录 例如：/abc/dd + * @return boolean + */ + public boolean copyLocalPath(String sourcePath, String targetPath) { + if (!StringUtils.isEmpty(sourcePath) && !StringUtils.isEmpty(targetPath)) { + sourcePath = formatPath(sourcePath); + if (sourcePath.endsWith(FILE_SEPARATOR)) { + sourcePath = sourcePath.substring(MagicNumConstant.ZERO, sourcePath.lastIndexOf(FILE_SEPARATOR)); + } + targetPath = formatPath(targetPath); + LogUtil.info(LogEnum.NFS_UTIL, "复制文件夹 原路径: {} , 目标路径 ： {} ", sourcePath, targetPath); + File[] files = new File(sourcePath).listFiles(); + LogUtil.info(LogEnum.NFS_UTIL, "需要复制的文件数量为: {}", files.length); + if (files.length != 0) { + for (File file : files) { + try { + if (file.isDirectory()) { + LogUtil.info(LogEnum.NFS_UTIL, "需要复制夹: {}", file.getAbsolutePath()); + LogUtil.info(LogEnum.NFS_UTIL, "目标文件夹: {}", targetPath + FILE_SEPARATOR + file.getName()); + File fileDir = new File(targetPath + FILE_SEPARATOR + file.getName()); + if(!fileDir.exists()){ + fileDir.mkdirs(); + } + copyLocalPath(sourcePath + FILE_SEPARATOR + file.getName(), targetPath + FILE_SEPARATOR + file.getName()); + } + if (file.isFile()) { + File fileTargetPath = new File(targetPath); + if(!fileTargetPath.exists()){ + fileTargetPath.mkdirs(); + } + LogUtil.info(LogEnum.NFS_UTIL, "需要复制文件: {}", file.getAbsolutePath()); + LogUtil.info(LogEnum.NFS_UTIL, "需要复制文件名称: {}", file.getName()); + copyLocalFile(file.getAbsolutePath() , targetPath + FILE_SEPARATOR + file.getName()); + } + }catch (Exception e){ + LogUtil.error(LogEnum.NFS_UTIL, "复制文件夹失败: {}", e); + return false; + } + } + } + return true; + } + return false; + } + + /** + * 解压前清理同路径下其他文件(目前只支持路径下无文件夹，文件均为zip文件) + * 上传路径垃圾文件清理 + * + * @param zipFilePath zip源文件 例如：/abc/z.zip + * @param path 文件夹 例如：/abc/ + * @return boolean + */ + public boolean cleanPath(String zipFilePath, String path) { + if (!StringUtils.isEmpty(zipFilePath) && !StringUtils.isEmpty(path) && zipFilePath.toLowerCase().endsWith(ZIP)) { + zipFilePath = formatPath(zipFilePath); + path = formatPath(path); + Nfs3File nfs3Files = getNfs3File(path); + try { + String zipName = zipFilePath.substring(zipFilePath.lastIndexOf(FILE_SEPARATOR) + MagicNumConstant.ONE); + if (!StringUtils.isEmpty(zipName)) { + List nfs3FilesList = nfs3Files.listFiles(); + if (!CollectionUtils.isEmpty(nfs3FilesList)) { + for (Nfs3File nfs3File : nfs3FilesList) { + if (!zipName.equals(nfs3File.getName())) { + nfs3File.delete(); + } + } + return true; + } + } + } catch (Exception e) { + LogUtil.error(LogEnum.NFS_UTIL, "路径{}清理失败，错误原因为：{} ", path, e); + return false; + } finally { + nfsPool.revertNfs(nfs3Files.getNfs()); + } + } + return false; + } + + /** + * zip解压并删除压缩文件 + * + * @param sourcePath zip源文件 例如：/abc/z.zip + * @param targetPath 解压后的目标文件夹 例如：/abc/ + * @return boolean + */ + public boolean unzip(String sourcePath, String targetPath) { + if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { + return false; + } + sourcePath = formatPath(sourcePath); + targetPath = formatPath(targetPath); + if (!sourcePath.toLowerCase().endsWith(ZIP)) { + return false; + } + ArchiveInputStream zIn = null; + Nfs3File sourceNfsFile = getNfs3File(sourcePath); + try { + zIn = new ZipArchiveInputStream(new BufferedInputStream(new NfsFileInputStream(sourceNfsFile)), CHARACTER_GBK, false, true); + //判断压缩文件编码方式,并重新获取NFS对象流 + try { + zIn.getNextEntry(); + zIn.close(); + zIn = new ZipArchiveInputStream(new BufferedInputStream(new NfsFileInputStream(sourceNfsFile)), CHARACTER_GBK, false, true); + } catch (Exception e) { + zIn.close(); + zIn = new ZipArchiveInputStream(new BufferedInputStream(new NfsFileInputStream(sourceNfsFile)), CHARACTER_UTF_8, false, true); + } + ZipEntry entry; + while ((entry = (ZipEntry) zIn.getNextEntry()) != null) { + if (entry.isDirectory()) { + createDir(targetPath + FILE_SEPARATOR + entry.getName()); + } else { + //若文件夹未创建则创建文件夹 + if (entry.getName().contains(FILE_SEPARATOR)) { + String entryName = entry.getName(); + String zipDirName = entryName.substring(MagicNumConstant.ZERO, entryName.lastIndexOf(FILE_SEPARATOR)); + createDir(targetPath + FILE_SEPARATOR + zipDirName); + } + Nfs3File nfs3File = getNfs3File(targetPath + FILE_SEPARATOR + entry.getName()); + try { + if (!nfs3File.exists()) { + nfs3File.createNewFile(); + } + BufferedOutputStream bos = new BufferedOutputStream(new NfsFileOutputStream(nfs3File)); + IOUtils.copyLarge(zIn, bos); + bos.flush(); + bos.close(); + } finally { + nfsPool.revertNfs(nfs3File.getNfs()); + } + } + } + sourceNfsFile.delete(); + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "解压失败: ", e); + return false; + } finally { + nfsPool.revertNfs(sourceNfsFile.getNfs()); + if (zIn != null) { + try { + zIn.close(); + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "输入流关闭失败: ", e); + } + } + } + + } + + /** + * NFS 解压压缩包 包含目录与子目录 + * + * @param sourcePath 需要复制的文件 例如：/abc/def/aaa.rar + * @return boolean + */ + public boolean unZip(String sourcePath) { + sourcePath = formatPath(sourcePath); + if (StringUtils.isEmpty(sourcePath)) { + return false; + } + String fileDir = sourcePath.substring(MagicNumConstant.ZERO, sourcePath.lastIndexOf(FILE_SEPARATOR)); + ZipEntry zipEntry = null; + try (ZipInputStream zipInputStream = new ZipInputStream(new NfsFileInputStream(getNfs3File(sourcePath)))) { + while ((zipEntry = zipInputStream.getNextEntry()) != null) { + if (zipEntry.isDirectory()) { + createDir(fileDir + FILE_SEPARATOR + zipEntry.getName()); + continue; + } + Nfs3File targetNfsFileNew = getNfs3File(fileDir + FILE_SEPARATOR + zipEntry.getName()); + try (BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new NfsFileOutputStream(targetNfsFileNew))) { + targetNfsFileNew.createNewFile(); + IOUtils.copyLarge(zipInputStream, bufferedOutputStream); + } finally { + nfsPool.revertNfs(targetNfsFileNew.getNfs()); + } + } + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "解压文件失败 : ", e); + return false; + } + } + + /** + * 压缩NFS 目录 或者文件 到压缩包 + * + * @param dirOrFile 目录或者文件 例如： /abc/def/aaa.txt , /abc/def + * @param zipName 压缩包名称 例如： aa,bb,cc + * @return boolean + */ + public boolean zipDirOrFile(String dirOrFile, String zipName) { + Nfs3File nfs3File = getNfs3File(formatPath(dirOrFile)); + try (ZipOutputStream zipOutputStream = getFileZipOutputStream(getNfsFilePath(formatPath(dirOrFile)), zipName)) { + zipFiles(zipOutputStream, nfs3File); + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "压缩文件失败 : ", e); + return false; + } finally { + nfsPool.revertNfs(nfs3File.getNfs()); + } + } + + /** + * 获取NFS 文件压缩目录 + * + * @param dirOrFile + * @return String + */ + private String getNfsFilePath(String dirOrFile) throws IOException { + Nfs3File nfs3File = getNfs3File(formatPath(dirOrFile)); + if (nfs3File.isFile()) { + nfsPool.revertNfs(nfs3File.getNfs()); + return dirOrFile.substring(MagicNumConstant.ZERO, dirOrFile.lastIndexOf(FILE_SEPARATOR)); + } + return dirOrFile; + } + + /** + * 根据文件路劲 获取Zip文件流 + * + * @param dirOrFile + * @param zipName + * @return ZipOutputStream + */ + private ZipOutputStream getFileZipOutputStream(String dirOrFile, String zipName) throws IOException { + Nfs3File targetNfsFileNew = getNfs3File(getNfsFilePath(formatPath(dirOrFile)) + FILE_SEPARATOR + zipName + ZIP); + targetNfsFileNew.createNewFile(); + return new ZipOutputStream(new NfsFileOutputStream(targetNfsFileNew)); + } + + /** + * 压缩文件和文件夹 + * + * @param zipOutputStream + * @param nfs3File + * @return boolean + */ + public boolean zipFiles(ZipOutputStream zipOutputStream, Nfs3File nfs3File) { + try { + if (nfs3File.isFile()) { + compressZip(zipOutputStream, nfs3File, ""); + } else { + List nfs3FileList = nfs3File.listFiles(); + if (!CollectionUtils.isEmpty(nfs3FileList)) { + for (Nfs3File nfs3FileChildren : nfs3FileList) { + zipFiles(zipOutputStream, nfs3FileChildren); + } + } + } + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "压缩文件失败 : ", e); + return false; + } + } + + /** + * 单个文件压缩 + * + * @param zipOutputStream + * @param nfs3File + */ + public void compressZip(ZipOutputStream zipOutputStream, Nfs3File nfs3File, String childPath) { + try (InputStream inputStream = new BufferedInputStream(new NfsFileInputStream(nfs3File))) { + if (StringUtils.isEmpty(childPath)) { + zipOutputStream.putNextEntry(new ZipEntry(nfs3File.getName())); + } else { + zipOutputStream.putNextEntry(new ZipEntry(childPath + FILE_SEPARATOR + nfs3File.getName())); + } + byte[] buffer = new byte[1024 * 10]; + int length; + while ((length = inputStream.read(buffer, MagicNumConstant.ZERO, buffer.length)) != -1) { + zipOutputStream.write(buffer, MagicNumConstant.ZERO, length); + } + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "解压单个文件异常： ", e); + } finally { + nfsPool.revertNfs(nfs3File.getNfs()); + } + } + + /** + * 复制文件夹(或文件)到另一个文件夹 + * + * @param sourcePath 复制文件夹 /abc/def 复制文件：/abc/def/dd.txt + * @param targetPath /abc/dd/def + * @return boolean + */ + public boolean copyDirs(String sourcePath, String targetPath) { + Nfs3File sourceNfsFile = getNfs3File(formatPath(sourcePath)); + try { + if (!sourceNfsFile.exists()) { + LogUtil.error(LogEnum.NFS_UTIL, "sourcePath不存在, 如下{} ", sourcePath); + return false; + } + if (sourceNfsFile.isFile()) { + return copyFile(sourcePath, targetPath); + } else if (sourceNfsFile.isDirectory()) { + targetPath = targetPath + FILE_SEPARATOR + sourceNfsFile.getName(); + boolean bool = createDir(formatPath(targetPath)); + if (!bool) { + LogUtil.error(LogEnum.NFS_UTIL, "{}文件夹创建失败... ", targetPath); + return false; + } + List files = sourceNfsFile.listFiles(); + for (Nfs3File file : files) { + copyDirs(file.getPath(), targetPath); + } + } + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "copyDirs失败, sourcePath为 , targetPath为 , 失败原因 ", sourcePath, targetPath, e); + } finally { + nfsPool.revertNfs(sourceNfsFile.getNfs()); + } + return false; + } + + /** + * 找到倒数第二新的文件夹 + * + * @param parentPath 父文件夹 + * @return + */ + public String find2ndNewDir(String parentPath) { + Nfs3File parentNfsFile = getNfs3File(formatPath(parentPath)); + try { + if (!parentNfsFile.exists() || parentNfsFile.isFile()) { + LogUtil.error(LogEnum.NFS_UTIL, "sourcePath不存在, 如下{} ", parentPath); + return ""; + } + List files = parentNfsFile.listFiles(); + List dirs = new ArrayList<>(); + for (Nfs3File file : files) { + if (file.isDirectory()) { + dirs.add(file); + } + } + if (dirs.size() < MagicNumConstant.TWO) { + return ""; + } + dirs.sort((o1, o2) -> { + try { + return (int) (o2.lastModified() - o1.lastModified()); + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "执行异常: {} ", e); + return MagicNumConstant.ZERO; + } + }); + return dirs.get(MagicNumConstant.ONE).getName(); + + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "findSecNewDir失败, parentPath为{}, 失败原因{}", parentPath, e); + } finally { + nfsPool.revertNfs(parentNfsFile.getNfs()); + } + return ""; + } + + /** + * 重命名文件夹 + * + * @param sourcePath 原文件夹 + * @param targetPath 目标文件夹 + * @return + */ + public void renameDir(String sourcePath, String targetPath) { + Nfs3File sourceNfsFile = getNfs3File(formatPath(sourcePath)); + Nfs3File targetNfsFile = getNfs3File(formatPath(targetPath)); + try { + if (!sourceNfsFile.exists()) { + LogUtil.error(LogEnum.NFS_UTIL, "sourcePath不存在, 如下{} ", sourcePath); + } + sourceNfsFile.rename(targetNfsFile); + } catch (IOException e) { + LogUtil.error(LogEnum.NFS_UTIL, "renameDir失败, sourcePath为{}, targetPath为{}, 失败原因{}", sourcePath, targetPath, e); + } finally { + nfsPool.revertNfs(sourceNfsFile.getNfs()); + nfsPool.revertNfs(targetNfsFile.getNfs()); + } + } + + + /** + * 替换路劲中多余的 "/" + * + * @param path + * @return String + */ + private String formatPath(String path) { + if (!StringUtils.isEmpty(path)) { + return path.replaceAll("///*", FILE_SEPARATOR); + } + return path; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java new file mode 100644 index 0000000..86c2977 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import org.dubhe.exception.NotebookBizException; + +import java.util.regex.Pattern; + +/** + * @description 数字验证工具 + * @date 2020-05-18 + */ +public class NumberUtil { + + private static final String REGEX = "^[0-9]*$"; + + private NumberUtil() { + + } + + /** + * 判断是否为数字格式不限制位数 + * + * @param object 待校验参数 + */ + public static void isNumber(Object object) { + if (!((Pattern.compile(REGEX)).matcher(String.valueOf(object)).matches())) { + throw new NotebookBizException("parameter is incorrect"); + } + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java new file mode 100644 index 0000000..8f791ab --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java @@ -0,0 +1,78 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import com.baomidou.mybatisplus.core.metadata.IPage; + +import java.util.*; +import java.util.function.Function; + +/** + * @description 分页工具 + * @date 2020-03-13 + */ +public class PageUtil extends cn.hutool.core.util.PageUtil { + + /** + * List 分页 + */ + public static List toPage(int page, int size, List list) { + int fromIndex = page * size; + int toIndex = page * size + size; + if (fromIndex > list.size()) { + return new ArrayList(); + } else if (toIndex >= list.size()) { + return list.subList(fromIndex, list.size()); + } else { + return list.subList(fromIndex, toIndex); + } + } + + /** + * Page 数据处理，预防redis反序列化报错 + */ + public static Map toPage(IPage page) { + return toPage(page, page.getRecords()); + } + + /** + * 自定义分页 + */ + public static Map toPage(IPage page, Function function) { + return toPage(page, function.apply(page.getRecords())); + } + + /** + * 自定义分页 + */ + public static Map toPage(IPage page, Collection data) { + Map map = new LinkedHashMap<>(2); + map.put("result", data); + map.put("page", buildPagination(page)); + return map; + } + + private static Map buildPagination(IPage page) { + Map map = new HashMap<>(2); + map.put("current", page.getCurrent()); + map.put("size", page.getSize()); + map.put("total", page.getTotal()); + return map; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java b/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java new file mode 100644 index 0000000..fb04133 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * @description 读文件工具类 + * @date 2020-5-6 + */ +public class ReadFile { + + public static List read(String path, Integer start, Integer length){ + if (start < 0) { + start = 0; + } + List ansList = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(path))){ + for (int i = 0; i < start+length; i++) { + String line = br.readLine(); + if (line == null) { + break; + } + if (start <= i && i < start+length) { + ansList.add(line); + } + } + } catch (IOException ex) { + ex.printStackTrace(); + return ansList; + } + return ansList; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java new file mode 100644 index 0000000..c5f1d5a --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java @@ -0,0 +1,707 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import org.dubhe.base.MagicNumConstant; +import org.dubhe.enums.LogEnum; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.RedisConnectionUtils; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.ScanOptions; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * @description redis工具类 + * @date 2020-03-13 + */ +@Component +@SuppressWarnings({"unchecked", "all"}) +public class RedisUtils { + + private RedisTemplate redisTemplate; + @Value("${jwt.online-key}") + private String onlineKey; + + public RedisUtils(RedisTemplate redisTemplate) { + this.redisTemplate = redisTemplate; + } + + // =============================common============================ + + /** + * 指定缓存失效时间 + * + * @param key 键 + * @param time 时间(秒) + */ + public boolean expire(String key, long time) { + try { + if (time > 0) { + redisTemplate.expire(key, time, TimeUnit.SECONDS); + } + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils expire key {} time {} error:{}",key,time,e.getMessage(),e); + return false; + } + return true; + } + + /** + * 根据 key 获取过期时间 + * + * @param key 键 不能为null + * @return 时间(秒) 返回0代表为永久有效 + */ + public long getExpire(Object key) { + return redisTemplate.getExpire(key, TimeUnit.SECONDS); + } + + /** + * 查找匹配key + * + * @param pattern key + * @return / + */ + public List scan(String pattern) { + ScanOptions options = ScanOptions.scanOptions().match(pattern).build(); + RedisConnectionFactory factory = redisTemplate.getConnectionFactory(); + RedisConnection rc = Objects.requireNonNull(factory).getConnection(); + Cursor cursor = rc.scan(options); + List result = new ArrayList<>(); + while (cursor.hasNext()) { + result.add(new String(cursor.next())); + } + try { + RedisConnectionUtils.releaseConnection(rc, factory,true); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils scan pattern {} error:{}",pattern,e.getMessage(),e); + } + return result; + } + + /** + * 分页查询 key + * + * @param patternKey key + * @param page 页码 + * @param size 每页数目 + * @return / + */ + public List findKeysForPage(String patternKey, int page, int size) { + ScanOptions options = ScanOptions.scanOptions().match(patternKey).build(); + RedisConnectionFactory factory = redisTemplate.getConnectionFactory(); + RedisConnection rc = Objects.requireNonNull(factory).getConnection(); + Cursor cursor = rc.scan(options); + List result = new ArrayList<>(size); + int tmpIndex = 0; + int fromIndex = page * size; + int toIndex = page * size + size; + while (cursor.hasNext()) { + if (tmpIndex >= fromIndex && tmpIndex < toIndex) { + result.add(new String(cursor.next())); + tmpIndex++; + continue; + } + // 获取到满足条件的数据后,就可以退出了 + if (tmpIndex >= toIndex) { + break; + } + tmpIndex++; + cursor.next(); + } + try { + RedisConnectionUtils.releaseConnection(rc, factory,true); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils findKeysForPage patternKey {} page {} size {} error:{}",patternKey,page,size,e.getMessage(),e); + } + return result; + } + + /** + * 判断key是否存在 + * + * @param key 键 + * @return true 存在 false不存在 + */ + public boolean hasKey(String key) { + try { + return redisTemplate.hasKey(key); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils hasKey key {} error:{}",key,e.getMessage(),e); + return false; + } + } + + /** + * 删除缓存 + * + * @param key 可以传一个值 或多个 + */ + public void del(String... key) { + if (key != null && key.length > 0) { + if (key.length == 1) { + redisTemplate.delete(key[0]); + } else { + redisTemplate.delete(CollectionUtils.arrayToList(key)); + } + } + } + + // ============================String============================= + + /** + * 普通缓存获取 + * + * @param key 键 + * @return 值 + */ + public Object get(String key) { + + return key == null ? null : redisTemplate.opsForValue().get(key); + } + + /** + * 批量获取 + * + * @param keys + * @return + */ + public List multiGet(List keys) { + Object obj = redisTemplate.opsForValue().multiGet(Collections.singleton(keys)); + return null; + } + + /** + * 普通缓存放入 + * + * @param key 键 + * @param value 值 + * @return true成功 false失败 + */ + public boolean set(String key, Object value) { + try { + redisTemplate.opsForValue().set(key, value); + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils set key {} value {} error:{}",key,value,e.getMessage(),e); + return false; + } + } + + /** + * 普通缓存放入并设置时间 + * + * @param key 键 + * @param value 值 + * @param time 时间(秒) time要大于0 如果time小于等于0 将设置无限期 + * @return true成功 false 失败 + */ + public boolean set(String key, Object value, long time) { + try { + if (time > 0) { + redisTemplate.opsForValue().set(key, value, time, TimeUnit.SECONDS); + } else { + set(key, value); + } + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils set key {} value {} time {} error:{}",key,value,time,e.getMessage(),e); + return false; + } + } + + /** + * 普通缓存放入并设置时间 + * + * @param key 键 + * @param value 值 + * @param time 时间 + * @param timeUnit 类型 + * @return true成功 false 失败 + */ + public boolean set(String key, Object value, long time, TimeUnit timeUnit) { + try { + if (time > 0) { + redisTemplate.opsForValue().set(key, value, time, timeUnit); + } else { + set(key, value); + } + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils set key {} value {} time {} timeUnit {} error:{}",key,value,time,timeUnit,e.getMessage(),e); + return false; + } + } + + // ================================Map================================= + + /** + * HashGet + * + * @param key 键 不能为null + * @param item 项 不能为null + * @return 值 + */ + public Object hget(String key, String item) { + return redisTemplate.opsForHash().get(key, item); + } + + /** + * 获取hashKey对应的所有键值 + * + * @param key 键 + * @return 对应的多个键值 + */ + public Map hmget(String key) { + return redisTemplate.opsForHash().entries(key); + + } + + /** + * HashSet + * + * @param key 键 + * @param map 对应多个键值 + * @return true 成功 false 失败 + */ + public boolean hmset(String key, Map map) { + try { + redisTemplate.opsForHash().putAll(key, map); + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils hmset key {} map {} error:{}",key,map,e.getMessage(),e); + return false; + } + } + + /** + * HashSet 并设置时间 + * + * @param key 键 + * @param map 对应多个键值 + * @param time 时间(秒) + * @return true成功 false失败 + */ + public boolean hmset(String key, Map map, long time) { + try { + redisTemplate.opsForHash().putAll(key, map); + if (time > 0) { + expire(key, time); + } + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils hmset key {} map {} time {} error:{}",key,map,time,e.getMessage(),e); + return false; + } + } + + /** + * 向一张hash表中放入数据,如果不存在将创建 + * + * @param key 键 + * @param item 项 + * @param value 值 + * @return true 成功 false失败 + */ + public boolean hset(String key, String item, Object value) { + try { + redisTemplate.opsForHash().put(key, item, value); + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils hset key {} item {} value {} error:{}",key,item,value,e.getMessage(),e); + return false; + } + } + + /** + * 向一张hash表中放入数据,如果不存在将创建 + * + * @param key 键 + * @param item 项 + * @param value 值 + * @param time 时间(秒) 注意:如果已存在的hash表有时间,这里将会替换原有的时间 + * @return true 成功 false失败 + */ + public boolean hset(String key, String item, Object value, long time) { + try { + redisTemplate.opsForHash().put(key, item, value); + if (time > 0) { + expire(key, time); + } + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils hset key {} item {} value {} time {} error:{}",key,item,value,time,e.getMessage(),e); + return false; + } + } + + /** + * 删除hash表中的值 + * + * @param key 键 不能为null + * @param item 项 可以使多个 不能为null + */ + public void hdel(String key, Object... item) { + redisTemplate.opsForHash().delete(key, item); + } + + /** + * 判断hash表中是否有该项的值 + * + * @param key 键 不能为null + * @param item 项 不能为null + * @return true 存在 false不存在 + */ + public boolean hHasKey(String key, String item) { + return redisTemplate.opsForHash().hasKey(key, item); + } + + /** + * hash递增 如果不存在,就会创建一个 并把新增后的值返回 + * + * @param key 键 + * @param item 项 + * @param by 要增加几(大于0) + * @return + */ + public double hincr(String key, String item, double by) { + return redisTemplate.opsForHash().increment(key, item, by); + } + + /** + * hash递减 + * + * @param key 键 + * @param item 项 + * @param by 要减少记(小于0) + * @return + */ + public double hdecr(String key, String item, double by) { + return redisTemplate.opsForHash().increment(key, item, -by); + } + + // ============================set============================= + + /** + * 根据key获取Set中的所有值 + * + * @param key 键 + * @return + */ + public Set sGet(String key) { + try { + return redisTemplate.opsForSet().members(key); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils sGet key {} error:{}",key,e.getMessage(),e); + return null; + } + } + + /** + * 根据value从一个set中查询,是否存在 + * + * @param key 键 + * @param value 值 + * @return true 存在 false不存在 + */ + public boolean sHasKey(String key, Object value) { + try { + return redisTemplate.opsForSet().isMember(key, value); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils sHasKey key {} value {} error:{}",key,value,e.getMessage(),e); + return false; + } + } + + /** + * 将数据放入set缓存 + * + * @param key 键 + * @param values 值 可以是多个 + * @return 成功个数 + */ + public long sSet(String key, Object... values) { + try { + return redisTemplate.opsForSet().add(key, values); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils sSet key {} values {} error:{}",key,values,e.getMessage(),e); + return 0; + } + } + + /** + * 将set数据放入缓存 + * + * @param key 键 + * @param time 时间(秒) + * @param values 值 可以是多个 + * @return 成功个数 + */ + public long sSetAndTime(String key, long time, Object... values) { + try { + Long count = redisTemplate.opsForSet().add(key, values); + if (time > 0) { + expire(key, time); + } + return count; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils sSetAndTime key {} time {} values {} error:{}",key,time,values,e.getMessage(),e); + return 0; + } + } + + /** + * 获取set缓存的长度 + * + * @param key 键 + * @return + */ + public long sGetSetSize(String key) { + try { + return redisTemplate.opsForSet().size(key); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils sGetSetSize key {} error:{}",key,e.getMessage(),e); + return 0; + } + } + + /** + * 移除值为value的 + * + * @param key 键 + * @param values 值 可以是多个 + * @return 移除的个数 + */ + public long setRemove(String key, Object... values) { + try { + Long count = redisTemplate.opsForSet().remove(key, values); + return count; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils setRemove key {} values {} error:{}",key,values,e.getMessage(),e); + return 0; + } + } + + // ===============================sorted set================================= + + /** + *将zSet数据放入缓存 + * + * @param key + * @param time + * @param values + * @return Boolean + */ + public Boolean zSet(String key, long time, Object value){ + try { + Boolean success = redisTemplate.opsForZSet().add(key, value,System.currentTimeMillis()); + if (success) { + expire(key, time); + } + return success; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils zSet key {} time {} value {} error:{}",key,time,value,e.getMessage(),e); + return false; + } + } + + /** + * 返回有序集合所有成员，从大到小排序 + * + * @param key + * @return Set + */ + public Set zGet(String key){ + try { + return redisTemplate.opsForZSet().reverseRange(key,Long.MIN_VALUE, Long.MAX_VALUE); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils zGet key {} error:{}",key,e.getMessage(),e); + return null; + } + } + + + + // ===============================list================================= + + /** + * 获取list缓存的内容 + * + * @param key 键 + * @param start 开始 + * @param end 结束 0 到 -1代表所有值 + * @return + */ + public List lGet(String key, long start, long end) { + try { + return redisTemplate.opsForList().range(key, start, end); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils lGetIndex key {} start {} end {} error:{}",key,start,end,e.getMessage(),e); + return null; + } + } + + /** + * 获取list缓存的长度 + * + * @param key 键 + * @return + */ + public long lGetListSize(String key) { + try { + return redisTemplate.opsForList().size(key); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils lGetListSize key {} error:{}",key,e.getMessage(),e); + return 0; + } + } + + /** + * 通过索引 获取list中的值 + * + * @param key 键 + * @param index 索引 index>=0时， 0 表头，1 第二个元素，依次类推；index<0时，-1，表尾，-2倒数第二个元素，依次类推 + * @return + */ + public Object lGetIndex(String key, long index) { + try { + return redisTemplate.opsForList().index(key, index); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils lGetIndex key {} index {} error:{}",key,index,e.getMessage(),e); + return null; + } + } + + /** + * 将list放入缓存 + * + * @param key 键 + * @param value 值 + * @return + */ + public boolean lSet(String key, Object value) { + try { + redisTemplate.opsForList().rightPush(key, value); + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils lSet key {} value {} error:{}",key,value,e.getMessage(),e); + return false; + } + } + + /** + * 将list放入缓存 + * + * @param key 键 + * @param value 值 + * @param time 时间(秒) + * @return + */ + public boolean lSet(String key, Object value, long time) { + try { + redisTemplate.opsForList().rightPush(key, value); + if (time > 0) { + expire(key, time); + } + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils lSet key {} value {} time {} error:{}",key,value,time,e.getMessage(),e); + return false; + } + } + + /** + * 将list放入缓存 + * + * @param key 键 + * @param value 值 + * @return + */ + public boolean lSet(String key, List value) { + try { + redisTemplate.opsForList().rightPushAll(key, value); + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils lSet key {} value {} error:{}",key,value,e.getMessage(),e); + return false; + } + } + + /** + * 将list放入缓存 + * + * @param key 键 + * @param value 值 + * @param time 时间(秒) + * @return + */ + public boolean lSet(String key, List value, long time) { + try { + redisTemplate.opsForList().rightPushAll(key, value); + if (time > 0) { + expire(key, time); + } + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils lSet key {} value {} time {} error:{}",key,value,time,e.getMessage(),e); + return false; + } + } + + /** + * 根据索引修改list中的某条数据 + * + * @param key 键 + * @param index 索引 + * @param value 值 + * @return / + */ + public boolean lUpdateIndex(String key, long index, Object value) { + try { + redisTemplate.opsForList().set(key, index, value); + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils lUpdateIndex key {} index {} value {} error:{}",key,index,value,e.getMessage(),e); + return false; + } + } + + /** + * 移除N个值为value + * + * @param key 键 + * @param count 移除多少个 + * @param value 值 + * @return 移除的个数 + */ + public long lRemove(String key, long count, Object value) { + try { + return redisTemplate.opsForList().remove(key, count, value); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR,"RedisUtils lRemove key {} count {} value {} error:{}",key,count,value,e.getMessage(),e); + return 0; + } + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java new file mode 100644 index 0000000..c5ea13b --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import java.lang.reflect.Field; +import java.util.*; + +/** + * @description 反射工具类 + * @date 2020-05-29 + **/ +public class ReflectionUtils { + + /** + * 获取所有字段集合 + * + * @param clazz 反射对象 + * @return List 字段集合 + **/ + public static List getFieldNames(Class clazz) { + Field[] fields = clazz.getDeclaredFields(); + List fieldNameList = new ArrayList<>(); + for (Field field : fields) { + fieldNameList.add(field.getName()); + } + return fieldNameList; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java new file mode 100644 index 0000000..0d97bd2 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java @@ -0,0 +1,63 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import lombok.extern.slf4j.Slf4j; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @description: 正则匹配工具类 + * @create: 2020/4/23 13:51 + */ +@Slf4j +public class RegexUtil { + private static final String DIGIT = "^[0-9]*$"; + /** + * str待匹配文本 + * regex 正则表达式 + *返回str中匹配regex的第一个子串 + */ + public static String getMatcher(String str,String regex) { + try{ + if (StringUtils.isEmpty(str) || StringUtils.isEmpty(regex)){ + return ""; + } + Pattern p = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + Matcher matcher = p.matcher(str); + matcher.find(); + return matcher.group(); + }catch (IllegalStateException e){ + log.error(e.getMessage(), e); + return ""; + } + } + + /** + * 数字匹配 + * @param str + * @return + */ + public static boolean isDigits(String str){ + if (StringUtils.isEmpty(str)){ + return false; + } + return str.matches(DIGIT); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java b/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java new file mode 100644 index 0000000..8bf7444 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java @@ -0,0 +1,91 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +/** + * @description 上下文工具类 + * @date 2020-03-25 + */ +@Slf4j +public class SpringContextHolder implements ApplicationContextAware, DisposableBean { + + private static ApplicationContext applicationContext = null; + + /** + * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型. + */ + @SuppressWarnings("unchecked") + public static T getBean(String name) { + assertContextInjected(); + return (T) applicationContext.getBean(name); + } + + /** + * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型. + */ + public static T getBean(Class requiredType) { + assertContextInjected(); + return applicationContext.getBean(requiredType); + } + + /** + * 检查ApplicationContext不为空. + */ + private static void assertContextInjected() { + if (applicationContext == null) { + throw new IllegalStateException("applicaitonContext属性未注入, 请在applicationContext" + + ".xml中定义SpringContextHolder或在SpringBoot启动类中注册SpringContextHolder."); + } + } + + /** + * 清除SpringContextHolder中的ApplicationContext为Null. + */ + private static void clearHolder() { + log.debug("清除SpringContextHolder中的ApplicationContext:" + + applicationContext); + applicationContext = null; + } + + @Override + public void destroy() { + SpringContextHolder.clearHolder(); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + if (SpringContextHolder.applicationContext != null) { + log.warn("SpringContextHolder中的ApplicationContext被覆盖, 原有ApplicationContext为:" + SpringContextHolder.applicationContext); + } + SpringContextHolder.applicationContext = applicationContext; + } + + /** + * 获取当前环境 + * + * @return + */ + public static String getActiveProfile(){ + return applicationContext.getEnvironment().getActiveProfiles()[0]; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java new file mode 100644 index 0000000..6c0267e --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +/** + * @description sql语句转换的工具类 + * @date 2020-07-06 + */ + +public class SqlUtil { + + /** + * 将数组转换成in('1','2')的形式 + * + * @param objs + * @return + */ + public static String integerlistToString(Integer[] objs) { + + if (objs != null && objs.length > 0) { + StringBuilder sb = new StringBuilder("("); + for (Object obj : objs) { + if (obj != null) { + sb.append("'" + obj.toString() + "',"); + } + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } + return ""; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java new file mode 100644 index 0000000..5bfa187 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java @@ -0,0 +1,276 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import eu.bitwalker.useragentutils.Browser; +import eu.bitwalker.useragentutils.UserAgent; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.constant.SymbolConstant; + +import javax.servlet.http.HttpServletRequest; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Calendar; +import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @description 字符串工具类, 继承org.apache.commons.lang3.StringUtils类 + * @date 2020-03-25 + */ +public class StringUtils extends org.apache.commons.lang3.StringUtils { + + private static final char SEPARATOR = '_'; + + private static final String UNKNOWN = "unknown"; + + private static Pattern linePattern = Pattern.compile("_(\\w)"); + + + /** + * 驼峰命名法工具 + * + * @return toCamelCase(" hello_world ") == "helloWorld" + * toCapitalizeCamelCase("hello_world") == "HelloWorld" + * toUnderScoreCase("helloWorld") = "hello_world" + */ + public static String toCamelCase(String s) { + if (s == null) { + return null; + } + + s = s.toLowerCase(); + + StringBuilder sb = new StringBuilder(s.length()); + boolean upperCase = false; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + + if (c == SEPARATOR) { + upperCase = true; + } else if (upperCase) { + sb.append(Character.toUpperCase(c)); + upperCase = false; + } else { + sb.append(c); + } + } + + return sb.toString(); + } + + /** + * 驼峰命名法工具 + * + * @return toCamelCase(" hello_world ") == "helloWorld" + * toCapitalizeCamelCase("hello_world") == "HelloWorld" + * toUnderScoreCase("helloWorld") = "hello_world" + */ + public static String toCapitalizeCamelCase(String s) { + if (s == null) { + return null; + } + s = toCamelCase(s); + return s.substring(0, 1).toUpperCase() + s.substring(1); + } + + /** + * 驼峰命名法工具 + * + * @return toCamelCase(" hello_world ") == "helloWorld" + * toCapitalizeCamelCase("hello_world") == "HelloWorld" + * toUnderScoreCase("helloWorld") = "hello_world" + */ + static String toUnderScoreCase(String s) { + if (s == null) { + return null; + } + + StringBuilder sb = new StringBuilder(); + boolean upperCase = false; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + + boolean nextUpperCase = true; + + if (i < (s.length() - 1)) { + nextUpperCase = Character.isUpperCase(s.charAt(i + 1)); + } + + if ((i > 0) && Character.isUpperCase(c)) { + if (!upperCase || !nextUpperCase) { + sb.append(SEPARATOR); + } + upperCase = true; + } else { + upperCase = false; + } + + sb.append(Character.toLowerCase(c)); + } + + return sb.toString(); + } + + + /** + * 获取ip地址 + * + * @param request + * @return + */ + public static String getIp(HttpServletRequest request) { + String ip = request.getHeader("x-forwarded-for"); + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + ip = request.getHeader("Proxy-Client-IP"); + } + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + ip = request.getHeader("WL-Proxy-Client-IP"); + } + if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { + ip = request.getRemoteAddr(); + } + String comma = ","; + String localhost = "127.0.0.1"; + if (ip.contains(comma)) { + ip = ip.split(",")[0]; + } + if (localhost.equals(ip)) { + // 获取本机真正的ip地址 + try { + ip = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + } + return ip; + } + + public static String getBrowser(HttpServletRequest request) { + UserAgent userAgent = UserAgent.parseUserAgentString(request.getHeader("User-Agent")); + Browser browser = userAgent.getBrowser(); + return browser.getName(); + } + + + /** + * 获得当天是周几 + */ + public static String getWeekDay() { + String[] weekDays = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; + Calendar cal = Calendar.getInstance(); + cal.setTime(new Date()); + + int w = cal.get(Calendar.DAY_OF_WEEK) - 1; + if (w < 0) { + w = 0; + } + return weekDays[w]; + } + + /** + * 字符串匹配工具类(正则表达式) + * + * @param str + * @param regexp + * @return + */ + public static boolean stringMatch(String str, String regexp) { + Pattern pattern = Pattern.compile(regexp); + return pattern.matcher(str).find(); + } + + /** + * 字符串长度不够补位 + * + * @param sourceStr 源字符串 + * @param length 最终长度 + * @param type 补位方式 0:左边 1:右边 + * @return + */ + public static String stringFillIn(String sourceStr, int length, int type) { + if (sourceStr.length() > length) { + return sourceStr; + } + StringBuilder stringBuilder = new StringBuilder(""); + for (int i = 0; i < length - sourceStr.length(); i++) { + stringBuilder.append("0"); + } + + if (type == MagicNumConstant.ZERO) { + return stringBuilder.toString() + sourceStr; + } else { + return sourceStr + stringBuilder.toString(); + } + } + + + /** + * 从头截取string字符串 + * + * @param str 被截取字符串 + * @param truncationIndex 0 -> 截取长度 + * @return + */ + public static String truncationString(String str, int truncationIndex) { + if (str == null) { + return SymbolConstant.BLANK; + } else if (truncationIndex < 1 + || str.length() <= truncationIndex) { + return str; + } + return str.substring(0, truncationIndex); + } + + /** + * 字符串驼峰转下划线 + * + * @param str 被转字符串 + * @return + */ + public static String humpToLine(String str) { + StringBuilder stringBuilder = new StringBuilder(); + char[] chars = str.toCharArray(); + for (char charactor : chars) { + if (Character.isUpperCase(charactor)) { + stringBuilder.append("_"); + charactor = Character.toLowerCase(charactor); + } + stringBuilder.append(charactor); + } + return stringBuilder.toString(); + } + + /** + * 字符串下划线转驼峰 + * + * @param str 被转字符串 + * @return + */ + public static String lineToHump(String str) { + str = str.toLowerCase(); + Matcher matcher = linePattern.matcher(str); + StringBuffer sb = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(sb, matcher.group(1).toUpperCase()); + } + matcher.appendTail(sb); + return sb.toString(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java new file mode 100644 index 0000000..0423246 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + + +import lombok.extern.slf4j.Slf4j; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +/** + * @description: UTC时间转换CST时间工具类 + * @create: 2020/5/20 12:10 + */ +@Slf4j +public class TimeTransferUtil { + /** + * @param utcTime + * @return cstTime + */ + public static String cstTransfer(String utcTime){ + Date utcDate = null; + /**2020-05-20T03:13:22Z 对应的时间格式 yyyy-MM-dd'T'HH:mm:ss'Z'**/ + SimpleDateFormat utcSimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + + try { + utcDate = utcSimpleDateFormat.parse(utcTime); + } catch (ParseException e) { + log.info(e.getMessage()); + return null; + } + /**System.out.println("UTC时间："+date);**/ + Calendar calendar = Calendar.getInstance(); + calendar.setTime(utcDate); + calendar.set(Calendar.HOUR,calendar.get(Calendar.HOUR)+8); + SimpleDateFormat cstSimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Date cstDate = calendar.getTime(); + String cstTime = cstSimpleDateFormat.format(calendar.getTime()); + return cstTime; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java b/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java new file mode 100644 index 0000000..1a8eec7 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java @@ -0,0 +1,131 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @description: 唯一码生成器 （依赖时间轴） + * + * @date 2020.05.08 + */ +public class UniqueKeyGenerator { + + private UniqueKeyGenerator(){ + + } + + /** + * 累加上限 + */ + private static final int ACCUMULATOR_MAX = 1000; + + /** + * 累加器 format + */ + private static final String ACCUMULATOR_FORMAT = "%0" + (ACCUMULATOR_MAX+"").length() + "d"; + + /** + * 时钟 format + */ + private static final String DATE_FORMAT = "yyMMddHHmmssSSS"; + + /** + * 累加计数器仓库 + * + * key:prefix + * value:累加计数器 + */ + private static Map prefixAccumulatorStorage = new ConcurrentHashMap<>(); + + /** + * 时间点仓库 + * + * key:prefix + * value:DATE_FORMAT 时钟 + */ + private static Map prefixDateStorage = new ConcurrentHashMap<>(); + + private static String getCurDateStr(){ + DateFormat df = new SimpleDateFormat(DATE_FORMAT); + return df.format(new Date()); + } + + /** + * 数字前位自动补0 + * + * @param atomicInteger + * @return + */ + private static String getAccumulatorStr(AtomicInteger atomicInteger){ + return String.format(ACCUMULATOR_FORMAT,atomicInteger.intValue()); + } + + private static String generateKey(String prefix,String curDateStr,AtomicInteger accumulator){ + return prefix + curDateStr + getAccumulatorStr(accumulator); + } + + /** + * 生成唯一key + * + * @param prefix + * @return + */ + public static String generateKey(final String prefix){ + if (prefix == null){ + return null; + } + synchronized(prefix){ + String curDateStr = getCurDateStr(); + String prevDateStr = prefixDateStorage.get(prefix); + if (curDateStr.equals(prevDateStr)){ + AtomicInteger accumulator = prefixAccumulatorStorage.get(prefix); + if (accumulator.incrementAndGet() >= ACCUMULATOR_MAX){ + return generateKey(prefix); + }else { + updateStorage(prefix,curDateStr,accumulator); + return generateKey(prefix,curDateStr,accumulator); + } + }else { + AtomicInteger accumulator = new AtomicInteger(0); + updateStorage(prefix,curDateStr,accumulator); + return generateKey(prefix,curDateStr,accumulator); + } + } + } + + + /** + * 更新仓库 + * + * @param prefix + * @param curDateStr + * @param accumulator + */ + private static void updateStorage(String prefix, String curDateStr, AtomicInteger accumulator) { + prefixAccumulatorStorage.put(prefix,accumulator); + prefixDateStorage.put(prefix,curDateStr); + } + + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java b/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java new file mode 100644 index 0000000..a303e19 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java @@ -0,0 +1,163 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import cn.hutool.core.util.ObjectUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.dubhe.annotation.Query; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +/** + * @description 构建Wrapper + * @date 2020-03-15 13:52:30 + */ +@Slf4j +public class WrapperHelp { + + public static QueryWrapper getWrapper(T query) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + if (query == null) { + return queryWrapper; + } + try { + List fields = getAllFields(query.getClass(), new ArrayList<>()); + for (Field field : fields) { + field.setAccessible(true); + Query q = field.getAnnotation(Query.class); + if (q != null) { + String propName = q.propName(); + String attributeName = isBlank(propName) ? field.getName() : propName; + Object val = field.get(query); + if (val == null) { + continue; + } + // 模糊多字段 + String blurry = q.blurry(); + if (ObjectUtil.isNotEmpty(blurry)) { + String[] blurrys = blurry.split(","); + queryWrapper.and(qw -> { + for (int i = 0; i < blurrys.length; i++) { + if (i == 0) { + qw.like(blurrys[i], val); + } else { + qw.or().like(blurrys[i], val); + } + } + }); + continue; + } + switch (q.type()) { + case EQ: + queryWrapper = queryWrapper.eq(attributeName, val); + break; + case NE: + queryWrapper = queryWrapper.ne(attributeName, val); + break; + case GE: + queryWrapper = queryWrapper.ge(attributeName, val); + break; + case GT: + queryWrapper = queryWrapper.gt(attributeName, val); + break; + case LT: + queryWrapper = queryWrapper.lt(attributeName, val); + break; + case LE: + queryWrapper = queryWrapper.le(attributeName, val); + break; + case BETWEEN: + List between = new ArrayList<>((List) val); + queryWrapper = queryWrapper.between(attributeName, between.get(0), between.get(1)); + break; + case NOT_BETWEEN: + List notBetween = new ArrayList<>((List) val); + queryWrapper = queryWrapper.notBetween(attributeName, notBetween.get(0), notBetween.get(1)); + break; + case LIKE: + queryWrapper = queryWrapper.like(attributeName, val); + break; + case NOT_LIKE: + queryWrapper = queryWrapper.notLike(attributeName, val); + break; + case LIkE_LEFT: + queryWrapper = queryWrapper.likeLeft(attributeName, val); + break; + case LIKE_RIGHT: + queryWrapper = queryWrapper.likeRight(attributeName, val); + break; + case IS_NULL: + queryWrapper = queryWrapper.isNull(attributeName); + break; + case IS_NOT_NULL: + queryWrapper = queryWrapper.isNotNull(attributeName); + break; + case IN: + queryWrapper = queryWrapper.in(attributeName, (Collection) val); + break; + case NOT_IN: + queryWrapper = queryWrapper.notIn(attributeName, (Collection) val); + break; + case INSQL: + queryWrapper = queryWrapper.inSql(attributeName, String.valueOf(val)); + break; + case NOT_INSQL: + queryWrapper = queryWrapper.notInSql(attributeName, String.valueOf(val)); + break; + case ORDER_BY: + queryWrapper = queryWrapper.last(" ORDER BY " + val); + break; + default: + break; + } + } + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } + return queryWrapper; + } + + private static boolean isBlank(final CharSequence cs) { + int strLen; + if (cs == null || (strLen = cs.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(cs.charAt(i))) { + return false; + } + } + return true; + } + + private static List getAllFields(Class clazz, List fields) { + if (clazz != null) { + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + getAllFields(clazz.getSuperclass(), fields); + } + return fields; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java new file mode 100644 index 0000000..a958000 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import net.lingala.zip4j.core.ZipFile; +import net.lingala.zip4j.exception.ZipException; +import org.dubhe.enums.LogEnum; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + +/** + * @description zip Util + * @date 2020-5-22 + */ +@Component +public class ZipCommonUtil { + public static boolean unzipLocal(String source) { + if (source == null) { + return false; + } + String[] sourceArr = source.split(Pattern.quote(File.separator)); + int length = sourceArr.length; + if (length <= 1) { + return false; + } + List destList = new ArrayList<>(Arrays.asList(sourceArr).subList(0, length - 1)); + String destination = String.join(File.separator, destList); + + try { + ZipFile zipFile = new ZipFile(source); + zipFile.extractAll(destination); + } catch (ZipException e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "解压失败: ", e); + return false; + } + return true; + } +} diff --git a/dubhe-server/common/src/main/resources/i18n/messages_en.properties b/dubhe-server/common/src/main/resources/i18n/messages_en.properties new file mode 100644 index 0000000..a26af0d --- /dev/null +++ b/dubhe-server/common/src/main/resources/i18n/messages_en.properties @@ -0,0 +1,51 @@ +#\u56fd\u9645\u5316\u4fe1\u606f +HTTPCODE_207=Your operation frequency is too fast, Please try again later! +HTTPCODE_200=Http request succeed +HTTPCODE_303=Login failed +HTTPCODE_400=Request parameter error +HTTPCODE_401=Unauthorized, please login +HTTPCODE_403=Forbidden +HTTPCODE_404=Page not found +HTTPCODE_408=Request timeout +HTTPCODE_409=Conflict +HTTPCODE_410=Gone +HTTPCODE_423=Locked +HTTPCODE_500=Server error +EMAIL.SET_HOST=Setting system property: mail.smtp.host={} +EMAIL.PRE_TALK=Ready to get email session object +EMAIL.ERROR_TALK=Error occurred during getting email session object +EMAIL.PRE_MIME=Ready to create MIME email object +EMAIL.ERROR_MIME=Create MIME email object failed +EMAIL.SET_AUTH=Setting smtp authorization :mail.smtp.auth={} +EMAIL.SET_SUBJECT=Setting email subject[{}]! +EMAIL.ERROR_SUBJECT=Error occurred while setting email subject +EMAIL.ERROR_BODY=Error occured while setting email body +EMAIL.ADD_ATTEND=Add attach [{}]! +EMAIL.SET_TO=Setting sendto : [{}]! +EMAIL.SET_COPYTO=Setting copyto [{}]! +EMAIL.SENDING=Email is sending... +EMAIL.SEND_SUCC=Email send succeed +EMAIL.SEND_ERR=Email send error +THIRDPARTY.LOGIN.NOTOKEN=Thirdparty token : %s get failed +IDCARD_ILLEGAL=Idcard is illegal +EMAIL_ILLEGAL=Email is illegal +MOBILE_ILLEGAL=Mobile is illegal +TASKID_IS_NULL=TaskId is null +LOGIN_FAIL=Incorrect username or password. +ACCOUNT_IS_NULL=Account is null +ACCOUNT_LENGTH=Account allowed length: {}-{}. +PASSWORD_IS_NULL=Password is null +USER_ID_IS_NULL=UserId is null +USER_IS_NULL=UserId[%1$s]error. +ID_IS_NULL=Id is null +TASKGROUP_IS_NULL=Taskgroup is null +TASKNAME_IS_NULL=Taskname is null +KEY_IS_NULL=Key is null +MENUID_IS_NULL=Menuid is null +NULL_POINTER_EXCEPTION_HAPPENS_IN_SERVICE_CALL=NULL POINTER EXCEPTION HAPPENS IN SERVICE CALL +SYS.EIXST_ROLE_CODE=ROLE CODE ALREADY EXISTS +DELETE_ROLE_ERROR=This role has user, delete failed +DEPARTMENT_ALREADY_EXIST=Department already exist +EXIST_SUBDEPARTMENT_OF_THIS_DEPARTMENT=Exist subdeptment of this department +EXIST_USER_OF_THIS_DEPARTMENT=Exist user of this department +HAS_SUB_LOCATIONS=Exist subLocation of this location diff --git a/dubhe-server/common/src/main/resources/i18n/messages_zh.properties b/dubhe-server/common/src/main/resources/i18n/messages_zh.properties new file mode 100644 index 0000000..0980220 --- /dev/null +++ b/dubhe-server/common/src/main/resources/i18n/messages_zh.properties @@ -0,0 +1,51 @@ +#\u56fd\u9645\u5316\u4fe1\u606f +HTTPCODE_207=\u60a8\u7684\u64cd\u4f5c\u9891\u7387\u8fc7\u5feb\uff0c\u8bf7\u7a0d\u540e\u518d\u8bd5! +HTTPCODE_200=\u8bf7\u6c42\u6210\u529f +HTTPCODE_303=\u767b\u5f55\u5931\u8d25 +HTTPCODE_400=\u8bf7\u6c42\u53c2\u6570\u51fa\u9519 +HTTPCODE_401=\u6ca1\u6709\u767b\u5f55 +HTTPCODE_403=\u6ca1\u6709\u6743\u9650 +HTTPCODE_404=\u627e\u4e0d\u5230\u9875\u9762 +HTTPCODE_408=\u8bf7\u6c42\u8d85\u65f6 +HTTPCODE_409=\u53d1\u751f\u51b2\u7a81 +HTTPCODE_410=\u5df2\u88ab\u5220\u9664 +HTTPCODE_423=\u5df2\u88ab\u9501\u5b9a +HTTPCODE_500=\u670d\u52a1\u5668\u51fa\u9519 +EMAIL.SET_HOST=\u8bbe\u7f6e\u7cfb\u7edf\u5c5e\u6027\uff1amail.smtp.host={} +EMAIL.PRE_TALK=\u51c6\u5907\u83b7\u53d6\u90ae\u4ef6\u4f1a\u8bdd\u5bf9\u8c61\uff01 +EMAIL.ERROR_TALK=\u83b7\u53d6\u90ae\u4ef6\u4f1a\u8bdd\u5bf9\u8c61\u65f6\u53d1\u751f\u9519\u8bef\uff01 +EMAIL.PRE_MIME=\u51c6\u5907\u521b\u5efaMIME\u90ae\u4ef6\u5bf9\u8c61\uff01 +EMAIL.ERROR_MIME=\u521b\u5efaMIME\u90ae\u4ef6\u5bf9\u8c61\u5931\u8d25\uff01 +EMAIL.SET_AUTH=\u8bbe\u7f6esmtp\u8eab\u4efd\u8ba4\u8bc1\uff1amail.smtp.auth={} +EMAIL.SET_SUBJECT=\u8bbe\u7f6e\u90ae\u4ef6\u4e3b\u9898[{}]\uff01 +EMAIL.ERROR_SUBJECT=\u8bbe\u7f6e\u90ae\u4ef6\u4e3b\u9898\u53d1\u751f\u9519\u8bef\uff01 +EMAIL.ERROR_BODY=\u8bbe\u7f6e\u90ae\u4ef6\u6b63\u6587\u65f6\u53d1\u751f\u9519\u8bef\uff01 +EMAIL.ADD_ATTEND=\u589e\u52a0\u90ae\u4ef6\u9644\u4ef6[{}]\uff01 +EMAIL.SET_TO=\u8bbe\u7f6e\u6536\u4fe1\u4eba[{}]\uff01 +EMAIL.SET_COPYTO=\u8bbe\u7f6e\u6284\u9001\u4eba[{}]\uff01 +EMAIL.SENDING=\u6b63\u5728\u53d1\u9001\u90ae\u4ef6.... +EMAIL.SEND_SUCC=\u53d1\u9001\u90ae\u4ef6\u6210\u529f\uff01 +EMAIL.SEND_ERR=\u90ae\u4ef6\u53d1\u9001\u5931\u8d25\uff01 +THIRDPARTY.LOGIN.NOTOKEN=\u672a\u83b7\u53d6\u5230%s\u767b\u5f55\u4ee4\u724c\uff01 +IDCARD_ILLEGAL=\u8eab\u4efd\u8bc1\u53f7\u7801\u9519\u8bef. +EMAIL_ILLEGAL=\u90ae\u7bb1\u683c\u5f0f\u9519\u8bef. +MOBILE_ILLEGAL=\u624b\u673a\u53f7\u683c\u5f0f\u9519\u8bef. +TASKID_IS_NULL=\u5b9a\u65f6\u4efb\u52a1Id\u4e0d\u80fd\u4e3a\u7a7a. +LOGIN_FAIL=\u7528\u6237\u540d\u6216\u5bc6\u7801\u9519\u8bef. +ACCOUNT_IS_NULL=\u5e10\u53f7\u4e0d\u80fd\u4e3a\u7a7a. +ACCOUNT_LENGTH=\u5e10\u53f7\u5141\u8bb8\u957f\u5ea6\u8303\u56f4{}-{}. +PASSWORD_IS_NULL=\u5bc6\u7801\u4e0d\u80fd\u4e3a\u7a7a. +USER_ID_IS_NULL=\u7528\u6237ID\u4e0d\u80fd\u4e3a\u7a7a. +USER_IS_NULL=\u7528\u6237Id[%1$s]\u9519\u8bef. +ID_IS_NULL=Id\u4e0d\u80fd\u4e3a\u7a7a. +TASKGROUP_IS_NULL=\u4efb\u52a1\u7ec4\u540d\u79f0\u4e0d\u80fd\u4e3a\u7a7a. +TASKNAME_IS_NULL=\u4efb\u52a1\u540d\u79f0\u4e0d\u80fd\u4e3a\u7a7a. +KEY_IS_NULL=\u952e\u503c\u4e3a\u7a7a +MENUID_IS_NULL=\u6b63\u5728\u64cd\u4f5c\u83dc\u5355id\u4e0d\u80fd\u4e3a\u7a7a +DELETE_ROLE_ERROR=\u8be5\u89d2\u8272\u5df2\u88ab\u7528\u6237\u5173\u8054\u4e0d\u652f\u6301\u5220\u9664 +NULL_POINTER_EXCEPTION_HAPPENS_IN_SERVICE_CALL=\u670d\u52a1\u8c03\u7528\u53d1\u751f\u7a7a\u6307\u9488\u5f02\u5e38 +SYS.EIXST_ROLE_CODE=\u89d2\u8272\u7f16\u7801\u91cd\u590d +DEPARTMENT_ALREADY_EXIST=\u90e8\u95e8\u5df2\u5b58\u5728 +EXIST_SUBDEPARTMENT_OF_THIS_DEPARTMENT=\u8be5\u90e8\u95e8\u4e0b\u5b58\u5728\u5b50\u90e8\u95e8 +EXIST_USER_OF_THIS_DEPARTMENT=\u8be5\u90e8\u95e8\u4e0b\u5b58\u5728\u7528\u6237 +HAS_SUB_LOCATIONS=\u8be5\u5730\u70b9\u4e0b\u542b\u6709\u5b50\u8282\u70b9 diff --git a/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java b/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java new file mode 100644 index 0000000..6a5f963 --- /dev/null +++ b/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import org.junit.Test; + +import static org.dubhe.utils.EncryptUtils.desDecrypt; +import static org.dubhe.utils.EncryptUtils.desEncrypt; +import static org.junit.Assert.assertEquals; +/** + * @description 对称加密|解密 + * @date 2020-03-25 + */ +public class EncryptUtilsTest { + + /** + * 对称加密 + */ + @Test + public void testDesEncrypt() { + try { + assertEquals("7772841DC6099402", desEncrypt("123456")); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 对称解密 + */ + @Test + public void testDesDecrypt() { + try { + assertEquals("123456", desDecrypt("7772841DC6099402")); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java b/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java new file mode 100644 index 0000000..da5488a --- /dev/null +++ b/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import org.junit.Test; +import org.springframework.mock.web.MockMultipartFile; + +import static org.dubhe.utils.FileUtil.*; +import static org.junit.Assert.assertEquals; +/** + * @description 测试类 + * @date 2020-03-25 + */ +public class FileUtilTest { + + @Test + public void testToFile() { + long retval = toFile(new MockMultipartFile("foo", (byte[]) null)).getTotalSpace(); + assertEquals(500695072768L, retval); + } + + @Test + public void testGetExtensionName() { + assertEquals("foo", getExtensionName("foo")); + assertEquals("exe", getExtensionName("bar.exe")); + } + + @Test + public void testGetFileNameNoEx() { + assertEquals("foo", getFileNameNoEx("foo")); + assertEquals("bar", getFileNameNoEx("bar.txt")); + } + + @Test + public void testGetSize() { + assertEquals("1000B ", getSize(1000)); + assertEquals("1.00KB ", getSize(1024)); + assertEquals("1.00MB ", getSize(1048576)); + assertEquals("1.00GB ", getSize(1073741824)); + } +} diff --git a/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java b/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java new file mode 100644 index 0000000..5403c02 --- /dev/null +++ b/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java @@ -0,0 +1,42 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import org.junit.Assert; +import org.junit.Test; + +import static org.dubhe.utils.HttpUtils.isSuccess; + +/** + * @description: HttpUtil + * @date 2020.04.30 + */ +public class HttpUtilsTest { + + @Test + public void testIsSuccess() { + Assert.assertTrue(isSuccess("200")); + Assert.assertFalse(isSuccess("2020")); + Assert.assertFalse(isSuccess("401")); + Assert.assertFalse(isSuccess(null)); + Assert.assertFalse(isSuccess("")); + Assert.assertTrue(isSuccess(200)); + } + + +} diff --git a/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java b/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java new file mode 100644 index 0000000..2e2d785 --- /dev/null +++ b/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import org.dubhe.utils.StringUtils; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import static org.dubhe.utils.StringUtils.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +/** + * @description 测试类 + * @date 2020-03-25 + */ +public class StringUtilsTest { + + @Test + public void testToCamelCase() { + assertNull(toCamelCase(null)); + } + + @Test + public void testToCapitalizeCamelCase() { + assertNull(StringUtils.toCapitalizeCamelCase(null)); + assertEquals("HelloWorld", toCapitalizeCamelCase("hello_world")); + } + + @Test + public void testGetWeekDay() { + SimpleDateFormat simpleDateformat = new SimpleDateFormat("E"); + assertEquals(simpleDateformat.format(new Date()), getWeekDay()); + } + + @Test + public void testGetIp() { + assertEquals("127.0.0.1", getIp(new MockHttpServletRequest())); + } + + @Test + public void truncationString() { + assertEquals("", StringUtils.truncationString(null,6)); + assertEquals("012345", StringUtils.truncationString("012345",0)); + assertEquals("0", StringUtils.truncationString("012345",1)); + assertEquals("012345", StringUtils.truncationString("012345",6)); + } +} diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java new file mode 100644 index 0000000..b45f320 --- /dev/null +++ b/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import org.junit.Assert; +import org.junit.Test; + +/** + * @desc + * @date 2020.06.01 + */ +public class AesUtilTest { + + @Test + public void encryptAndDecrypt(){ + String data = "20200601163000"; + String key = "123412asdsad"; + String val = AesUtil.encrypt(data,key); + System.out.println(val); + Assert.assertEquals(data, AesUtil.decrypt(val,key)); + } +} diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java new file mode 100644 index 0000000..27bef50 --- /dev/null +++ b/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import com.alibaba.fastjson.JSON; +import lombok.extern.slf4j.Slf4j; +import org.dubhe.exception.NotebookBizException; +import org.junit.Test; + +/** + * @desc + * @date 2020.06.16 + */ +@Slf4j +public class NotebookBizExceptionTest { + + @Test + public void toStringTest() { + try { + throw new NotebookBizException("NotebookBizException toString Test"); + }catch (NotebookBizException e){ + log.error("{}", e); + } + } +} diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java new file mode 100644 index 0000000..ab0ffec --- /dev/null +++ b/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @description + * @create: 2020-05-07 + */ +public class ReadFileTest { + + @Test + public void testRead(){ + String path = "/Users/lizhe/hello2.log"; + List ans = Arrays.asList("lin1", "l2", "l3", "l4"); + List ans2 = ans.subList(1, 3); + assertEquals(ans, ReadFile.read(path,-2, 10)); + assertEquals(ans2, ReadFile.read(path, 1, 2)); + } +} diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java new file mode 100644 index 0000000..87dae06 --- /dev/null +++ b/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java @@ -0,0 +1,31 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import org.junit.Test; +/** + * @description + * @date 2020-05-25 + */ +public class ZipUtilTest { + @Test + public void testUnzip(){ + ZipCommonUtil.unzipLocal("Z:\\dubhe-dev\\undefined\\a1.zip"); + } +} + diff --git a/dubhe-server/dubhe-admin/pom.xml b/dubhe-server/dubhe-admin/pom.xml new file mode 100644 index 0000000..8db6e5f --- /dev/null +++ b/dubhe-server/dubhe-admin/pom.xml @@ -0,0 +1,90 @@ + + + + dubheplatform + zhejianglab + 1.0 + + 4.0.0 + + dubhe-admin + 平台管理 + + 0.10.6 + dubhe + + + + zhejianglab + common + 1.0 + + + zhejianglab + dubhe-system + 1.0 + + + zhejianglab + common + + + + + zhejianglab + dubhe-data + 1.0 + + + zhejianglab + common + + + + + zhejianglab + dubhe-model + 1.0 + + + zhejianglab + common + + + + + zhejianglab + dubhe-k8s + 1.0 + + + zhejianglab + common + + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + false + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java new file mode 100644 index 0000000..cdbe26a --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java @@ -0,0 +1,76 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import org.dubhe.utils.SpringContextHolder; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; +import org.springframework.boot.web.servlet.server.ServletWebServerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * @description 访问首页提示 + * @date 2020-03-05 + */ +@EnableAsync +@RestController +@SpringBootApplication +@EnableTransactionManagement +@EnableScheduling +@MapperScan("org.dubhe.**.dao") +public class AppRun { + + public static void main(String[] args) { + SpringApplication.run(AppRun.class, args); + } + + @Bean + public SpringContextHolder springContextHolder() { + return new SpringContextHolder(); + } + + @Bean + public ServletWebServerFactory webServerFactory() { + TomcatServletWebServerFactory fa = new TomcatServletWebServerFactory(); + fa.addConnectorCustomizers(connector -> connector.setProperty("relaxedQueryChars", "[]{}")); + return fa; + } + + @Bean + public BCryptPasswordEncoder passwordEncoder() { + return new BCryptPasswordEncoder(); + } + + /** + * 访问首页提示 + * + * @return / + */ + @GetMapping("/") + public String index() { + return "Backend service started successfully"; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java new file mode 100644 index 0000000..8d851a2 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.config; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Component; + +import java.sql.Timestamp; + +/** + * @description: 转换时间戳类型 + * @date 2020-05-22 + */ +@Component +public class TimestampConverter implements Converter { + + /** + * Convert string to Timestamp type + * @param timeStr + * @return + */ + @Override + public Timestamp convert(String timeStr) { + + Timestamp timestamp = null; + if (StringUtils.isNotBlank(timeStr)) { + long time = Long.valueOf(timeStr); + timestamp = new Timestamp(time); + } + + return timestamp; + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TrainPoolConfig.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TrainPoolConfig.java new file mode 100644 index 0000000..1a3d281 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TrainPoolConfig.java @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * @description 线程池配置类 + * @date 2020-07-17 + */ +@Configuration +public class TrainPoolConfig { + + @Value("${basepool.corePoolSize:40}") + private Integer corePoolSize; + @Value("${basepool.maximumPoolSize:60}") + private Integer maximumPoolSize; + @Value("${basepool.keepAliveTime:120}") + private Integer keepAliveTime; + @Value("${basepool.blockQueueSize:20}") + private Integer blockQueueSize; + + /** + * 训练任务异步处理线程池 + * @return Executor 线程实例 + */ + @Bean + public Executor trainJobAsyncExecutor() { + ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); + //核心线程数 + taskExecutor.setCorePoolSize(corePoolSize); + taskExecutor.setAllowCoreThreadTimeOut(true); + //最大线程数 + taskExecutor.setMaxPoolSize(maximumPoolSize); + //超时时间 + taskExecutor.setKeepAliveSeconds(keepAliveTime); + //配置队列大小 + taskExecutor.setQueueCapacity(blockQueueSize); + //配置线程池前缀 + taskExecutor.setThreadNamePrefix("async-train-job-"); + //拒绝策略 + taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); + taskExecutor.initialize(); + return taskExecutor; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java new file mode 100644 index 0000000..4f977e7 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.dubhe.domain.entity.NoteBook; + +import java.util.List; + +/** + * @description notebook mapper + * @date 2020-04-28 + */ +public interface NoteBookMapper extends BaseMapper { + + /** + * 根据名称查询 + * + * @param name + * @param userId + * @param status + * @return NoteBook + */ + @Select("select * from notebook where notebook_name = #{name} and user_id = #{userId} and status != #{status} and deleted = 0 limit 1") + NoteBook findByNameAndUserId(@Param("name") String name, @Param("userId") long userId, @Param("status") Integer status); + + /** + * 查询正在运行的notebook数量 + * + * @param userId + * @param status + * @return int + */ + @Select("select count(1) from notebook where user_id = #{userId} and status = #{status} and deleted = 0") + int selectRunNoteBookNum(@Param("userId") long userId, @Param("status") Integer status); + + /** + * 根据namespace + resourceName查询 + * + * @param namespace + * @param resourceName + * @param status + * @return NoteBook + */ + @Select("select * from notebook where k8s_namespace = #{namespace} and k8s_resource_name = #{resourceName} and status != #{status} and deleted = 0 limit 1") + NoteBook findByNamespaceAndResourceName(@Param("namespace") String namespace, @Param("resourceName") String resourceName, @Param("status") Integer status); + + /** + * 查询已经运行并且没有URL的notebook + * + * @param page + * @param status + * @return List + */ + @Select("select * from notebook where deleted = 0 and status = #{status} and (url is null or url = '')") + List selectRunNotUrlList(@Param("page") Page page, @Param("status") Integer status); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java new file mode 100644 index 0000000..4b0281f --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Select; +import org.dubhe.domain.entity.NoteBookModel; + +import java.util.List; + +/** + * @description : notebook 模板 Mapper + * @date 2020-06-01 + */ +public interface NoteBookModelMapper extends BaseMapper { + + /** + * 根据名称查询 + * + * @return NoteBookModel + */ + @Select("select * from notebook_model where deleted = 0 order by model_type") + List selectAllNoteBookModel(); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtDatasetMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtDatasetMapper.java new file mode 100644 index 0000000..8154436 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtDatasetMapper.java @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.dubhe.domain.PtDataset; + +/** + * @description 数据集 + * @date 2020-03-30 + */ +public interface PtDatasetMapper extends BaseMapper { +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtDevEnvsMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtDevEnvsMapper.java new file mode 100644 index 0000000..272f17b --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtDevEnvsMapper.java @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.dubhe.domain.PtDevEnvs; + +/** + * @description 开发环境 + * @date 2020-03-30 + */ +public interface PtDevEnvsMapper extends BaseMapper { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java new file mode 100644 index 0000000..d9255e3 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java @@ -0,0 +1,30 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.dubhe.domain.entity.PtImage; + +/** + * @description 镜像 Mapper 接口 + * @date 2020-04-27 + */ +public interface PtImageMapper extends BaseMapper { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java new file mode 100644 index 0000000..ea05002 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import org.dubhe.domain.entity.PtJobParam; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * @description job运行参数及结果 + * @date 2020-04-27 + */ +public interface PtJobParamMapper extends BaseMapper { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtStorageMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtStorageMapper.java new file mode 100644 index 0000000..a7b4596 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtStorageMapper.java @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.dubhe.domain.PtStorage; + +/** + * @description 存储管理 + * @date 2020-03-30 + */ +public interface PtStorageMapper extends BaseMapper { +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java new file mode 100644 index 0000000..1828c2c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.dubhe.domain.entity.PtTrainAlgorithm; + +import java.util.List; + +/** + * @description 训练算法Mapper + * @date 2020-04-27 + */ +public interface PtTrainAlgorithmMapper extends BaseMapper { + + /** + * 根据算法id查询算法信息 + * @param id 算法id + * @return PtTrainAlgorithm 算法信息 + */ + @Select("select * from pt_train_algorithm where id= #{id}") + PtTrainAlgorithm selectAllById(@Param("id") Long id); + + /** + * 根据算法id集合查询对应的算法信息 + * @param ids 算法集合id + * @return List 算法信息集合 + */ + @Select({ + "" + }) + List selectAllBatchIds(@Param("ids") List ids); + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java new file mode 100644 index 0000000..95d68bb --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import org.dubhe.domain.entity.PtTrainAlgorithmUsage; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * + * 用户辅助信息Mapper 接口 + *

+ * + * @since 2020-06-23 + */ +public interface PtTrainAlgorithmUsageMapper extends BaseMapper { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java new file mode 100644 index 0000000..5d9edd7 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java @@ -0,0 +1,59 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.dubhe.domain.entity.PtTrainJob; +import org.dubhe.domain.vo.PtTrainVO; + +/** + * @description 训练作业job Mapper 接口 + * @date 2020-04-27 + */ +public interface PtTrainJobMapper extends BaseMapper { + + /** + * 获取训练列表，并进行分页。 + * + * @param page 页 + * @param createUserId 用户id + * @param trainStatus 训练状态 + * @param trainName 训练名称 + * @param sort 排序字段 + * @param order 排序方式 + * + * @return PtTrainVO + */ + Page getPageTrain(Page page, @Param("createUserId") Long createUserId, + @Param("trainStatus") Integer trainStatus, @Param("trainName") String trainName, @Param("sort") String sort, + @Param("order") String order); + + /** + * 根据状态进行统计数量 + * + * @param userId 当前用户id + * @param param sql片段 + * @return 统计的数量 + */ + @Select("select count(1) from pt_train_job t1 inner join pt_train t2 on t1.train_id = t2.id where t1.create_user_id= #{userId} and t1.train_status in ${param} and t1.deleted= 0 and t2.deleted = 0 ") + Integer selectCountByStatus(@Param("userId") Long userId, @Param("param") String param); + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java new file mode 100644 index 0000000..886b966 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.dubhe.domain.entity.PtTrainJobSpecs; + +/** + * @description 训练作业规格 Mapper 接口 + * @date 2020-05-06 + */ +public interface PtTrainJobSpecsMapper extends BaseMapper { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java new file mode 100644 index 0000000..c70db46 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.dubhe.domain.entity.PtTrain; + +/** + * @description 训练作业主 Mapper 接口 + * @date 2020-04-27 + */ +public interface PtTrainMapper extends BaseMapper { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java new file mode 100644 index 0000000..9ceff8e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import org.dubhe.domain.entity.PtTrainParam; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * @description 任务参数 Mapper 接口 + * @date 2020-04-27 + */ +public interface PtTrainParamMapper extends BaseMapper { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/PtDataset.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/PtDataset.java new file mode 100644 index 0000000..4cee51f --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/PtDataset.java @@ -0,0 +1,71 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.bean.copier.CopyOptions; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import org.dubhe.base.BaseEntity; +import org.dubhe.domain.entity.Team; +import org.dubhe.domain.entity.User; + +import javax.validation.constraints.NotBlank; + +/** + * @description 数据集 + * @date 2020-03-17 + */ +@Data +@TableName("pt_dataset") +public class PtDataset extends BaseEntity { + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @TableField(value = "name") + @NotBlank + private String name; + + @TableField(value = "remark") + private String remark; + + @TableField(value = "type") + @NotBlank + private String type; + /** + * 团队 + */ + @TableField(exist = false) + private Team team; + /** + * 创建用户 + */ + @TableField(exist = false) + private User createUser; + + public void copy(PtDataset source) { + BeanUtil.copyProperties(source, this, CopyOptions.create().setIgnoreNullValue(true)); + } + + public @interface Update { + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/PtDevEnvs.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/PtDevEnvs.java new file mode 100644 index 0000000..776bf9f --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/PtDevEnvs.java @@ -0,0 +1,114 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.bean.copier.CopyOptions; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import org.dubhe.base.BaseEntity; +import org.dubhe.domain.entity.PtImage; +import org.dubhe.domain.entity.Team; +import org.dubhe.domain.entity.User; + +import javax.validation.constraints.NotBlank; +import java.sql.Timestamp; + +/** + * @description 开发环境 + * @date 2020-03-17 + */ + +@Data +@TableName("pt_dev_envs") +public class PtDevEnvs extends BaseEntity { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @NotBlank + private String name; + + @TableField(value = "remark") + private String remark; + + @TableField(value = "type") + @NotBlank + private String type; + + @TableField(value = "pod_num") + private Integer podNum; + + @TableField(value = "gpu_num") + private Integer gpuNum; + + @TableField(value = "mem_num") + private Integer memNum; + + @TableField(value = "cpu_num") + private Integer cpuNum; + + @TableField(value = "duration") + private Integer duration; + + @TableField(value = "start_time") + private Timestamp startTime; + + @TableField(value = "close_time") + private Timestamp closeTime; + + /** + * 数据集 + */ + @TableField(exist = false) + private PtDataset dataset; + + /** + * 镜像 + */ + @TableField(exist = false) + private PtImage image; + + /** + * 存储 + */ + @TableField(exist = false) + private PtStorage storage; + + /** + * 团队 + */ + @TableField(exist = false) + private Team team; + + /** + * 创建用户 + */ + @TableField(exist = false) + private User createUser; + + + public void copy(PtDevEnvs source) { + BeanUtil.copyProperties(source, this, CopyOptions.create().setIgnoreNullValue(true)); + } + + public @interface Update { + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/PtStorage.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/PtStorage.java new file mode 100644 index 0000000..75a41fd --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/PtStorage.java @@ -0,0 +1,69 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.bean.copier.CopyOptions; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import org.dubhe.base.BaseEntity; +import org.dubhe.domain.entity.Team; +import org.dubhe.domain.entity.User; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +/** + * @description 存储 + * @date 2020-03-17 + */ +@Data +@TableName("pt_storage") +public class PtStorage extends BaseEntity { + @TableId(value = "id", type = IdType.AUTO) + @NotNull(groups = {Update.class}) + private Long id; + + @TableField(value = "name") + @NotBlank + private String name; + + @TableField(value = "size") + private Integer size; + + @TableField(value = "storageclass") + @NotBlank + private String storageclass; + + @TableField(exist = false) + private Team team; + + @TableField(exist = false) + private User createUser; + + + public void copy(PtStorage source) { + BeanUtil.copyProperties(source, this, CopyOptions.create().setIgnoreNullValue(true)); + } + + public @interface Update { + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java new file mode 100644 index 0000000..fa63396 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import com.alibaba.fastjson.JSONObject; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.domain.entity.PtTrainJobSpecs; + +import java.io.Serializable; + +/** + * @description 创建训练任务的数据包 + * @date 2020-07-15 + */ +@Data +@Accessors(chain = true) +public class BaseTrainJobDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + private JSONObject runParams; + private String jobName; + private String dataSourcePath; + private PtTrainJobSpecs ptTrainJobSpecs; + private String outPath; + private String logPath; + private String visualizedLogPath; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java new file mode 100644 index 0000000..29d4b94 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.dubhe.utils.NotebookUtil; + +import javax.validation.constraints.*; + +/** + * @description 创建notebook请求体 + * @date 2020-06-28 + */ +@Data +public class NoteBookCreateDTO { + + @NotBlank + @Pattern(regexp = NotebookUtil.K8S_NOTEBOOK_REGEX, message = "Notebook 名称仅支持字母、数字、汉字、英文横杠和下划线") + @Size(max = 30, message = "名称内容超长") + @ApiModelProperty(value = "notebook 名称") + private String noteBookName; + + @Size(max = 255, message = "描述内容超长") + @ApiModelProperty(value = "notebook 描述") + private String description; + + @ApiModelProperty(hidden = true) + private String url; + + @ApiModelProperty(hidden = true) + private Integer totalRunMin; + + @NotNull + @Min(value = NotebookUtil.CPU_MIN_NUMBER, message = "最少需要一个CPU") + @Max(value = NotebookUtil.CPU_MAX_NUMBER, message = "CPU数量超过限制，不可大于8") + @ApiModelProperty(value = "cpu数量") + private Integer cpuNum; + + @Min(value = NotebookUtil.GPU_MIN_NUMBER, message = "最少需要一个GPU") + @Max(value = NotebookUtil.GPU_MAX_NUMBER, message = "GPU数量超过限制，不可大于4") + @ApiModelProperty(value = "gpu数量") + private Integer gpuNum; + + @NotNull + @Min(value = NotebookUtil.MEMORY_MIN_NUMBER, message = "最少需要1G内存") + @Max(value = NotebookUtil.MEMORY_MAX_NUMBER, message = "内存超出限制，不可大于8") + @ApiModelProperty(value = "内存大小") + private Integer memNum; + + @NotNull + @Min(value = NotebookUtil.DISK_MEMORY_MIN_NUMBER, message = "至少需要1G硬盘内存") + @Max(value = NotebookUtil.DISK_MEMORY_MAX_NUMBER, message = "硬盘内存超出限制，不可大于1024") + @ApiModelProperty(value = "硬盘内存大小") + private Integer diskMemNum; + + @ApiModelProperty("镜像名称") + @NotNull(message = "请选择镜像") + private String k8sImageName; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java new file mode 100644 index 0000000..485d3d3 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.dubhe.annotation.Query; + +import java.io.Serializable; + +/** + * @description 查询请求体 + * @date 2020-06-28 + */ +@Data +public class NoteBookListQueryDTO implements Serializable { + + @Query(propName = "status", type = Query.Type.EQ) + @ApiModelProperty("0运行中，1停止, 2删除, 3启动中，4停止中，5删除中，6运行异常（暂未启用）") + private Integer status; + + @Query(propName = "notebook_name", type = Query.Type.LIKE) + @ApiModelProperty("notebook名称") + private String noteBookName; + + @Query(propName = "user_id", type = Query.Type.EQ) + @ApiModelProperty(value = "所属用户ID", hidden = true) + private Long userId; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java new file mode 100644 index 0000000..93712f8 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java @@ -0,0 +1,93 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; +import org.dubhe.annotation.Query; +import org.dubhe.enums.NoteBookStatusEnum; +import org.dubhe.utils.K8sCallBackTool; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; + +/** + * @description 查询notebook请求体 + * @date 2020-04-28 + */ +@Data +public class NoteBookQueryDTO implements Serializable { + + @Query(propName = "status", type = Query.Type.NE) + private Integer neStatus; + + @Query(propName = "status", type = Query.Type.IN) + private List statusList; + + @Query(propName = "k8s_pvc_path", type = Query.Type.EQ) + private String k8sPvcPath; + + @Query(propName = "user_id", type = Query.Type.EQ) + private Long userId; + + @Query(propName = "last_operation_timeout", type = Query.Type.LT) + private Long lastOperationTimeout; + + + private static final NoteBookQueryDTO QUERY_CRITERIA; + + public NoteBookQueryDTO() { + } + + public NoteBookQueryDTO(Integer neStatus, String k8sPvcPath, Long userId) { + this.neStatus = neStatus; + this.k8sPvcPath = k8sPvcPath; + this.userId = userId; + } + + static { + QUERY_CRITERIA = new NoteBookQueryDTO(); + QUERY_CRITERIA.setNeStatus(NoteBookStatusEnum.DELETE.getCode()); + } + + /** + * 获取需要刷新状态的查询条件 + * + * @return + */ + public static NoteBookQueryDTO getToRefreshCriteria() { + NoteBookQueryDTO refreshCriteria = new NoteBookQueryDTO(); + refreshCriteria.setStatusList(Arrays.asList( + NoteBookStatusEnum.STARTING.getCode() + , NoteBookStatusEnum.STOPPING.getCode() + , NoteBookStatusEnum.DELETING.getCode())); + refreshCriteria.setLastOperationTimeout(K8sCallBackTool.getCurrentSecondLong()); + return refreshCriteria; + } + + /** + * 获取查询列表需要的查询条件 + * + * @return + */ + public static NoteBookQueryDTO getListCriteria() { + return QUERY_CRITERIA; + } + + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java new file mode 100644 index 0000000..196a973 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description note book status + * @date 2020-05-14 + */ +@Data +public class NoteBookStatusDTO implements Serializable { + + @ApiModelProperty("状态编码") + private Integer statusCode; + + @ApiModelProperty("状态名称") + private String statusName; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java new file mode 100644 index 0000000..76b06fa --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description 数据集 + * @date 2020-03-17 + */ +@Data +public class PtDatasetDTO implements Serializable { + private Long id; + private String name; + private String remark; + private String type; + private TeamSmallDTO team; + /** + * 创建用户 + */ + private UserSmallDTO createUser; + private Timestamp createTime; + private Timestamp updateTime; + private Boolean deleted; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetQueryCriteria.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetQueryCriteria.java new file mode 100644 index 0000000..d15dd2a --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetQueryCriteria.java @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; + +/** + * @description 数据集查询条件 + * @date 2020-03-17 + */ +@Data +public class PtDatasetQueryCriteria { +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java new file mode 100644 index 0000000..3f5b30b --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; + +import java.io.Serializable; + +/** + * @description 数据集 + * @date 2020-03-17 + */ +@Data +public class PtDatasetSmallDTO implements Serializable { + private Long id; + private String name; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java new file mode 100644 index 0000000..ea6cf40 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description 开发环境 + * @date 2020-03-17 + */ +@Data +public class PtDevEnvsDTO implements Serializable { + + private Long id; + private String name; + private String remark; + private String type; + private Integer podNum; + private Integer gpuNum; + private Integer memNum; + private Integer cpuNum; + private Integer duration; + private Timestamp startTime; + private Timestamp closeTime; + private PtDatasetSmallDTO dataset; + private PtImageSmallDTO image; + private PtStorageSmallDTO storage; + private TeamSmallDTO team; + private UserSmallDTO createUser; + private Timestamp createTime; + private Timestamp updateTime; + private Boolean deleted; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java new file mode 100644 index 0000000..46973ec --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; + +/** + * @description 开发环境查询条件 + * @date 2020-03-17 + */ +@Data +public class PtDevEnvsQueryCriteria { +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java new file mode 100644 index 0000000..85e0d41 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java @@ -0,0 +1,69 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.dubhe.domain.entity.Team; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description： 镜像 + * @date 2020-03-17 + */ +@Data +public class PtImageDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("镜像ID") + private Long id; + + @ApiModelProperty("镜像名称") + private String name; + + @ApiModelProperty("镜像描述") + private String remark; + + @ApiModelProperty("类型") + private String type; + + @ApiModelProperty("标签") + private String label; + + @ApiModelProperty("地址") + private String uri; + + @ApiModelProperty("团队") + private Team team; + + @ApiModelProperty("创建人") + private UserSmallDTO createUser; + + @ApiModelProperty("创建时间") + private Timestamp createTime; + + @ApiModelProperty("更新时间") + private Timestamp updateTime; + + @ApiModelProperty("删除(0正常，1已删除)") + private Boolean deleted; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java new file mode 100644 index 0000000..e1bc556 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.base.PageQueryBase; + +import java.io.Serializable; + +/** + * @description 镜像查询 + * @date 2020-05-09 + */ +@Data +@Accessors(chain = true) +public class PtImageQueryDTO extends PageQueryBase implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") + private Integer imageResource; + + @ApiModelProperty(value = "镜像状态，0为制作中，1位制作成功，2位制作失败") + private Integer imageStatus; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java new file mode 100644 index 0000000..8d3c7a1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description： 镜像 + * @date 2020-03-17 + */ +@Data +public class PtImageSmallDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("镜像ID") + private Long id; + + @ApiModelProperty("镜像名称") + private String name; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java new file mode 100644 index 0000000..a3ae025 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import java.io.Serializable; + + +/** + * @description 上传镜像 + * @date 2020-06-11 + */ +@Data +@Accessors(chain = true) +public class PtImageUploadDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + + @ApiModelProperty(value = "镜像文件路径", required = true) + @NotBlank(message = "镜像文件路径不能为空") + private String imagePath; + + @ApiModelProperty(value = "镜像名称", required = true) + @NotBlank(message = "源镜像名称不能为空") + @Length(min = TrainUtil.NUMBER_ONE, max = TrainUtil.NUMBER_SIXTY_FOUR, message = "镜像名称长度在1-64个字符") + @Pattern(regexp = TrainUtil.REGEXP_NAME, message = "镜像名称支持字母、数字、英文横杠和下划线") + private String imageName; + + @ApiModelProperty(value = "镜像版本号", required = true) + @NotBlank(message = "镜像版本号不能为空") + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "镜像版本号长度在1-32个字符") + @Pattern(regexp = TrainUtil.REGEXP_TAG, message = "镜像版本号支持字母、数字、英文横杠、英文.号和下划线") + private String imageTag; + + @ApiModelProperty("备注") + @Length(max = TrainUtil.NUMBER_ONE_THOUSAND_AND_TWENTY_FOUR, message = "备注-输入长度不能超过1024个字符") + private String remark; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageDTO.java new file mode 100644 index 0000000..8354adc --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageDTO.java @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description 存储 + * @date 2020-03-17 + */ +@Data +public class PtStorageDTO implements Serializable { + + private Long id; + private String name; + private Integer size; + private String storageclass; + private TeamSmallDTO team; + private UserSmallDTO createUser; + private Timestamp createTime; + private Timestamp updateTime; + private Boolean deleted; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageQueryCriteria.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageQueryCriteria.java new file mode 100644 index 0000000..4f5528a --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageQueryCriteria.java @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; + +/** + * @description 存储 + * @date 2020-03-17 + */ +@Data +public class PtStorageQueryCriteria { +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java new file mode 100644 index 0000000..4544556 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; + +import java.io.Serializable; + +/** + * @description 存储 + * @date 2020-03-17 + */ +@Data +public class PtStorageSmallDTO implements Serializable { + private Long id; + private String name; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java new file mode 100644 index 0000000..4109495 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java @@ -0,0 +1,88 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import java.io.Serializable; + +/** + * @desciption 创建算法条件 + * @date 2020-04-29 + */ +@Data +@Accessors(chain = true) +public class PtTrainAlgorithmCreateDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "算法名称，输入长度不能超过32个字符", required = true) + @NotBlank(message = "算法名称不能为空") + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "算法名称-输入长度不能超过32个字符") + @Pattern(regexp = TrainUtil.REGEXP, message = "算法名称支持字母、数字、汉字、英文横杠和下划线") + private String algorithmName; + + @ApiModelProperty("算法描述，输入长度不能超过256个字符") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "算法描述-输入长度不能超过256个字符") + private String description; + + @ApiModelProperty(value = "镜像版本") + private String imageTag; + + @ApiModelProperty(value = "镜像名称") + private String imageName; + + @ApiModelProperty(value = "创建算法来源(true:由fork创建算法，false：其它创建算法方式)") + private Boolean fork; + + @ApiModelProperty(value = "代码目录（路径规则：/algorithm-manage/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/用户上传的算法具体文件(zip文件）名称或从notebook跳转时为/notebook/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/）", required = true) + @NotBlank(message = "代码目录不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "代码目录-输入长度不能超过128个字符") + private String codeDir; + + @ApiModelProperty(value = "运行命令") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") + private String runCommand; + + @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") + private JSONObject runParams; + + @ApiModelProperty("算法用途，输入长度不能超过128个字符") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "算法用途-输入长度不能超过128个字符") + private String algorithmUsage; + + @ApiModelProperty("是否输出训练结果，不填则默认为true") + private Boolean isTrainOut; + + @ApiModelProperty("是否输出训练日志，不填则默认为true") + private Boolean isTrainLog; + + @ApiModelProperty("是否输出可视化日志，不填则默认为false") + private Boolean isVisualizedLog; + + @ApiModelProperty("noteBookId") + private Long noteBookId; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java new file mode 100644 index 0000000..08daa8d --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java @@ -0,0 +1,42 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; +import java.util.Set; + +/** + * @description 算法删除 + * @date 2020-07-02 + */ +@Data +@Accessors(chain = true) +public class PtTrainAlgorithmDeleteDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为空") + private Set ids; + +} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java new file mode 100644 index 0000000..21bec33 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.annotation.Query; +import org.dubhe.base.PageQueryBase; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import java.io.Serializable; + +/** + * @description 查询算法条件 + * @date 2020-04-29 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +public class PtTrainAlgorithmQueryDTO extends PageQueryBase implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "算法来源(1为我的算法, 2为预置算法)") + @Min(value = TrainUtil.NUMBER_ONE, message = "算法来源错误") + @Max(value = TrainUtil.NUMBER_TWO, message = "算法来源错误") + @Query(propName = "algorithm_source", type = Query.Type.EQ) + private Integer algorithmSource; + + @ApiModelProperty(value = "算法名称或者id") + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "算法名称或者id有误") + private String algorithmName; + + @ApiModelProperty(value = "算法用途") + private String algorithmUsage; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java new file mode 100644 index 0000000..9c83be2 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java @@ -0,0 +1,64 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; +import java.io.Serializable; + +/** + * @description 修改算法 + * @date 2020-06-19 + */ +@Data +@Accessors(chain = true) +public class PtTrainAlgorithmUpdateDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为null") + @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") + private Long id; + + @ApiModelProperty(value = "算法名称，输入长度不能超过32个字符") + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "算法名称-输入长度不能超过32个字符") + @Pattern(regexp = TrainUtil.REGEXP, message = "算法名称支持字母、数字、汉字、英文横杠和下划线") + private String algorithmName; + + @ApiModelProperty("算法描述，输入长度不能超过256个字符") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "算法描述-输入长度不能超过256个字符") + private String description; + + @ApiModelProperty("算法用途，输入长度不能超过128个字符") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "算法用途-输入长度不能超过128个字符") + private String algorithmUsage; + + @ApiModelProperty("是否输出训练日志") + private Boolean isTrainLog; + + @ApiModelProperty("是否输出可视化日志") + private Boolean isVisualizedLog; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java new file mode 100644 index 0000000..4dba0f8 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import java.io.Serializable; + +/** + * @description 创建算法用途条件 + * @date 2020-06-23 + */ +@Data +@Accessors(chain = true) +public class PtTrainAlgorithmUsageCreateDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "名称输入长度不能超过32个字符", required = true) + @NotBlank(message = "名称不能为空") + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "名称-输入长度不能超过32个字符") + @Pattern(regexp = TrainUtil.REGEXP, message = "名称支持字母、数字、汉字、英文横杠和下划线") + private String auxInfo; + + @ApiModelProperty(value = "类型", hidden = true) + private String type; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java new file mode 100644 index 0000000..e2e3abd --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 算法用途删除 + * @date 2020/7/2 + */ +@Data +@Accessors(chain = true) +public class PtTrainAlgorithmUsageDeleteDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为空") + private Long[] ids; + +} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java new file mode 100644 index 0000000..f3612f1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotNull; + +import org.dubhe.base.PageQueryBase; + +/** + * @description 查询算法用途条件 + * @date 2020-06-23 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +public class PtTrainAlgorithmUsageQueryDTO extends PageQueryBase { + + @ApiModelProperty(value = "类型", hidden = true) + private String type; + + @ApiModelProperty(value = "是否包含默认值 0：不包含 1包含", required = true) + @NotNull + private Boolean isContainDefault; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java new file mode 100644 index 0000000..6c12f7e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 算法用途修改 + * @date 2020-06-23 + */ +@Data +@Accessors(chain = true) +public class PtTrainAlgorithmUsageUpdateDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为null") + @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") + private Long id; + + @ApiModelProperty("描述, 长度不能超过20个字符") + @Length(max = TrainUtil.NUMBER_TWENTY, message = "名称长度不能超过20个字符") + private String auxInfo; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java new file mode 100644 index 0000000..5d43e8e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; +import java.util.List; + +/** + * @description 查询数据集对应训练状态查询条件 + * @date 2020-05-21 + */ +@Data +@Accessors(chain = true) +public class PtTrainDataSourceStatusQueryDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("数据集路径") + private List dataSourcePath; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java new file mode 100644 index 0000000..d380647 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java @@ -0,0 +1,96 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseImageDTO; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.*; + +/** + * @description 创建训练任务 + * @date 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +public class PtTrainJobCreateDTO extends BaseImageDTO { + + @ApiModelProperty(value = "训练作业名, 长度在1-32个字符", required = true) + @NotNull(message = "训练作业名不能为空") + @Length(min = TrainUtil.NUMBER_ONE, max = TrainUtil.NUMBER_THIRTY_TWO, message = "训练作业名长度在1-32个字符") + @Pattern(regexp = TrainUtil.REGEXP, message = "训练作业名称支持字母、数字、汉字、英文横杠和下划线") + private String trainName; + + @ApiModelProperty("描述, 长度不能超过255个字符") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "描述长度不能超过255个字符") + private String description; + + @ApiModelProperty(value = "算法来源id", required = true) + @NotNull(message = "algorithmId不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "algorithmId必须不小于1") + private Long algorithmId; + + @ApiModelProperty(value = "运行命令,输入长度不能超过128个字符", required = true) + @NotBlank(message = "运行命令不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") + private String runCommand; + + @ApiModelProperty(value = "数据来源名称, 长度在1-127个字符", required = true) + @NotNull(message = "数据来源名称不能为空") + @Length(min = TrainUtil.NUMBER_ONE, max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据来源名称长度在1-127个字符") + private String dataSourceName; + + @ApiModelProperty(value = "数据来源路径, 长度在1-127个字符", required = true) + @NotNull(message = "数据来源路径不能为空") + @Length(min = TrainUtil.NUMBER_ONE, max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据来源路径长度在1-127个字符") + private String dataSourcePath; + + @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") + private JSONObject runParams; + + @ApiModelProperty(value = "类型(0为CPU，1为GPU)", required = true) + @Min(value = TrainUtil.NUMBER_ZERO, message = "类型错误") + @Max(value = TrainUtil.NUMBER_ONE, message = "类型错误") + @NotNull(message = "类型(0为CPU，1为GPU)不能为空") + private Integer resourcesPoolType; + + @ApiModelProperty(value = "规格类型Id", required = true) + @NotNull(message = "规格类型id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "规格类型id必须不小于1") + private Integer trainJobSpecsId; + + @ApiModelProperty("true代表保存作业参数") + private Boolean saveParams; + + @ApiModelProperty("作业参数名称，当saveParams为true的时候传递, 输入长度不能超过32个字符") + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "作业参数名称-输入长度不能超过32个字符") + @Pattern(regexp = TrainUtil.REGEXP, message = "作业参数名称支持字母、数字、汉字、英文横杠和下划线") + private String trainParamName; + + @ApiModelProperty("作业参数描述，当saveParams为true的时候传递, 长度不能超过255个字符") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "作业参数描述-输入长度不能超过255个字符") + private String trainParamDesc; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java new file mode 100644 index 0000000..306dca3 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description： 训练任务 + * @date 2020-03-17 + */ +@Data +public class PtTrainJobDTO implements Serializable { + private Long id; + private String name; + private String remark; + private PtDatasetSmallDTO dataset; + private PtImageSmallDTO image; + private PtStorageSmallDTO storage; + private TeamSmallDTO team; + private String selectorLabel; + private Integer podNum; + private Integer gpuNum; + private Integer memNum; + private Integer cpuNum; + private Integer level; + private Timestamp startTime; + private Timestamp closeTime; + private String cmd; + private String status; + private UserSmallDTO confirmUser; + private UserSmallDTO createUser; + private Timestamp createTime; + private Timestamp updateTime; + private Boolean deleted; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java new file mode 100644 index 0000000..6bfdc09 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java @@ -0,0 +1,48 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 删除训练任务 + * @date 2020-04-27 + */ +@Data +@Accessors(chain = true) +public class PtTrainJobDeleteDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "trainId,如果只传递trainId,代表删除该trainId下的所有job", required = true) + @NotNull(message = "trainId不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "trainId数值不合法") + private Long trainId; + + @ApiModelProperty(value = "jobId") + @Min(value = TrainUtil.NUMBER_ONE, message = "jobId数值不合法") + private Long id; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java new file mode 100644 index 0000000..ba044de --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 根据jobId查询训练任务详情查询条件 + * @date 2020-06-12 + */ +@Data +@Accessors(chain = true) +public class PtTrainJobDetailQueryDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为null") + @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") + private Long id; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java new file mode 100644 index 0000000..2637c7c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description: 恢复训练 + * @date: 2020-04-27 + */ +@Data +@Accessors(chain = true) +public class PtTrainJobResumeDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "id数值不合法") + private Long id; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java new file mode 100644 index 0000000..f217fbb --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description： 训练作业规格查询 + * @date 2020-05-06 + */ +@Data +public class PtTrainJobSpecsQueryDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "规格类型(0为CPU, 1为GPU)") + private Integer resourcesPoolType; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java new file mode 100644 index 0000000..c953cdf --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java @@ -0,0 +1,48 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 停止训练 + * @date 2020-04-27 + */ +@Data +@Accessors(chain = true) +public class PtTrainJobStopDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "trainId,如果只传递trainId,代表删除该trainId下的所有job", required = true) + @NotNull(message = "trainId不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "trainId数值不合法") + private Long trainId; + + @ApiModelProperty(value = "jobId") + @Min(value = TrainUtil.NUMBER_ONE, message = "jobId数值不合法") + private Long id; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java new file mode 100644 index 0000000..401f454 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java @@ -0,0 +1,86 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseImageDTO; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +/** + * @description 训练任务修改 + * @date 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +public class PtTrainJobUpdateDTO extends BaseImageDTO { + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为null") + @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") + private Long id; + + @ApiModelProperty("描述, 长度不能超过255个字符") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "描述长度不能超过255个字符") + private String description; + + @ApiModelProperty("算法id") + @NotNull(message = "algorithmId不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "algorithmId必须大于1") + private Long algorithmId; + + @ApiModelProperty(value = "运行命令,输入长度不能超过128个字符", required = true) + @NotBlank(message = "运行命令不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") + private String runCommand; + + @ApiModelProperty(value = "数据集来源路径,输入长度不能超过127个字符", required = true) + @NotBlank(message = "数据集来源路径不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源路径-输入长度不能超过127个字符") + private String dataSourcePath; + + @ApiModelProperty(value = "数据集来源名称,输入长度不能超过127个字符", required = true) + @NotBlank(message = "数据集来源名称不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源名称-输入长度不能超过127个字符") + private String dataSourceName; + + @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") + private JSONObject runParams; + + @ApiModelProperty(value = "类型(0为CPU，1为GPU)", required = true) + @Min(value = TrainUtil.NUMBER_ZERO, message = "类型错误") + @Max(value = TrainUtil.NUMBER_ONE, message = "类型错误") + @NotNull(message = "类型(0为CPU，1为GPU)不能为空") + private Integer resourcesPoolType; + + @ApiModelProperty(value = "规格类型Id", required = true) + @NotNull(message = "规格类型id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "规格类型id必须不小于1") + private Integer trainJobSpecsId; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java new file mode 100644 index 0000000..cbe33b1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java @@ -0,0 +1,48 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.base.PageQueryBase; +import org.dubhe.utils.TrainUtil; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 训练任务版本查询 + * @date 2020-06-30 + */ +@Data +@Accessors(chain = true) +public class PtTrainJobVersionQueryDTO extends PageQueryBase implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "训练作业id", required = true) + @NotNull(message = "训练作业id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "训练作业id必须不小于1") + private Long trainId; + + @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知,6为删除，7为创建失败") + private Integer trainStatus; + +} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java new file mode 100644 index 0000000..3be8dec --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 训练日志 查询DTO + * @date 2020-05-08 + */ +@Data +@Accessors(chain = true) +public class PtTrainLogQueryDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("训练job的id") + @NotNull(message = "训练job的id不能为空") + private Integer jobId; + + @ApiModelProperty(value = "起始行") + @Min(value = TrainUtil.NUMBER_ZERO, message = "startLine不能小于0") + private Integer startLine; + + @ApiModelProperty(value = "行数") + @Min(value = TrainUtil.NUMBER_ONE, message = "lines不能小于1") + @Max(value = TrainUtil.NUMBER_ONE_THOUSAND, message = "lines不能大于1000") + private Integer lines; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java new file mode 100644 index 0000000..1b9e636 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java @@ -0,0 +1,84 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseImageDTO; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.*; + +/** + * @description 任务参数创建条件 + * @date 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +public class PtTrainParamCreateDTO extends BaseImageDTO { + + @ApiModelProperty(value = "任务参数名称,输入长度不能超过32个字符", required = true) + @NotBlank(message = "任务参数名称不能为空") + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "任务参数名称-输入长度不能超过32个字符") + @Pattern(regexp = TrainUtil.REGEXP, message = "任务参数名称支持字母、数字、汉字、英文横杠和下划线") + private String paramName; + + @ApiModelProperty("描述，输入长度不能超过256个字符") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "任务参数描述-输入长度不能超过256个字符") + private String description; + + @ApiModelProperty(value = "算法id,算法id不能小于1", required = true) + @NotNull(message = "算法id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "算法id不能小于1") + private Long algorithmId; + + @ApiModelProperty(value = "运行命令,输入长度不能超过128个字符", required = true) + @NotBlank(message = "运行命令不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") + private String runCommand; + + @ApiModelProperty(value = "数据集来源路径,输入长度不能超过127个字符", required = true) + @NotBlank(message = "数据集来源路径不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源路径-输入长度不能超过127个字符") + private String dataSourcePath; + + @ApiModelProperty(value = "数据集来源名称,输入长度不能超过127个字符", required = true) + @NotBlank(message = "数据集来源名称不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源名称-输入长度不能超过127个字符") + private String dataSourceName; + + @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") + private JSONObject runParams; + + @ApiModelProperty(value = "类型(0为CPU，1为GPU)", required = true) + @Min(value = TrainUtil.NUMBER_ZERO, message = "类型错误") + @Max(value = TrainUtil.NUMBER_ONE, message = "类型错误") + @NotNull(message = "类型(0为CPU，1为GPU)不能为空") + private Integer resourcesPoolType; + + @ApiModelProperty(value = "规格类型Id", required = true) + @NotNull(message = "规格类型id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "规格类型id必须不小于1") + private Integer trainJobSpecsId; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java new file mode 100644 index 0000000..b0635f2 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java @@ -0,0 +1,42 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; +import java.util.Set; + +/** + * @description 训练模板删除 + * @date 2020-07-02 + */ +@Data +@Accessors(chain = true) +public class PtTrainParamDeleteDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为空") + private Set ids; + +} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java new file mode 100644 index 0000000..c470f37 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.base.PageQueryBase; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import java.io.Serializable; + +/** + * @description 任务参数查询条件 + * @date 2020-4-27 + */ +@Data +@Accessors(chain = true) +public class PtTrainParamQueryDTO extends PageQueryBase implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("任务参数名称,输入长度不能超过128个字符") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "任务参数名称-输入长度不能超过128个字符") + private String paramName; + + @ApiModelProperty(value = "规格类型(0为CPU, 1为GPU)") + private Integer resourcesPoolType; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java new file mode 100644 index 0000000..59675e0 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java @@ -0,0 +1,89 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseImageDTO; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.*; + +/** + * @description 任务参数修改条件 + * @date 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +public class PtTrainParamUpdateDTO extends BaseImageDTO { + + @ApiModelProperty(value = "任务参数id,任务参数id不能小于1", required = true) + @NotNull(message = "任务参数id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "任务参数id不能小于1") + private Long id; + + @ApiModelProperty(value = "任务参数名称,输入长度不能超过32个字符", required = true) + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "任务参数名称-输入长度不能超过32个字符") + @NotBlank(message = "任务参数名称不能为空") + @Pattern(regexp = TrainUtil.REGEXP, message = "任务参数名称支持字母、数字、汉字、英文横杠和下划线") + private String paramName; + + @ApiModelProperty("描述,输入长度不能超过256个字符") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "任务参数描述-输入长度不能超过256个字符") + private String description; + + @ApiModelProperty(value = "算法id,算法id不能小于1", required = true) + @NotNull(message = "算法id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "算法id不能小于1") + private Long algorithmId; + + @ApiModelProperty(value = "运行命令,输入长度不能超过128个字符", required = true) + @NotBlank(message = "运行命令不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") + private String runCommand; + + @ApiModelProperty(value = "数据集来源路径,输入长度不能超过127个字符", required = true) + @NotBlank(message = "数据集来源路径不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源路径-输入长度不能超过127个字符") + private String dataSourcePath; + + @ApiModelProperty(value = "数据集来源名称,输入长度不能超过127个字符", required = true) + @NotBlank(message = "数据集来源名称不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源名称-输入长度不能超过127个字符") + private String dataSourceName; + + @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") + private JSONObject runParams; + + @ApiModelProperty(value = "类型(0为CPU，1为GPU)", required = true) + @Min(value = TrainUtil.NUMBER_ZERO, message = "类型错误") + @Max(value = TrainUtil.NUMBER_ONE, message = "类型错误") + @NotNull(message = "类型(0为CPU，1为GPU)不能为空") + private Integer resourcesPoolType; + + @ApiModelProperty(value = "规格类型Id", required = true) + @NotNull(message = "规格类型id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "规格类型id必须不小于1") + private Integer trainJobSpecsId; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java new file mode 100644 index 0000000..f50746d --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.base.PageQueryBase; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import java.io.Serializable; + +/** + * @description 查询训练 + * @date 2020-04-27 + */ +@Data +@Accessors(chain = true) +public class PtTrainQueryDTO extends PageQueryBase implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("训练作业名称或者id") + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "训练作业名称或者id有误") + private String trainName; + + @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败") + @Min(value = TrainUtil.NUMBER_ZERO, message = "trainStatus错误") + @Max(value = TrainUtil.NUMBER_SEVEN, message = "trainStatus错误") + private Integer trainStatus; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java new file mode 100644 index 0000000..eff7890 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 第三方创建NoteBook请求对象 + * + * @date 2020.05.15 + */ +@Data +public class SourceNoteBookDTO implements Serializable { + + @ApiModelProperty("第三方源主键") + @NotNull + private Long sourceId; + + @ApiModelProperty("第三方源资源路径") + @NotBlank + private String sourceFilePath; + + @ApiModelProperty(value = "当前登录人", hidden = true) + private Long curUserId; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java new file mode 100644 index 0000000..2d822a9 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java @@ -0,0 +1,202 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonIgnore; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.*; +import java.io.Serializable; +import java.util.Date; + +/** + * @description note book 实体 + * @date 2020-04-28 + */ +@Data +@TableName("notebook") +public class NoteBook implements Serializable { + + @TableId(value = "id", type = IdType.AUTO) + @ApiModelProperty(hidden = true) + private Long id; + + @TableField(value = "user_id") + @ApiModelProperty(hidden = true) + private Long userId; + + @TableField(value = "name") + @JsonIgnore + private String name; + + @TableField(value = "notebook_name") + private String noteBookName; + + @TableField(value = "description") + private String description; + + @TableField(value = "url") + @ApiModelProperty(hidden = true) + private String url; + + @TableField(value = "total_run_min") + @ApiModelProperty(hidden = true) + private Integer totalRunMin; + + @TableField(value = "cpu_num") + @ApiModelProperty(value = "cpu数量") + private Integer cpuNum; + + @TableField(value = "gpu_num") + @ApiModelProperty(value = "gpu数量") + private Integer gpuNum; + + @TableField(value = "mem_num") + @ApiModelProperty(value = "内存大小") + private Integer memNum; + + @TableField(value = "disk_mem_num") + @ApiModelProperty(value = "硬盘内存大小") + private Integer diskMemNum; + + /** + * 0运行中，1停止, 2删除, 3启动中，4停止中，5删除中，6运行异常（暂未启用） + */ + @TableField(value = "status") + @ApiModelProperty(hidden = true) + private Integer status; + + @TableField(value = "last_start_time") + @ApiModelProperty(hidden = true) + private Date lastStartTime; + + @TableField(value = "last_operation_timeout") + @ApiModelProperty(hidden = true) + private Long lastOperationTimeout; + + /** + * 0 - notebook 创建 1- 其它系统创建 + */ + @TableField(value = "create_resource") + @ApiModelProperty(hidden = true) + private Integer createResource; + + @TableField(value = "k8s_status_code") + @ApiModelProperty(hidden = true) + private String k8sStatusCode; + + @TableField(value = "k8s_status_info") + @ApiModelProperty(hidden = true) + private String k8sStatusInfo; + + @TableField(value = "k8s_namespace") + @ApiModelProperty(hidden = true) + private String k8sNamespace; + + @TableField(value = "k8s_resource_name") + @ApiModelProperty(hidden = true) + private String k8sResourceName; + + @TableField(value = "k8s_image_name") + @ApiModelProperty("镜像名称") + @NotNull(message = "请选择镜像") + private String k8sImageName; + + @TableField(value = "k8s_mount_path") + @ApiModelProperty(hidden = true) + private String k8sMountPath; + + @TableField(value = "k8s_pvc_path") + @ApiModelProperty(hidden = true) + private String k8sPvcPath; + + @TableField(value = "create_time") + @ApiModelProperty(hidden = true) + private Date createTime; + + @TableField(value = "create_user_id") + @ApiModelProperty(hidden = true) + private Long createUserId; + + @TableField(value = "update_time") + @ApiModelProperty(hidden = true) + private Date updateTime; + + @TableField(value = "update_user_id") + @ApiModelProperty(hidden = true) + private Long updateUserId; + + @TableField(value = "deleted") + @ApiModelProperty(hidden = true) + private Integer deleted; + + @TableField(value = "data_source_name") + @ApiModelProperty(hidden = true) + @Size(max = 255, message = "数据集名称超长") + private String dataSourceName; + + @TableField(value = "data_source_path") + @Size(max = 255, message = "数据集路径超长") + @ApiModelProperty(hidden = true) + private String dataSourcePath; + + @TableField(value = "algorithm_id") + @ApiModelProperty(hidden = true) + private Long algorithmId; + + @Override + public String toString() { + return "NoteBook{" + + "id=" + id + + ", userId=" + userId + + ", name='" + name + '\'' + + ", noteBookName='" + noteBookName + '\'' + + ", description='" + description + '\'' + + ", url='" + url + '\'' + + ", totalRunMin=" + totalRunMin + + ", cpuNum=" + cpuNum + + ", gpuNum=" + gpuNum + + ", memNum=" + memNum + + ", diskMemNum=" + diskMemNum + + ", status=" + status + + ", lastStartTime=" + lastStartTime + + ", lastOperationTimeout=" + lastOperationTimeout + + ", createResource=" + createResource + + ", k8sStatusCode='" + k8sStatusCode + '\'' + + ", k8sStatusInfo='" + k8sStatusInfo + '\'' + + ", k8sNamespace='" + k8sNamespace + '\'' + + ", k8sResourceName='" + k8sResourceName + '\'' + + ", k8sImageName='" + k8sImageName + '\'' + + ", k8sMountPath='" + k8sMountPath + '\'' + + ", k8sPvcPath='" + k8sPvcPath + '\'' + + ", createTime=" + createTime + + ", createUserId=" + createUserId + + ", updateTime=" + updateTime + + ", updateUserId=" + updateUserId + + ", deleted=" + deleted + + ", dataSourceName='" + dataSourceName + '\'' + + ", dataSourcePath='" + dataSourcePath + '\'' + + ", algorithmId=" + algorithmId + + '}'; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java new file mode 100644 index 0000000..41fb51c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.dubhe.base.BaseEntity; + +/** + * @description notebook 模板 + * @date 2020-06-01 + */ +@Data +@TableName("notebook_model") +public class NoteBookModel extends BaseEntity { + + @TableField(value = "model_type") + private String modelType; + + @TableField(value = "cpu_num") + @ApiModelProperty("cpu数量") + private Integer cpuNum; + + @TableField(value = "gpu_num") + @ApiModelProperty("gpu数量") + private Integer gpuNum; + + @TableField(value = "mem_num") + @ApiModelProperty("内存大小") + private Integer memNum; + + @TableField(value = "spec") + @ApiModelProperty("GPU规格") + private String spec; + + @TableField(value = "disk_mem_num") + @ApiModelProperty("硬盘内存大小") + private Integer diskMemNum; + + @TableField(value = "default_status") + @ApiModelProperty("默认值状态 0 - 非默认 1- 默认") + private Integer defaultStatus; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java new file mode 100644 index 0000000..cc64ce1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java @@ -0,0 +1,87 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseEntity; + +/** + * @description 镜像 + * @date 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +@TableName(value = "pt_image") +public class PtImage extends BaseEntity { + + /** + * 主键 + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 镜像名称 + */ + @TableField(value = "image_name") + private String imageName; + + /** + * 镜像地址 + */ + @TableField(value = "image_url") + private String imageUrl; + + /** + * 镜像版本 + */ + @TableField(value = "image_tag") + private String imageTag; + + /** + * 镜像描述 + */ + @TableField(value = "remark") + private String remark; + + /** + * projectName + */ + @TableField(value = "project_name") + private String projectName; + + /** + * 镜像来源 + */ + @TableField(value = "image_resource") + private Integer imageResource; + + + /** + * 镜像状态 + */ + @TableField(value = "image_status") + private Integer imageStatus; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java new file mode 100644 index 0000000..4cba152 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java @@ -0,0 +1,101 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseEntity; + +/** + * @description job运行参数及结果 + * @date 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +@TableName(value = "pt_job_param", autoResultMap = true) +public class PtJobParam extends BaseEntity { + + /** + * 主键id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 训练作业jobId + */ + @TableField(value = "train_job_id") + private Long trainJobId; + + /** + * 算法来源id + */ + @TableField(value = "algorithm_id") + private Long algorithmId; + + /** + * 运行命令 + */ + @TableField(value = "run_command") + private String runCommand; + + /** + * 镜像名称 + */ + @TableField(value = "image_name") + private String imageName; + + /** + * 运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数) + */ + @TableField(value = "run_params", typeHandler = FastjsonTypeHandler.class) + private JSONObject runParams; + + /** + * F1值 + */ + @TableField(value = "param_f1") + private String paramF1; + + /** + * 召回率 + */ + @TableField(value = "param_callback") + private String paramCallback; + + /** + * 精确率 + */ + @TableField(value = "param_precise") + private String paramPrecise; + + /** + * 准确率 + */ + @TableField(value = "param_accuracy") + private String paramAccuracy; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java new file mode 100644 index 0000000..9f6251c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseEntity; + +/** + * @description 训练作业主 + * @date 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +@TableName("pt_train") +public class PtTrain extends BaseEntity { + + /** + * 主键id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 训练作业名 + */ + @TableField(value = "train_name") + private String trainName; + + /** + * 训练作业名 + */ + @TableField(value = "train_key") + private String trainKey; + + /** + * 训练作业有效job版本数量 + */ + @TableField(value = "version_num") + private Integer versionNum; + + /** + * 训练作业job总版本数量 + */ + @TableField(value = "total_num") + private Integer totalNum; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java new file mode 100644 index 0000000..8f1f6b4 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java @@ -0,0 +1,129 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseEntity; + +import javax.validation.constraints.NotNull; + +/** + * @description 算法 + * @date 2020-04-29 + */ + +@EqualsAndHashCode(callSuper = true) +@Data +@TableName(value = "pt_train_algorithm", autoResultMap = true) +@Accessors(chain = true) +public class PtTrainAlgorithm extends BaseEntity { + + /** + * 主键 + */ + @TableId(value = "id", type = IdType.AUTO) + @NotNull(groups = {Update.class}) + private Long id; + + /** + * 算法名称 + */ + @TableField(value = "algorithm_name") + private String algorithmName; + + /** + * 算法描述 + */ + @TableField(value = "description") + private String description; + + /** + * 算法来源(1为我的算法，2为预置算法) + */ + @TableField(value = "algorithm_source") + private Integer algorithmSource; + + /** + * 环境镜像名称 + */ + @TableField(value = "image_name") + private String imageName; + + /** + * 代码目录 + */ + @TableField(value = "code_dir") + private String codeDir; + + /** + * 运行命令 + */ + @TableField(value = "run_command") + private String runCommand; + + /** + * 运行参数 + */ + @TableField(value = "run_params", typeHandler = FastjsonTypeHandler.class) + private JSONObject runParams; + + /** + * 算法用途 + */ + @TableField(value = "algorithm_usage") + private String algorithmUsage; + + /** + * 算法精度 + */ + @TableField(value = "accuracy") + private String accuracy; + + /** + * P4推理速度（ms） + */ + @TableField(value = "p4_inference_speed") + private Integer p4InferenceSpeed; + + /** + * 输出结果（1是，0否） + */ + @TableField(value = "is_train_out") + private Boolean isTrainOut; + + /** + * 输出日志（1是，0否） + */ + @TableField(value = "is_train_log") + private Boolean isTrainLog; + + /** + * 可视化日志（1是，0否） + */ + @TableField(value = "is_visualized_log") + private Boolean isVisualizedLog; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java new file mode 100644 index 0000000..9739e7a --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseEntity; + +import javax.validation.constraints.NotNull; + +/** + * @description 算法 + * @date 2020-06-23 + */ + +@EqualsAndHashCode(callSuper = true) +@Data +@TableName("pt_auxiliary_info") +@Accessors(chain = true) +public class PtTrainAlgorithmUsage extends BaseEntity { + + /** + * 主键 + */ + @TableId(value = "id", type = IdType.AUTO) + @NotNull(groups = {Update.class}) + private Long id; + + /** + * 用户id + */ + @TableField(value = "user_id") + private Long userId; + + /** + * 类型 + */ + @TableField(value = "type") + private String type; + + /** + * 算法来源(1为我的算法，2为预置算法) + */ + @TableField(value = "aux_info") + private String auxInfo; + + /** + * 是否为默认值(0否，1是默认值) + */ + @TableField(value = "is_default") + private Boolean isDefault; + + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java new file mode 100644 index 0000000..2e7f752 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java @@ -0,0 +1,148 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseEntity; + +/** + * @description 训练作业job + * @date 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +@TableName(value = "pt_train_job", autoResultMap = true) +public class PtTrainJob extends BaseEntity { + + /** + * 主键id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 训练作业id + */ + @TableField(value = "train_id") + private Long trainId; + + /** + * job版本 + */ + @TableField(value = "train_version") + private String trainVersion; + + /** + * job父版本 + */ + @TableField(value = "parent_train_version") + private String parentTrainVersion; + + /** + * 任务名称 + */ + @TableField(value = "job_name") + private String jobName; + + /** + * 描述 + */ + @TableField(value = "description") + private String description; + + /** + * 数据来源路径 + */ + @TableField(value = "data_source_path") + private String dataSourcePath; + + + /** + * 数据来源名称 + */ + @TableField(value = "data_source_name") + private String dataSourceName; + + /** + * 训练时长 + */ + @TableField(value = "runtime") + private String runtime; + + /** + * 训练输出位置 + */ + @TableField(value = "out_path") + private String outPath; + + /** + * 作业日志路径 + */ + @TableField(value = "log_path") + private String logPath; + + /** + * 规格Id + */ + @TableField(value = "train_job_specs_id") + private Integer trainJobSpecsId; + + /** + * 类型(0为CPU，1为GPU) + */ + @TableField(value = "resources_pool_type") + private Integer resourcesPoolType; + + /** + * 规格 + */ + @TableField(value = "resources_pool_specs") + private String resourcesPoolSpecs; + + /** + * 节点个数 + */ + @TableField(value = "resources_pool_node") + private Integer resourcesPoolNode; + + /** + * 训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败 + */ + @TableField(value = "train_status") + private Integer trainStatus; + + /** + * 可视化日志路径 + */ + @TableField(value = "visualized_log_path") + private String visualizedLogPath; + + /** + * k8s返回的job名称 + */ + @TableField(value = "k8s_job_name") + private String k8sJobName; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java new file mode 100644 index 0000000..9ca2eae --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseEntity; + +/** + * @decription 训练作业规格 + * @date 2020-05-06 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +@TableName(value = "pt_train_job_specs", autoResultMap = true) +public class PtTrainJobSpecs extends BaseEntity { + + /** + * 主键id + */ + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + + /** + * 规格名称 + */ + @TableField(value = "specs_name") + private String specsName; + + /** + * 规格信息 + */ + @TableField(value = "specs_info", typeHandler = FastjsonTypeHandler.class) + private JSONObject specsInfo; + + /** + * 规格类型 + */ + @TableField(value = "resources_pool_type") + private Integer resourcesPoolType; + +} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java new file mode 100644 index 0000000..34543e6 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java @@ -0,0 +1,130 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseEntity; + +/** + * @descrption 任务参数 + * @date 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Accessors(chain = true) +@TableName(value = "pt_train_param", autoResultMap = true) +public class PtTrainParam extends BaseEntity { + + /** + * 主键 + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 任务参数名称 + */ + @TableField(value = "param_name") + private String paramName; + + /** + * 描述 + */ + @TableField(value = "description") + private String description; + + /** + * 算法id + */ + @TableField(value = "algorithm_id") + private Long algorithmId; + + /** + * 运行命令 + */ + @TableField(value = "run_command") + private String runCommand; + + /** + * 镜像名称 + */ + @TableField(value = "image_name") + private String imageName; + + /** + * 数据集来源名称 + */ + @TableField(value = "data_source_name") + private String dataSourceName; + + /** + * 数据集来源路径 + */ + @TableField(value = "data_source_path") + private String dataSourcePath; + + /** + * 输出路径 + */ + @TableField(value = "out_path") + private String outPath; + + /** + * 运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数) + */ + @TableField(value = "run_params", typeHandler = FastjsonTypeHandler.class) + private JSONObject runParams; + + /** + * 算法来源(1为我的算法，2为预置算法) + */ + @TableField(value = "algorithm_source") + private Integer algorithmSource; + + /** + * 日志输出路径 + */ + @TableField(value = "log_path") + private String logPath; + + /** + * 规格Id + */ + @TableField(value = "train_job_specs_id") + private Integer trainJobSpecsId; + + /** + * 类型(0为CPU，1为GPU) + */ + @TableField(value = "resources_pool_type") + private Integer resourcesPoolType; + + /** + * 节点个数 + */ + @TableField(value = "resources_pool_node") + private Integer resourcesPoolNode; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java new file mode 100644 index 0000000..65c1470 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java @@ -0,0 +1,115 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import cn.hutool.core.date.DatePattern; +import com.baomidou.mybatisplus.annotation.TableField; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnore; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.Size; +import java.io.Serializable; +import java.util.Date; + +/** + * @description 返回前端请求体 + * @create 2020/4/28 + */ +@Data +@ApiModel("NoteBookDTO 响应") +public class NoteBookVO implements Serializable { + + @ApiModelProperty("ID") + private Long id; + + @ApiModelProperty("所属用户") + private Long userId; + + @ApiModelProperty("NoteBook 名称") + @JsonIgnore + private String name; + + @ApiModelProperty("NoteBook 名称") + private String noteBookName; + + @ApiModelProperty("备注描述") + private String description; + + @ApiModelProperty("可访问jupyter地址") + private String url; + + @JsonIgnore + private Integer totalRunMin; + + @ApiModelProperty("CPU数量") + private Integer cpuNum; + + @ApiModelProperty("GPU数量") + private Integer gpuNum; + + @ApiModelProperty("内存大小（G）") + private Integer memNum; + + @ApiModelProperty("硬盘内存大小（G）") + private Integer diskMemNum; + + @ApiModelProperty("0运行，1停止, 2删除, 3启动中，4停止中，5删除中，6运行异常（暂未启用）") + private Integer status; + + @ApiModelProperty("k8s响应状态码") + private String k8sStatusCode; + + @ApiModelProperty("k8s响应状态信息") + private String k8sStatusInfo; + + @JsonIgnore + private String k8sNamespace; + + @JsonIgnore + private String k8sResourceName; + + private String k8sImageName; + + @ApiModelProperty("k8s中pvc存储路径") + private String k8sPvcPath; + + @JsonFormat(pattern = DatePattern.NORM_DATETIME_MS_PATTERN) + private Date createTime; + + @JsonIgnore + private Long createUserId; + + @JsonFormat(pattern = DatePattern.NORM_DATETIME_MS_PATTERN) + private Date updateTime; + + @JsonIgnore + private Long updateUserId; + + @ApiModelProperty("数据集名称") + private String dataSourceName; + + @ApiModelProperty("数据集路径") + private String dataSourcePath; + + @ApiModelProperty("算法ID") + private Long algorithmId; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java new file mode 100644 index 0000000..9f7669f --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import lombok.Data; +import lombok.experimental.Accessors; + +/** + * @description: doJob返回封装 + * @Date:2020-07-03 + */ +@Data +@Accessors(chain = true) +public class PtDoJobResultVO { + /** + * 返回是否成功 + */ + private Boolean isSuccess; + + /** + * 返回k8s返回的JobName + */ + private String k8sJobName; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java new file mode 100644 index 0000000..124efdd --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; + +/** + * @description: 镜像 + * @date 2020-04-27 + */ +@Data +@Accessors(chain = true) +public class PtImageAndAlgorithmVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("镜像名称") + private String imageName; + + @ApiModelProperty("镜像地址") + private String imageUrl; + + @ApiModelProperty("代码目录") + private String codeDir; + + @ApiModelProperty("运行命令") + private String runCommand; + + @ApiModelProperty("输出目录") + private Boolean isTrainOut; + + @ApiModelProperty("输出日志") + private Boolean isTrainLog; + + @ApiModelProperty("输出可视化日志") + private Boolean isVisualizedLog; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java new file mode 100644 index 0000000..033a79b --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description: 返回镜像查询结果 + * @date: 2020-04-27 + */ +@Data +public class PtImageQueryVO implements Serializable { + + private static final long serialVersionUID = 1L; + @ApiModelProperty("镜像ID") + private Long id; + + @ApiModelProperty("镜像项目名") + private String projectName; + + @ApiModelProperty("镜像名称") + private String imageName; + + @ApiModelProperty("镜像版本") + private String imageTag; + + @ApiModelProperty("镜像状态(0:上传中,1:上传成功,2:上传失败)") + private Integer imageStatus; + + @ApiModelProperty("备注") + private String remark; + + @ApiModelProperty("创建时间") + private Timestamp createTime; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java new file mode 100644 index 0000000..a367563 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; + +/** + * @description Pod Metrics Grafana url + * @date 2020-06-30 + */ +@Data +@Accessors(chain = true) +public class PtJobMetricsGrafanaVO implements Serializable { + + @ApiModelProperty("job监控地址") + private String jobMetricsGrafanaUrl; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java new file mode 100644 index 0000000..cdb2278 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java @@ -0,0 +1,87 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description: 训练算法返回列表 + * @date 2020-04-27 + */ +@Data +public class PtTrainAlgorithmQueryVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "算法ID") + private Long id; + + @ApiModelProperty(value = "算法名称") + private String algorithmName; + + @ApiModelProperty(value = "描述信息") + private String description; + + @ApiModelProperty(value = "算法来源") + private Integer algorithmSource; + + @ApiModelProperty(value = "镜像名称") + private String imageName; + + @ApiModelProperty(value = "镜像Project") + private String imageNameProject; + + @ApiModelProperty(value = "镜像版本") + private String imageTag; + + @ApiModelProperty(value = "代码目录") + private String codeDir; + + @ApiModelProperty(value = "运行命令") + private String runCommand; + + @ApiModelProperty(value = "运行参数") + private JSONObject runParams; + + @ApiModelProperty(value = "算法用途") + private String algorithmUsage; + + @ApiModelProperty(value = "精度") + private String accuracy; + + @ApiModelProperty(value = "P4推理速度") + private Integer p4InferenceSpeed; + + @ApiModelProperty(value = "创建时间") + private Timestamp createTime; + + @ApiModelProperty(value = "输出结果（1是，0否）") + private Boolean isTrainOut; + + @ApiModelProperty(value = "输出日志（1是，0否)") + private Boolean isTrainLog; + + @ApiModelProperty(value = "可视化日志（1是，0否）") + private Boolean isVisualizedLog; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java new file mode 100644 index 0000000..b5e7db9 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java @@ -0,0 +1,59 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description: 算法用途返回列表 + * @date 2020-06-23 + */ +@Data +public class PtTrainAlgorithmUsageQueryVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "ID") + private Long id; + + @ApiModelProperty(value = "用户ID") + private Long userId; + + @ApiModelProperty(value = "类型") + private String type; + + @ApiModelProperty(value = "辅助信息") + private String auxInfo; + + @ApiModelProperty(value = "辅助信息") + private Boolean deleted; + + @ApiModelProperty(value = "创建时间") + private Timestamp createTime; + + @ApiModelProperty(value = "更新时间") + private Timestamp updateTime; + + @ApiModelProperty(value = "是否为默认值") + private Boolean isDefault; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java new file mode 100644 index 0000000..712b9eb --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.Setter; + +import java.io.Serializable; + +/** + * @description: 查询数据集状态查询结果 + * @date: 2020-05-21 + */ +@Data +@Setter +public class PtTrainDataSourceStatusQueryVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("数据集名称") + private String dataSourceName; + + @ApiModelProperty("数据集路径") + private String dataSourcePath; + + @ApiModelProperty("数据集状态(true为当前没有训练在使用，可以删除；false为有训练使用,不可以删除)") + private Boolean status; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java new file mode 100644 index 0000000..1a03ee0 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description: 返回删除训练任务结果 + * @date: 2020-04-28 + */ +@Data +public class PtTrainJobDeleteVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("trainID,如果只传递trainID,代表删除该trainID下的所有job") + private Long trainId; + + @ApiModelProperty("训练作业jobID") + private Long id; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java new file mode 100644 index 0000000..6228c3e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java @@ -0,0 +1,135 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description: 根据jobId查询训练任务详情返回结果 + * @date: 2020-06-12 + */ +@Data +@Accessors(chain = true) +public class PtTrainJobDetailQueryVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("jobID") + private Long id; + + @ApiModelProperty("训练作业ID") + private Long trainId; + + @ApiModelProperty("训练作业job版本") + private String trainVersion; + + @ApiModelProperty("训练作业job父版本") + private String parentTrainVersion; + + @ApiModelProperty("训练作业jobName") + private String jobName; + + @ApiModelProperty("描述信息") + private String description; + + @ApiModelProperty("数据集名称") + private String dataSourceName; + + @ApiModelProperty("数据集路径") + private String dataSourcePath; + + @ApiModelProperty("训练时长") + private String runtime; + + @ApiModelProperty("训练输出位置") + private String outPath; + + @ApiModelProperty("日志输出路径") + private String logPath; + + @ApiModelProperty("可视化日志路径") + private String visualizedLogPath; + + @ApiModelProperty("规格ID") + private Integer trainJobSpecsId; + + @ApiModelProperty("类型(0为CPU，1为GPU)") + private Integer resourcesPoolType; + + @ApiModelProperty("节点个数") + private Integer resourcesPoolNode; + + @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败") + private Integer trainStatus; + + @ApiModelProperty("创建时间") + private Timestamp createTime; + + @ApiModelProperty("算法ID") + private Long algorithmId; + + @ApiModelProperty("镜像名称") + private String imageName; + + @ApiModelProperty(value = "镜像Project") + private String imageNameProject; + + @ApiModelProperty(value = "镜像版本") + private String imageTag; + + @ApiModelProperty("运行命令,输入长度不能超过128个字符") + private String runCommand; + + @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") + private JSONObject runParams; + + @ApiModelProperty("F1值") + private String paramF1; + + @ApiModelProperty("召回率") + private String paramCallback; + + @ApiModelProperty("精确率") + private String paramPrecise; + + @ApiModelProperty("准确率") + private String paramAccuracy; + + @ApiModelProperty("算法名称") + private String algorithmName; + + @ApiModelProperty("算法来源(1为我的算法，2为预置算法)") + private Integer algorithmSource; + + @ApiModelProperty("算法用途") + private String algorithmUsage; + + @ApiModelProperty("算法精度") + private String accuracy; + + @ApiModelProperty("P4推理速度（ms）") + private Integer p4InferenceSpeed; + + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java new file mode 100644 index 0000000..29987c6 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java @@ -0,0 +1,137 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description: 返回训练版本查询详情 + * @date 2020-04-27 + */ +@Data +@Accessors(chain = true) +public class PtTrainJobDetailVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("训练作业名") + private String trainName; + + @ApiModelProperty("jobID") + private Long id; + + @ApiModelProperty("训练作业ID") + private Long trainId; + + @ApiModelProperty("训练作业job版本") + private String trainVersion; + + @ApiModelProperty("训练作业job父版本") + private String parentTrainVersion; + + @ApiModelProperty("训练作业jobName") + private String jobName; + + @ApiModelProperty("描述信息") + private String description; + + @ApiModelProperty("数据集名称") + private String dataSourceName; + + @ApiModelProperty("数据集路径") + private String dataSourcePath; + + @ApiModelProperty("训练时长") + private String runtime; + + @ApiModelProperty("训练输出位置") + private String outPath; + + @ApiModelProperty("日志输出路径") + private String logPath; + + @ApiModelProperty("可视化日志路径") + private String visualizedLogPath; + + @ApiModelProperty("规格ID") + private Integer trainJobSpecsId; + + @ApiModelProperty("类型(0为CPU，1为GPU)") + private Integer resourcesPoolType; + + @ApiModelProperty("节点个数") + private Integer resourcesPoolNode; + + @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败") + private Integer trainStatus; + + @ApiModelProperty("创建时间") + private Timestamp createTime; + + @ApiModelProperty("算法ID") + private Long algorithmId; + + @ApiModelProperty("镜像名称") + private String imageName; + + @ApiModelProperty(value = "镜像Project") + private String imageNameProject; + + @ApiModelProperty(value = "镜像版本") + private String imageTag; + + @ApiModelProperty("运行命令,输入长度不能超过128个字符") + private String runCommand; + + @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") + private JSONObject runParams; + + @ApiModelProperty("F1值") + private String paramF1; + + @ApiModelProperty("召回率") + private String paramCallback; + + @ApiModelProperty("精确率") + private String paramPrecise; + + @ApiModelProperty("准确率") + private String paramAccuracy; + + @ApiModelProperty("算法名称") + private String algorithmName; + + @ApiModelProperty("算法来源(1为我的算法，2为预置算法)") + private Integer algorithmSource; + + @ApiModelProperty("算法用途") + private String algorithmUsage; + + @ApiModelProperty("算法精度") + private String accuracy; + + @ApiModelProperty("P4推理速度（ms）") + private Integer p4InferenceSpeed; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java new file mode 100644 index 0000000..23b0bc1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; + +/** + * @description: 训练作业规格 + * @date 2020-05-06 + */ +@Data +@Accessors(chain = true) +public class PtTrainJobSpecsQueryVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("训练作业规格ID") + private Integer id; + + @ApiModelProperty("训练作业规格名称") + private String specsName; + + @ApiModelProperty(value = "规格类型(0为CPU, 1为GPU)") + private Integer resourcesPoolType; + + @ApiModelProperty(value = "规格信息") + private JSONObject specsInfo; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java new file mode 100644 index 0000000..686e39e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java @@ -0,0 +1,42 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description: 我的训练任务统计 + * @date: 2020-07-15 + */ +@Data +@ApiModel(description = "我的训练任务统计结果") +public class PtTrainJobStatisticsMineVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("运行中的任务") + private Integer runJobCount; + + @ApiModelProperty("已经完成的任务") + private Integer finishJobCount; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java new file mode 100644 index 0000000..f05900e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description: 返回停止训练任务结果 + * @date: 2020-04-28 + */ +@Data +public class PtTrainJobStopVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("trainID,如果只传递trainID,代表停止该trainID下的所有job") + private Long trainId; + + @ApiModelProperty("训练作业jobID") + private Long id; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java new file mode 100644 index 0000000..2409a48 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.util.List; + +/** + * @description: 训练日志 查询VO + * @date 2020-05-08 + */ +@Data +@Accessors(chain = true) +public class PtTrainLogQueryVO { + private static final long serialVersionUID = 1L; + + @ApiModelProperty("log内容") + private List content; + + @ApiModelProperty(value = "任务名称") + private String jobName; + + @ApiModelProperty(value = "起始行") + private Integer startLine; + + @ApiModelProperty(value = "结束行") + private Integer endLine; + + @ApiModelProperty(value = "lines") + private Integer lines; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java new file mode 100644 index 0000000..0db9ef8 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java @@ -0,0 +1,86 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.dubhe.base.BaseVO; + +import java.io.Serializable; + +/** + * @description: 任务参数查询返回查询结果 + * @date: 2020-04-27 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class PtTrainParamQueryVO extends BaseVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("任务参数ID") + private Long id; + + @ApiModelProperty("任务参数名称") + private String paramName; + + @ApiModelProperty("描述") + private String description; + + @ApiModelProperty("镜像名称") + private String imageName; + + @ApiModelProperty(value = "镜像Project") + private String imageNameProject; + + @ApiModelProperty(value = "镜像版本") + private String imageTag; + + @ApiModelProperty("算法ID") + private Long algorithmId; + + @ApiModelProperty("运行命令,输入长度不能超过128个字符") + private String runCommand; + + @ApiModelProperty("算法名称") + private String algorithmName; + + @ApiModelProperty("算法来源(1为我的算法，2为预置算法)") + private Integer algorithmSource; + + @ApiModelProperty("数据来源路径") + private String dataSourcePath; + + @ApiModelProperty("数据来源名称") + private String dataSourceName; + + @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") + private JSONObject runParams; + + @ApiModelProperty("规格ID") + private Integer trainJobSpecsId; + + @ApiModelProperty("类型(0为CPU，1为GPU)") + private Integer resourcesPoolType; + + @ApiModelProperty("节点个数") + private Integer resourcesPoolNode; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java new file mode 100644 index 0000000..910632e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description: 训练查询结果 + * @date 2020-04-27 + */ +@Data +@Accessors(chain = true) +public class PtTrainVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("训练作业jobID") + private Long jobId; + + @ApiModelProperty("训练作业ID") + private Long trainId; + + @ApiModelProperty("训练作业名") + private String trainName; + + @ApiModelProperty("训练作业jobName") + private String jobName; + + @ApiModelProperty("训练作业版本数") + private Integer versionNum; + + @ApiModelProperty("训练时长") + private String runtime; + + @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败") + private Integer trainStatus; + + @ApiModelProperty("创建时间") + private Timestamp createTime; + + @ApiModelProperty("数据来源路径") + private String dataSourcePath; + + @ApiModelProperty("数据来源名称") + private String dataSourceName; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/HarborResourceEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/HarborResourceEnum.java new file mode 100644 index 0000000..445f5d2 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/HarborResourceEnum.java @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.enums; + +/** + * @description harbor镜像来源枚举 + * @date 2020-07-16 + */ +public enum HarborResourceEnum { + + NOTEBOOK(0, "notebook预置"), + TRAIN(1, "train预置"), + TRAIN_SYNC(2, "train同步镜像"); + + /** + * 编码 + */ + private Integer code; + + /** + * 描述 + */ + private String description; + + HarborResourceEnum(Integer code, String description) { + this.code = code; + this.description = description; + } + + public Integer getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageSourceEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageSourceEnum.java new file mode 100644 index 0000000..471bf71 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageSourceEnum.java @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.enums; + +/** + * @description 镜像来源枚举类 + * @date 2020-07-15 + */ +public enum ImageSourceEnum { + MINE(0, "我的镜像"), + PRE(1, "预置镜像"); + + + /** + * 编码 + */ + private Integer code; + + /** + * 描述 + */ + private String description; + + ImageSourceEnum(int code, String description) { + this.code = code; + this.description = description; + } + + public Integer getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageStateEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageStateEnum.java new file mode 100644 index 0000000..127e747 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageStateEnum.java @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + +/** + * @description 镜像运行状态枚举 + * @date 2020-07-15 + **/ +public enum ImageStateEnum { + + MAKING(0, "制作中"), + SUCCESS(1, "制作成功"), + FAIL(2, "制作失败"); + + + /** + * 编码 + */ + private Integer code; + + /** + * 描述 + */ + private String description; + + ImageStateEnum(int code, String description) { + this.code = code; + this.description = description; + } + + public Integer getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java new file mode 100644 index 0000000..9ccc3f7 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java @@ -0,0 +1,153 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + + +import org.dubhe.base.MagicNumConstant; +import org.dubhe.k8s.enums.PodPhaseEnum; + +import java.util.ArrayList; +import java.util.List; + +/** + * @description notebook 状态枚举 + * @create 2020-4-28 + */ +public enum NoteBookStatusEnum { + /** + * 运行 + */ + RUN(0, "运行中", 0), + /** + * 停止 + */ + STOP(1, "停止", 0), + /** + * 删除 + */ + DELETE(2, "删除", 0), + /** + * 启动中 + */ + STARTING(3, "启动中", 1), + /** + * 停止中 + */ + STOPPING(4, "停止中", 1), + /** + * 删除中 + */ + DELETING(5, "删除中", 1), + /** + * 运行异常 + */ + ERROR(6, "运行异常", 0); + /** + * 编码 + */ + private Integer code; + /** + * 描述 + */ + private String description; + /** + * 删除标识 + */ + private Integer deleteType; + + public Integer getCode() { + return code; + } + + public void setCode(Integer code) { + this.code = code; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Integer getDeleteType() { + return deleteType; + } + + public void setDeleteType(Integer deleteType) { + this.deleteType = deleteType; + } + + NoteBookStatusEnum(int code, String description, int deleteType) { + this.code = code; + this.description = description; + this.deleteType = deleteType; + } + + public static String getDescription(Integer code) { + if (code != null) { + for (NoteBookStatusEnum en : NoteBookStatusEnum.values()) { + if (en.getCode().equals(code)) { + return en.getDescription(); + } + } + } + return null; + } + + /** + * k8s状态转换成NoteBook状态 + * 当查询不到k8s状态时即为删除 + * + * @param bizPodPhase + * @return + */ + public static NoteBookStatusEnum convert(String bizPodPhase) { + if (bizPodPhase == null + || PodPhaseEnum.DELETED.getPhase().equals(bizPodPhase) + || PodPhaseEnum.FAILED.getPhase().equals(bizPodPhase)) { + return STOP; + } else if (PodPhaseEnum.PENDING.getPhase().equals(bizPodPhase)) { + return STARTING; + } else if (PodPhaseEnum.RUNNING.getPhase().equals(bizPodPhase)) { + return RUN; + } else if (PodPhaseEnum.SUCCEEDED.getPhase().equals(bizPodPhase)) { + return STOPPING; + } else { + return ERROR; + } + } + + + /** + * 查找可以删除的状态集合 + * + * @return List + */ + public static List getCanDeleteStatus() { + List list = new ArrayList<>(); + for (NoteBookStatusEnum en : NoteBookStatusEnum.values()) { + if (en.getDeleteType().equals(MagicNumConstant.ONE)) { + list.add(en.getCode()); + } + } + return list; + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java new file mode 100644 index 0000000..32a7640 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.enums; + +/** + *@description 规格类型 + *@date: 2020-07-15 + + */ +public enum ResourcesPoolTypeEnum { + + CPU(0, "CPU"), + GPU(1, "GPU"); + + + /** + * 编码 + */ + private Integer code; + + /** + * 描述 + */ + private String description; + + ResourcesPoolTypeEnum(int code, String description) { + this.code = code; + this.description = description; + } + + public Integer getCode() { + return code; + } + + public String getDescription() { + return description; + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java new file mode 100644 index 0000000..0043e36 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java @@ -0,0 +1,126 @@ +/** Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import cn.hutool.core.util.StrUtil; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.client.RestTemplate; + +import javax.servlet.http.HttpServletRequest; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * @description 代理转发 + * @date 2020-06-23 + */ + +@RestController +public class ForwardController { + @Value("${dubhe-proxy.visual.keyword}") + private String visual; + @Value("${dubhe-proxy.visual.server}") + private String visualServer; + @Value("${dubhe-proxy.visual.port}") + private String visualPort; + @Value("${dubhe-proxy.refine.keyword}") + private String refine; + @Value("${dubhe-proxy.refine.server}") + private String refineServer; + @Value("${dubhe-proxy.refine.port}") + private String refinePort; + + RestTemplate restTemplate = new RestTemplate(); + + /** + * 根据不同的请求拼上对应的转发路径 + * + * @param request http请求 + * @return URI 用于restTemplate的请求路径 + **/ + private URI getURI(HttpServletRequest request) throws URISyntaxException { + String requestURI = request.getRequestURI(); + String server = null; + String prefix = ""; + int port = 0; + if (requestURI.startsWith(StrUtil.SLASH + visual)) { + prefix = visual; + server = visualServer; + port = Integer.parseInt(visualPort); + } else if (requestURI.startsWith(StrUtil.SLASH + refine)) { + prefix = refine; + server = refineServer; + port = Integer.parseInt(refinePort); + } + + return new URI("http", null, server, port, requestURI.substring(prefix.length() + 1), request.getQueryString(), null); + } + + /** + * 获取请求中的Cookie + * + * @param request http请求 + * @return HttpHeaders 用于restTemplate的header + **/ + private HttpHeaders getHeader(HttpServletRequest request) { + String cookie = request.getHeader("Cookie"); + HttpHeaders httpHeaders = new HttpHeaders(); + if (null != cookie) { + httpHeaders.set("Cookie", cookie); + } + return httpHeaders; + } + + /** + * 转发get请求 + * + * @param request http请求 + * @return ResponseEntity 返回给前端的响应实体 + **/ + @GetMapping({StrUtil.SLASH + "${dubhe-proxy.visual.keyword}" + StrUtil.SLASH + "**", StrUtil.SLASH + "${dubhe-proxy.refine.keyword}" + StrUtil.SLASH + "**"}) + @ResponseBody + public ResponseEntity mirrorRest(HttpServletRequest request) throws URISyntaxException { + URI uri = getURI(request); + HttpHeaders httpHeaders = getHeader(request); + ResponseEntity responseEntity = + restTemplate.exchange(uri, HttpMethod.GET, new HttpEntity(httpHeaders), String.class); + return responseEntity; + } + + /** + * 转发get请求 + * + * @param request http请求 + * @param method 请求方法 + * @param body 请求体 + * @return ResponseEntity 返回给前端的响应实体 + **/ + @RequestMapping({StrUtil.SLASH + "${dubhe-proxy.visual.keyword}" + StrUtil.SLASH + "**", StrUtil.SLASH + "${dubhe-proxy.refine.keyword}" + StrUtil.SLASH + "**"}) + @ResponseBody + public ResponseEntity mirrorRest(HttpMethod method, HttpServletRequest request, @RequestBody String body) throws URISyntaxException { + URI uri = getURI(request); + HttpHeaders httpHeaders = getHeader(request); + ResponseEntity responseEntity = + restTemplate.exchange(uri, method, new HttpEntity(body, httpHeaders), String.class); + return responseEntity; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java new file mode 100644 index 0000000..87f05f4 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java @@ -0,0 +1,82 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import javax.annotation.Resource; + +import org.dubhe.base.DataResponseBody; +import org.dubhe.factory.DataResponseFactory; +import org.dubhe.dto.callback.AlgorithmK8sPodCallbackCreateDTO; +import org.dubhe.dto.callback.NotebookK8sPodCallbackCreateDTO; +import org.dubhe.service.PodCallbackAsyncService; +import org.dubhe.utils.K8sCallBackTool; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; + +/** + * @description k8s Pod 异步回调处理类 + * + * @date 2020-05-28 + */ +@Api(tags = "k8s回调：Pod") +@RestController +@RequestMapping("/api/k8s/callback/pod") +public class K8sCallbackPodController { + + @Resource(name = "noteBookAsyncServiceImpl") + private PodCallbackAsyncService noteBookAsyncService; + @Resource(name = "algorithmAsyncServiceImpl") + private PodCallbackAsyncService algorithmAsyncServiceImpl; + + /** + * notebook异步回调 + * + * @param k8sToken + * @param k8sPodCallbackReq + * @return + */ + @PostMapping(value = "/notebook") + @ApiOperation("模型管理 pod 回调") + public DataResponseBody notebookPodCallBack(@ApiParam(type = "head") @RequestHeader(name= K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken + ,@Validated @RequestBody NotebookK8sPodCallbackCreateDTO k8sPodCallbackReq) { + noteBookAsyncService.podCallBack(k8sPodCallbackReq); + return DataResponseFactory.success("notebook正在异步处理pod中。"); + } + + + /** + * algorithm异步回调 + * + * @param k8sToken + * @param k8sPodCallbackReq + * @return + */ + @PostMapping(value = "/algorithm") + @ApiOperation("算法管理 pod 回调") + public DataResponseBody notebookPodCallBack(@ApiParam(type = "head") @RequestHeader(name= K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken + ,@Validated @RequestBody AlgorithmK8sPodCallbackCreateDTO k8sPodCallbackReq) { + algorithmAsyncServiceImpl.podCallBack(k8sPodCallbackReq); + return DataResponseFactory.success("算法管理异步回调处理方法中"); + } + + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java new file mode 100644 index 0000000..1076cc7 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java @@ -0,0 +1,163 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.swagger.annotations.*; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.base.DataResponseBody; +import org.dubhe.base.ResponseCode; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.entity.NoteBook; +import org.dubhe.domain.dto.NoteBookListQueryDTO; +import org.dubhe.domain.dto.SourceNoteBookDTO; +import org.dubhe.domain.dto.NoteBookCreateDTO; +import org.dubhe.enums.BizNfsEnum; +import org.dubhe.factory.DataResponseFactory; +import org.dubhe.service.NoteBookService; +import org.dubhe.utils.NotebookUtil; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import java.util.List; +import java.util.Set; + +/** + * @description notebook controller + * @date 2020-04-27 + */ +@Api(tags = "模型开发：Notebook") +@RestController +@RequestMapping("/api/{version}/notebook") +public class NoteBookController { + + @Autowired + private NoteBookService noteBookService; + + @ApiOperation("新增加notebook") + @PostMapping(value = "/notebooks") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody createNoteBook(@Validated @RequestBody NoteBookCreateDTO noteBookCreateDTO) { + long userId = NotebookUtil.getCurUserId(); + NoteBook noteBook = new NoteBook(); + BeanUtils.copyProperties(noteBookCreateDTO, noteBook); + noteBook.setCreateUserId(userId); + noteBook.setUserId(userId); + noteBook.setCreateResource(BizNfsEnum.NOTEBOOK.getCreateResource()); + return new DataResponseBody(noteBookService.createNoteBook(noteBook)); + } + + @ApiOperation("查询notebook") + @GetMapping(value = "/notebooks") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody getNoteBookList(Page page, NoteBookListQueryDTO noteBookListQueryDTO) { + noteBookListQueryDTO.setUserId(NotebookUtil.getCurUserId()); + return new DataResponseBody(noteBookService.getNoteBookList(page, noteBookListQueryDTO)); + } + + @ApiOperation("异步批量删除notebook") + @DeleteMapping + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody deleteNoteBook(@RequestBody Set noteBookIds) { + List noteBookList = noteBookService.validateDeleteNoteBook(noteBookIds); + noteBookService.deleteNoteBooks(noteBookList); + return new DataResponseBody(ResponseCode.SUCCESS, "正在异步删除NoteBook中.."); + } + + @ApiOperation("启动notebook") + @PutMapping(value = "/start") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody startNotebook(@RequestParam(required = true) Long noteBookId) { + String resultInfo = noteBookService.startNoteBook(noteBookId); + return new DataResponseBody( + NotebookUtil.validateFailedInfo(resultInfo) ? ResponseCode.ERROR : ResponseCode.SUCCESS + , resultInfo); + } + + @ApiOperation("停止notebook") + @PutMapping(value = "/stop") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody stopNotebook(@RequestParam(required = true) Long noteBookId) { + String resultInfo = noteBookService.stopNoteBook(noteBookId); + return new DataResponseBody( + NotebookUtil.validateFailedInfo(resultInfo) ? ResponseCode.ERROR : ResponseCode.SUCCESS + , resultInfo); + } + + + @ApiOperation("开启notebook") + @GetMapping(value = "/{id}") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody openNotebook(@PathVariable(name = "id", required = true) Long noteBookId) { + return DataResponseFactory.success("获取成功", noteBookService.openNoteBook(noteBookId)); + } + + + @ApiOperation("第三方业务创建notebook") + @PostMapping(value = "/create/{source}") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody createByThirdParty(@PathVariable int source, @Validated @RequestBody SourceNoteBookDTO sourceNoteBookDTO) { + BizNfsEnum bizNfsEnum = BizNfsEnum.getByCreateResource(source); + if (bizNfsEnum == null || BizNfsEnum.NOTEBOOK == bizNfsEnum) { + return new DataResponseBody(ResponseCode.BADREQUEST, "不支持该渠道创建Notebook！"); + } + sourceNoteBookDTO.setCurUserId(NotebookUtil.getCurUserId()); + return DataResponseFactory.success("操作成功", noteBookService.createNoteBookByThirdParty(bizNfsEnum, sourceNoteBookDTO)); + } + + @ApiOperation("获取编辑地址") + @GetMapping(value = "/{id}/get-address") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody getAddress(@PathVariable(name = "id", required = true) Long noteBookId) { + return new DataResponseBody(noteBookService.getAddress(noteBookId)); + } + + @ApiOperation("获取状态") + @GetMapping(value = "/status") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody getNoteBookStatus() { + return new DataResponseBody(noteBookService.getNoteBookStatus()); + } + + + @ApiOperation("获取正在运行的notebook数量") + @GetMapping(value = "/run-number") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody getNoteBookRunNumber() { + return new DataResponseBody(noteBookService.getNoteBookRunNumber(NotebookUtil.getCurUserId())); + } + + + @ApiOperation("获取notebook模板") + @GetMapping(value = "/notebook-model") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody getNoteBookModel() { + return new DataResponseBody(noteBookService.getNoteBookModel()); + } + + + @ApiOperation("获取notebook详情") + @PostMapping(value = "/detail") + @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) + public DataResponseBody getNotebookDetail(@RequestBody Set noteBookIds) { + return new DataResponseBody(noteBookService.getNotebookDetail(noteBookIds)); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtDatasetController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtDatasetController.java new file mode 100644 index 0000000..486e34c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtDatasetController.java @@ -0,0 +1,89 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.dubhe.base.DataResponseBody; +import org.dubhe.domain.PtDataset; +import org.dubhe.domain.dto.PtDatasetQueryCriteria; +import org.dubhe.service.PtDatasetService; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * @description dataset管理 + * @date 2020-03-17 + */ +@Api(tags = "dataset管理") +@ApiIgnore +@RestController +@RequestMapping("/api/{version}/pt_dataset") +public class PtDatasetController { + + private final PtDatasetService ptDatasetService; + + public PtDatasetController(PtDatasetService ptDatasetService) { + this.ptDatasetService = ptDatasetService; + } + + + @ApiOperation("导出数据") + @GetMapping(value = "/download") + @PreAuthorize("@el.check('ptDataset:list')") + public void download(HttpServletResponse response, PtDatasetQueryCriteria criteria) throws IOException { + ptDatasetService.download(ptDatasetService.queryAll(criteria), response); + } + + @GetMapping + @ApiOperation("查询dataset") + @PreAuthorize("@el.check('ptDataset:list')") + public DataResponseBody getPtDatasets(PtDatasetQueryCriteria criteria, Page page) { + return new DataResponseBody(ptDatasetService.queryAll(criteria, page)); + } + + @PostMapping + @ApiOperation("新增dataset") + @PreAuthorize("@el.check('ptDataset:add')") + public DataResponseBody create(@Validated @RequestBody PtDataset resources) { + return new DataResponseBody(ptDatasetService.create(resources)); + } + + @PutMapping + @ApiOperation("修改dataset") + @PreAuthorize("@el.check('ptDataset:edit')") + public DataResponseBody update(@Validated @RequestBody PtDataset resources) { + ptDatasetService.update(resources); + return new DataResponseBody(); + } + + @ApiOperation("删除dataset") + @PreAuthorize("@el.check('ptDataset:del')") + @DeleteMapping + public DataResponseBody deleteAll(@RequestBody Long[] ids) { + ptDatasetService.deleteAll(ids); + return new DataResponseBody(); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtDevEnvsController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtDevEnvsController.java new file mode 100644 index 0000000..620d167 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtDevEnvsController.java @@ -0,0 +1,88 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.dubhe.base.DataResponseBody; +import org.dubhe.domain.PtDevEnvs; +import org.dubhe.domain.dto.PtDevEnvsQueryCriteria; +import org.dubhe.service.PtDevEnvsService; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * @description devEnvs管理 + * @date 2020-03-17 + */ +@Api(tags = "devEnvs管理") +@ApiIgnore +@RestController +@RequestMapping("/api/{version}/pt_ev_envs") +public class PtDevEnvsController { + + private final PtDevEnvsService ptDevEnvsService; + + public PtDevEnvsController(PtDevEnvsService ptDevEnvsService) { + this.ptDevEnvsService = ptDevEnvsService; + } + + @ApiOperation("导出数据") + @GetMapping(value = "/download") + @PreAuthorize("@el.check('ptDevEnvs:list')") + public void download(HttpServletResponse response, PtDevEnvsQueryCriteria criteria) throws IOException { + ptDevEnvsService.download(ptDevEnvsService.queryAll(criteria), response); + } + + @GetMapping + @ApiOperation("查询devEnvs") + @PreAuthorize("@el.check('ptDevEnvs:list')") + public DataResponseBody getPtDevEnvss(PtDevEnvsQueryCriteria criteria, Page page) { + return new DataResponseBody(ptDevEnvsService.queryAll(criteria, page)); + } + + @PostMapping + @ApiOperation("新增devEnvs") + @PreAuthorize("@el.check('ptDevEnvs:add')") + public DataResponseBody create(@Validated @RequestBody PtDevEnvs resources) { + return new DataResponseBody(ptDevEnvsService.create(resources)); + } + + @PutMapping + @ApiOperation("修改devEnvs") + @PreAuthorize("@el.check('ptDevEnvs:edit')") + public DataResponseBody update(@Validated @RequestBody PtDevEnvs resources) { + ptDevEnvsService.update(resources); + return new DataResponseBody(); + } + + @ApiOperation("删除devEnvs") + @PreAuthorize("@el.check('ptDevEnvs:del')") + @DeleteMapping + public DataResponseBody deleteAll(@RequestBody Long[] ids) { + ptDevEnvsService.deleteAll(ids); + return new DataResponseBody(); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java new file mode 100644 index 0000000..792b32d --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.annotation.ApiVersion; +import org.dubhe.base.DataResponseBody; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.dto.PtImageQueryDTO; +import org.dubhe.domain.dto.PtImageUploadDTO; +import org.dubhe.service.PtImageService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +/** + * @description 镜像 + * @date 2020-04-27 + */ +@Api(tags = "镜像：镜像管理") +@RestController +@ApiVersion(1) +@RequestMapping("/api/{version}/ptImage") +public class PtImageController { + + @Autowired + private PtImageService ptImageService; + + @GetMapping("/info") + @ApiOperation("查询镜像") + @RequiresPermissions(Permissions.TRAINING_IMAGE) + public DataResponseBody getImage(PtImageQueryDTO ptImageQueryDTO) { + return new DataResponseBody(ptImageService.getImage(ptImageQueryDTO)); + } + + @ApiOperation("通过projectName查询镜像") + @GetMapping + @RequiresPermissions(Permissions.TRAINING_IMAGE) + public DataResponseBody getTagsByImageName(@RequestParam String imageName) { + return new DataResponseBody(ptImageService.searchImages(imageName)); + } + + @PostMapping("uploadImage") + @ApiOperation("上传镜像包到harbor") + @RequiresPermissions(Permissions.TRAINING_IMAGE) + public DataResponseBody uploadImage(@Validated @RequestBody PtImageUploadDTO ptImageUploadDTO) { + ptImageService.uploadImage(ptImageUploadDTO); + return new DataResponseBody(); + } + + @GetMapping("/project") + @ApiOperation("查询镜像project列表") + @RequiresPermissions(Permissions.TRAINING_IMAGE) + public DataResponseBody getHarborProjectList() { + return new DataResponseBody(ptImageService.getHarborProjectList()); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtStorageController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtStorageController.java new file mode 100644 index 0000000..408e28a --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtStorageController.java @@ -0,0 +1,87 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.dubhe.base.DataResponseBody; +import org.dubhe.domain.PtStorage; +import org.dubhe.domain.dto.PtStorageQueryCriteria; +import org.dubhe.service.PtStorageService; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * @description storage管理 + * @date 2020-03-17 + */ +@Api(tags = "storage管理") +@ApiIgnore +@RestController +@RequestMapping("/api/{version}/pt_storage") +public class PtStorageController { + + private final PtStorageService ptStorageService; + + public PtStorageController(PtStorageService ptStorageService) { + this.ptStorageService = ptStorageService; + } + + @ApiOperation("导出数据") + @GetMapping(value = "/download") + @PreAuthorize("@el.check('ptStorage:list')") + public void download(HttpServletResponse response, PtStorageQueryCriteria criteria) throws IOException { + ptStorageService.download(ptStorageService.queryAll(criteria), response); + } + + @GetMapping + @ApiOperation("查询storage") + @PreAuthorize("@el.check('ptStorage:list')") + public DataResponseBody getPtStorages(PtStorageQueryCriteria criteria, Page page) { + return new DataResponseBody(ptStorageService.queryAll(criteria, page)); + } + + @PostMapping + @ApiOperation("新增storage") + @PreAuthorize("@el.check('ptStorage:add')") + public DataResponseBody create(@Validated @RequestBody PtStorage resources) { + return new DataResponseBody(ptStorageService.create(resources)); + } + + @PutMapping + @ApiOperation("修改storage") + @PreAuthorize("@el.check('ptStorage:edit')") + public DataResponseBody update(@Validated @RequestBody PtStorage resources) { + ptStorageService.update(resources); + return new DataResponseBody(); + } + + @ApiOperation("删除storage") + @PreAuthorize("@el.check('ptStorage:del')") + @DeleteMapping + public DataResponseBody deleteAll(@RequestBody Long[] ids) { + ptStorageService.deleteAll(ids); + return new DataResponseBody(); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java new file mode 100644 index 0000000..8f17cbd --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java @@ -0,0 +1,84 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.annotation.ApiVersion; +import org.dubhe.base.DataResponseBody; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.dto.PtTrainAlgorithmCreateDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmDeleteDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmQueryDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUpdateDTO; +import org.dubhe.service.PtTrainAlgorithmService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +/** + * @description 训练算法 + * @date 2020-04-27 + */ +@Api(tags = "训练：算法管理") +@RestController +@ApiVersion(1) +@RequestMapping("/api/{version}/algorithm") +public class PtTrainAlgorithmController { + + @Autowired + private PtTrainAlgorithmService ptTrainAlgorithmService; + + @GetMapping + @ApiOperation("查询算法") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody getAlgorithms(@Validated PtTrainAlgorithmQueryDTO ptTrainAlgorithmQueryDTO) { + return new DataResponseBody(ptTrainAlgorithmService.queryAll(ptTrainAlgorithmQueryDTO)); + } + + @GetMapping("/myAlgorithmCount") + @ApiOperation("查询当前用户的算法个数") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody getAlgorithmCount() { + return new DataResponseBody(ptTrainAlgorithmService.getAlgorithmCount()); + } + + @PostMapping + @ApiOperation("新增算法") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody create(@Validated @RequestBody PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO) { + return new DataResponseBody(ptTrainAlgorithmService.create(ptTrainAlgorithmCreateDTO)); + } + + @PutMapping + @ApiOperation("修改算法") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody update(@Validated @RequestBody PtTrainAlgorithmUpdateDTO ptTrainAlgorithmUpdateDTO) { + return new DataResponseBody(ptTrainAlgorithmService.update(ptTrainAlgorithmUpdateDTO)); + } + + @DeleteMapping + @ApiOperation("删除算法") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody deleteAll(@Validated @RequestBody PtTrainAlgorithmDeleteDTO ptTrainAlgorithmDeleteDTO) { + ptTrainAlgorithmService.deleteAll(ptTrainAlgorithmDeleteDTO); + return new DataResponseBody(); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java new file mode 100644 index 0000000..928dc31 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java @@ -0,0 +1,85 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.annotation.ApiVersion; +import org.dubhe.base.DataResponseBody; +import org.dubhe.constant.Permissions; +import org.dubhe.constant.UserAuxiliaryInfoConstant; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageDeleteDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageCreateDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageQueryDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageUpdateDTO; +import org.dubhe.factory.DataResponseFactory; +import org.dubhe.service.PtTrainAlgorithmUsageService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +/** + * @description 算法用途管理 + * @date 2020-06-19 + */ +@Api(tags = "训练：算法用途管理") +@RestController +@ApiVersion(1) +@RequestMapping("/api/{version}/algorithmUsage") +public class PtTrainAlgorithmUsageController { + + @Autowired + private PtTrainAlgorithmUsageService ptTrainAlgorithmUsageService; + + @GetMapping + @ApiOperation("算法用途列表展示") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody queryAll(@Validated PtTrainAlgorithmUsageQueryDTO ptTrainAlgorithmUsageQueryDTO) { + ptTrainAlgorithmUsageQueryDTO.setType(UserAuxiliaryInfoConstant.ALGORITHM_USAGE); + return DataResponseFactory + .success(ptTrainAlgorithmUsageService.queryAll(ptTrainAlgorithmUsageQueryDTO)); + } + + @PostMapping + @ApiOperation("新增算法用途") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody create( + @Validated @RequestBody PtTrainAlgorithmUsageCreateDTO ptTrainAlgorithmUsageCreateDTO) { + ptTrainAlgorithmUsageCreateDTO.setType(UserAuxiliaryInfoConstant.ALGORITHM_USAGE); + return DataResponseFactory.success(ptTrainAlgorithmUsageService.create(ptTrainAlgorithmUsageCreateDTO)); + } + + @DeleteMapping + @ApiOperation("删除算法用途") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody deleteAll(@Validated @RequestBody PtTrainAlgorithmUsageDeleteDTO ptTrainAlgorithmUsageDeleteDTO) { + ptTrainAlgorithmUsageService.deleteAll(ptTrainAlgorithmUsageDeleteDTO); + return new DataResponseBody(); + } + + @PutMapping + @ApiOperation("修改算法用途") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody update( + @Validated @RequestBody PtTrainAlgorithmUsageUpdateDTO ptTrainAlgorithmUsageUpdateDTO) { + ptTrainAlgorithmUsageService.update(ptTrainAlgorithmUsageUpdateDTO); + return new DataResponseBody(); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java new file mode 100644 index 0000000..04f025e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java @@ -0,0 +1,135 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.Logical; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.annotation.ApiVersion; +import org.dubhe.base.DataResponseBody; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.dto.*; +import org.dubhe.service.PtTrainJobService; +import org.dubhe.service.PtTrainJobSpecsService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + + +/** + * @description 训练作业job + * @date 2020-04-27 + */ +@Api(tags = "训练：任务管理") +@RestController +@ApiVersion(1) +@RequestMapping("/api/{version}/trainJob") +public class PtTrainJobController { + + @Autowired + private PtTrainJobService ptTrainJobService; + + @Autowired + private PtTrainJobSpecsService ptTrainJobSpecsService; + + @GetMapping + @ApiOperation("作业列表展示") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody getTrainJob(@Validated PtTrainQueryDTO ptTrainQueryDTO) { + return new DataResponseBody(ptTrainJobService.getTrainJob(ptTrainQueryDTO)); + } + + @GetMapping("/trainJobSpecs") + @ApiOperation("规格展示") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody getTrainJobSpecs(@Validated PtTrainJobSpecsQueryDTO ptTrainJobSpecsQueryDTO) { + return new DataResponseBody(ptTrainJobSpecsService.getTrainJobSpecs(ptTrainJobSpecsQueryDTO)); + } + + @GetMapping("/jobDetail") + @ApiOperation("根据jobId查询训练任务详情") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody getTrainJobDetail(@Validated PtTrainJobDetailQueryDTO ptTrainJobDetailQueryDTO) { + return new DataResponseBody(ptTrainJobService.getTrainJobDetail(ptTrainJobDetailQueryDTO)); + } + + @GetMapping("/mine") + @ApiOperation(value = "我的训练任务统计", notes = "运行中的任务:PENDDING,RUNNING；完成的任务:其他状态") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody statisticsMine() { + return new DataResponseBody(ptTrainJobService.statisticsMine()); + } + + @GetMapping("/trainJobVersionDetail") + @ApiOperation("作业不同版本任务列表展示") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody getTrainJobVersion(@Validated PtTrainJobVersionQueryDTO ptTrainJobVersionQueryDTO) { + return new DataResponseBody(ptTrainJobService.getTrainJobVersion(ptTrainJobVersionQueryDTO)); + } + + @GetMapping("/dataSourceStatus") + @ApiOperation("数据集状态展示") + @RequiresPermissions(value = {Permissions.TRAINING_JOB, Permissions.DATA}, logical = Logical.OR) + public DataResponseBody getTrainDataSourceStatus(@Validated PtTrainDataSourceStatusQueryDTO ptTrainDataSourceStatusQueryDTO) { + return new DataResponseBody(ptTrainJobService.getTrainDataSourceStatus(ptTrainDataSourceStatusQueryDTO)); + } + + @PostMapping + @ApiOperation("创建训练任务") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody createTrainJob(@Validated @RequestBody PtTrainJobCreateDTO ptTrainJobCreateDTO) { + return new DataResponseBody(ptTrainJobService.createTrainJobVersion(ptTrainJobCreateDTO)); + } + + @PutMapping + @ApiOperation("修改训练任务") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody updateTrainJob(@Validated @RequestBody PtTrainJobUpdateDTO ptTrainJobUpdateDTO) { + return new DataResponseBody(ptTrainJobService.updateTrainJob(ptTrainJobUpdateDTO)); + } + + @DeleteMapping + @ApiOperation("删除训练任务") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody deleteTrainJob(@Validated @RequestBody PtTrainJobDeleteDTO ptTrainJobDeleteDTO) { + return new DataResponseBody(ptTrainJobService.deleteTrainJob(ptTrainJobDeleteDTO)); + } + + @PostMapping("/stop") + @ApiOperation("停止训练任务") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody stopTrainJob(@Validated @RequestBody PtTrainJobStopDTO ptTrainJobStopDTO) { + return new DataResponseBody(ptTrainJobService.stopTrainJob(ptTrainJobStopDTO)); + } + + @PostMapping("/resume") + @ApiOperation("恢复训练任务") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody resumeTrainJob(@Validated @RequestBody PtTrainJobResumeDTO ptTrainJobResumeDTO) { + ptTrainJobService.resumeTrainJob(ptTrainJobResumeDTO); + return new DataResponseBody(); + } + + @GetMapping("/grafanaUrl/{jobId}") + @ApiOperation("获取job在grafana监控的地址") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody getGrafanaUrl( @PathVariable Long jobId) { + return new DataResponseBody(ptTrainJobService.getGrafanaUrl(jobId)); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java new file mode 100644 index 0000000..add7226 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.annotation.ApiVersion; +import org.dubhe.base.DataResponseBody; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.dto.PtTrainLogQueryDTO; +import org.dubhe.domain.vo.PtTrainLogQueryVO; +import org.dubhe.service.PtTrainLogService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * @description 训练日志 控制器入口 + * @date 2020-05-08 + */ +@Api(tags = "训练：训练日志") +@RestController +@ApiVersion(1) +@RequestMapping("/api/{version}/trainLog") +public class PtTrainLogController { + + @Autowired + private PtTrainLogService ptTrainLogService; + + @GetMapping + @ApiOperation("训练日志查询") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody getTrainLog(@Validated PtTrainLogQueryDTO ptTrainLogQueryDTO) { + return new DataResponseBody(ptTrainLogService.queryTrainLog(ptTrainLogQueryDTO)); + } + + @GetMapping("/download") + @ApiOperation("训练日志下载") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody downLoadTrainLog(@Validated PtTrainLogQueryDTO ptTrainLogQueryDTO) { + + ptTrainLogQueryDTO.setStartLine(MagicNumConstant.ONE).setLines(MagicNumConstant.MILLION); + + PtTrainLogQueryVO ptTrainLogQueryVO = ptTrainLogService.queryTrainLog(ptTrainLogQueryDTO); + return new DataResponseBody(ptTrainLogService.getTrainLogString(ptTrainLogQueryVO.getContent())); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java new file mode 100644 index 0000000..7b75ae9 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java @@ -0,0 +1,77 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.annotation.ApiVersion; +import org.dubhe.base.DataResponseBody; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.dto.PtTrainParamCreateDTO; +import org.dubhe.domain.dto.PtTrainParamDeleteDTO; +import org.dubhe.domain.dto.PtTrainParamQueryDTO; +import org.dubhe.domain.dto.PtTrainParamUpdateDTO; +import org.dubhe.service.PtTrainParamService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +/** + * @description 任务参数 + * @date 2020-04-27 + */ +@Api(tags = "训练：任务参数管理") +@RestController +@ApiVersion(1) +@RequestMapping("/api/{version}/trainParams") +public class PtTrainParamController { + + @Autowired + private PtTrainParamService ptTrainParamService; + + @GetMapping + @ApiOperation("任务参数列表展示") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody getTrainParam(@Validated PtTrainParamQueryDTO ptTrainParamQueryDTO) { + return new DataResponseBody(ptTrainParamService.getTrainParam(ptTrainParamQueryDTO)); + } + + @PostMapping + @ApiOperation("保存任务参数") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody createTrainParam(@Validated @RequestBody PtTrainParamCreateDTO ptTrainParamCreateDTO) { + return new DataResponseBody(ptTrainParamService.createTrainParam(ptTrainParamCreateDTO)); + } + + @PutMapping + @ApiOperation("修改任务参数") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody updateTrainParam(@Validated @RequestBody PtTrainParamUpdateDTO ptTrainParamUpdateDTO) { + return new DataResponseBody(ptTrainParamService.updateTrainParam(ptTrainParamUpdateDTO)); + } + + @DeleteMapping + @ApiOperation("删除任务参数") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody deleteTrainParam(@Validated @RequestBody PtTrainParamDeleteDTO ptTrainParamDeleteDTO) { + ptTrainParamService.deleteTrainParam(ptTrainParamDeleteDTO); + return new DataResponseBody(); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java new file mode 100644 index 0000000..abfe52d --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java @@ -0,0 +1,207 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.domain.entity.NoteBook; +import org.dubhe.domain.entity.NoteBookModel; +import org.dubhe.domain.dto.*; +import org.dubhe.domain.vo.NoteBookVO; +import org.dubhe.enums.BizNfsEnum; +import org.dubhe.enums.NoteBookStatusEnum; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @description notebook服务接口 + * @date 2020-04-28 + */ +public interface NoteBookService { + /** + * 分页查询所有notebook记录 + * + * @param page + * @param noteBookListQueryDTO + * @return Map + */ + Map getNoteBookList(Page page, NoteBookListQueryDTO noteBookListQueryDTO); + + /** + * 查询所有notebook记录 + * + * @param page + * @param noteBookQueryDTO + * @return List + */ + List getList(Page page, NoteBookQueryDTO noteBookQueryDTO); + + /** + * 新增加notebook + * + * @param noteBook + * @return NoteBookVO + */ + NoteBookVO createNoteBook(NoteBook noteBook); + + /** + * 验证notebook是否可删除 + * + * @param noteBookIds + * @return List + */ + List validateDeleteNoteBook(Set noteBookIds); + + /** + * 删除notebook异步方法 + * + * @param noteBookList + */ + void deleteNoteBooks(List noteBookList); + + + /** + * 启动notebook + * + * @param noteBookId + * @return String + */ + String startNoteBook(Long noteBookId); + + /** + * 停止notebook + * + * @param noteBookId + * @return String + */ + String stopNoteBook(Long noteBookId); + + /** + * update by ID + * + * @param noteBook + * @return noteBook + */ + NoteBook updateById(NoteBook noteBook); + + /** + * 打开notebook + * + * @param noteBookId + * @return String + */ + String openNoteBook(Long noteBookId); + + /** + * 获取notebook可访问URL + * + * @param noteBook + * @return String + */ + String getJupyterUrl(NoteBook noteBook); + + /** + * 查询notebook在K8s对应的状态 + * + * @param noteBook + * @return NoteBookStatusEnum + */ + NoteBookStatusEnum getStatus(NoteBook noteBook); + + /** + * 第三方创建notebook + * + * @param bizNfsEnum + * @param sourceNoteBookDTO + * @return NoteBookDTO + */ + NoteBookVO createNoteBookByThirdParty(BizNfsEnum bizNfsEnum, SourceNoteBookDTO sourceNoteBookDTO); + + /** + * 获取编辑地址 + * + * @param noteBookId + * @return String + */ + String getAddress(Long noteBookId); + + /** + * 删除PVC + * + * @param noteBook + * @return String + */ + String deletePvc(NoteBook noteBook); + + /** + * 获取状态编码 + * + * @return List + */ + List getNoteBookStatus(); + + /** + * 获取正在运行的notebook数量 + * + * @param curUserId + * @return int + */ + int getNoteBookRunNumber(long curUserId); + + /** + * 获取notebook模板 + * + * @return Map> + */ + Map> getNoteBookModel(); + + /** + * 刷新notebook对象状态 + * + * @param statusEnum + * @param noteBook + * @return true 无需更新 + */ + boolean refreshNoteBookStatus(NoteBookStatusEnum statusEnum, NoteBook noteBook); + + /** + * 根据notebook id更新训练id + * + * @param noteBookId + * @param algorithmId + * @return boolean + */ + boolean updateTrainIdByNoteBookId(Long noteBookId, Long algorithmId); + + /** + * 根据ID查询notebook详情 + * + * @param noteBookIds + * @return List + */ + List getNotebookDetail(Set noteBookIds); + + /** + * 获取已经运行却没有URL的notebook + * + * @param page + * @return List + */ + List getRunNotUrlList(Page page); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java new file mode 100644 index 0000000..6fd353f --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; +import org.springframework.scheduling.annotation.Async; + +/** + * @description Pod 异步回调处理接口 + * @date 2020-05-28 + */ +public interface PodCallbackAsyncService { + + /** + * pod 异步回调 + * @param k8sPodCallbackCreateDTO + */ + @Async + void podCallBack (R k8sPodCallbackCreateDTO); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtDatasetService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtDatasetService.java new file mode 100644 index 0000000..2fed740 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtDatasetService.java @@ -0,0 +1,91 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.domain.PtDataset; +import org.dubhe.domain.dto.PtDatasetDTO; +import org.dubhe.domain.dto.PtDatasetQueryCriteria; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * @description 查询数据 + * @date 2020-03-17 + */ +public interface PtDatasetService { + + /** + * 查询数据分页 + * + * @param criteria 条件 + * @param page 分页参数 + * @return Map + */ + Map queryAll(PtDatasetQueryCriteria criteria, Page page); + + /** + * 查询所有数据不分页 + * + * @param criteria 条件参数 + * @return List + */ + List queryAll(PtDatasetQueryCriteria criteria); + + /** + * 根据ID查询 + * + * @param id ID + * @return PtDatasetDto + */ + PtDatasetDTO findById(Long id); + + /** + * 创建 + * + * @param resources / + * @return PtDatasetDto + */ + PtDatasetDTO create(PtDataset resources); + + /** + * 编辑 + * + * @param resources / + */ + void update(PtDataset resources); + + /** + * 多选删除 + * + * @param ids / + */ + void deleteAll(Long[] ids); + + /** + * 导出数据 + * + * @param all 待导出的数据 + * @param response / + * @throws IOException / + */ + void download(List all, HttpServletResponse response) throws IOException; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtDevEnvsService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtDevEnvsService.java new file mode 100644 index 0000000..4eaa17d --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtDevEnvsService.java @@ -0,0 +1,91 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.domain.PtDevEnvs; +import org.dubhe.domain.dto.PtDevEnvsDTO; +import org.dubhe.domain.dto.PtDevEnvsQueryCriteria; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * @description 查询数据 + * @date 2020-03-17 + */ +public interface PtDevEnvsService { + + /** + * 查询数据分页 + * + * @param criteria 条件 + * @param page 分页参数 + * @return Map + */ + Map queryAll(PtDevEnvsQueryCriteria criteria, Page page); + + /** + * 查询所有数据不分页 + * + * @param criteria 条件参数 + * @return List + */ + List queryAll(PtDevEnvsQueryCriteria criteria); + + /** + * 根据ID查询 + * + * @param id ID + * @return PtDevEnvsDto + */ + PtDevEnvsDTO findById(Long id); + + /** + * 创建 + * + * @param resources / + * @return PtDevEnvsDto + */ + PtDevEnvsDTO create(PtDevEnvs resources); + + /** + * 编辑 + * + * @param resources / + */ + void update(PtDevEnvs resources); + + /** + * 多选删除 + * + * @param ids / + */ + void deleteAll(Long[] ids); + + /** + * 导出数据 + * + * @param all 待导出的数据 + * @param response / + * @throws IOException / + */ + void download(List all, HttpServletResponse response) throws IOException; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java new file mode 100644 index 0000000..f549ffd --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java @@ -0,0 +1,71 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.domain.dto.PtImageQueryDTO; +import org.dubhe.domain.dto.PtImageUploadDTO; +import org.dubhe.domain.entity.HarborProject; + +import java.util.List; +import java.util.Map; + +/** + * @description 镜像服务service + * @date 2020-06-22 + */ +public interface PtImageService { + + /** + * 查询镜像 + * + * @param ptImageQueryDTO 查询条件 + * @return Map 镜像列表分页信息 + **/ + Map getImage(PtImageQueryDTO ptImageQueryDTO); + + + /** + * 上传镜像到harbor + * + * @param ptImageUploadDTO 上传条件 + */ + void uploadImage(PtImageUploadDTO ptImageUploadDTO); + + + /** + * 定时到harbor同步imageName + */ + void harborImageNameSync(); + + + /** + * 通过imageName查询所含镜像版本信息 + * + * @param imageName 镜像名 + * @return List 镜像集合 + */ + List searchImages(String imageName); + + + /** + * 查询harbor镜像列表 + * + * @return List harbor镜像集合 + **/ + List getHarborProjectList(); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtStorageService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtStorageService.java new file mode 100644 index 0000000..ebbf862 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtStorageService.java @@ -0,0 +1,92 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.domain.PtStorage; +import org.dubhe.domain.dto.PtStorageDTO; +import org.dubhe.domain.dto.PtStorageQueryCriteria; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * @description 查询数据 + * @date 2020-03-17 + */ +public interface PtStorageService { + + /** + * 查询数据分页 + * + * @param criteria 条件 + * @param page 分页参数 + * @return Map + */ + Map queryAll(PtStorageQueryCriteria criteria, Page page); + + /** + * 查询所有数据不分页 + * + * @param criteria 条件参数 + * @return List + */ + List queryAll(PtStorageQueryCriteria criteria); + + /** + * 根据ID查询 + * + * @param id ID + * @return PtStorageDto + */ + PtStorageDTO findById(Long id); + + /** + * 创建 + * + * @param resources / + * @return PtStorageDto + */ + PtStorageDTO create(PtStorage resources); + + /** + * 编辑 + * + * @param resources / + */ + void update(PtStorage resources); + + /** + * 多选删除 + * + * @param ids / + */ + void deleteAll(Long[] ids); + + /** + * 导出数据 + * + * @param all 待导出的数据 + * @param response / + * @throws IOException / + */ + void download(List all, HttpServletResponse response) throws IOException; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java new file mode 100644 index 0000000..62e4d07 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java @@ -0,0 +1,71 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.domain.dto.PtTrainAlgorithmCreateDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmDeleteDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmQueryDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUpdateDTO; + +import java.util.List; +import java.util.Map; + +/** + * @description 训练算法 服务类 + * @date 2020-04-27 + */ +public interface PtTrainAlgorithmService { + + /** + * 查询数据分页 + * + * @param criteria 分页参数条件 + * @return Map map + */ + Map queryAll(PtTrainAlgorithmQueryDTO criteria); + + /** + * 新增算法 + * + * @param resources 新增算法条件 + * @return PtTrainAlgorithmCreateVO 新建训练算法 + */ + List create(PtTrainAlgorithmCreateDTO resources); + + /** + * 修改算法 + * + * @param resources 修改算法条件 + * @return PtTrainAlgorithmUpdateVO 修改训练算法 + */ + List update(PtTrainAlgorithmUpdateDTO resources); + + /** + * 删除算法 + * + * @param ptTrainAlgorithmDeleteDTO 删除算法条件 + */ + void deleteAll(PtTrainAlgorithmDeleteDTO ptTrainAlgorithmDeleteDTO); + + /** + * 查询当前用户的算法个数 + */ + Map getAlgorithmCount(); + + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java new file mode 100644 index 0000000..ac8c2d3 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java @@ -0,0 +1,62 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.domain.dto.PtTrainAlgorithmUsageDeleteDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageCreateDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageQueryDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageUpdateDTO; + +import java.util.List; +import java.util.Map; + +/** + * @description 算法用途 服务类 + * @date 2020-06-23 + */ +public interface PtTrainAlgorithmUsageService { + + /** + * 查询算法用途 + * + * @param ptTrainAlgorithmUsageQueryDTO 查询算法用途参数 + */ + Map queryAll(PtTrainAlgorithmUsageQueryDTO ptTrainAlgorithmUsageQueryDTO); + + /** + * 新增算法用途 + * + * @param ptTrainAlgorithmUsageCreateDTO 新增算法用途参数 + */ + List create(PtTrainAlgorithmUsageCreateDTO ptTrainAlgorithmUsageCreateDTO); + + /** + * 删除算法用途 + * + * @param ptTrainAlgorithmUsageDeleteDTO 删除算法用途参数 + */ + void deleteAll(PtTrainAlgorithmUsageDeleteDTO ptTrainAlgorithmUsageDeleteDTO); + + /** + * 更新算法用途 + * + * @param ptTrainAlgorithmUsageUpdateDTO 更新算法用途参数 + */ + void update(PtTrainAlgorithmUsageUpdateDTO ptTrainAlgorithmUsageUpdateDTO); + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java new file mode 100644 index 0000000..d42cb19 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java @@ -0,0 +1,125 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.domain.dto.*; +import org.dubhe.domain.vo.*; + +import java.util.List; +import java.util.Map; + +/** + * @description 训练作业job 服务类 + * @date 2020-04-27 + */ +public interface PtTrainJobService { + + + /** + * 作业列表展示 + * + * @param ptTrainQueryDTO 查询作业列表参数 + * @return Map 作业列表分页数据 + **/ + Map getTrainJob(PtTrainQueryDTO ptTrainQueryDTO); + + + /** + * 作业不同版本job列表展示 + * + * @param ptTrainJobVersionQueryDTO 查询作业不同版本job列表参数 + * @return List 训练版本查询详情集合 + **/ + List getTrainJobVersion(PtTrainJobVersionQueryDTO ptTrainJobVersionQueryDTO); + + + /** + * 创建训练job + * + * @param ptTrainJobCreateDTO 创建训练job参数 + * @return List id集合 + */ + List createTrainJobVersion(PtTrainJobCreateDTO ptTrainJobCreateDTO); + + + /** + * 修改训练job + * + * @param ptTrainJobUpdateDTO 修改训练job参数 + * @return List id集合 + **/ + List updateTrainJob(PtTrainJobUpdateDTO ptTrainJobUpdateDTO); + + + /** + * 删除训练job + * + * @param ptTrainJobDeleteDTO 删除训练job参数 + * @return PtTrainJobDeleteVO 返回删除训练任务结果 + **/ + PtTrainJobDeleteVO deleteTrainJob(PtTrainJobDeleteDTO ptTrainJobDeleteDTO); + + + /** + * 停止训练job + * + * @param ptTrainJobStopDTO 停止训练job参数 + * @return PtTrainJobStopVO 停止训练任务结果 + **/ + PtTrainJobStopVO stopTrainJob(PtTrainJobStopDTO ptTrainJobStopDTO); + + + /** + * 查询训练作业job状态 + * + * @param ptTrainDataSourceStatusQueryDTO 查询训练作业job状态参数 + * @return HashedMap 数据集路径-是否可以删除 的map集合 + **/ + Map getTrainDataSourceStatus(PtTrainDataSourceStatusQueryDTO ptTrainDataSourceStatusQueryDTO); + + + /** + * 我的训练任务统计 + * + * @return PtTrainJobStatisticsMineVO 统计信息 + **/ + PtTrainJobStatisticsMineVO statisticsMine(); + + /** + * 根据jobId查询训练任务详情查询 + * + * @param ptTrainJobDetailQueryDTO 根据jobId查询训练任务详情查询条件 + * @return PtTrainQueryJobDetailVO 根据jobId查询训练任务详情返回结果 + */ + PtTrainJobDetailQueryVO getTrainJobDetail(PtTrainJobDetailQueryDTO ptTrainJobDetailQueryDTO); + + /** + * 恢复训练 + * + * @param ptTrainJobResumeDTO 恢复训练请求参数 + */ + void resumeTrainJob(PtTrainJobResumeDTO ptTrainJobResumeDTO); + + /** + * 获取job在grafana监控的地址 + * + * @param jobId 任务ID + * @return PtJobMetricsGrafanaVO Pod Metrics Grafana url + */ + PtJobMetricsGrafanaVO getGrafanaUrl(Long jobId); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java new file mode 100644 index 0000000..ca069b1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.domain.dto.PtTrainJobSpecsQueryDTO; +import org.dubhe.domain.vo.PtTrainJobSpecsQueryVO; + +import java.util.List; + +/** + * @description 训练作业规格服务类 + * @date 2020-05-06 + */ +public interface PtTrainJobSpecsService { + + + /** + * 查询规格表 + * + * @param ptTrainJobSpecsQueryDTO 查询规格表参数 + * @return List list + **/ + List getTrainJobSpecs(PtTrainJobSpecsQueryDTO ptTrainJobSpecsQueryDTO); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java new file mode 100644 index 0000000..b4abf6d --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import java.util.List; + +import org.dubhe.domain.dto.PtTrainLogQueryDTO; +import org.dubhe.domain.vo.PtTrainLogQueryVO; + +/** + * @description 训练日志服务类 + * @date 2020-05-08 + */ +public interface PtTrainLogService { + + /** + * 查询训练任务运行日志 + * + * @param ptTrainLogQueryDTO 训练日志查询 + * @return PtTrainLogQueryVO 返回训练日志查询 + **/ + PtTrainLogQueryVO queryTrainLog(PtTrainLogQueryDTO ptTrainLogQueryDTO); + + /** + * + * 字符串换行 + * @param content 个数 + * @return String 字符串 + */ + String getTrainLogString(List content); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java new file mode 100644 index 0000000..232b22a --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.domain.dto.PtTrainParamCreateDTO; +import org.dubhe.domain.dto.PtTrainParamDeleteDTO; +import org.dubhe.domain.dto.PtTrainParamQueryDTO; +import org.dubhe.domain.dto.PtTrainParamUpdateDTO; + +import java.util.List; +import java.util.Map; + +/** + * @description 任务参数 服务类 + * @date 2020-04-27 + */ +public interface PtTrainParamService { + + /** + * 任务参数列表展示 + * + * @param ptTrainParamQueryDTO 任务参数列表展示条件 + * @return Map 任务参数列表分页数据 + **/ + Map getTrainParam(PtTrainParamQueryDTO ptTrainParamQueryDTO); + + /** + * 保存任务参数 + * + * @param ptTrainParamCreateDTO 保存任务参数条件 + * @return List 保存任务参数id集合 + **/ + List createTrainParam(PtTrainParamCreateDTO ptTrainParamCreateDTO); + + /** + * 修改任务参数 + * + * @param ptTrainParamUpdateDTO 修改任务参数条件 + * @return List 修改任务参数id集合 + **/ + List updateTrainParam(PtTrainParamUpdateDTO ptTrainParamUpdateDTO); + + /** + * 删除任务参数 + * + * @param ptTrainParamDeleteDTO 删除任务参数条件 + **/ + void deleteTrainParam(PtTrainParamDeleteDTO ptTrainParamDeleteDTO); + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java new file mode 100644 index 0000000..612816b --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java @@ -0,0 +1,82 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.abstracts; + +import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; +import org.dubhe.enums.LogEnum; +import org.dubhe.service.PodCallbackAsyncService; +import org.dubhe.utils.K8sCallBackTool; +import org.dubhe.utils.LogUtil; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * @description pod 异步回调抽象处理类 + * @date 2020-05-29 + */ +public abstract class AbstractPodCallback implements PodCallbackAsyncService { + + @Autowired + private K8sCallBackTool k8sCallBackTool; + + /** + * 公共 失败重试策略 + * + * @param k8sPodCallbackCreateDTO + * @param + */ + @Override + public void podCallBack(R k8sPodCallbackCreateDTO) { + int tryTime = 1; + while (!doCallback(tryTime,k8sPodCallbackCreateDTO)){ + if (k8sCallBackTool.continueRetry(++tryTime)){ + // 继续重试 tryTime重试次数+1 + try { + Thread.sleep(tryTime * 1000); + continue; + } catch (InterruptedException e) { + LogUtil.error(LogEnum.NOTE_BOOK,"AbstractPodCallback podCallBack InterruptedException", e); + // Restore interrupted state...       + Thread.currentThread().interrupt(); + } + }else { + // 重试超限 tryTime重试次数+1未尝试，因此需要tryTime重试次数-1 + callbackFailed(--tryTime,k8sPodCallbackCreateDTO); + break; + } + } + } + + /** + * pod 异步回调具体实现处理类 + * @param times 第n次处理 + * @param k8sPodCallbackCreateDTO k8s回调实体类 + * @param BaseK8sPodCallbackReq k8s回调基类 + * @return true：处理成功 false：处理失败 + */ + public abstract boolean doCallback(int times,R k8sPodCallbackCreateDTO); + + + /** + * pod 异步回调具体实现处理类 + * @param retryTimes 总处理次数 + * @param k8sPodCallbackCreateDTO k8s回调实体类 + * @param BaseK8sPodCallbackReq k8s回调基类 + */ + public abstract void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO); + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java new file mode 100644 index 0000000..4c1cef1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.convert; + +import org.dubhe.base.BaseConvert; +import org.dubhe.domain.entity.NoteBook; +import org.dubhe.domain.vo.NoteBookVO; +import org.mapstruct.Mapper; +import org.mapstruct.ReportingPolicy; + +/** + * @description notebook 转化器 + * @create 2020-04-28 + */ +@Mapper(componentModel = "spring", unmappedTargetPolicy = ReportingPolicy.IGNORE) +public interface NoteBookConvert extends BaseConvert { +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtDatasetConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtDatasetConvert.java new file mode 100644 index 0000000..87c986d --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtDatasetConvert.java @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.convert; + +import org.dubhe.base.BaseConvert; +import org.dubhe.domain.PtDataset; +import org.dubhe.domain.dto.PtDatasetDTO; +import org.mapstruct.Mapper; +import org.mapstruct.ReportingPolicy; + +/** + * @description 数据集 + * @date 2020-03-17 + */ +@Mapper(componentModel = "spring", unmappedTargetPolicy = ReportingPolicy.IGNORE) +public interface PtDatasetConvert extends BaseConvert { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtDevEnvsConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtDevEnvsConvert.java new file mode 100644 index 0000000..81f7688 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtDevEnvsConvert.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.convert; + + +import org.dubhe.base.BaseConvert; +import org.dubhe.domain.PtDevEnvs; +import org.dubhe.domain.dto.PtDevEnvsDTO; +import org.mapstruct.Mapper; +import org.mapstruct.ReportingPolicy; + +/** + * @description 开发环境 + * @date 2020-03-17 + */ +@Mapper(componentModel = "spring", unmappedTargetPolicy = ReportingPolicy.IGNORE) +public interface PtDevEnvsConvert extends BaseConvert { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtImageConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtImageConvert.java new file mode 100644 index 0000000..8cbd6af --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtImageConvert.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.convert; + + +import org.dubhe.base.BaseConvert; +import org.dubhe.domain.entity.PtImage; +import org.dubhe.domain.dto.PtImageDTO; +import org.mapstruct.Mapper; +import org.mapstruct.ReportingPolicy; + +/** + * @description 镜像 + * @date 2020-03-17 + */ +@Mapper(componentModel = "spring", unmappedTargetPolicy = ReportingPolicy.IGNORE) +public interface PtImageConvert extends BaseConvert { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java new file mode 100644 index 0000000..9c371e8 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.convert; + +import org.dubhe.base.MagicNumConstant; +import org.dubhe.enums.BizEnum; +import org.dubhe.k8s.domain.bo.PtJupyterResourceBO; +import org.dubhe.domain.entity.NoteBook; +import org.dubhe.utils.K8sNameTool; + +/** + * @description PtJupyterResourceBO 转换器 + * @date 2020-04-30 + */ +public class PtJupyterResourceConvert { + + private PtJupyterResourceConvert(){ + + } + + /** + * NoteBook 转换为 PtJupyterResourceBO + * + * @param noteBook + * @return PtJupyterResourceBO + */ + public static PtJupyterResourceBO toPtJupyterResourceBo(NoteBook noteBook,K8sNameTool k8sNameTool){ + if (noteBook == null){ + return null; + } + PtJupyterResourceBO bo = new PtJupyterResourceBO(); + bo.setNamespace(noteBook.getK8sNamespace()) + .setName(noteBook.getK8sResourceName()) + .setCpuNum(noteBook.getCpuNum()* MagicNumConstant.ONE_THOUSAND) + .setGpuNum(noteBook.getGpuNum()< MagicNumConstant.ONE?null:noteBook.getGpuNum()) + .setMemNum(noteBook.getMemNum()* MagicNumConstant.ONE_THOUSAND_TWENTY_FOUR) + .setImage(noteBook.getK8sImageName()) + .setWorkspaceDir(k8sNameTool.getAbsoluteNfsPath(noteBook.getK8sPvcPath())) + .setWorkspaceMountPath(noteBook.getK8sMountPath()) + // request和limit先一致 + .setWorkspaceRequest(noteBook.getDiskMemNum()*MagicNumConstant.ONE_THOUSAND_TWENTY_FOUR+"Mi") + .setWorkspaceLimit(noteBook.getDiskMemNum()*MagicNumConstant.ONE_THOUSAND_TWENTY_FOUR+"Mi") + .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.NOTEBOOK)) + .setDatasetDir(k8sNameTool.getAbsoluteNfsPath(noteBook.getDataSourcePath())) + .setDatasetMountPath(k8sNameTool.getDatasetPath()) + .setDatasetReadOnly(true) + ; + return bo; + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtStorageConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtStorageConvert.java new file mode 100644 index 0000000..4b39370 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtStorageConvert.java @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.convert; + + +import org.dubhe.base.BaseConvert; +import org.dubhe.domain.PtStorage; +import org.dubhe.domain.dto.PtStorageDTO; +import org.mapstruct.Mapper; +import org.mapstruct.ReportingPolicy; + +/** + * @description 存储类转化 + * @date 2020-03-17 + */ +@Mapper(componentModel = "spring", unmappedTargetPolicy = ReportingPolicy.IGNORE) +public interface PtStorageConvert extends BaseConvert { +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtTrainJobConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtTrainJobConvert.java new file mode 100644 index 0000000..433c3af --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtTrainJobConvert.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.convert; + + +import org.dubhe.base.BaseConvert; +import org.dubhe.domain.entity.PtTrainJob; +import org.dubhe.domain.dto.PtTrainJobDTO; +import org.mapstruct.Mapper; +import org.mapstruct.ReportingPolicy; + +/** + * @description 训练任务转化 + * @date 2020-03-17 + */ +@Mapper(componentModel = "spring", unmappedTargetPolicy = ReportingPolicy.IGNORE) +public interface PtTrainJobConvert extends BaseConvert { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java new file mode 100644 index 0000000..34fb1ae --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java @@ -0,0 +1,121 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.enums.TrainJobStatusEnum; +import org.dubhe.dao.PtTrainJobMapper; +import org.dubhe.domain.entity.PtTrainJob; +import org.dubhe.dto.callback.AlgorithmK8sPodCallbackCreateDTO; +import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; +import org.dubhe.enums.LogEnum; +import org.dubhe.service.PodCallbackAsyncService; +import org.dubhe.service.abstracts.AbstractPodCallback; +import org.dubhe.utils.K8sNameTool; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.TrainUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.concurrent.TimeUnit; + +/** + * @description 训练任务回调 + * @date 2020-06-03 + */ +@Service(value = "algorithmAsyncServiceImpl") +public class AlgorithmAsyncServiceImpl extends AbstractPodCallback implements PodCallbackAsyncService { + + @Autowired + private PtTrainJobMapper ptTrainJobMapper; + @Autowired + private K8sNameTool k8sNameTool; + + /** + * pod 异步回调具体实现处理类 + * + * @param times 第n次处理 + * @param k8sPodCallbackCreateDTO k8s回调实体类 + * @param BaseK8sPodCallbackReq k8s回调基类 + * @return boolean true：处理成功 false：处理失败 + */ + @Override + public boolean doCallback(int times, R k8sPodCallbackCreateDTO) { + // 强制转型 + AlgorithmK8sPodCallbackCreateDTO req = (AlgorithmK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; + LogUtil.info(LogEnum.BIZ_TRAIN, "Thread {} try {} time.Request: {}", Thread.currentThread(), times, req.toString()); + // 根据namespace和podName找到job + QueryWrapper queryTrainJonWrapper = new QueryWrapper<>(); + Long userId = k8sNameTool.getUserIdFromNameSpace(req.getNamespace()); + String podName = req.getPodName(); + if (null == podName || podName.length() <= MagicNumConstant.SIX) { + LogUtil.error(LogEnum.BIZ_TRAIN, "podName={} too short", podName); + return false; + } + String k8sJobName = podName.substring(MagicNumConstant.ZERO, podName.length() - MagicNumConstant.SIX); + queryTrainJonWrapper.eq("k8s_job_name", k8sJobName). + eq("create_user_id", userId); + PtTrainJob ptTrainJob = ptTrainJobMapper.selectOne(queryTrainJonWrapper); + if (null == ptTrainJob) { + LogUtil.error(LogEnum.BIZ_TRAIN, "k8s_job_name={} not found", k8sJobName); + return false; + } + String phase = req.getPhase(); + // 对于当前状态为结束状态或上报状态为删除的任务不做处理 + if (TrainJobStatusEnum.isEnd(ptTrainJob.getTrainStatus()) || TrainJobStatusEnum.DELETED.getMessage().equalsIgnoreCase(phase)) { + return true; + } + PtTrainJob updatePtTrainJob = new PtTrainJob(); + + // 更新job运行时间和状态 + updatePtTrainJob.setId(ptTrainJob.getId()) + .setTrainStatus(TrainJobStatusEnum.get(phase).getStatus()); + // 如果上报状态是结束状态并没指定过运行时间，则更新运行时间 + if (TrainJobStatusEnum.isEnd(phase) && "".equals(ptTrainJob.getRuntime())) { + long timeDelta = System.currentTimeMillis() - ptTrainJob.getCreateTime().getTime(); + String runTime = String.format(TrainUtil.RUNTIME, + TimeUnit.MILLISECONDS.toHours(timeDelta), + TimeUnit.MILLISECONDS.toMinutes(timeDelta) % TimeUnit.HOURS.toMinutes(1), + TimeUnit.MILLISECONDS.toSeconds(timeDelta) % TimeUnit.MINUTES.toSeconds(1) + ); + updatePtTrainJob.setRuntime(runTime); + } + int updateResult = ptTrainJobMapper.updateById(updatePtTrainJob); + if (updateResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "update trainJob_id={} failed, phase={}", ptTrainJob.getId(), req.getPhase()); + return false; + } + + return true; + } + + /** + * pod 异步回调具体实现处理类 + * + * @param retryTimes 总处理次数 + * @param k8sPodCallbackCreateDTO k8s回调实体类 + * @param BaseK8sPodCallbackReq k8s回调基类 + */ + @Override + public void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO) { + // 强制转型 + AlgorithmK8sPodCallbackCreateDTO req = (AlgorithmK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; + LogUtil.info(LogEnum.BIZ_TRAIN, "Thread {}try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java new file mode 100644 index 0000000..146c63b --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java @@ -0,0 +1,80 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import org.dubhe.dao.NoteBookMapper; +import org.dubhe.domain.entity.NoteBook; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.NoteBookStatusEnum; +import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; +import org.dubhe.dto.callback.NotebookK8sPodCallbackCreateDTO; +import org.dubhe.service.NoteBookService; +import org.dubhe.service.PodCallbackAsyncService; +import org.dubhe.service.abstracts.AbstractPodCallback; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.NotebookUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * @description 异步刪除接口实现 + * @date 2020-05-28 + */ +@Service(value = "noteBookAsyncServiceImpl") +public class NoteBookAsyncServiceImpl extends AbstractPodCallback implements PodCallbackAsyncService { + + @Autowired + private NoteBookService noteBookService; + + @Autowired + private NoteBookMapper noteBookMapper; + + + @Override + public boolean doCallback(int times, R k8sPodCallbackCreateDTO) { + try { + // 强制转型 + NotebookK8sPodCallbackCreateDTO req = (NotebookK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; + LogUtil.info(LogEnum.NOTE_BOOK, "NoteBookAsyncServiceImpl try {} time.Request: {}", times, req.toString()); + NoteBook notebook = noteBookMapper.findByNamespaceAndResourceName(req.getNamespace(), req.getResourceName(),NoteBookStatusEnum.DELETE.getCode()); + if (notebook == null) { + LogUtil.warn(LogEnum.NOTE_BOOK, "Cannot find notebook! Request: {}", Thread.currentThread(), times, req.toString()); + return true; + } + NoteBookStatusEnum statusEnum = NoteBookStatusEnum.convert(k8sPodCallbackCreateDTO.getPhase()); + if (noteBookService.refreshNoteBookStatus(statusEnum, notebook)) { + return true; + } + notebook.setK8sStatusCode(req.getPhase()); + notebook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(req.getMessages())); + noteBookService.updateById(notebook); + return true; + } catch (Exception e) { + LogUtil.error(LogEnum.NOTE_BOOK, "NoteBook doCallback error!", e); + return false; + } + } + + @Override + public void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO) { + // 强制转型 + NotebookK8sPodCallbackCreateDTO req = (NotebookK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; + LogUtil.info(LogEnum.NOTE_BOOK, "Thread {}try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); + // 目前利用定时补偿补充处理，无需做callbackFailed + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java new file mode 100644 index 0000000..0537a24 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java @@ -0,0 +1,739 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.util.RandomUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.commons.lang3.StringUtils; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.constant.SymbolConstant; +import org.dubhe.dao.NoteBookMapper; +import org.dubhe.dao.NoteBookModelMapper; +import org.dubhe.domain.dto.NoteBookListQueryDTO; +import org.dubhe.domain.dto.NoteBookQueryDTO; +import org.dubhe.domain.dto.NoteBookStatusDTO; +import org.dubhe.domain.dto.SourceNoteBookDTO; +import org.dubhe.domain.entity.NoteBook; +import org.dubhe.domain.entity.NoteBookModel; +import org.dubhe.domain.vo.NoteBookVO; +import org.dubhe.enums.BizEnum; +import org.dubhe.enums.BizNfsEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.NoteBookStatusEnum; +import org.dubhe.exception.NotebookBizException; +import org.dubhe.harbor.api.HarborApi; +import org.dubhe.k8s.api.PodApi; +import org.dubhe.k8s.api.JupyterResourceApi; +import org.dubhe.k8s.api.NamespaceApi; +import org.dubhe.k8s.domain.PtBaseResult; +import org.dubhe.k8s.domain.resource.BizNamespace; +import org.dubhe.k8s.domain.resource.BizPod; +import org.dubhe.k8s.domain.vo.PtJupyterDeployVO; +import org.dubhe.k8s.enums.K8sResponseEnum; +import org.dubhe.service.HarborProjectService; +import org.dubhe.service.NoteBookService; +import org.dubhe.service.convert.NoteBookConvert; +import org.dubhe.service.convert.PtJupyterResourceConvert; +import org.dubhe.utils.HttpUtils; +import org.dubhe.utils.K8sNameTool; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.NotebookUtil; +import org.dubhe.utils.NumberUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.WrapperHelp; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * @description notebook服务实现 + * @date 2020-04-28 + */ +@Service +public class NoteBookServiceImpl implements NoteBookService { + + @Autowired + private NoteBookMapper noteBookMapper; + + @Autowired + private NoteBookModelMapper noteBookModelMapper; + + @Autowired + private NoteBookConvert noteBookConvert; + + @Autowired + private JupyterResourceApi jupyterResourceApi; + + @Autowired + private PodApi podApi; + + @Autowired + private NamespaceApi namespaceApi; + + @Autowired + private K8sNameTool k8sNameTool; + + @Autowired + private HarborApi harborApi; + + @Autowired + private HarborProjectService harborProjectService; + + private static final String BLANK = SymbolConstant.BLANK; + + /** + * 分页查询所有 notebook 记录 + * + * @param page + * @param noteBookListQueryDTO + * @return Map + */ + @Override + public Map getNoteBookList(Page page, NoteBookListQueryDTO noteBookListQueryDTO) { + IPage noteBookPage = noteBookMapper.selectPage(page, WrapperHelp.getWrapper(noteBookListQueryDTO) + .ne(true, "status", NoteBookStatusEnum.DELETE.getCode()) + .ne(true, "deleted", NoteBookStatusEnum.STOP.getCode()) + .orderBy(true, false, "id")); + return PageUtil.toPage(noteBookPage, noteBookConvert::toDto); + } + + /** + * 查询所有 notebook 记录 + * + * @param page + * @param noteBookQueryDTO + * @return List + */ + @Override + public List getList(Page page, NoteBookQueryDTO noteBookQueryDTO) { + return noteBookMapper.selectPage(page, WrapperHelp.getWrapper(noteBookQueryDTO)).getRecords(); + } + + /** + * 获取镜像路径 + * + * @param bizEnum + * @return String + */ + private String getDefaultImage(BizEnum bizEnum) { + if (bizEnum == null) { + throw new NotebookBizException("业务模块未识别！无法获取默认镜像。"); + } + List projectList = harborProjectService.getHarborProjects(bizEnum.getCreateResource()); + if (CollUtil.isEmpty(projectList)) { + throw new NotebookBizException("此模块" + bizEnum.getBizName() + "未配置Project！无法获取默认镜像。"); + } + List imageList = harborApi.searchImageNames(projectList); + if (CollUtil.isEmpty(imageList)) { + throw new NotebookBizException("此模块" + bizEnum.getBizName() + "未配置镜像！"); + } + return imageList.get(MagicNumConstant.ZERO); + } + + /** + * 新增加 notebook + * + * @param noteBook + * @return NoteBookVO + */ + @Override + @Transactional(rollbackFor = Exception.class) + public NoteBookVO createNoteBook(NoteBook noteBook) { + if (noteBookMapper.findByNameAndUserId(noteBook.getNoteBookName(), noteBook.getCreateUserId(),NoteBookStatusEnum.DELETE.getCode()) != null) { + throw new NotebookBizException("Notebook名称已使用过！请重新提交。"); + } + if (StringUtils.isEmpty(noteBook.getName())) { + noteBook.setName(k8sNameTool.getK8sName()); + } + noteBook.setK8sNamespace(k8sNameTool.generateNameSpace(noteBook.getCreateUserId())); + noteBook.setK8sResourceName(k8sNameTool.generateResourceName(BizEnum.NOTEBOOK, noteBook.getName())); + if (StringUtils.isBlank(noteBook.getK8sPvcPath())) { + //20200618 修改为 使用训练路劲 + noteBook.setK8sPvcPath(k8sNameTool.getNfsPath(BizNfsEnum.ALGORITHM, noteBook.getCreateUserId())); + } + noteBook.setK8sMountPath(NotebookUtil.getK8sMountPath()); + if (start(noteBook)) { + noteBook.setStatus(NoteBookStatusEnum.STARTING.getCode()); + } else { + noteBook.setStatus(NoteBookStatusEnum.STOP.getCode()); + } + noteBookMapper.insert(noteBook); + return noteBookConvert.toDto(noteBook); + } + + /** + * 初始化namespace + * + * @param noteBook + * @param labels + * @return boolean + */ + private boolean initNameSpace(NoteBook noteBook, Map labels) { + try { + BizNamespace result = namespaceApi.create(noteBook.getK8sNamespace(), labels); + noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); + return (HttpUtils.isSuccess(result.getCode()) + || K8sResponseEnum.EXISTS.getCode().equals(result.getCode())); + } catch (Exception e) { + LogUtil.error(LogEnum.NOTE_BOOK, "createNoteBook调用jupyterResourceApi.createWithPvc异常！", e); + noteBook.setK8sStatusCode(BLANK); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); + return false; + } + } + + /** + * 验证 notebook 是否可删除 + * + * @param noteBookIds + * @return List + */ + @Override + public List validateDeleteNoteBook(Set noteBookIds) { + for (Long noteBookId : noteBookIds) { + NumberUtil.isNumber(noteBookId); + } + List deleteTypeList = NoteBookStatusEnum.getCanDeleteStatus(); + List noteBookList = noteBookMapper.selectBatchIds(noteBookIds); + for (NoteBook noteBook : noteBookList) { + if (deleteTypeList.contains(noteBook.getStatus())) { + throw new NotebookBizException("不可删除正在运行的notebook！"); + } + } + return noteBookList; + } + + /** + * 删除notebook异步方法 + * + * @param noteBookList + */ + @Override + @Async + @Transactional(rollbackFor = Exception.class) + public void deleteNoteBooks(List noteBookList) { + if (CollUtil.isNotEmpty(noteBookList)) { + for (NoteBook noteBook : noteBookList) { + if (noteBook.getStatus().equals(NoteBookStatusEnum.STOP.getCode())) { + deleteNoteBook(noteBook); + } + } + } + } + + /** + * 删除notebook实现逻辑 + * + * @param noteBook + * @return String + */ + private String deleteNoteBook(NoteBook noteBook) { + if (noteBook == null) { + throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); + } + String returnStr; + NoteBookStatusEnum statusEnum = getStatus(noteBook); + if (NoteBookStatusEnum.STOP == statusEnum) { + noteBook.setK8sStatusCode(BLANK); + noteBook.setK8sStatusInfo(BLANK); + noteBook.setUrl(BLANK); + returnStr = this.deletePvc(noteBook); + } else { + try { + PtBaseResult result = jupyterResourceApi.delete(noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); + noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); + if (HttpUtils.isSuccess(result.getCode())) { + noteBook.setStatus(NoteBookStatusEnum.DELETING.getCode()); + // 添加超时时间点 + noteBook.setLastOperationTimeout(NotebookUtil.getTimeoutSecondLong()); + noteBook.setUrl(BLANK); + returnStr = NoteBookStatusEnum.DELETING.getDescription(); + } else if (K8sResponseEnum.REPEAT.getCode().equals(result.getCode())) { + // 重复提交停止指令，无需再次停止，直接删除PVC文件 + noteBook.setUrl(BLANK); + returnStr = deletePvc(noteBook); + } else { + // 其他失败编码 -> 删除失败,保留原状态 + returnStr = "删除失败"; + } + } catch (Exception e) { + LogUtil.error(LogEnum.NOTE_BOOK, "deleteNoteBook调用jupyterResourceApi.delete异常！", e); + noteBook.setK8sStatusCode(BLANK); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); + returnStr = "删除失败"; + } + } + this.updateById(noteBook); + return returnStr; + } + + /** + * 启动notebook + * + * @param noteBookId + * @return String + */ + @Override + @Transactional(rollbackFor = Exception.class) + public String startNoteBook(Long noteBookId) { + NumberUtil.isNumber(noteBookId); + NoteBook noteBook = noteBookMapper.selectById(noteBookId); + return startNoteBook(noteBook); + } + + /** + * 具体启动notebook实现 + * + * @param noteBook + * @return String + */ + private String startNoteBook(NoteBook noteBook) { + if (noteBook == null) { + throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); + } + if (NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { + return "notebook " + NoteBookStatusEnum.RUN.getDescription(); + } else if (NoteBookStatusEnum.STARTING.getCode().equals(noteBook.getStatus())) { + return "notebook " + NoteBookStatusEnum.STARTING.getDescription(); + } else if (!NoteBookStatusEnum.STOP.getCode().equals(noteBook.getStatus())) { + throw new NotebookBizException("notebook【" + noteBook.getName() + "】当前状态：" + NoteBookStatusEnum.getDescription(noteBook.getStatus()) + ",无法再次启动。"); + } + String returnStr; + if (start(noteBook)) { + noteBook.setStatus(NoteBookStatusEnum.STARTING.getCode()); + returnStr = NoteBookStatusEnum.STARTING.getDescription(); + } else { + // 重启notebook状态沿用历史状态 + returnStr = "启动" + NotebookUtil.FAILED; + } + this.updateById(noteBook); + return returnStr; + } + + /** + * 更新notebook + * + * @param noteBook + * @return NoteBook + */ + @Override + @Transactional(rollbackFor = Exception.class) + public NoteBook updateById(NoteBook noteBook) { + noteBook.setUpdateTime(null); + noteBook.setUpdateUserId(NotebookUtil.getCurUserId()); + noteBookMapper.updateById(noteBook); + return noteBook; + } + + /** + * 启动notebook + * + * @param noteBook + * @return true 启动成功；false 启动失败 + */ + private boolean start(NoteBook noteBook) { + // 添加启动时间 + noteBook.setLastStartTime(new Date()); + // 添加超时时间点 + noteBook.setLastOperationTimeout(NotebookUtil.getTimeoutSecondLong()); + if (initNameSpace(noteBook, null)) { + try { + //20200618 修改为 创建时不创建PVC + PtJupyterDeployVO result = jupyterResourceApi.create(PtJupyterResourceConvert.toPtJupyterResourceBo(noteBook, k8sNameTool)); + noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); + return HttpUtils.isSuccess(result.getCode()); + } catch (Exception e) { + LogUtil.error(LogEnum.NOTE_BOOK, "notebook调用jupyterResourceApi.createWithPvc异常！", e); + noteBook.setK8sStatusCode(BLANK); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); + return false; + } + } else { + return false; + } + } + + /** + * 停止notebook + * + * @param noteBookId + * @return String + */ + @Override + @Transactional(rollbackFor = Exception.class) + public String stopNoteBook(Long noteBookId) { + NumberUtil.isNumber(noteBookId); + NoteBook noteBook = noteBookMapper.selectById(noteBookId); + if (noteBook == null) { + throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); + } + if (!NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { + throw new NotebookBizException("notebook正在运行,不能停止"); + } + String returnStr; + NoteBookStatusEnum statusEnum = getStatus(noteBook); + if (NoteBookStatusEnum.STOP == statusEnum) { + noteBook.setK8sStatusCode(BLANK); + noteBook.setK8sStatusInfo(BLANK); + noteBook.setUrl(BLANK); + returnStr = "已停止"; + } else { + try { + PtBaseResult result = jupyterResourceApi.delete(noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); + noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); + if (HttpUtils.isSuccess(result.getCode())) { + noteBook.setStatus(NoteBookStatusEnum.STOPPING.getCode()); + // 添加超时时间点 + noteBook.setLastOperationTimeout(NotebookUtil.getTimeoutSecondLong()); + noteBook.setUrl(BLANK); + returnStr = NoteBookStatusEnum.STOPPING.getDescription(); + } else if (K8sResponseEnum.REPEAT.getCode().equals(result.getCode())) { + // 重复提交停止指令，无需再次停止，直接标记停止 + noteBook.setStatus(NoteBookStatusEnum.STOP.getCode()); + noteBook.setUrl(BLANK); + returnStr = NoteBookStatusEnum.STOP.getDescription(); + } else { + // 其他失败编码 -> 停止失败,保留原状态 + returnStr = "停止" + NotebookUtil.FAILED; + } + } catch (Exception e) { + LogUtil.error(LogEnum.NOTE_BOOK, "停止notebook调用jupyterResourceApi.delete异常！", e); + noteBook.setK8sStatusCode(BLANK); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); + returnStr = "停止" + NotebookUtil.FAILED; + } + } + this.updateById(noteBook); + return returnStr; + } + + /** + * 开启notebook + * + * @param noteBookId + * @return String + */ + @Override + public String openNoteBook(Long noteBookId) { + NumberUtil.isNumber(noteBookId); + NoteBook noteBook = noteBookMapper.selectById(noteBookId); + if (noteBook == null) { + throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); + } else if (NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { + if (NotebookUtil.checkUrlContainsToken(noteBook.getUrl())) { + return noteBook.getUrl(); + } else { + // 补偿:已启动notebook获取可访问地址 + String jupyterUrlWithToken = this.getJupyterUrl(noteBook); + if (NotebookUtil.checkUrlContainsToken(jupyterUrlWithToken)) { + noteBook.setUrl(jupyterUrlWithToken); + this.updateById(noteBook); + return noteBook.getUrl(); + } else { + throw new NotebookBizException("notebook已启动 获取URL失败！"); + } + } + } else { + throw new NotebookBizException("notebook 尚未启动成功,无法打开。"); + } + } + + /** + * 获取jupyter 地址 + * + * @param noteBook + * @return String + */ + @Override + public String getJupyterUrl(NoteBook noteBook) { + try { + return podApi.getUrlByResourceName(noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); + } catch (Exception e) { + LogUtil.error(LogEnum.NOTE_BOOK, "notebook nameSpace:{} resourceName:{} 获取URL失败！", noteBook.getK8sNamespace(), noteBook.getK8sResourceName(), e); + noteBook.setK8sStatusCode(BLANK); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); + return null; + } + } + + /** + * 获取notebook状态 + * + * @param noteBook + * @return NoteBookStatusEnum + */ + @Override + public NoteBookStatusEnum getStatus(NoteBook noteBook) { + try { + BizPod result = podApi.getWithResourceName(noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); + noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); + if (K8sResponseEnum.NOT_FOUND.getCode().equals(result.getCode())) { + // 结果不存在当已停止 + return NoteBookStatusEnum.STOP; + } else if (!HttpUtils.isSuccess(result.getCode())) { + LogUtil.warn(LogEnum.NOTE_BOOK, "notebook nameSpace:{} resourceName:{} 查询失败！", noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); + return null; + } + return NoteBookStatusEnum.convert(result.getPhase()); + } catch (Exception e) { + LogUtil.error(LogEnum.NOTE_BOOK, "notebook nameSpace:{} resourceName:{} 查询异常！", noteBook.getK8sNamespace(), noteBook.getK8sResourceName(), e); + noteBook.setK8sStatusCode(BLANK); + noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); + return null; + } + } + + /** + * 第三方创建notebook + * + * @param bizNfsEnum + * @param sourceNoteBookDTO + * @return NoteBookVO + */ + @Override + public NoteBookVO createNoteBookByThirdParty(BizNfsEnum bizNfsEnum, SourceNoteBookDTO sourceNoteBookDTO) { + String k8sPvcPath = sourceNoteBookDTO.getSourceFilePath(); + NoteBook noteBook = noteBookMapper.selectOne(WrapperHelp.getWrapper(new NoteBookQueryDTO( + NoteBookStatusEnum.DELETE.getCode(), + k8sPvcPath, + sourceNoteBookDTO.getCurUserId()) + )); + if (noteBook == null) { + return this.createNoteBook(initSourceReqNoteBook(bizNfsEnum, sourceNoteBookDTO, k8sPvcPath)); + } else { + if (!NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { + this.startNoteBook(noteBook); + } + return noteBookConvert.toDto(noteBook); + } + } + + /** + * 初始化第三方请求的notebook + * + * @param bizNfsEnum + * @param sourceNoteBookDTO + * @param k8sPvcPath + * @return NoteBook + */ + private NoteBook initSourceReqNoteBook(BizNfsEnum bizNfsEnum, SourceNoteBookDTO sourceNoteBookDTO, String k8sPvcPath) { + NoteBook noteBook = new NoteBook(); + noteBook.setCreateUserId(sourceNoteBookDTO.getCurUserId()); + noteBook.setUserId(sourceNoteBookDTO.getCurUserId()); + noteBook.setCreateResource(bizNfsEnum.getCreateResource()); + noteBook.setDescription(bizNfsEnum.getBizName()); + noteBook.setName(k8sNameTool.getK8sName()); + String notebookName = NotebookUtil.generateName(bizNfsEnum, sourceNoteBookDTO.getSourceId()); + if (noteBookMapper.findByNameAndUserId(notebookName, noteBook.getCreateUserId(),NoteBookStatusEnum.DELETE.getCode()) != null) { + // 重名随机符号拼接 + notebookName += RandomUtil.randomString(MagicNumConstant.TWO); + } + + noteBook.setNoteBookName(notebookName); + noteBook.setCpuNum(MagicNumConstant.ONE); + noteBook.setGpuNum(MagicNumConstant.ZERO); + noteBook.setMemNum(MagicNumConstant.ONE); + noteBook.setDiskMemNum(MagicNumConstant.ONE); + noteBook.setAlgorithmId(sourceNoteBookDTO.getSourceId()); + + noteBook.setK8sPvcPath(k8sPvcPath); + noteBook.setK8sImageName(getDefaultImage(BizEnum.NOTEBOOK)); + return noteBook; + } + + /** + * 获取地址 + * + * @param noteBookId + * @return String + */ + @Override + @Transactional(rollbackFor = Exception.class) + public String getAddress(Long noteBookId) { + NumberUtil.isNumber(noteBookId); + NoteBook noteBook = noteBookMapper.selectById(noteBookId); + if (noteBook == null) { + throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); + } else if (NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { + if (NotebookUtil.checkUrlContainsToken(noteBook.getUrl())) { + return noteBook.getUrl(); + } else { + // 补偿:已启动notebook获取可访问地址 + String jupyterUrlWithToken = this.getJupyterUrl(noteBook); + if (NotebookUtil.checkUrlContainsToken(jupyterUrlWithToken)) { + noteBook.setUrl(jupyterUrlWithToken); + this.updateById(noteBook); + return noteBook.getUrl(); + } + } + } + return null; + } + + /** + * 删除PVC + * + * @param noteBook + * @return String + */ + @Override + public String deletePvc(NoteBook noteBook) { + noteBook.setStatus(NoteBookStatusEnum.DELETE.getCode()); + noteBook.setDeleted(MagicNumConstant.ONE); + return NoteBookStatusEnum.DELETE.getDescription(); + } + + /** + * 获取notebook所有状态 + * + * @return List + */ + @Override + public List getNoteBookStatus() { + List noteBookStatusDtoList = new ArrayList<>(); + for (NoteBookStatusEnum noteBookStatusEnum : NoteBookStatusEnum.values()) { + if (noteBookStatusEnum != NoteBookStatusEnum.DELETE) { + NoteBookStatusDTO noteBookStatusDTO = new NoteBookStatusDTO(); + noteBookStatusDTO.setStatusCode(noteBookStatusEnum.getCode()); + noteBookStatusDTO.setStatusName(noteBookStatusEnum.getDescription()); + noteBookStatusDtoList.add(noteBookStatusDTO); + } + } + return noteBookStatusDtoList; + } + + /** + * 获取正在运行的notebook数量 + * + * @param curUserId + * @return int + */ + @Override + public int getNoteBookRunNumber(long curUserId) { + return noteBookMapper.selectRunNoteBookNum(curUserId,NoteBookStatusEnum.RUN.getCode()); + } + + /** + * 获取notebook配置的模式数据 + * + * @return Map> + */ + @Override + public Map> getNoteBookModel() { + List noteBookModelList = noteBookModelMapper.selectAllNoteBookModel(); + return CollUtil.isEmpty(noteBookModelList) ? + new HashMap<>(MagicNumConstant.EIGHT) : + noteBookModelList.stream().collect(Collectors.groupingBy(NoteBookModel::getModelType)); + } + + /** + * 刷新notebook状态 + * + * @param statusEnum + * @param noteBook + * @return boolean + */ + @Override + public boolean refreshNoteBookStatus(NoteBookStatusEnum statusEnum, NoteBook noteBook) { + if (NoteBookStatusEnum.RUN == statusEnum) { + if (NoteBookStatusEnum.STARTING.getCode().equals(noteBook.getStatus())) { + // append jupyter url+token + noteBook.setUrl(this.getJupyterUrl(noteBook)); + // 仅启动中可切换为启动状态 + noteBook.setStatus(statusEnum.getCode()); + } else { + return true; + } + } else if (NoteBookStatusEnum.STOP == statusEnum) { + if (NoteBookStatusEnum.DELETING.getCode().equals(noteBook.getStatus())) { + this.deletePvc(noteBook); + // deletePVC方法内部已设置状态，无需这边设置 + } else { + noteBook.setStatus(statusEnum.getCode()); + } + noteBook.setUrl(BLANK); + } + return false; + } + + /** + * 支持双向更新，如果不为0 ，则是绑定关联关系，反之解除绑定关系 + * + * @param noteBookId + * @param algorithmId + * @return boolean + */ + @Override + @Transactional(rollbackFor = Exception.class) + public boolean updateTrainIdByNoteBookId(Long noteBookId, Long algorithmId) { + if (noteBookId != null) { + NumberUtil.isNumber(noteBookId); + NoteBook noteBook = noteBookMapper.selectById(noteBookId); + if (noteBook != null) { + noteBook.setAlgorithmId(algorithmId == null ? MagicNumConstant.ZERO : algorithmId); + noteBookMapper.updateById(noteBook); + return true; + } + } + return false; + } + + /** + * 获取notebook详情 + * + * @param noteBookIds + * @return List + */ + @Override + public List getNotebookDetail(Set noteBookIds) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.in("id",noteBookIds); + queryWrapper.eq("user_id",NotebookUtil.getCurUserId()); + queryWrapper.ne("status",NoteBookStatusEnum.DELETE.getCode()); + List noteBookList = noteBookMapper.selectList(queryWrapper); + return noteBookConvert.toDto(noteBookList); + } + + /** + * 获取正在运行却没有URL的notebook + * + * @param page + * @return List + */ + @Override + public List getRunNotUrlList(Page page) { + return noteBookMapper.selectRunNotUrlList(page,NoteBookStatusEnum.RUN.getCode()); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtDatasetServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtDatasetServiceImpl.java new file mode 100644 index 0000000..28fcced --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtDatasetServiceImpl.java @@ -0,0 +1,122 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.dao.PtDatasetMapper; +import org.dubhe.domain.PtDataset; +import org.dubhe.domain.dto.PtDatasetDTO; +import org.dubhe.domain.dto.PtDatasetQueryCriteria; +import org.dubhe.service.PtDatasetService; +import org.dubhe.service.convert.PtDatasetConvert; +import org.dubhe.utils.FileUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.StringUtils; +import org.dubhe.utils.WrapperHelp; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheConfig; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * @description 数据集管理 + * @date 2020-03-17 + */ +@Service +@CacheConfig(cacheNames = "ptDataset") +@Transactional(propagation = Propagation.SUPPORTS, readOnly = true, rollbackFor = Exception.class) +public class PtDatasetServiceImpl implements PtDatasetService { + + @Autowired + private PtDatasetMapper ptDatasetMapper; + @Autowired + private PtDatasetConvert ptDatasetConvert; + + + @Override + @Cacheable + public Map queryAll(PtDatasetQueryCriteria criteria, Page page) { + IPage ptDatasets = ptDatasetMapper.selectPage(page, WrapperHelp.getWrapper(criteria)); + return PageUtil.toPage(ptDatasets, ptDatasetConvert::toDto); + } + + @Override + @Cacheable + public List queryAll(PtDatasetQueryCriteria criteria) { + return ptDatasetConvert.toDto(ptDatasetMapper.selectList(WrapperHelp.getWrapper(criteria))); + } + + @Override + @Cacheable(key = "#p0") + public PtDatasetDTO findById(Long id) { + PtDataset ptDataset = ptDatasetMapper.selectById(id); + return ptDatasetConvert.toDto(ptDataset); + } + + @Override + @CacheEvict(allEntries = true) + @Transactional(rollbackFor = Exception.class) + public PtDatasetDTO create(PtDataset resources) { + ptDatasetMapper.insert(resources); + return ptDatasetConvert.toDto(resources); + } + + @Override + @CacheEvict(allEntries = true) + @Transactional(rollbackFor = Exception.class) + public void update(PtDataset resources) { + PtDataset ptDataset = ptDatasetMapper.selectById(resources.getId()); + ptDataset.copy(resources); + ptDatasetMapper.updateById(ptDataset); + } + + @Override + @CacheEvict(allEntries = true) + public void deleteAll(Long[] ids) { + for (Long id : ids) { + ptDatasetMapper.deleteById(id); + } + } + + @Override + public void download(List all, HttpServletResponse response) throws IOException { + List> list = new ArrayList<>(); + for (PtDatasetDTO ptDataset : all) { + Map map = new LinkedHashMap<>(); + map.put(" name", ptDataset.getName()); + map.put(" remark", ptDataset.getRemark()); + map.put(" type", ptDataset.getType()); + map.put(" team", ptDataset.getTeam()); + map.put(" createUser", ptDataset.getCreateUser()); + map.put(" createTime", ptDataset.getCreateTime()); + list.add(map); + } + FileUtil.downloadExcel(list, response); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtDevEnvsServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtDevEnvsServiceImpl.java new file mode 100644 index 0000000..09d43f9 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtDevEnvsServiceImpl.java @@ -0,0 +1,124 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.dao.PtDevEnvsMapper; +import org.dubhe.domain.PtDevEnvs; +import org.dubhe.domain.dto.PtDevEnvsDTO; +import org.dubhe.domain.dto.PtDevEnvsQueryCriteria; +import org.dubhe.service.PtDevEnvsService; +import org.dubhe.service.convert.PtDevEnvsConvert; +import org.dubhe.utils.FileUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.StringUtils; +import org.dubhe.utils.WrapperHelp; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheConfig; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * @description 开发环境 + * @date 2020-03-17 + */ +@Service +@CacheConfig(cacheNames = "ptDevEnvs") +@Transactional(propagation = Propagation.SUPPORTS, readOnly = true, rollbackFor = Exception.class) +public class PtDevEnvsServiceImpl implements PtDevEnvsService { + + @Autowired + private PtDevEnvsMapper ptDevEnvsMapper; + private PtDevEnvsConvert ptDevEnvsConvert; + + + @Override + @Cacheable + public Map queryAll(PtDevEnvsQueryCriteria criteria, Page page) { + IPage ptDevEnvss = ptDevEnvsMapper.selectPage(page, WrapperHelp.getWrapper(criteria)); + return PageUtil.toPage(ptDevEnvss, ptDevEnvsConvert::toDto); + } + + @Override + @Cacheable + public List queryAll(PtDevEnvsQueryCriteria criteria) { + return ptDevEnvsConvert.toDto(ptDevEnvsMapper.selectList(WrapperHelp.getWrapper(criteria))); + } + + @Override + @Cacheable(key = "#p0") + public PtDevEnvsDTO findById(Long id) { + PtDevEnvs ptDevEnvs = ptDevEnvsMapper.selectById(id); + return ptDevEnvsConvert.toDto(ptDevEnvs); + } + + @Override + @CacheEvict(allEntries = true) + @Transactional(rollbackFor = Exception.class) + public PtDevEnvsDTO create(PtDevEnvs resources) { + ptDevEnvsMapper.insert(resources); + return ptDevEnvsConvert.toDto(resources); + } + + @Override + @CacheEvict(allEntries = true) + @Transactional(rollbackFor = Exception.class) + public void update(PtDevEnvs resources) { + PtDevEnvs ptDevEnvs = ptDevEnvsMapper.selectById(resources.getId()); + ptDevEnvs.copy(resources); + ptDevEnvsMapper.updateById(ptDevEnvs); + } + + @Override + @CacheEvict(allEntries = true) + public void deleteAll(Long[] ids) { + for (Long id : ids) { + ptDevEnvsMapper.deleteById(id); + } + } + + @Override + public void download(List all, HttpServletResponse response) throws IOException { + List> list = new ArrayList<>(); + for (PtDevEnvsDTO ptDevEnvs : all) { + Map map = new LinkedHashMap<>(); + map.put(" name", ptDevEnvs.getName()); + map.put(" remark", ptDevEnvs.getRemark()); + map.put(" podnum", ptDevEnvs.getPodNum()); + map.put(" gpunum", ptDevEnvs.getGpuNum()); + map.put(" memnum", ptDevEnvs.getMemNum()); + map.put(" cpunum", ptDevEnvs.getCpuNum()); + map.put(" duration", ptDevEnvs.getDuration()); + map.put(" startTime", ptDevEnvs.getStartTime()); + map.put(" closeTime", ptDevEnvs.getCloseTime()); + list.add(map); + } + FileUtil.downloadExcel(list, response); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java new file mode 100644 index 0000000..05040fc --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java @@ -0,0 +1,284 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import cn.hutool.core.util.StrUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.base.ResponseCode; +import org.dubhe.config.TrainHarborConfig; +import org.dubhe.config.NfsConfig; +import org.dubhe.dao.HarborProjectMapper; +import org.dubhe.dao.PtImageMapper; +import org.dubhe.data.constant.Constant; +import org.dubhe.domain.dto.PtImageQueryDTO; +import org.dubhe.domain.dto.PtImageUploadDTO; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.HarborProject; +import org.dubhe.domain.entity.PtImage; +import org.dubhe.domain.vo.PtImageQueryVO; +import org.dubhe.enums.HarborResourceEnum; +import org.dubhe.enums.ImageSourceEnum; +import org.dubhe.enums.ImageStateEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.harbor.api.HarborApi; +import org.dubhe.service.PtImageService; +import org.dubhe.task.HarborImagePushAsync; +import org.dubhe.utils.*; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * @description 镜像服务实现类 + * @date 2020-06-22 + */ +@Service +public class PtImageServiceImpl implements PtImageService { + + @Autowired + private PtImageMapper ptImageMapper; + + @Autowired + private HarborProjectMapper harborProjectMapper; + + @Autowired + private HarborApi harborApi; + + @Autowired + private NfsConfig nfsConfig; + + @Autowired + private HarborImagePushAsync imagePushAsync; + + @Autowired + private TrainHarborConfig trainHarborConfig; + + public final static List filedNames; + + static { + filedNames = ReflectionUtils.getFieldNames(PtImageQueryVO.class); + } + + /** + * 查询镜像 + * + * @param ptImageQueryDTO 查询镜像条件 + * @return Map 返回镜像分页数据 + **/ + @Override + public Map getImage(PtImageQueryDTO ptImageQueryDTO) { + + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} query image list display begins, and the received parameters are :{}", user.getId(), ptImageQueryDTO); + Page page = ptImageQueryDTO.toPage(); + //镜像来源判断，默认为0(我的镜像) + if (ptImageQueryDTO.getImageResource() == null) { + ptImageQueryDTO.setImageResource(ImageSourceEnum.MINE.getCode()); + } + QueryWrapper query = new QueryWrapper<>(); + if (ImageSourceEnum.MINE.getCode().equals(ptImageQueryDTO.getImageResource())) { + query.eq("create_user_id", user.getId()); + } + if (ptImageQueryDTO.getImageStatus() != null) { + query.eq("image_status", ptImageQueryDTO.getImageStatus()); + } + query.eq("image_resource", ptImageQueryDTO.getImageResource()); + //排序 + IPage ptImages; + try { + if (ptImageQueryDTO.getSort() != null && filedNames.contains(ptImageQueryDTO.getSort())) { + if (Constant.SORT_ASC.equalsIgnoreCase(ptImageQueryDTO.getOrder())) { + query.orderByAsc(StringUtils.humpToLine(ptImageQueryDTO.getSort())); + } else { + query.orderByDesc(StringUtils.humpToLine(ptImageQueryDTO.getSort())); + } + } else { + query.orderByDesc(Constant.ID); + } + ptImages = ptImageMapper.selectPage(page, query); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} query mirror list display exception :{}, request information :{}", e, ptImageQueryDTO); + throw new BusinessException("查询镜像列表展示异常"); + } + List ptImageQueryResult = ptImages.getRecords().stream().map(x -> { + PtImageQueryVO ptImageQueryVO = new PtImageQueryVO(); + BeanUtils.copyProperties(x, ptImageQueryVO); + return ptImageQueryVO; + }).collect(Collectors.toList()); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query mirror list display ends, the result is {}", user.getId(), ptImageQueryResult); + return PageUtil.toPage(page, ptImageQueryResult); + } + + /** + * 上传镜像到harbor + * + * @param ptImageUploadDTO 上传条件 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void uploadImage(PtImageUploadDTO ptImageUploadDTO) { + LogUtil.info(LogEnum.BIZ_TRAIN, "Upload image to harbor to receive parameters :{}", ptImageUploadDTO); + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + QueryWrapper query = new QueryWrapper<>(); + query.eq("image_name", ptImageUploadDTO.getImageName()); + Integer harborProjectCountResult = harborProjectMapper.selectCount(query); + if (harborProjectCountResult < 1) { + LogUtil.info(LogEnum.BIZ_TRAIN, "The imageName for uploading the image is [{}] not configured", ptImageUploadDTO.getImageName()); + + throw new BusinessException(ResponseCode.SUCCESS, "上传镜像的harborProject未配置!"); + } + + //校验用户自定义镜像不能和预置镜像重名 + List resList = checkUploadImage(ptImageUploadDTO, currentUser, ImageSourceEnum.MINE.getCode()); + if (!CollectionUtils.isEmpty(resList)) { + LogUtil.info(LogEnum.BIZ_TRAIN, "Preset image [{}:{}] already exists, no repeat allowed!", ptImageUploadDTO.getImageName(), ptImageUploadDTO.getImageTag()); + throw new BusinessException(ResponseCode.SUCCESS, "不允许和预置镜像信息重复，请重新上传!"); + } + + //同一用户上传镜像的(userId+imageName+imageTag)存在的情况下是不能重复上传的 + List imageList = checkUploadImage(ptImageUploadDTO, currentUser, ImageSourceEnum.PRE.getCode()); + if (!CollectionUtils.isEmpty(imageList) && ImageStateEnum.SUCCESS.getCode().equals(imageList.get(0).getImageStatus())) { + LogUtil.info(LogEnum.BIZ_TRAIN, "The mirror [id:{}] already exists", imageList.get(0).getId()); + throw new BusinessException(ResponseCode.SUCCESS, "镜像信息已存在，不允许重复上传!"); + } + + String harborImagePath = trainHarborConfig.getModelName() + StrUtil.SLASH + ptImageUploadDTO.getImageName() + + StrUtil.COLON + ptImageUploadDTO.getImageTag() + StrUtil.DASHED + currentUser.getId(); + //存储镜像信息 + PtImage ptImage = new PtImage(); + ptImage.setImageName(ptImageUploadDTO.getImageName()) + .setProjectName(trainHarborConfig.getModelName()) + .setImageUrl(harborImagePath) + .setImageResource(ImageSourceEnum.MINE.getCode()) + .setImageStatus(ImageStateEnum.MAKING.getCode()) + .setRemark(ptImageUploadDTO.getRemark()) + .setImageTag(ptImageUploadDTO.getImageTag()) + .setCreateUserId(currentUser.getId()); + int count = ptImageMapper.insert(ptImage); + if (count < 1) { + imagePushAsync.updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} failed to store image information!", currentUser.getUsername()); + throw new BusinessException("内部错误!"); + } + //shell脚本上传镜像 + try { + String imagePath = nfsConfig.getRootDir() + nfsConfig.getBucket().substring(1) + ptImageUploadDTO.getImagePath(); + String imageNameAndTag = ptImageUploadDTO.getImageName() + StrUtil.COLON + ptImageUploadDTO.getImageTag() + StrUtil.DASHED + currentUser.getId(); + imagePushAsync.execShell(imagePath, imageNameAndTag, ptImage); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "Image upload exception :{}", e); + throw new BusinessException("镜像上传失败!"); + } + } + + /** + *定时到harbor同步projectName + */ + @Override + public void harborImageNameSync() { + //每天晚上11点定时去harbor同步项目名到表harbor_project + QueryWrapper query = new QueryWrapper<>(); + List imageNames = harborApi.searchImageByProjects(Arrays.asList(trainHarborConfig.getModelName())); + Set imageList = new HashSet<>(); + imageNames.forEach(image -> { + imageList.add((String) image.get("imageName")); + }); + query.in("image_name", imageList); + List harborProjects = harborProjectMapper.selectList(query); + harborProjects.forEach(harborProject -> { + imageList.removeIf(image -> image.contains(harborProject.getImageName())); + }); + + HarborProject project = new HarborProject(); + project.setCreateResource(HarborResourceEnum.TRAIN_SYNC.getCode()); + project.setSyncStatus(TrainUtil.NUMBER_ONE); + imageList.forEach(imageName -> { + project.setImageName(imageName); + harborProjectMapper.insert(project); + }); + } + + /** + * 查询所含镜像版本信息 + * + * @param imageName 镜像名 + * @return List 通过imageName查询所含镜像版本信息 + */ + @Override + public List searchImages(String imageName) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("image_name", imageName); + queryWrapper.eq("image_status", ImageStateEnum.SUCCESS.getCode()); + List ptImages = ptImageMapper.selectList(queryWrapper); + if (CollectionUtils.isEmpty(ptImages)) { + LogUtil.info(LogEnum.BIZ_TRAIN, "No mirroring information for imageName is :{}", imageName); + throw new BusinessException(ResponseCode.SUCCESS, "未查询到镜像信息!"); + } + List list = new ArrayList<>(); + ptImages = ptImages.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> + new TreeSet<>(Comparator.comparing(PtImage::getImageTag))), ArrayList::new)); + ptImages.stream().forEach(ptImage -> { + list.add(ptImage.getImageTag()); + }); + return list; + } + + /** + * 查询项目列表 + * + * @return List 获取Harbor + **/ + @Override + public List getHarborProjectList() { + LogUtil.info(LogEnum.BIZ_TRAIN, "Query the mirror project list..."); + QueryWrapper queryWrapper = new QueryWrapper<>(); + List list = Arrays.asList(TrainUtil.NUMBER_ONE, TrainUtil.NUMBER_TWO); + queryWrapper.in("create_resource", list); + return harborProjectMapper.selectList(queryWrapper); + } + + /** + * @param ptImageUploadDTO 镜像上传逻辑校验 + * @param user 用户 + * @param source 来源 + * @return List 镜像列表 + **/ + private List checkUploadImage(PtImageUploadDTO ptImageUploadDTO, UserDTO user, int source) { + + QueryWrapper queryWrapper = new QueryWrapper<>(); + if (ImageSourceEnum.PRE.getCode().equals(source)) { + queryWrapper.eq("create_user_id", user.getId()); + } else { + queryWrapper.eq("image_resource", ImageSourceEnum.PRE.getCode()); + } + queryWrapper.eq("image_name", ptImageUploadDTO.getImageName()); + queryWrapper.eq("image_tag", ptImageUploadDTO.getImageTag()); + List imageList = ptImageMapper.selectList(queryWrapper); + return imageList; + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtStorageServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtStorageServiceImpl.java new file mode 100644 index 0000000..8426079 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtStorageServiceImpl.java @@ -0,0 +1,123 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.dao.PtStorageMapper; +import org.dubhe.domain.PtStorage; +import org.dubhe.domain.dto.PtStorageDTO; +import org.dubhe.domain.dto.PtStorageQueryCriteria; +import org.dubhe.service.PtStorageService; +import org.dubhe.service.convert.PtStorageConvert; +import org.dubhe.utils.FileUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.StringUtils; +import org.dubhe.utils.WrapperHelp; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.CacheConfig; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * @description 存储服务类 + * @date 2020-03-17 + */ +@Service +@CacheConfig(cacheNames = "ptStorage") +@Transactional(propagation = Propagation.SUPPORTS, readOnly = true, rollbackFor = Exception.class) +public class PtStorageServiceImpl implements PtStorageService { + + @Autowired + private PtStorageMapper ptStorageMapper; + + @Autowired + private PtStorageConvert ptStorageConvert; + + + @Override + @Cacheable + public Map queryAll(PtStorageQueryCriteria criteria, Page page) { + IPage ptStorages = ptStorageMapper.selectPage(page, WrapperHelp.getWrapper(criteria)); + return PageUtil.toPage(ptStorages, ptStorageConvert::toDto); + } + + @Override + @Cacheable + public List queryAll(PtStorageQueryCriteria criteria) { + return ptStorageConvert.toDto(ptStorageMapper.selectList(WrapperHelp.getWrapper(criteria))); + } + + @Override + @Cacheable(key = "#p0") + public PtStorageDTO findById(Long id) { + PtStorage ptStorage = ptStorageMapper.selectById(id); + return ptStorageConvert.toDto(ptStorage); + } + + @Override + @CacheEvict(allEntries = true) + @Transactional(rollbackFor = Exception.class) + public PtStorageDTO create(PtStorage resources) { + ptStorageMapper.insert(resources); + return ptStorageConvert.toDto(resources); + } + + @Override + @CacheEvict(allEntries = true) + @Transactional(rollbackFor = Exception.class) + public void update(PtStorage resources) { + PtStorage ptStorage = ptStorageMapper.selectById(resources.getId()); + ptStorage.copy(resources); + ptStorageMapper.updateById(ptStorage); + } + + @Override + @CacheEvict(allEntries = true) + public void deleteAll(Long[] ids) { + for (Long id : ids) { + ptStorageMapper.deleteById(id); + } + } + + @Override + public void download(List all, HttpServletResponse response) throws IOException { + List> list = new ArrayList<>(); + for (PtStorageDTO ptStorage : all) { + Map map = new LinkedHashMap<>(); + map.put(" name", ptStorage.getName()); + map.put(" size", ptStorage.getSize()); + map.put(" storageclass", ptStorage.getStorageclass()); + map.put(" createUser", ptStorage.getCreateUser()); + map.put(" createTime", ptStorage.getCreateTime()); + list.add(map); + } + FileUtil.downloadExcel(list, response); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java new file mode 100644 index 0000000..999dfbe --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java @@ -0,0 +1,462 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import cn.hutool.core.util.RandomUtil; +import cn.hutool.core.util.StrUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.map.HashedMap; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.base.ResponseCode; +import org.dubhe.config.NfsConfig; +import org.dubhe.constant.AlgorithmSourceEnum; +import org.dubhe.constant.TrainAlgorithmConstant; +import org.dubhe.dao.NoteBookMapper; +import org.dubhe.dao.PtImageMapper; +import org.dubhe.dao.PtTrainAlgorithmMapper; +import org.dubhe.data.constant.Constant; +import org.dubhe.domain.dto.*; +import org.dubhe.domain.entity.NoteBook; +import org.dubhe.domain.entity.PtImage; +import org.dubhe.domain.entity.PtTrainAlgorithm; +import org.dubhe.domain.vo.PtTrainAlgorithmQueryVO; +import org.dubhe.enums.BizNfsEnum; +import org.dubhe.enums.ImageSourceEnum; +import org.dubhe.enums.ImageStateEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.service.NoteBookService; +import org.dubhe.service.PtTrainAlgorithmService; +import org.dubhe.utils.*; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * @description 训练算法 服务实现类 + * @date 2020-04-27 + */ +@Service +public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { + + @Autowired + private PtTrainAlgorithmMapper ptTrainAlgorithmMapper; + + @Autowired + private PtImageMapper ptImageMapper; + + @Autowired + private NfsUtil nfsUtil; + + @Autowired + private K8sNameTool k8sNameTool; + + @Autowired + private NfsConfig nfsConfig; + + @Autowired + private TrainAlgorithmConstant trainAlgorithmConstant; + + @Autowired + private NoteBookService noteBookService; + + @Autowired + private NoteBookMapper noteBookMapper; + + public final static List filedNames; + + static { + filedNames = ReflectionUtils.getFieldNames(PtTrainAlgorithmQueryVO.class); + } + + /** + * 查询数据分页 + * + * @param ptTrainAlgorithmQueryDTO 条件 + * @return Map 返回查询数据 + */ + @Override + public Map queryAll(PtTrainAlgorithmQueryDTO ptTrainAlgorithmQueryDTO) { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "The display of user {} query training algorithm list begins, and the parameters received are {}.", user.getId(), ptTrainAlgorithmQueryDTO); + //当算法来源为空时，设置默认算法来源 + if (ptTrainAlgorithmQueryDTO.getAlgorithmSource() == null) { + ptTrainAlgorithmQueryDTO.setAlgorithmSource(trainAlgorithmConstant.getAlgorithmSource()); + } + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("algorithm_source", ptTrainAlgorithmQueryDTO.getAlgorithmSource()); + //判断算法来源 + if (AlgorithmSourceEnum.MINE.getStatus().equals(ptTrainAlgorithmQueryDTO.getAlgorithmSource())) { + wrapper.eq("create_user_id", user.getId()); + } + //根据算法用途筛选 + if (ptTrainAlgorithmQueryDTO.getAlgorithmUsage() != null) { + wrapper.like("algorithm_usage", ptTrainAlgorithmQueryDTO.getAlgorithmUsage()); + } + if (!StringUtils.isEmpty(ptTrainAlgorithmQueryDTO.getAlgorithmName())) { + wrapper.and(qw -> qw.eq("id", ptTrainAlgorithmQueryDTO.getAlgorithmName()).or().like("algorithm_name", + ptTrainAlgorithmQueryDTO.getAlgorithmName())); + } + + Page page = ptTrainAlgorithmQueryDTO.toPage(); + IPage ptTrainAlgorithms; + try { + if (ptTrainAlgorithmQueryDTO.getSort() != null && filedNames.contains(ptTrainAlgorithmQueryDTO.getSort())) { + if (Constant.SORT_ASC.equalsIgnoreCase(ptTrainAlgorithmQueryDTO.getOrder())) { + wrapper.orderByAsc(StringUtils.humpToLine(ptTrainAlgorithmQueryDTO.getSort())); + } else { + wrapper.orderByDesc(StringUtils.humpToLine(ptTrainAlgorithmQueryDTO.getSort())); + } + } else { + wrapper.orderByDesc(Constant.ID); + } + ptTrainAlgorithms = ptTrainAlgorithmMapper.selectPage(page, wrapper); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "Query training algorithm list display exceptions :{}, request information :{}", e, + ptTrainAlgorithmQueryDTO); + throw new BusinessException("查询训练算法列表展示异常"); + } + List ptTrainAlgorithmQueryResult = ptTrainAlgorithms.getRecords().stream().map(x -> { + PtTrainAlgorithmQueryVO ptTrainAlgorithmQueryVO = new PtTrainAlgorithmQueryVO(); + BeanUtils.copyProperties(x, ptTrainAlgorithmQueryVO); + //获取镜像名称与版本 + getImageNameAndImageTag(x, ptTrainAlgorithmQueryVO); + return ptTrainAlgorithmQueryVO; + }).collect(Collectors.toList()); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query training algorithm list display ends, the result is {}", user.getUsername(), ptTrainAlgorithmQueryResult); + return PageUtil.toPage(page, ptTrainAlgorithmQueryResult); + } + + /** + * 新增算法 + * + * @param ptTrainAlgorithmCreateDTO 新增算法条件 + * @return idList 返回新增算法 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public List create(PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO) { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "Save the new algorithm and receive the parameter {}", ptTrainAlgorithmCreateDTO); + // 校验path + if (!(k8sNameTool.validateBizNfsPath(ptTrainAlgorithmCreateDTO.getCodeDir(), BizNfsEnum.ALGORITHM))) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} passed in the path {} is not valid", user.getUsername(), ptTrainAlgorithmCreateDTO.getCodeDir()); + throw new BusinessException("路径名称不合法"); + } + //获取镜像url + if (StringUtils.isNotBlank(ptTrainAlgorithmCreateDTO.getImageName()) && StringUtils.isNotBlank(ptTrainAlgorithmCreateDTO.getImageTag())) { + ptTrainAlgorithmCreateDTO.setImageName(getImages(ptTrainAlgorithmCreateDTO, user)); + } + //创建算法校验DTO并设置默认值 + setAlgorithmDtoDefault(ptTrainAlgorithmCreateDTO); + //算法路径 + String path = nfsConfig.getBucket() + ptTrainAlgorithmCreateDTO.getCodeDir(); + if (nfsUtil.fileOrDirIsEmpty(path)) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} upload path {} does not exist", user.getUsername(), path); + throw new BusinessException("算法文件或路径不存在"); + } + //保存算法 + PtTrainAlgorithm ptTrainAlgorithm = new PtTrainAlgorithm(); + BeanUtils.copyProperties(ptTrainAlgorithmCreateDTO, ptTrainAlgorithm); + ptTrainAlgorithm.setAlgorithmSource(AlgorithmSourceEnum.MINE.getStatus()) + .setCreateUserId(user.getId()); + + //算法名称校验 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("algorithm_name", ptTrainAlgorithmCreateDTO.getAlgorithmName()); + queryWrapper.eq("create_user_id", user.getId()); + Integer countResult = ptTrainAlgorithmMapper.selectCount(queryWrapper); + //如果是通过【保存至算法】接口创建算法，名称重复可用随机数生成新算法名，待后续客户自主修改 + if (countResult > 0) { + if (ptTrainAlgorithmCreateDTO.getNoteBookId() != null) { + String randomStr = RandomUtil.randomNumbers(MagicNumConstant.FOUR); + ptTrainAlgorithm.setAlgorithmName(ptTrainAlgorithmCreateDTO.getAlgorithmName() + randomStr); + } else { + LogUtil.error(LogEnum.BIZ_TRAIN, "The algorithm name ({}) already exists", ptTrainAlgorithmCreateDTO.getAlgorithmName()); + throw new BusinessException("算法名称已存在，请重新输入"); + } + } + //校验path是否带有压缩文件，如有，则解压至当前文件夹并删除压缩文件 + if (path.toLowerCase().endsWith(Constant.COMPRESS_ZIP)) { + unZip(user, path, ptTrainAlgorithm); + } + //校验创建算法来源(true:由fork创建算法，false：其它创建算法方式),若为true则拷贝预置算法文件至新路径 + if (ptTrainAlgorithmCreateDTO.getFork()) { + //生成算法相对路径 + String algorithmPath = k8sNameTool.getNfsPath(BizNfsEnum.ALGORITHM, user.getId()); + //拷贝预置算法文件夹 + boolean copyResult = nfsUtil.copyPath(path, nfsConfig.getBucket() + algorithmPath); + if (!copyResult) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} copied the preset algorithm path {} successfully", user.getUsername(), path); + throw new BusinessException("内部错误"); + } + ptTrainAlgorithm.setCodeDir(algorithmPath); + } + + try { + //算法未保存成功，抛出异常，并返回失败信息 + ptTrainAlgorithmMapper.insert(ptTrainAlgorithm); + + //保存算法根据notbookId更新算法id + if (ptTrainAlgorithmCreateDTO.getNoteBookId() != null) { + LogUtil.info(LogEnum.BIZ_TRAIN, "Save algorithm Update algorithm ID :{} according to notBookId:{}", ptTrainAlgorithmCreateDTO.getNoteBookId(), ptTrainAlgorithm.getId()); + noteBookService.updateTrainIdByNoteBookId(ptTrainAlgorithmCreateDTO.getNoteBookId(), ptTrainAlgorithm.getId()); + } + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} saving algorithm was not successful. Failure reason :{}", user.getUsername(), e.getMessage()); + throw new BusinessException("算法未保存成功"); + } + + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} saves the new algorithm and returns the new algorithm id as {}.", user.getUsername(), ptTrainAlgorithm.getId()); + return Collections.singletonList(ptTrainAlgorithm.getId()); + } + + /** + * 修改算法 + * + * @param ptTrainAlgorithmUpdateDTO 修改算法条件 + * @return PtTrainAlgorithmUpdateVO 返回修改算法 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public List update(PtTrainAlgorithmUpdateDTO ptTrainAlgorithmUpdateDTO) { + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} modifies the algorithm and receives {} as the parameter", currentUser.getUsername(), ptTrainAlgorithmUpdateDTO); + //权限校验 + PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(ptTrainAlgorithmUpdateDTO.getId()); + if (null == ptTrainAlgorithm || ptTrainAlgorithm.getCreateUserId().compareTo(currentUser.getId()) != 0) { + LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for the user {} to modify the algorithm with id {}", currentUser.getUsername(), ptTrainAlgorithmUpdateDTO.getId()); + throw new BusinessException(ResponseCode.SUCCESS, "您修改的算法不存在或已被删除"); + } + PtTrainAlgorithm updatePtAlgorithm = new PtTrainAlgorithm(); + updatePtAlgorithm.setId(ptTrainAlgorithm.getId()).setUpdateUserId(currentUser.getId()); + //判断是否修改算法名称 + if (StringUtils.isNotBlank(ptTrainAlgorithmUpdateDTO.getAlgorithmName())) { + //算法名称校验 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("algorithm_name", ptTrainAlgorithmUpdateDTO.getAlgorithmName()) + .eq("create_user_id", currentUser.getId()) + .ne("id", ptTrainAlgorithmUpdateDTO.getId()); + Integer countResult = ptTrainAlgorithmMapper.selectCount(queryWrapper); + if (countResult > 0) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The algorithm name ({}) already exists", ptTrainAlgorithmUpdateDTO.getAlgorithmName()); + throw new BusinessException("算法名称已存在，请重新输入"); + } + updatePtAlgorithm.setAlgorithmName(ptTrainAlgorithmUpdateDTO.getAlgorithmName()); + } + //判断是否修改算法描述 + if (ptTrainAlgorithmUpdateDTO.getDescription() != null) { + updatePtAlgorithm.setDescription(ptTrainAlgorithmUpdateDTO.getDescription()); + } + //判断是否修改算法用途 + if (ptTrainAlgorithmUpdateDTO.getAlgorithmUsage() != null) { + updatePtAlgorithm.setAlgorithmUsage(ptTrainAlgorithmUpdateDTO.getAlgorithmUsage()); + } + //判断是否修改输出日志 + if (ptTrainAlgorithmUpdateDTO.getIsTrainLog() != null) { + updatePtAlgorithm.setIsTrainLog(ptTrainAlgorithmUpdateDTO.getIsTrainLog()); + } + //判断是否修改可视化日志 + if (ptTrainAlgorithmUpdateDTO.getIsVisualizedLog() != null) { + updatePtAlgorithm.setIsVisualizedLog(ptTrainAlgorithmUpdateDTO.getIsVisualizedLog()); + } + try { + //算法未修改成功，抛出异常，并返回失败信息 + ptTrainAlgorithmMapper.updateById(updatePtAlgorithm); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to modify the algorithm. Pt_train_algorithm table modification operation failed. Failure reason :{}", currentUser.getUsername(), e.getMessage()); + throw new BusinessException("修改失败"); + } + LogUtil.info(LogEnum.BIZ_TRAIN, "End of user {} modification algorithm, return modification algorithm ID={}", currentUser.getUsername(), ptTrainAlgorithm.getId()); + return Collections.singletonList(ptTrainAlgorithm.getId()); + } + + /** + * 解压缩zip压缩包 + * + * @param user 用户 + * @param path 文件路径 + * @param ptTrainAlgorithm 算法参数 + */ + private void unZip(UserDTO user, String path, PtTrainAlgorithm ptTrainAlgorithm) { + String[] pathArray = path.split(StrUtil.SLASH); + String pathSuffix = pathArray[pathArray.length - 1]; + String targetPath = path.replace(pathSuffix, ""); + //上传路径垃圾文件清理 + Boolean aBoolean = nfsUtil.cleanPath(path, targetPath); + if (!aBoolean) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to clean up {} garbage", user.getUsername(), targetPath); + } + Boolean unzip = nfsUtil.unzip(path, targetPath); + if (!unzip) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to unzip", user.getUsername()); + throw new BusinessException("内部错误"); + } + //算法路径 + ptTrainAlgorithm.setCodeDir(StrUtil.SLASH + path.replace(nfsConfig.getBucket(), "").replace(pathSuffix, "")); + } + + /** + * 删除算法 + * + * @param ptTrainAlgorithmDeleteDTO 删除算法条件 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void deleteAll(PtTrainAlgorithmDeleteDTO ptTrainAlgorithmDeleteDTO) { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} delete algorithm, the parameter received is {}", user.getUsername(), ptTrainAlgorithmDeleteDTO.getIds()); + Set idList = ptTrainAlgorithmDeleteDTO.getIds(); + //权限校验 + QueryWrapper query = new QueryWrapper<>(); + query.eq("create_user_id", user.getId()); + query.in("id", idList); + Integer queryCountResult = ptTrainAlgorithmMapper.selectCount(query); + if (queryCountResult < idList.size()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} delete algorithm failed, no permission to delete the corresponding data in the algorithm table", user.getUsername()); + throw new BusinessException(ResponseCode.SUCCESS, "您删除的ID不存在或已被删除"); + } + int deleteCountResult = ptTrainAlgorithmMapper.deleteBatchIds(idList); + if (deleteCountResult < idList.size()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} deletion algorithm failed, and the algorithm table deletion operation based on the ID array {} failed", user.getUsername(), ptTrainAlgorithmDeleteDTO.getIds()); + throw new BusinessException(ResponseCode.SUCCESS, "删除算法未成功"); + } + //同步更新noteBook表中algorithmId=0 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("user_id", user.getId()); + queryWrapper.in("algorithm_id", idList); + List noteBookList = noteBookMapper.selectList(queryWrapper); + if (!CollectionUtils.isEmpty(noteBookList)) { + noteBookList.forEach(noteBook -> { + noteBookService.updateTrainIdByNoteBookId(noteBook.getId(), null); + }); + } + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} delete algorithm end, delete algorithm ID array IDS ={}", user.getUsername(), idList); + } + + /** + * 查询算法个数 + * + * @return count 返回个数 + */ + @Override + public Map getAlgorithmCount() { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} queries his algorithm number", user.getUsername()); + QueryWrapper wrapper = new QueryWrapper(); + wrapper.eq("create_user_id", user.getId()); + wrapper.eq("algorithm_source", AlgorithmSourceEnum.MINE.getStatus()); + Integer countResult = ptTrainAlgorithmMapper.selectCount(wrapper); + return new HashedMap() {{ + put("count", countResult); + }}; + } + + /** + * 获取镜像名称与版本 + * + * @param trainAlgorithm 镜像URL + * @param ptTrainAlgorithmQueryVO 镜像名称与版本 + */ + private void getImageNameAndImageTag(PtTrainAlgorithm trainAlgorithm, PtTrainAlgorithmQueryVO ptTrainAlgorithmQueryVO) { + if (StringUtils.isNotBlank(trainAlgorithm.getImageName())) { + String imageNameSuffix = trainAlgorithm.getImageName().substring(trainAlgorithm.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); + String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); + ptTrainAlgorithmQueryVO.setImageName(imageNameSuffixArray[0]); + ptTrainAlgorithmQueryVO.setImageTag(imageNameSuffixArray[1]); + } + } + + + /** + * 创建算法DTO校验并设置默认值 + * + * @param dto 校验DTO + **/ + private void setAlgorithmDtoDefault(PtTrainAlgorithmCreateDTO dto) { + + //设置fork默认值 + if (dto.getFork() == null) { + dto.setFork(trainAlgorithmConstant.getFork()); + } + //设置是否输出训练日志 + if (dto.getIsTrainLog() == null) { + dto.setIsTrainLog(trainAlgorithmConstant.getIsTrainLog()); + } + //设置是否输出训练结果 + if (dto.getIsTrainOut() == null) { + dto.setIsTrainOut(trainAlgorithmConstant.getIsTrainOut()); + } + //设置是否输出可视化日志 + if (dto.getIsVisualizedLog() == null) { + dto.setIsVisualizedLog(trainAlgorithmConstant.getIsVisualizedLog()); + } + } + + /** + * 获取镜像url + * + * @param ptTrainAlgorithmCreateDTO 获取镜像 + * @param user 用户 + * @return String 返回镜像路径 + **/ + private String getImages(PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO, UserDTO user) { + //获取镜像url + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("image_name", ptTrainAlgorithmCreateDTO.getImageName()); + queryWrapper.eq("image_tag", ptTrainAlgorithmCreateDTO.getImageTag()); + queryWrapper.eq("image_status", ImageStateEnum.SUCCESS.getCode()); + List ptImages = ptImageMapper.selectList(queryWrapper); + if (CollectionUtils.isEmpty(ptImages)) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} gets image ,the imageName is {}, the imageTag is {}, and the result of query image table (PT_image) is empty", user.getUsername(), ptTrainAlgorithmCreateDTO.getImageName(), ptTrainAlgorithmCreateDTO.getImageTag()); + throw new BusinessException("镜像不存在"); + } + //获取镜像为用户自定义镜像或预置镜像，且两者自身不能重复 + if (ptImages.size() > MagicNumConstant.TWO) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} got more images than scheduled, the imageName provided is {} and the imageTag is {}. The parameters are illegal", user.getUsername(), ptTrainAlgorithmCreateDTO.getImageName(), ptTrainAlgorithmCreateDTO.getImageTag()); + throw new BusinessException("镜像不合法"); + } + for (PtImage ptImage : ptImages) { + if (ImageSourceEnum.PRE.getCode().equals(ptImage.getImageResource())) { + ptTrainAlgorithmCreateDTO.setImageName(ptImage.getImageUrl()); + } else if (user.getId().equals(ptImage.getCreateUserId())) { + ptTrainAlgorithmCreateDTO.setImageName(ptImage.getImageUrl()); + } + } + if (StringUtils.isBlank(ptTrainAlgorithmCreateDTO.getImageName())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} gets image, the imageName provided is {} and the imageTag is {}. The parameters are illegal", user.getUsername(), ptTrainAlgorithmCreateDTO.getImageName(), ptTrainAlgorithmCreateDTO.getImageTag()); + throw new BusinessException("镜像不合法"); + } + return ptTrainAlgorithmCreateDTO.getImageName(); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java new file mode 100644 index 0000000..0a209cd --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java @@ -0,0 +1,177 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.base.ResponseCode; +import org.dubhe.dao.PtTrainAlgorithmUsageMapper; +import org.dubhe.domain.dto.*; +import org.dubhe.domain.entity.PtTrainAlgorithmUsage; +import org.dubhe.domain.vo.PtTrainAlgorithmUsageQueryVO; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.service.PtTrainAlgorithmUsageService; +import org.dubhe.utils.JwtUtils; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.WrapperHelp; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @description 用途管理 服务实现类 + * @date 2020-06-23 + */ +@Service +public class PtTrainAlgorithmUsageServiceImpl implements PtTrainAlgorithmUsageService { + + @Autowired + private PtTrainAlgorithmUsageMapper ptTrainAlgorithUsagemMapper; + + /** + * 算法用途 + * + * @param ptTrainAlgorithmUsageQueryDTO 查询算法用途参数 + * @return Map 返回查询算法用途分页 + */ + @Override + public Map queryAll(PtTrainAlgorithmUsageQueryDTO ptTrainAlgorithmUsageQueryDTO) { + + // 从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + QueryWrapper wrapper = WrapperHelp.getWrapper(ptTrainAlgorithmUsageQueryDTO); + + Page page = ptTrainAlgorithmUsageQueryDTO.toPage(); + IPage ptTrainAlgorithms = null; + + if (ptTrainAlgorithmUsageQueryDTO.getIsContainDefault()) { + wrapper.and(qw -> qw.eq("user_id", user.getId()).or().eq("is_default", + ptTrainAlgorithmUsageQueryDTO.getIsContainDefault())); + } else { + wrapper.eq("user_id", user.getId()); + } + + wrapper.eq("type", ptTrainAlgorithmUsageQueryDTO.getType()); + + ptTrainAlgorithms = ptTrainAlgorithUsagemMapper.selectPage(page, wrapper); + + List ptTrainAlgorithmUsageQueryResult = ptTrainAlgorithms.getRecords().stream() + .map(x -> { + PtTrainAlgorithmUsageQueryVO ptTrainAlgorithmUsageQueryVO = new PtTrainAlgorithmUsageQueryVO(); + BeanUtils.copyProperties(x, ptTrainAlgorithmUsageQueryVO); + return ptTrainAlgorithmUsageQueryVO; + }).collect(Collectors.toList()); + + return PageUtil.toPage(page, ptTrainAlgorithmUsageQueryResult); + } + + /** + * 新增算法用途 + * + * @param ptTrainAlgorithmUsageCreateDTO 新增算法用途参数 + * @return List 返回新增算法用途 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public List create(PtTrainAlgorithmUsageCreateDTO ptTrainAlgorithmUsageCreateDTO) { + UserDTO user = JwtUtils.getCurrentUserDto(); + PtTrainAlgorithmUsage ptTrainAlgorithmUsage = new PtTrainAlgorithmUsage(); + ptTrainAlgorithmUsage.setAuxInfo(ptTrainAlgorithmUsageCreateDTO.getAuxInfo()) + .setType(ptTrainAlgorithmUsageCreateDTO.getType()).setUserId(user.getId()); + + int insertResult = ptTrainAlgorithUsagemMapper.insert(ptTrainAlgorithmUsage); + + if (insertResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} secondary information was not saved successfully", user.getUsername()); + throw new BusinessException("用户辅助信息未保存成功"); + } + return Collections.singletonList(ptTrainAlgorithmUsage.getId()); + } + + /** + * 删除算法用途 + * + * @param ptTrainAlgorithmUsageDeleteDTO 删除算法用途参数 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void deleteAll(PtTrainAlgorithmUsageDeleteDTO ptTrainAlgorithmUsageDeleteDTO) { + UserDTO user = JwtUtils.getCurrentUserDto(); + Set idList = Stream.of(ptTrainAlgorithmUsageDeleteDTO.getIds()).collect(Collectors.toSet()); + QueryWrapper query = new QueryWrapper<>(); + query.eq("user_id", user.getId()); + query.in("id", idList); + Integer queryCountResult = ptTrainAlgorithUsagemMapper.selectCount(query); + + if (queryCountResult < idList.size()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete user secondary. No permissions to delete corresponding data in user secondary table", user.getUsername()); + throw new BusinessException(ResponseCode.SUCCESS, "您删除的ID不存在或已被删除"); + } + int deleteCountResult = ptTrainAlgorithUsagemMapper.deleteBatchIds(idList); + + if (deleteCountResult < idList.size()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete user assistance information. User service deletion based on id array {} failed", user.getUsername(), ptTrainAlgorithmUsageDeleteDTO.getIds()); + throw new BusinessException(ResponseCode.SUCCESS, "内部错误"); + } + + } + + /** + *更新算法用途 + * + * @param ptTrainAlgorithmUsageUpdateDTO 更新算法用途参数 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void update(PtTrainAlgorithmUsageUpdateDTO ptTrainAlgorithmUsageUpdateDTO) { + UserDTO user = JwtUtils.getCurrentUserDto(); + + QueryWrapper query = new QueryWrapper<>(); + query.eq("user_id", user.getId()); + query.in("id", ptTrainAlgorithmUsageUpdateDTO.getId()); + Integer queryIntResult = ptTrainAlgorithUsagemMapper.selectCount(query); + + if (queryIntResult < MagicNumConstant.ONE) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to modify user auxiliary information and has no permission to modify the corresponding data in the algorithm table", user.getUsername()); + throw new BusinessException(ResponseCode.SUCCESS, "您修改的ID不存在或已被删除,请重新输入"); + } + PtTrainAlgorithmUsage ptTrainAlgorithmUsage = new PtTrainAlgorithmUsage(); + ptTrainAlgorithmUsage.setId(ptTrainAlgorithmUsageUpdateDTO.getId()); + ptTrainAlgorithmUsage.setAuxInfo(ptTrainAlgorithmUsageUpdateDTO.getAuxInfo()); + + int updateResult = ptTrainAlgorithUsagemMapper.updateById(ptTrainAlgorithmUsage); + if (updateResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to modify user assistance information", user.getUsername()); + throw new BusinessException("内部错误"); + } + + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java new file mode 100644 index 0000000..8393072 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java @@ -0,0 +1,1123 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.util.StrUtil; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.map.HashedMap; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.base.ResponseCode; +import org.dubhe.config.TrainHarborConfig; +import org.dubhe.constant.AlgorithmSourceEnum; +import org.dubhe.constant.TrainJobConstant; +import org.dubhe.dao.*; +import org.dubhe.data.constant.Constant; +import org.dubhe.domain.dto.*; +import org.dubhe.domain.entity.*; +import org.dubhe.domain.vo.*; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.TrainJobStatusEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.k8s.api.PersistentVolumeClaimApi; +import org.dubhe.k8s.api.PodApi; +import org.dubhe.k8s.api.TrainJobApi; +import org.dubhe.k8s.domain.PtBaseResult; +import org.dubhe.k8s.domain.resource.BizPod; +import org.dubhe.service.PtTrainJobService; +import org.dubhe.task.TransactionAsyncManager; +import org.dubhe.utils.*; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * @description 训练作业job服务实现类 + * @date 2020-04-27 + */ +@Service +public class PtTrainJobServiceImpl implements PtTrainJobService { + + @Autowired + private PtTrainMapper ptTrainMapper; + + @Autowired + private PtTrainJobMapper ptTrainJobMapper; + + @Autowired + private PtJobParamMapper ptJobParamMapper; + + @Autowired + private PtTrainParamMapper ptTrainParamMapper; + + @Autowired + private PtTrainAlgorithmMapper ptTrainAlgorithmMapper; + + @Autowired + private TrainJobApi trainJobApi; + + @Autowired + private PersistentVolumeClaimApi persistentVolumeClaimApi; + + @Autowired + private PtTrainJobSpecsMapper ptTrainJobSpecsMapper; + + @Autowired + private PodApi podApi; + + @Autowired + private TrainJobConstant trainJobConstant; + + @Autowired + private NfsUtil nfsUtil; + + @Autowired + private TrainHarborConfig trainHarborConfig; + + @Autowired + private K8sNameTool k8sNameTool; + + @Autowired + private ImageUtil imageUtil; + + @Autowired + private TransactionAsyncManager asyncManager; + + public final static List filedNames; + + static { + filedNames = ReflectionUtils.getFieldNames(PtTrainVO.class); + } + + /** + * 作业列表展示 + * + * @param ptTrainQueryDTO 查询作业列表参数 + * @return Map 作业列表分页信息 + **/ + @Override + public Map getTrainJob(@NonNull PtTrainQueryDTO ptTrainQueryDTO) { + Page pageTrainResult; + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query job list display, received parameters are {}", currentUser.getUsername(), ptTrainQueryDTO); + + Page page = ptTrainQueryDTO.toPage(); + String order; + String sort; + try { + //排序方式 + order = Constant.SORT_ASC.equalsIgnoreCase(ptTrainQueryDTO.getOrder()) ? Constant.SORT_ASC : Constant.SORT_DESC; + //排序字段 + String sortField = filedNames.contains(ptTrainQueryDTO.getSort()) ? ptTrainQueryDTO.getSort() : Constant.ID; + sort = StringUtils.humpToLine(sortField); + pageTrainResult = ptTrainJobMapper.getPageTrain(page, currentUser.getId(), ptTrainQueryDTO.getTrainStatus(), ptTrainQueryDTO.getTrainName(), sort, order); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "Query job list shows exception {}, receive parameter {}", e, ptTrainQueryDTO); + throw new BusinessException("查询作业列表展示异常"); + } + List trainResult = pageTrainResult.getRecords(); + if (CollectionUtils.isNotEmpty(trainResult)) { + LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} query job list is displayed and the result is as follows {}.", currentUser.getUsername(), trainResult); + } + return PageUtil.toPage(page, trainResult); + + } + + /** + * 计算job训练时长 + * + * @param bizPod pod信息 + * @return String 训练时长 + */ + private String calculateRuntime(BizPod bizPod) { + return calculateRuntime(bizPod, (x) -> { + }); + } + + /** + * 计算job训练时长 + * + * @param bizPod + * @param consumer pod已经完成状态的回调函数 + * @return res 返回训练时长 + */ + private String calculateRuntime(BizPod bizPod, Consumer consumer) { + Long completedTime; + if (StringUtils.isBlank(bizPod.getStartTime())) { + return ""; + } + Long startTime = transformTime(bizPod.getStartTime()); + boolean hasCompleted = StringUtils.isNotBlank(bizPod.getCompletedTime()); + completedTime = hasCompleted ? transformTime(bizPod.getCompletedTime()) : LocalDateTime.now().toEpochSecond(ZoneOffset.of(trainJobConstant.getPlusEight())); + Long time = completedTime - startTime; + String res = DubheDateUtil.convert2Str(time); + if (hasCompleted) { + consumer.accept(res); + } + return res; + } + + /** + * 时间转换 + * + * @param time 时间 + * @return Long 时间戳 + */ + private Long transformTime(String time) { + LocalDateTime localDateTime = LocalDateTime.parse(time, DateTimeFormatter.ISO_OFFSET_DATE_TIME); + //没有根据时区做处理, 默认当前为东八区 + localDateTime = localDateTime.plusHours(Long.valueOf(trainJobConstant.getEight())); + return localDateTime.toEpochSecond(ZoneOffset.of(trainJobConstant.getPlusEight())); + } + + /** + * 作业不同版本job列表展示 + * + * @param ptTrainJobVersionQueryDTO 查询作业不同版本job列表参数 + * @return List 训练详情集合 + **/ + @Override + public List getTrainJobVersion(PtTrainJobVersionQueryDTO ptTrainJobVersionQueryDTO) { + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} queries different versions of job list display, received parameter trainId is {}", currentUser.getUsername(), ptTrainJobVersionQueryDTO.getTrainId()); + + //校验参数 + checkTrainId(ptTrainJobVersionQueryDTO.getTrainId(), currentUser); + String sort = null == ptTrainJobVersionQueryDTO.getSort() ? Constant.ID : ptTrainJobVersionQueryDTO.getSort(); + + QueryWrapper queryTrainJonWrapper = new QueryWrapper<>(); + queryTrainJonWrapper.eq("train_id", ptTrainJobVersionQueryDTO.getTrainId()); + //根据训练状态筛选 + if (ptTrainJobVersionQueryDTO.getTrainStatus() != null) { + queryTrainJonWrapper.eq("train_status", ptTrainJobVersionQueryDTO.getTrainStatus()); + } + if (Constant.SORT_ASC.equals(ptTrainJobVersionQueryDTO.getOrder())) { + queryTrainJonWrapper.orderByAsc(StringUtils.humpToLine(sort)); + } else { + queryTrainJonWrapper.orderByDesc(StringUtils.humpToLine(sort)); + } + //按照trainId查找 + List ptTrainJobs = ptTrainJobMapper.selectList(queryTrainJonWrapper); + if (CollectionUtils.isEmpty(ptTrainJobs)) { + LogUtil.info(LogEnum.BIZ_TRAIN, "No training task with trainId of {} and training status of {} was found by user {}", currentUser.getUsername(), ptTrainJobVersionQueryDTO.getTrainId(), ptTrainJobVersionQueryDTO.getTrainStatus()); + return Collections.emptyList(); + } + Set jobIds = ptTrainJobs.stream().map(PtTrainJob::getId).collect(Collectors.toSet()); + QueryWrapper queryJobParamWrapper = new QueryWrapper<>(); + queryJobParamWrapper.in("train_job_id", jobIds); + //找出所有训练参数 + List ptJobParams = ptJobParamMapper.selectList(queryJobParamWrapper); + List algorithmIds = ptJobParams.stream().map(PtJobParam::getAlgorithmId).distinct().collect(Collectors.toList()); + List ptTrainAlgorithms = ptTrainAlgorithmMapper.selectAllBatchIds(algorithmIds); + //获取训练信息 + + PtTrain ptTrain = ptTrainMapper.selectById(ptTrainJobVersionQueryDTO.getTrainId()); + //结果集处理 + List list = getTrainJobDetail(ptTrainJobs, ptJobParams, ptTrainAlgorithms, ptTrain); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query different version of job list display completed, return result {}", currentUser.getUsername(), list); + return list; + } + + /** + * 结果集处理 拼装PtTrainJobDetailVO + * + * @param ptTrainJobs 训练任务集合 + * @param ptJobParams 训练参数集合 + * @param ptTrainAlgorithms 训练算法集合 + * @param ptTrain 训练 + * @return List 训练版本查询详情集合 + */ + private List getTrainJobDetail(List ptTrainJobs, List ptJobParams, List ptTrainAlgorithms, PtTrain ptTrain) { + List list = new ArrayList<>(); + Map jobParamMap = new HashedMap<>(); + ptTrainJobs.forEach(x -> { + PtTrainJobDetailVO ptTrainJobDetailVO = new PtTrainJobDetailVO(); + BeanUtil.copyProperties(x, ptTrainJobDetailVO); + list.add(ptTrainJobDetailVO); + jobParamMap.put(x.getId(), list.size()); + }); + + ptJobParams.forEach(x -> { + PtTrainJobDetailVO ptTrainJobDetailVO = list.get(jobParamMap.get(x.getTrainJobId()) - 1); + if (null != ptTrainJobDetailVO) { + + ptTrainJobDetailVO.setAlgorithmId(x.getAlgorithmId()).setRunCommand(x.getRunCommand()).setImageName(x.getImageName()) + .setRunParams(x.getRunParams()) + .setParamF1(x.getParamF1()).setParamCallback(x.getParamCallback()) + .setParamPrecise(x.getParamPrecise()).setParamAccuracy(x.getParamAccuracy()); + //image信息拼装 + if (StringUtils.isNotBlank(x.getImageName())) { + String imageNameSuffix = x.getImageName().substring(x.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); + String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); + ptTrainJobDetailVO.setImageName(imageNameSuffixArray[0]); + ptTrainJobDetailVO.setImageTag(imageNameSuffixArray[1]); + } + } + }); + + Map algorithmMap = new HashedMap<>(); + ptTrainAlgorithms.forEach(x -> algorithmMap.put(x.getId(), x)); + + for (PtTrainJobDetailVO ptTrainJobDetailVO : list) { + PtTrainAlgorithm ptTrainAlgorithm = algorithmMap.get(ptTrainJobDetailVO.getAlgorithmId()); + if (null != ptTrainAlgorithm) { + ptTrainJobDetailVO.setAlgorithmName(ptTrainAlgorithm.getAlgorithmName()) + .setAlgorithmSource(ptTrainAlgorithm.getAlgorithmSource()) + .setAlgorithmUsage(ptTrainAlgorithm.getAlgorithmUsage()) + .setAccuracy(ptTrainAlgorithm.getAccuracy()) + .setP4InferenceSpeed(ptTrainAlgorithm.getP4InferenceSpeed()); + } + } + + list.forEach(x -> x.setTrainName(ptTrain.getTrainName())); + return list; + } + + /** + * 校验请求不同版本job所传参数是否合法 + * + * @param trainId 训练ID + * @param currentUser 当前用户 + */ + private void checkTrainId(Long trainId, UserDTO currentUser) { + if (null == trainId || trainId < 1) { + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} queries different versions of job list display, request parameter trainId is {}, it is illegal", currentUser.getUsername(), trainId); + throw new BusinessException("参数不合法"); + } + PtTrain ptTrain = ptTrainMapper.selectById(trainId); + if (null == ptTrain || !currentUser.getId().equals(ptTrain.getCreateUserId())) { + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} queries different versions of job list display, request parameter trainId is {}, it is illegal", currentUser.getUsername(), trainId); + throw new BusinessException("参数不合法"); + } + } + + /** + * 创建训练job + * + * @param ptTrainJobCreateDTO 创建训练job参数 + * @return List id集合 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public List createTrainJobVersion(PtTrainJobCreateDTO ptTrainJobCreateDTO) { + + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} creates a training job and receives {} as an argument", currentUser.getUsername(), ptTrainJobCreateDTO); + // 判断当前trainName是否已经存在 + checkTrainName(ptTrainJobCreateDTO.getTrainName(), currentUser.getId()); + // 校验trainParamName是否存在 + if (ptTrainJobCreateDTO.getSaveParams() != null && ptTrainJobCreateDTO.getSaveParams()) { + checkTrainParamName(ptTrainJobCreateDTO, currentUser.getId()); + // 保存任务参数到数据库 + saveParamToDb(ptTrainJobCreateDTO, currentUser); + } + // 获取镜像和算法目录 + PtImageAndAlgorithmVO ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId(), + currentUser.getId()); + //使用用户创建训练时提供的镜像与运行命令 + String images = imageUtil.getImages(ptTrainJobCreateDTO, currentUser); + ptImageAndAlgorithmVO.setImageName(trainHarborConfig.getAddress() + StrUtil.SLASH + images).setRunCommand(ptTrainJobCreateDTO.getRunCommand()); + + // 获取规格 + PtTrainJobSpecs ptTrainJobSpecs = getSpecs(ptTrainJobCreateDTO.getTrainJobSpecsId(), currentUser); + //jobKey + String trainKey = KeyUtil.generateTrainKey(currentUser.getId()); + //版本 + String version = trainJobConstant.getVersionLabel() + String.format(TrainUtil.FOUR_DECIMAL, 1); + //生成k8s 的job名称 + String jobName = trainKey + trainJobConstant.getSeparator() + version; + + BaseTrainJobDTO baseTrainJobDTO = new BaseTrainJobDTO(); + BeanUtil.copyProperties(ptTrainJobCreateDTO, baseTrainJobDTO); + baseTrainJobDTO.setJobName(jobName); + baseTrainJobDTO.setPtTrainJobSpecs(ptTrainJobSpecs); + + //结果集处理 + PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, currentUser, images, trainKey, jobName); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} creates training job, returns result {}", currentUser.getUsername(), ptTrainJob.getTrainId()); + + // 提交job + asyncManager.execute(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); + return Collections.singletonList(ptTrainJob.getTrainId()); + } + + /** + * 保存训练任务数据 + * + * @param ptTrainJobCreateDTO 创建训练任务DTO + * @param currentUser 用户 + * @param imageName 镜像名称 + * @param trainKey 训练key + * @param jobName 训练名称 + * @return PtTrain 训练 + */ + private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserDTO currentUser, String imageName, String trainKey, String jobName) { + // 添加train表 + PtTrain ptTrain = new PtTrain(); + ptTrain.setTrainName(ptTrainJobCreateDTO.getTrainName()) + .setTrainKey(trainKey) + .setCreateUserId(currentUser.getId()); + int trainResult = ptTrainMapper.insert(ptTrain); + if (trainResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} creates training job, pt Train table insert data failed", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + + // 添加train_job表 + PtTrainJob ptTrainJob = new PtTrainJob(); + BeanUtil.copyProperties(ptTrainJobCreateDTO, ptTrainJob); + ptTrainJob.setTrainId(ptTrain.getId()) + .setTrainVersion(trainJobConstant.getVersionLabel().toUpperCase() + String.format(TrainUtil.FOUR_DECIMAL, 1)) + .setJobName(jobName) + .setCreateUserId(currentUser.getId()); + int jobResult = ptTrainJobMapper.insert(ptTrainJob); + if (jobResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training Job, failed to insert data in train_job table", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + + // 保存job参数 + PtJobParam ptJobParam = new PtJobParam(); + ptJobParam.setTrainJobId(ptTrainJob.getId()) + .setAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId()) + .setRunCommand(ptTrainJobCreateDTO.getRunCommand()) + .setImageName(imageName) + .setRunParams(ptTrainJobCreateDTO.getRunParams()) + .setCreateUserId(currentUser.getId()); + int jobParamResult = ptJobParamMapper.insert(ptJobParam); + if (jobParamResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training job, pT_job_parAM table insert data failed", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + return ptTrainJob; + } + + /** + * 保存任务参数到数据库 + * + * @param ptTrainJobCreateDTO 创建训练任务DTO + * @param currentUser 用户 + */ + private void saveParamToDb(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserDTO currentUser) { + PtTrainParam ptTrainParam = new PtTrainParam(); + BeanUtil.copyProperties(ptTrainJobCreateDTO, ptTrainParam); + //获取镜像url + String images = imageUtil.getImages(ptTrainJobCreateDTO, currentUser); + ptTrainParam.setImageName(images); + ptTrainParam.setParamName(ptTrainJobCreateDTO.getTrainParamName()) + .setDescription(ptTrainJobCreateDTO.getTrainParamDesc()) + .setRunParams(ptTrainJobCreateDTO.getRunParams()) + .setCreateUserId(currentUser.getId()); + int trainParamResult = ptTrainParamMapper.insert(ptTrainParam); + if (trainParamResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training job, pT_param_param table failed to insert data", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + } + + /** + * 获取训练作业规格 + * + * @param trainJobSpecsId 规格类型Id + * @param currentUser 用户 + * @return PtTrainJobSpecs 训练作业规格 + */ + private PtTrainJobSpecs getSpecs(Integer trainJobSpecsId, UserDTO currentUser) { + + PtTrainJobSpecs ptTrainJobSpecs = ptTrainJobSpecsMapper.selectById(trainJobSpecsId); + if (ptTrainJobSpecs == null) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The PT_train_job_specs table specification type Id for the user {} query does not exist", currentUser.getUsername(), trainJobSpecsId); + throw new BusinessException("内部错误"); + } + return ptTrainJobSpecs; + } + + /** + * 获取镜像和算法目录 + * + * @param algorithmId 算法ID + * @param userId 用户ID + * @return PtImageAndAlgorithmVO 镜像 + */ + private PtImageAndAlgorithmVO getPtImageByAlgorithmId(Long algorithmId, Long userId) { + + PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(algorithmId); + if (null == ptTrainAlgorithm || StringUtils.isBlank(ptTrainAlgorithm.getCodeDir())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The record with algorithm training ID {} has no corresponding image or algorithm directory configuration", algorithmId); + throw new BusinessException(ResponseCode.SUCCESS, "该id的记录没有相应的镜像或者算法目录配置"); + } + if (!(userId.equals(ptTrainAlgorithm.getCreateUserId()) || AlgorithmSourceEnum.PRE.getStatus().equals(ptTrainAlgorithm.getAlgorithmSource()))) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The data {} does not belong to the user {}!", ptTrainAlgorithm, userId); + throw new BusinessException(ResponseCode.SUCCESS, "该数据不属于该用户!"); + } + + PtImageAndAlgorithmVO ptImageAndAlgorithmVO = new PtImageAndAlgorithmVO(); + BeanUtil.copyProperties(ptTrainAlgorithm, ptImageAndAlgorithmVO); + + return ptImageAndAlgorithmVO; + } + + /** + * 校验trainName + * + * @param trainName 校验trainName + * @param userId 返回用户ID + */ + private void checkTrainName(String trainName, Long userId) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("train_name", trainName) + .eq("create_user_id", userId); + Integer ptTrainCountResult = ptTrainMapper.selectCount(queryWrapper); + if (ptTrainCountResult > 0) { + throw new BusinessException("当前训练名称已经存在"); + } + } + + /** + * 校验trainParamName + * + * @param ptTrainJobCreateDTO 创建训练DTO + * @param userId 用户ID + */ + private void checkTrainParamName(PtTrainJobCreateDTO ptTrainJobCreateDTO, Long userId) { + + String trainParamName = ptTrainJobCreateDTO.getTrainParamName(); + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("param_name", trainParamName) + .eq("create_user_id", userId); + Integer ptTrainParamCountResult = ptTrainParamMapper.selectCount(queryWrapper); + if (ptTrainParamCountResult > 0) { + throw new BusinessException("当前trainParamName已经存在"); + } + } + + /** + * 生成k8s 的job名称 + * + * @param ptTrain 训练 + * @return String 版本 + */ + private String buildVersion(PtTrain ptTrain) { + return ptTrain.getTrainKey() + trainJobConstant.getSeparator() + trainJobConstant.getVersionLabel() + String.format(TrainUtil.FOUR_DECIMAL, ptTrain.getTotalNum() + 1); + } + + /** + * 修改训练job + * + * @param ptTrainJobUpdateDTO 修改训练job参数 + * @return List id集合 + **/ + @Override + @Transactional(rollbackFor = Exception.class) + public List updateTrainJob(PtTrainJobUpdateDTO ptTrainJobUpdateDTO) { + + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} modifies the training job and receives the parameter {}", ptTrainJobUpdateDTO); + + PtTrainJob existPtTrainJob = ptTrainJobMapper.selectById(ptTrainJobUpdateDTO.getId()); + if (null == existPtTrainJob || !currentUser.getId().equals(existPtTrainJob.getCreateUserId())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for a user {} to modify a training job, jobId, to {}", currentUser.getUsername(), ptTrainJobUpdateDTO.getId()); + throw new BusinessException(ResponseCode.SUCCESS, "您输入的id不存在或已被删除"); + } + + //获取算法 + PtImageAndAlgorithmVO ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId(), currentUser.getId()); + //使用用户修改训练时提供的镜像与运行命令 + //获取镜像url + String images = imageUtil.getImages(ptTrainJobUpdateDTO, currentUser); + ptImageAndAlgorithmVO.setImageName(trainHarborConfig.getAddress() + StrUtil.SLASH + images).setRunCommand(ptTrainJobUpdateDTO.getRunCommand()); + + //获取规格 + PtTrainJobSpecs ptTrainJobSpecs = getSpecs(ptTrainJobUpdateDTO.getTrainJobSpecsId(), currentUser); + + PtTrain ptTrain = ptTrainMapper.selectById(existPtTrainJob.getTrainId()); + + String jobName = buildVersion(ptTrain); + + BaseTrainJobDTO baseTrainJobDTO = new BaseTrainJobDTO(); + BeanUtil.copyProperties(ptTrainJobUpdateDTO, baseTrainJobDTO); + baseTrainJobDTO.setJobName(jobName); + baseTrainJobDTO.setPtTrainJobSpecs(ptTrainJobSpecs); + //结果集处理 + PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, currentUser, existPtTrainJob, images, ptTrain, jobName); + //提交job + asyncManager.execute(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); + + return Collections.singletonList(ptTrainJob.getId()); + } + + /** + * 结果集处理 + * + * @param ptTrainJobUpdateDTO 更新训练任务DTO + * @param currentUser 当前用户 + * @param existPtTrainJob 存在的训练任务 + * @param imageName 镜像名称 + * @param ptTrain 训练 + * @param jobName 任务名称 + * @return PtTrainJob 训练任务 + */ + private PtTrainJob updateTrainJobTableData(PtTrainJobUpdateDTO ptTrainJobUpdateDTO, UserDTO + currentUser, PtTrainJob existPtTrainJob, String imageName, PtTrain ptTrain, String jobName) { + //添加train_job表 + PtTrainJob ptTrainJob = new PtTrainJob(); + BeanUtil.copyProperties(ptTrainJobUpdateDTO, ptTrainJob); + ptTrainJob.setTrainId(ptTrain.getId()).setTrainVersion(trainJobConstant.getVersionLabel().toUpperCase() + String.format(TrainUtil.FOUR_DECIMAL, ptTrain.getTotalNum() + 1)) + .setJobName(jobName).setParentTrainVersion(existPtTrainJob.getTrainVersion()) + .setCreateUserId(currentUser.getId()); + int jobResult = ptTrainJobMapper.insert(ptTrainJob); + if (jobResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training Job, failed to insert data in train_job table", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + + //保存job参数 + PtJobParam ptJobParam = new PtJobParam(); + ptJobParam.setTrainJobId(ptTrainJob.getId()) + .setAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId()) + .setRunCommand(ptTrainJobUpdateDTO.getRunCommand()) + .setImageName(imageName) + .setRunParams(ptTrainJobUpdateDTO.getRunParams()) + .setCreateUserId(currentUser.getId()); + int jobParamResult = ptJobParamMapper.insert(ptJobParam); + if (jobParamResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training job, pT_job_parAM table insert data failed", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + + //更新pt_train + PtTrain updatePtTrain = new PtTrain(); + updatePtTrain.setId(ptTrain.getId()).setVersionNum(ptTrain.getVersionNum() + 1) + .setTotalNum(ptTrain.getTotalNum() + 1).setUpdateUserId(currentUser.getId()); + int updateResult = ptTrainMapper.updateById(updatePtTrain); + if (updateResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training job, pT_train table failed to update version number", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + return ptTrainJob; + } + + /** + * 删除训练job + * + * @param ptTrainJobDeleteDTO 删除训练job参数 + * @return PtTrainJobDeleteVO 删除训练任务结果 + **/ + @Override + @Transactional(rollbackFor = Exception.class) + public PtTrainJobDeleteVO deleteTrainJob(PtTrainJobDeleteDTO ptTrainJobDeleteDTO) { + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} deletes the training job and receives the parameter {}", currentUser.getUsername(), ptTrainJobDeleteDTO); + + List jobList = new ArrayList<>(); + PtTrain ptTrain = checkAndReturnPtTrain(ptTrainJobDeleteDTO, currentUser, jobList); + + Collection jobIdList = new ArrayList<>(); + if (null != ptTrainJobDeleteDTO.getId()) { + + //删除job + deleteJobs(currentUser, jobList); + + int jobResult = ptTrainJobMapper.deleteById(ptTrainJobDeleteDTO.getId()); + if (jobResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pT_train_job table failed to delete data", currentUser.getUsername()); + throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); + } + + PtTrain updatePtTrain = new PtTrain(); + updatePtTrain.setVersionNum(ptTrain.getVersionNum() - 1); + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", ptTrain.getId()).eq("version_num", ptTrain.getVersionNum()); + int updateResult = ptTrainMapper.update(updatePtTrain, updateWrapper); + if (updateResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted the training job and updated the version_num in the PT_train table failed", currentUser.getUsername()); + throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); + } + + if (ptTrain.getVersionNum() == 1) { + int trainResult = ptTrainMapper.deleteById(ptTrain.getId()); + if (trainResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pt Train table deleted data failed", currentUser.getUsername()); + throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); + } + } + jobIdList.add(ptTrainJobDeleteDTO.getId()); + } else { + deleteTrainAndJob(ptTrainJobDeleteDTO, currentUser, jobList, ptTrain, jobIdList); + } + + //删除pt_job_param表中相关数据 + UpdateWrapper updateJobParamWrapper = new UpdateWrapper<>(); + updateJobParamWrapper.in("train_job_id", jobIdList); + int jobParamResult = ptJobParamMapper.delete(updateJobParamWrapper); + if (jobParamResult < jobIdList.size()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pT_job_param table failed to delete data", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + + PtTrainJobDeleteVO ptTrainJobDeleteVO = new PtTrainJobDeleteVO(); + BeanUtil.copyProperties(ptTrainJobDeleteDTO, ptTrainJobDeleteVO); + LogUtil.info(LogEnum.BIZ_TRAIN, "Deleting the training job completes and the result returned is {}", ptTrainJobDeleteVO); + return ptTrainJobDeleteVO; + } + + /** + * 删除训练和任务 + * + * @param ptTrainJobDeleteDTO 删除训练任务DTO + * @param currentUser 用户 + * @param jobList 任务集合 + * @param ptTrain 训练 + * @param jobIdList 任务ID集合 + */ + private void deleteTrainAndJob(PtTrainJobDeleteDTO ptTrainJobDeleteDTO, UserDTO + currentUser, List jobList, PtTrain ptTrain, Collection jobIdList) { + QueryWrapper query = new QueryWrapper<>(); + query.eq("train_id", ptTrainJobDeleteDTO.getTrainId()); + List ptTrainJobs = ptTrainJobMapper.selectList(query); + if (ptTrainJobs.size() < 1) { + throw new BusinessException(ResponseCode.SUCCESS, "没有待删除的训练任务"); + } + ptTrainJobs.forEach(x -> { + jobList.add(x); + jobIdList.add(x.getId()); + }); + + //删除job + deleteJobs(currentUser, jobList); + + PtTrain updatePtTrain = new PtTrain(); + updatePtTrain.setVersionNum(0); + UpdateWrapper updateTrainWrapper = new UpdateWrapper<>(); + updateTrainWrapper.eq("id", ptTrain.getId()).eq("version_num", ptTrain.getVersionNum()); + int updateResult = ptTrainMapper.update(updatePtTrain, updateTrainWrapper); + if (updateResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted the training job and updated the version_num in the PT_train table failed", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + + int trainResult = ptTrainMapper.deleteById(ptTrain.getId()); + if (trainResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pt Train table deleted data failed", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + + UpdateWrapper updateJobWrapper = new UpdateWrapper<>(); + updateJobWrapper.eq("train_id", ptTrain.getId()); + int jobResult = ptTrainJobMapper.delete(updateJobWrapper); + if (jobResult < jobIdList.size()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pT_train_job table failed to delete data", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + } + + /** + * 校验并返回PtTrain + * + * @param ptTrainJobDeleteDTO 删除训练任务DTO + * @param currentUser 用户 + * @param jobList 任务集合 + * @return PtTrain 训练 + */ + private PtTrain checkAndReturnPtTrain(PtTrainJobDeleteDTO ptTrainJobDeleteDTO, UserDTO currentUser, List jobList) { + PtTrain ptTrain = ptTrainMapper.selectById(ptTrainJobDeleteDTO.getTrainId()); + if (null == ptTrain || !ptTrain.getCreateUserId().equals(currentUser.getId())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete training job, invalid parameter, as follows {}", currentUser.getUsername(), ptTrainJobDeleteDTO); + throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); + } + + if (null != ptTrainJobDeleteDTO.getId()) { + PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(ptTrainJobDeleteDTO.getId()); + if (null == ptTrainJob || !ptTrainJob.getTrainId().equals(ptTrainJobDeleteDTO.getTrainId())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete training job, invalid parameter, as follows {}", currentUser.getUsername(), ptTrainJobDeleteDTO); + throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); + } + jobList.add(ptTrainJob); + } + return ptTrain; + } + + /** + * 检测停止训练任务DTO + * + * @param ptTrainJobStopDTO 停止训练DTO + * @param currentUser 用户 + * @param jobList 任务集合 + */ + private void checkAndReturnPtTrain(PtTrainJobStopDTO ptTrainJobStopDTO, UserDTO currentUser, List jobList) { + PtTrain ptTrain = ptTrainMapper.selectById(ptTrainJobStopDTO.getTrainId()); + if (null == ptTrain || !ptTrain.getCreateUserId().equals(currentUser.getId())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} stopped the training job failed, the parameter is illegal, the training does not exist, as follows {}", currentUser.getUsername(), ptTrainJobStopDTO); + throw new BusinessException(ResponseCode.SUCCESS, "参数不合法，该训练不存在"); + } + + if (null != ptTrainJobStopDTO.getId()) { + PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(ptTrainJobStopDTO.getId()); + if (null == ptTrainJob || !ptTrainJob.getTrainId().equals(ptTrainJobStopDTO.getTrainId()) || + TrainJobStatusEnum.checkStopStatus(ptTrainJob.getTrainStatus())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} stopped training job failed, invalid parameter, as follows {}", currentUser.getUsername(), ptTrainJobStopDTO); + throw new BusinessException(ResponseCode.SUCCESS, "参数不合法，该训练不存在"); + } + jobList.add(ptTrainJob); + } + } + + /** + * 删除任务 + * + * @param currentUser 用户 + * @param jobList 任务集合 + */ + private void deleteJobs(UserDTO currentUser, List jobList) { + String namespace = k8sNameTool.generateNameSpace(currentUser.getId()); + try { + for (PtTrainJob job : jobList) { + if (TrainJobStatusEnum.STOP.getStatus().equals(job.getTrainStatus())) { + boolean bool = trainJobApi.delete(namespace, job.getJobName()); + if (!bool) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deletes the training job and K8s fails to execute the delete() method, namespace为{}, resourceName为{}", + currentUser.getUsername(), namespace, job.getJobName()); + } + } + PtBaseResult ptBaseResult = persistentVolumeClaimApi.recycle(namespace, job.getJobName()); + if (null == ptBaseResult || !ptBaseResult.isSuccess()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, k8s failed to implement the recycle() method, namespace为{}, resourceName为{}", + currentUser.getUsername(), namespace, job.getJobName()); + } + } + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} delete training job, k8s delete failed,exception:{}", e); + throw new BusinessException("内部错误"); + } + } + + /** + * 停止训练job + * + * @param ptTrainJobStopDTO 停止训练job参数 + * @return PtTrainJobStopVO 停止训练任务结果 + **/ + @Override + @Transactional(rollbackFor = Exception.class) + public PtTrainJobStopVO stopTrainJob(PtTrainJobStopDTO ptTrainJobStopDTO) { + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} stops training Job and receives the parameter {}", currentUser.getUsername(), ptTrainJobStopDTO); + //训练job名称集合 用于停止k8s + List jobList = new ArrayList<>(); + checkAndReturnPtTrain(ptTrainJobStopDTO, currentUser, jobList); + + if (null != ptTrainJobStopDTO.getId()) { + //停止job + stopJobs(currentUser, jobList); + + } else if (null != ptTrainJobStopDTO.getTrainId()) { + QueryWrapper queryTrainJonWrapper = new QueryWrapper<>(); + queryTrainJonWrapper.eq("train_id", ptTrainJobStopDTO.getTrainId()); + List trainJobs = ptTrainJobMapper.selectList(queryTrainJonWrapper); + + for (PtTrainJob trainJob : trainJobs) { + if (!TrainJobStatusEnum.checkStopStatus(trainJob.getTrainStatus())) { + jobList.add(trainJob); + } + } + + if (jobList.size() < 1) { + throw new BusinessException(ResponseCode.SUCCESS, "没有待停止的job"); + } + + //停止job + stopJobs(currentUser, jobList); + } + + //更新job状态 + updateJobStatus(currentUser, jobList); + + PtTrainJobStopVO ptTrainJobStopVO = new PtTrainJobStopVO(); + ptTrainJobStopVO.setTrainId(ptTrainJobStopDTO.getTrainId()); + ptTrainJobStopVO.setId(ptTrainJobStopDTO.getId()); + return ptTrainJobStopVO; + } + + /** + * 更新训练状态 + * + * @param currentUser 用户 + * @param jobList 任务集合 + */ + private void updateJobStatus(UserDTO currentUser, List jobList) { + for (PtTrainJob ptTrainJob : jobList) { + PtTrainJob updateTrainJob = new PtTrainJob(); + updateTrainJob.setId(ptTrainJob.getId()).setRuntime(ptTrainJob.getRuntime()).setTrainStatus(TrainJobStatusEnum.STOP.getStatus()); + int updateResult = ptTrainJobMapper.updateById(updateTrainJob); + if (updateResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} stops training job, pT_train_job table fails to update status, the information is as follows {}", currentUser.getUsername(), updateTrainJob); + throw new BusinessException(ResponseCode.SUCCESS, "没有待停止的job"); + } + } + } + + /** + * 任务统计 + * + * @return PtTrainJobStatisticsMineVO 我的训练任务统计结果 + **/ + @Override + public PtTrainJobStatisticsMineVO statisticsMine() { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + // 获取运行中的任务 + Integer runCount = ptTrainJobMapper.selectCountByStatus(userDTO.getId(), + SqlUtil.integerlistToString(new Integer[]{TrainJobStatusEnum.RUNNING.getStatus()})); + + // 已经完成的任务 + Integer finishCount = ptTrainJobMapper.selectCountByStatus(userDTO.getId(), + SqlUtil.integerlistToString( + new Integer[]{TrainJobStatusEnum.FAILED.getStatus(), TrainJobStatusEnum.STOP.getStatus(), + TrainJobStatusEnum.SUCCEEDED.getStatus(), TrainJobStatusEnum.UNKNOWN.getStatus()})); + + PtTrainJobStatisticsMineVO vo = new PtTrainJobStatisticsMineVO(); + vo.setRunJobCount(runCount); + vo.setFinishJobCount(finishCount); + return vo; + } + + + /** + * 停止任务 + * + * @param currentUser 用户 + * @param jobList 任务集合 + */ + void stopJobs(UserDTO currentUser, List jobList) { + String namespace = k8sNameTool.generateNameSpace(currentUser.getId()); + jobList.forEach(job -> { + BizPod bizPod = podApi.getWithResourceName(namespace, job.getJobName()); + if (!bizPod.isSuccess()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} stops training Job return code:{},message:{}", currentUser.getUsername(), Integer.valueOf(bizPod.getCode()), bizPod.getMessage()); + } + boolean bool = trainJobApi.delete(namespace, job.getJobName()); + if (!bool) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} stops training Job and K8S fails in the stop process, namespace为{}, resourceName为{}", + currentUser.getUsername(), namespace, job.getJobName()); + } + job.setRuntime(calculateRuntime(bizPod)); + }); + } + + /** + * 查询训练作业job状态 + * + * @param ptTrainDataSourceStatusQueryDTO 查询训练作业job状态参数 + * @return HashedMap 数据集路径-是否可以删除 的map集合 + **/ + @Override + public Map getTrainDataSourceStatus(PtTrainDataSourceStatusQueryDTO ptTrainDataSourceStatusQueryDTO) { + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} queries the state of the dataset starting with the received parameter {}", currentUser.getUsername(), ptTrainDataSourceStatusQueryDTO); + if (CollectionUtils.isEmpty(ptTrainDataSourceStatusQueryDTO.getDataSourcePath())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The dataset set {} is empty", ptTrainDataSourceStatusQueryDTO.getDataSourcePath()); + throw new BusinessException("传入参数为空，请重新输入"); + } + //去重 + List dataSourceList = ptTrainDataSourceStatusQueryDTO.getDataSourcePath().stream().distinct().collect(Collectors.toList()); + QueryWrapper query = new QueryWrapper<>(); + query.in("data_source_path", dataSourceList); + List ptTrainJobs = ptTrainJobMapper.selectList(query); + //结果集处理 + List ptTrainDataSourceStatusQueryList = ptTrainJobs.stream().map(x -> { + PtTrainDataSourceStatusQueryVO ptTrainDataSourceStatusQuery = new PtTrainDataSourceStatusQueryVO(); + ptTrainDataSourceStatusQuery.setDataSourcePath(x.getDataSourcePath()); + ptTrainDataSourceStatusQuery.setStatus(x.getTrainStatus() >= TrainJobStatusEnum.SUCCEEDED.getStatus()); + return ptTrainDataSourceStatusQuery; + }).distinct().collect(Collectors.toList()); + //结果去重 + HashMap map = new HashMap<>(); + for (PtTrainDataSourceStatusQueryVO ptTrainDataSourceStatusQuery : ptTrainDataSourceStatusQueryList) { + if (map.containsKey(ptTrainDataSourceStatusQuery.getDataSourcePath())) { + if (!ptTrainDataSourceStatusQuery.getStatus()) { + map.put(ptTrainDataSourceStatusQuery.getDataSourcePath(), ptTrainDataSourceStatusQuery.getStatus()); + } + } else { + map.put(ptTrainDataSourceStatusQuery.getDataSourcePath(), ptTrainDataSourceStatusQuery.getStatus()); + } + } + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query data set state ends, return result is {}", currentUser.getUsername(), map); + return map; + } + + /** + * 根据jobId查询训练任务详情查询 + * + * @param ptTrainJobDetailQueryDTO 根据jobId查询训练任务详情查询条件 + * @return PtTrainQueryJobDetailVO 根据jobId查询训练任务详情返回结果 + */ + @Override + public PtTrainJobDetailQueryVO getTrainJobDetail(PtTrainJobDetailQueryDTO ptTrainJobDetailQueryDTO) { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} starts by querying jobId={} for training task details", user.getUsername(), ptTrainJobDetailQueryDTO.getId()); + QueryWrapper trainJobQuery = new QueryWrapper<>(); + trainJobQuery.eq("create_user_id", user.getId()); + trainJobQuery.eq("id", ptTrainJobDetailQueryDTO.getId()); + //获取训练 + PtTrainJob ptTrainJob = ptTrainJobMapper.selectOne(trainJobQuery); + if (ptTrainJob == null) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The jobId for the user {} query does not exist", user.getUsername()); + throw new BusinessException(ResponseCode.SUCCESS, "您查询的id不存在或已被删除"); + } + QueryWrapper jobParamQuery = new QueryWrapper<>(); + jobParamQuery.eq("train_job_id", ptTrainJob.getId()); + //获取训练任务参数 + PtJobParam ptJobParam = ptJobParamMapper.selectOne(jobParamQuery); + if (ptJobParam == null || ptJobParam.getAlgorithmId() < MagicNumConstant.ONE) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The algorithm ID corresponding to the jobId={} query by the user {} does not exist", user.getUsername(), ptTrainJobDetailQueryDTO.getId()); + throw new BusinessException(ResponseCode.SUCCESS, "您查询的jobId对应的算法id不存在或已被删除"); + } + //获取算法 + PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectAllById(ptJobParam.getAlgorithmId()); + //拼装job Detail信息 + PtTrainJobDetailQueryVO ptTrainJobDetailQueryVO = new PtTrainJobDetailQueryVO(); + BeanUtils.copyProperties(ptTrainJob, ptTrainJobDetailQueryVO); + ptTrainJobDetailQueryVO.setAlgorithmId(ptJobParam.getAlgorithmId()).setRunCommand(ptJobParam.getRunCommand()).setRunParams(ptJobParam.getRunParams()) + .setParamF1(ptJobParam.getParamF1()).setParamCallback(ptJobParam.getParamCallback()).setParamPrecise(ptJobParam.getParamPrecise()).setParamAccuracy(ptJobParam.getParamAccuracy()); + //拼装镜像信息 + if (StringUtils.isNotBlank(ptJobParam.getImageName())) { + String imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); + String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); + ptTrainJobDetailQueryVO.setImageName(imageNameSuffixArray[0]); + ptTrainJobDetailQueryVO.setImageTag(imageNameSuffixArray[1]); + } + //拼装算法信息 + if (ptTrainAlgorithm != null) { + ptTrainJobDetailQueryVO.setAlgorithmName(ptTrainAlgorithm.getAlgorithmName()) + .setAlgorithmSource(ptTrainAlgorithm.getAlgorithmSource()) + .setAlgorithmUsage(ptTrainAlgorithm.getAlgorithmUsage()) + .setAccuracy(ptTrainAlgorithm.getAccuracy()) + .setP4InferenceSpeed(ptTrainAlgorithm.getP4InferenceSpeed()); + } + + return ptTrainJobDetailQueryVO; + } + + /** + * 恢复训练 + * + * @param ptTrainJobResumeDTO 恢复训练请求参数 + */ + @Override + public void resumeTrainJob(PtTrainJobResumeDTO ptTrainJobResumeDTO) { + //从会话中获取用户信息 + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "user {} according to jobId={} started to check the training task details", currentUser.getUsername(), ptTrainJobResumeDTO.getId()); + // 检查jobId是否合法 + PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(ptTrainJobResumeDTO.getId()); + if (null == ptTrainJob || !currentUser.getId().equals(ptTrainJob.getCreateUserId())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for user {} to resume training job and jobId to be {}", currentUser.getUsername(), ptTrainJobResumeDTO.getId()); + throw new BusinessException(ResponseCode.SUCCESS, "您输入的id不存在或已被删除，请重新输入"); + } + // 获取算法id和运行参数 + QueryWrapper jobParamQuery = new QueryWrapper<>(); + jobParamQuery.eq("train_job_id", ptTrainJob.getId()); + PtJobParam ptJobParam = ptJobParamMapper.selectOne(jobParamQuery); + if (ptJobParam == null || ptJobParam.getAlgorithmId() < MagicNumConstant.ONE) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The algorithm ID corresponding to the jobId={} query by the user {} does not exist", currentUser.getUsername(), ptTrainJobResumeDTO.getId()); + throw new BusinessException(ResponseCode.SUCCESS, "您查询的jobId对应的算法id不存在"); + } + //获取镜像 + PtImageAndAlgorithmVO ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptJobParam.getAlgorithmId(), currentUser.getId()); + //使用用户训练时提供的镜像与运行命令 + ptImageAndAlgorithmVO.setImageName(ptJobParam.getImageName()).setRunCommand(ptJobParam.getRunCommand()); + //获取规格 + PtTrainJobSpecs ptTrainJobSpecs = getSpecs(ptTrainJob.getTrainJobSpecsId(), currentUser); + + //处理目录问题 + String commonPath = nfsUtil.getNfsConfig().getBucket() + trainJobConstant.getManage() + StrUtil.SLASH + + currentUser.getId() + StrUtil.SLASH + ptTrainJob.getJobName(); + String outPath = commonPath + StrUtil.SLASH + trainJobConstant.getOutPath(); + String loadPath = commonPath + StrUtil.SLASH + trainJobConstant.getLoadPath(); + String modelLoadDirName = nfsUtil.find2ndNewDir(outPath); + if ("".equals(modelLoadDirName)) { + LogUtil.error(LogEnum.BIZ_TRAIN, "outPath: {}", outPath); + throw new BusinessException(ResponseCode.ERROR, "该任务没有前序结果可以继续训练"); + } + nfsUtil.deleteDirOrFile(loadPath); + nfsUtil.renameDir(outPath, loadPath); + + // 拼load路径 + JSONObject runParams = ptJobParam.getRunParams(); + runParams.put(trainJobConstant.getLoadKey(), trainJobConstant.getDockerTrainPath() + StrUtil.SLASH + + trainJobConstant.getLoadPath() + StrUtil.SLASH + modelLoadDirName); + BaseTrainJobDTO baseTrainJobDTO = new BaseTrainJobDTO(); + BeanUtil.copyProperties(ptTrainJob, baseTrainJobDTO); + baseTrainJobDTO.setPtTrainJobSpecs(ptTrainJobSpecs); + baseTrainJobDTO.setRunParams(runParams); + + // 初始化训练时间和状态 + PtTrainJob updatePtTrainJob = new PtTrainJob(); + updatePtTrainJob.setId(ptTrainJob.getId()).setRuntime("").setTrainStatus(TrainJobStatusEnum.PENDING.getStatus()); + int updateResult = ptTrainJobMapper.updateById(updatePtTrainJob); + if (updateResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} resumed training job, pt train Job table update failed", currentUser.getUsername()); + throw new BusinessException("内部错误"); + } + // 提交job + asyncManager.execute(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); + } + + /** + * 获取job在grafana监控的地址 + * + * @param jobId 任务ID + * @return PtJobMetricsGrafanaVO grafana监控的地址信息 + */ + @Override + public PtJobMetricsGrafanaVO getGrafanaUrl(Long jobId) { + + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} gets grafanaUrl of Job and receives parameter [jobId] {}", currentUser.getUsername(), jobId); + + //通过jobId获取job相关信息 + PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(jobId); + if (null == ptTrainJob || !currentUser.getId().equals(ptTrainJob.getCreateUserId())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for user {} to get grafanaUrl on Job, jobId for {}", currentUser.getUsername(), jobId); + throw new BusinessException(ResponseCode.SUCCESS, "您输入的id不存在或已被删除，请重新输入"); + } + + String podMetricsGrafanaUrl = podApi + .getPodMetricsGrafanaUrl(k8sNameTool.generateNameSpace(currentUser.getId()), ptTrainJob.getJobName()); + + PtJobMetricsGrafanaVO ptJobMetricsGrafanaVO = new PtJobMetricsGrafanaVO(); + ptJobMetricsGrafanaVO.setJobMetricsGrafanaUrl(podMetricsGrafanaUrl); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} completes getting grafanaUrl on job, receives {} parameter [jobId], returns {} result", + currentUser.getUsername(), jobId, ptJobMetricsGrafanaVO); + return ptJobMetricsGrafanaVO; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java new file mode 100644 index 0000000..198449f --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java @@ -0,0 +1,69 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import org.dubhe.dao.PtTrainJobSpecsMapper; +import org.dubhe.domain.dto.PtTrainJobSpecsQueryDTO; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.PtTrainJobSpecs; +import org.dubhe.domain.vo.PtTrainJobSpecsQueryVO; +import org.dubhe.enums.LogEnum; +import org.dubhe.service.PtTrainJobSpecsService; +import org.dubhe.utils.JwtUtils; +import org.dubhe.utils.LogUtil; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * @description 训练作业规格服务实现类 + * @date 2020-05-06 + */ +@Service +public class PtTrainJobSpecsServiceImpl implements PtTrainJobSpecsService { + + @Autowired + private PtTrainJobSpecsMapper ptTrainJobSpecsMapper; + + /** + * 规格查询 + * + * @param ptTrainJobSpecsQueryDTO 查询规格表参数 + * @return List 返回规格查询结果 + **/ + @Override + public List getTrainJobSpecs(PtTrainJobSpecsQueryDTO ptTrainJobSpecsQueryDTO) { + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "user{}Query job list display, the received parameters are{}", currentUser.getUsername(), ptTrainJobSpecsQueryDTO); + QueryWrapper query = new QueryWrapper<>(); + if (ptTrainJobSpecsQueryDTO.getResourcesPoolType() != null) { + query.eq("resources_pool_type", ptTrainJobSpecsQueryDTO.getResourcesPoolType()); + } + List ptTrainJobSpecs = ptTrainJobSpecsMapper.selectList(query); + List ptTrainJobSpecsQueryList = ptTrainJobSpecs.stream().map(x -> { + PtTrainJobSpecsQueryVO ptTrainJobSpecsQueryVO = new PtTrainJobSpecsQueryVO(); + BeanUtils.copyProperties(x, ptTrainJobSpecsQueryVO); + return ptTrainJobSpecsQueryVO; + }).collect(Collectors.toList()); + return ptTrainJobSpecsQueryList; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java new file mode 100644 index 0000000..7a59754 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java @@ -0,0 +1,125 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import org.dubhe.constant.NumberConstant; +import org.dubhe.dao.PtTrainJobMapper; +import org.dubhe.domain.dto.PtTrainLogQueryDTO; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.PtTrainJob; +import org.dubhe.domain.vo.PtTrainLogQueryVO; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.k8s.api.LogMonitoringApi; +import org.dubhe.k8s.domain.bo.LogMonitoringBO; +import org.dubhe.k8s.domain.vo.LogMonitoringVO; +import org.dubhe.service.PtTrainLogService; +import org.dubhe.utils.JwtUtils; +import org.dubhe.utils.K8sNameTool; +import org.dubhe.utils.LogUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.List; + + +/** + * @description: 训练日志 服务实现类 + * @date 2020-05-08 + */ + +@Service +public class PtTrainLogServiceImpl implements PtTrainLogService { + + @Autowired + private LogMonitoringApi logMonitoringApi; + + @Autowired + private PtTrainJobMapper ptTrainJobMapper; + + @Autowired + private K8sNameTool k8sNameTool; + + /** + * 查询训练任务运行日志 + * + * @param ptTrainLogQueryDTO 训练日志查询 + * @return PtTrainLogQueryVO 返回训练日志查询 + **/ + @Override + public PtTrainLogQueryVO queryTrainLog(PtTrainLogQueryDTO ptTrainLogQueryDTO) { + LogUtil.info(LogEnum.BIZ_TRAIN, "Start viewing log parameters, received as {}", ptTrainLogQueryDTO); + + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + String nameSpace = k8sNameTool.generateNameSpace(user.getId()); + + PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(ptTrainLogQueryDTO.getJobId()); + if (null == ptTrainJob || !user.getId().equals(ptTrainJob.getCreateUserId())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for the user {} to look up the training job log with jobId as {}", user.getId(), ptTrainLogQueryDTO.getJobId()); + throw new BusinessException("内部错误"); + } + + /** 查询日志起始行 **/ + Integer startLine = null == ptTrainLogQueryDTO.getStartLine() ? NumberConstant.NUMBER_1 : ptTrainLogQueryDTO.getStartLine(); + /** 查询日志总行数 **/ + Integer lines = null == ptTrainLogQueryDTO.getLines() ? NumberConstant.NUMBER_50 : ptTrainLogQueryDTO.getLines(); + /** 拼接请求es的参数 **/ + LogMonitoringBO logMonitoringBo = new LogMonitoringBO(); + logMonitoringBo.setNamespace(nameSpace) + .setResourceName(ptTrainJob.getJobName()); + + PtTrainLogQueryVO ptTrainLogQueryVO = new PtTrainLogQueryVO(); + + LogMonitoringVO result = logMonitoringApi.searchLog(startLine, lines, logMonitoringBo); + List list = result.getLogs(); + + if (result == null || CollectionUtils.isEmpty(list)) { + ptTrainLogQueryVO.setContent(list); + ptTrainLogQueryVO.setStartLine(startLine); + ptTrainLogQueryVO.setEndLine(startLine - 1); + ptTrainLogQueryVO.setLines(0); + return ptTrainLogQueryVO; + } + + ptTrainLogQueryVO.setContent(list).setStartLine(startLine) + .setEndLine(Long.valueOf(startLine + result.getTotalLogs()).intValue() - 1) + .setLines(Long.valueOf(result.getTotalLogs()).intValue()).setJobName(ptTrainJob.getJobName()); + LogUtil.info(LogEnum.BIZ_TRAIN, "Query log results are returned==>{}", ptTrainLogQueryVO); + return ptTrainLogQueryVO; + } + + /** + * 字符串换行 + * + * @param content 内容 + * @return strContent 返回字符串内容 + */ + @Override + public String getTrainLogString(List content) { + String strContent = ""; + if (content != null) { + for (String str : content) { + strContent = strContent.concat(str).concat("\r\n"); + } + } + + return strContent; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java new file mode 100644 index 0000000..19d4d88 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java @@ -0,0 +1,319 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import cn.hutool.core.util.StrUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.base.ResponseCode; +import org.dubhe.constant.TrainJobConstant; +import org.dubhe.dao.PtTrainAlgorithmMapper; +import org.dubhe.dao.PtTrainParamMapper; +import org.dubhe.data.constant.Constant; +import org.dubhe.domain.dto.*; +import org.dubhe.domain.entity.PtTrainAlgorithm; +import org.dubhe.domain.entity.PtTrainParam; +import org.dubhe.domain.vo.PtTrainParamQueryVO; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.service.PtTrainParamService; +import org.dubhe.utils.*; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * @description 任务参数服务实现类 + * @date 2020-04-27 + */ +@Service +public class PtTrainParamServiceImpl implements PtTrainParamService { + + @Autowired + private PtTrainParamMapper ptTrainParamMapper; + + @Autowired + private PtTrainAlgorithmMapper ptTrainAlgorithmMapper; + + @Autowired + private ImageUtil imageUtil; + + /** + * 参数列表展示 + * + * @param ptTrainParamQueryDTO 任务参数列表展示条件 + * @return Map 任务参数列表分页数据 + **/ + @Override + public Map getTrainParam(PtTrainParamQueryDTO ptTrainParamQueryDTO) { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "The display of user {} query task parameter list begins. The parameters received are {}.", user.getUsername(), ptTrainParamQueryDTO); + Page page = ptTrainParamQueryDTO.toPage(); + //查询任务参数列表 + QueryWrapper query = new QueryWrapper<>(); + query.eq("create_user_id", user.getId()); + //根据任务参数名称模糊搜索 + if (ptTrainParamQueryDTO.getParamName() != null) { + query.like("param_name", ptTrainParamQueryDTO.getParamName()); + } + //根据类型筛选 + if (ptTrainParamQueryDTO.getResourcesPoolType() != null) { + query.eq("resources_pool_type", ptTrainParamQueryDTO.getResourcesPoolType()); + } + IPage ptTrainParams; + try { + if (ptTrainParamQueryDTO.getSort() == null || ptTrainParamQueryDTO.getSort().equalsIgnoreCase(TrainJobConstant.ALGORITHM_NAME)) { + query.orderByDesc(Constant.ID); + } else { + if (Constant.SORT_ASC.equalsIgnoreCase(ptTrainParamQueryDTO.getOrder())) { + query.orderByAsc(StringUtils.humpToLine(ptTrainParamQueryDTO.getSort())); + } else { + query.orderByDesc(StringUtils.humpToLine(ptTrainParamQueryDTO.getSort())); + } + } + ptTrainParams = ptTrainParamMapper.selectPage(page, query); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "Query task parameter list shows exception {}", e); + throw new BusinessException("内部错误"); + } + //结果集的处理 + //获取算法id + List algorithmIds = ptTrainParams.getRecords().stream().map(PtTrainParam::getAlgorithmId).collect(Collectors.toList()); + if (algorithmIds.size() < 1) { + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query task parameter list is empty, the result is {}", user.getUsername(), ptTrainParams); + List ptTrainParamQueryResult = new ArrayList<>(); + return PageUtil.toPage(page, ptTrainParamQueryResult); + } + List ptTrainAlgorithms = ptTrainAlgorithmMapper.selectAllBatchIds(algorithmIds); + //获取算法id对应的算法名称并封装至map集合中 + Map ptTrainAlgorithmMap = ptTrainAlgorithms.stream().collect(Collectors.toMap(PtTrainAlgorithm::getId, PtTrainAlgorithm::getAlgorithmName, (o, n) -> n)); + List ptTrainParamQueryResult = ptTrainParams.getRecords().stream().map(x -> { + PtTrainParamQueryVO ptTrainParamQueryVO = new PtTrainParamQueryVO(); + BeanUtils.copyProperties(x, ptTrainParamQueryVO); + ptTrainParamQueryVO.setAlgorithmName(ptTrainAlgorithmMap.get(x.getAlgorithmId())); + //获取镜像名称与版本 + getImageNameAndImageTag(x, ptTrainParamQueryVO); + return ptTrainParamQueryVO; + }).collect(Collectors.toList()); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query task parameter list display ends, the result is {}", user.getUsername(), ptTrainParamQueryResult); + return PageUtil.toPage(page, ptTrainParamQueryResult); + } + + /** + * 保存任务参数 + * + * @param ptTrainParamCreateDTO 保存任务参数条件 + * @return List 保存任务参数id集合 + **/ + @Override + @Transactional(rollbackFor = Exception.class) + public List createTrainParam(PtTrainParamCreateDTO ptTrainParamCreateDTO) { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} saves the task parameter and receives the parameter {}.", user.getUsername(), ptTrainParamCreateDTO); + //参数校验 + PtTrainAlgorithm ptTrainAlgorithm = checkCreateTrainParam(ptTrainParamCreateDTO, user); + //获取算法来源（1为我的算法，2为预置算法） + Integer algorithmSource = ptTrainAlgorithm.getAlgorithmSource(); + //保存任务参数 + PtTrainParam ptTrainParam = new PtTrainParam(); + BeanUtils.copyProperties(ptTrainParamCreateDTO, ptTrainParam); + //获取镜像 + String images = imageUtil.getImages(ptTrainParamCreateDTO, user); + ptTrainParam.setImageName(images).setAlgorithmSource(algorithmSource).setCreateUserId(user.getId()); + int insertResult = ptTrainParamMapper.insert(ptTrainParam); + //任务参数未保存成功，抛出异常，并返回失败信息 + if (insertResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} saved the task parameters successfully, and the pt_train_param table insert operation failed", user.getUsername()); + throw new BusinessException("内部错误"); + } + //返回新增任务参数id + LogUtil.info(LogEnum.BIZ_TRAIN, "End of user {} saving task parameters, return new task parameter ID ={}", user.getUsername(), ptTrainParam.getId()); + return Collections.singletonList(ptTrainParam.getId()); + } + + /** + * 修改任务参数 + * + * @param ptTrainParamUpdateDTO 修改任务参数条件 + * @return List 修改任务参数id集合 + **/ + @Override + @Transactional(rollbackFor = Exception.class) + public List updateTrainParam(PtTrainParamUpdateDTO ptTrainParamUpdateDTO) { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} modifies the task parameters and receives the parameters as {}.", user.getUsername(), ptTrainParamUpdateDTO); + //参数校验 + checkUpdateTrainParam(ptTrainParamUpdateDTO, user); + //修改任务参数 + PtTrainParam ptTrainParam = new PtTrainParam(); + BeanUtils.copyProperties(ptTrainParamUpdateDTO, ptTrainParam); + ptTrainParam.setUpdateUserId(user.getId()); + //获取镜像url + String images = imageUtil.getImages(ptTrainParamUpdateDTO, user); + //添加镜像url + ptTrainParam.setImageName(images); + try { + //任务参数未修改成功，抛出异常，并返回失败信息 + ptTrainParamMapper.updateById(ptTrainParam); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} failed to modify the task parameters. The modification operation of pt_train_param table failed. The failure reason :{}", user.getUsername(), e.getMessage()); + throw new BusinessException("内部错误"); + } + + //返回修改任务参数id + LogUtil.info(LogEnum.BIZ_TRAIN, "End of user {} saving task parameters, return modify task parameter ID ={}", user.getUsername(), ptTrainParam.getId()); + return Collections.singletonList(ptTrainParam.getId()); + } + + /** + * 删除任务参数 + * + * @param ptTrainParamDeleteDTO 删除任务参数条件 + **/ + @Override + @Transactional(rollbackFor = Exception.class) + public void deleteTrainParam(PtTrainParamDeleteDTO ptTrainParamDeleteDTO) { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.BIZ_TRAIN, "User {} deletes the task parameter and receives the parameter {}", user.getUsername(), ptTrainParamDeleteDTO.getIds()); + Set idList = ptTrainParamDeleteDTO.getIds(); + //参数校验 + checkDeleteTrainParam(ptTrainParamDeleteDTO, user, idList); + //删除任务参数 + int deleteCountResult = ptTrainParamMapper.deleteBatchIds(idList); + //任务参数未删除成功,抛出异常，并返回失败信息 + if (deleteCountResult < idList.size()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} failed to delete the task parameter, and the pt_train_param table deletion operation failed according to the id array {}", user.getUsername(), ptTrainParamDeleteDTO.getIds()); + throw new BusinessException("内部错误"); + } + //返回删除任务参数id数组 + LogUtil.info(LogEnum.BIZ_TRAIN, "End of user {} deleting task parameters,the array of delete task parameters IDS ={}", user.getUsername(), ptTrainParamDeleteDTO.getIds()); + } + + /** + * 获取镜像名称与版本 + * + * @param trainParam 镜像URL + * @param ptTrainParamQueryVO 镜像名称与版本 + */ + private void getImageNameAndImageTag(PtTrainParam trainParam, PtTrainParamQueryVO ptTrainParamQueryVO) { + if (StringUtils.isNotBlank(trainParam.getImageName())) { + String imageNameSuffix = trainParam.getImageName().substring(trainParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); + String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); + ptTrainParamQueryVO.setImageName(imageNameSuffixArray[0]); + ptTrainParamQueryVO.setImageTag(imageNameSuffixArray[1]); + } + } + + /** + * 创建训练参数模板参数校验 + * + * @param ptTrainParamCreateDTO 任务参数创建条件 + * @param user 用户 + * @return PtTrainAlgorithm 算法 + */ + private PtTrainAlgorithm checkCreateTrainParam(PtTrainParamCreateDTO ptTrainParamCreateDTO, UserDTO user) { + //算法id校验 + PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectAllById(ptTrainParamCreateDTO.getAlgorithmId()); + if (ptTrainAlgorithm == null) { + LogUtil.error(LogEnum.BIZ_TRAIN, "Algorithm ID ={} does not exist", ptTrainParamCreateDTO.getAlgorithmId()); + throw new BusinessException("算法不存在或已被删除"); + } + //任务参数名称校验 + QueryWrapper query = new QueryWrapper<>(); + query.eq("param_name", ptTrainParamCreateDTO.getParamName()); + query.eq("create_user_id", user.getId()); + Integer trainParamCountResult = ptTrainParamMapper.selectCount(query); + if (trainParamCountResult > 0) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The task parameter name ({}) already exists", ptTrainParamCreateDTO.getParamName()); + throw new BusinessException("任务参数名称已存在"); + } + return ptTrainAlgorithm; + } + + /** + * 修改训练参数模板参数校验 + * + * @param ptTrainParamUpdateDTO 训练参数模板参数 + * @param user 用户 + */ + private void checkUpdateTrainParam(PtTrainParamUpdateDTO ptTrainParamUpdateDTO, UserDTO user) { + //任务参数id校验 + if (ptTrainParamMapper.selectById(ptTrainParamUpdateDTO.getId()) == null) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The task parameter ID ={} does not exist", ptTrainParamUpdateDTO.getId()); + throw new BusinessException(ResponseCode.SUCCESS, "任务参数id不存在"); + } + //算法id校验 + PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(ptTrainParamUpdateDTO.getAlgorithmId()); + if (ptTrainAlgorithm == null) { + LogUtil.error(LogEnum.BIZ_TRAIN, "Algorithm ID ={} does not exist", ptTrainParamUpdateDTO.getAlgorithmId()); + throw new BusinessException(ResponseCode.SUCCESS, "算法id不存在"); + } + //权限校验 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("id", ptTrainParamUpdateDTO.getId()).eq("create_user_id", user.getId()); + Integer countResult = ptTrainParamMapper.selectCount(queryWrapper); + if (countResult < 1) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} failed to modify the task parameters and has no permission to modify the corresponding data in the pt_train_param table", user.getUsername()); + throw new BusinessException(ResponseCode.SUCCESS, "您修改的ID不存在或已被删除"); + } + //任务参数名称校验 + QueryWrapper query = new QueryWrapper<>(); + query.eq("param_name", ptTrainParamUpdateDTO.getParamName()).eq("create_user_id", user.getId()); + PtTrainParam trainParam = ptTrainParamMapper.selectOne(query); + if (trainParam != null && !ptTrainParamUpdateDTO.getId().equals(trainParam.getId())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The task parameter name ({}) already exists", ptTrainParamUpdateDTO.getParamName()); + throw new BusinessException(ResponseCode.SUCCESS, "任务参数名称已存在"); + } + } + + /** + * 删除训练参数模板参数校验 + * + * @param ptTrainParamDeleteDTO 训练参数模板参数 + * @param user 用户 + * @param idList 训练参数id集合 + **/ + private void checkDeleteTrainParam(PtTrainParamDeleteDTO ptTrainParamDeleteDTO, UserDTO user, Set idList) { + //id校验 + List ptTrainParams = ptTrainParamMapper.selectBatchIds(idList); + if (ptTrainParams.size() == 0 || ptTrainParams.size() != idList.size()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete the task parameters, request parameters IDS ={} cannot query the corresponding data in pt_train_param table, the parameters are illegal", user.getUsername(), ptTrainParamDeleteDTO.getIds()); + throw new BusinessException(ResponseCode.SUCCESS, "您删除的ID不存在或已被删除"); + } + //权限校验 + QueryWrapper query = new QueryWrapper<>(); + query.eq("create_user_id", user.getId()); + query.in("id", idList); + Integer queryCountResult = ptTrainParamMapper.selectCount(query); + if (queryCountResult < idList.size()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete the task parameters and has no permission to delete the corresponding data in the pt_train_param table", user.getUsername()); + throw new BusinessException(ResponseCode.SUCCESS, "您删除的ID不存在或已被删除"); + } + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/HarborImagePushAsync.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/HarborImagePushAsync.java new file mode 100644 index 0000000..9d39c6e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/HarborImagePushAsync.java @@ -0,0 +1,120 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.task; + +import cn.hutool.core.util.StrUtil; +import org.dubhe.base.ResponseCode; +import org.dubhe.config.TrainHarborConfig; +import org.dubhe.dao.PtImageMapper; +import org.dubhe.domain.entity.PtImage; +import org.dubhe.enums.ImageStateEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.harbor.api.HarborApi; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +/** + * @description 异步推送镜像到harbor + * @date 2020-07-10 + */ +@Component +public class HarborImagePushAsync { + + @Autowired + private TrainHarborConfig trainHarborConfig; + + @Autowired + private HarborApi harborApi; + + @Autowired + private PtImageMapper ptImageMapper; + + /** + * 组装shell脚本执行命令 + * + * @param imagePath + * @param imageNameandTag + **/ + @Async + public void execShell(String imagePath, String imageNameandTag, PtImage ptImage) { + try { + String imageResource = trainHarborConfig.getAddress() + StrUtil.SLASH + trainHarborConfig.getModelName() + + StrUtil.SLASH + imageNameandTag; + String cmdStr = "docker login --username=" + trainHarborConfig.getUsername() + " " + trainHarborConfig.getAddress() + " --password=" + trainHarborConfig.getPassword() + " ; docker " + + "load < " + imagePath + " |awk '{print $3}' |xargs -I str docker tag str " + imageResource + " ; docker push " + imageResource; + String[] cmd = {"/bin/bash", "-c", cmdStr}; + LogUtil.info(LogEnum.BIZ_TRAIN, "镜像上传执行脚本参数:{}", cmd); + + Process process = Runtime.getRuntime().exec(cmd); + //读取标准输出流 + BufferedReader brOut = new BufferedReader(new InputStreamReader(process.getInputStream())); + //读取标准错误流 + BufferedReader brErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String line; + String outMessage = ""; + String errMessage = ""; + while ((line = brOut.readLine()) != null) { + outMessage += line; + } + if (StringUtils.isNotEmpty(outMessage)) { + LogUtil.info(LogEnum.BIZ_TRAIN, "shell上传镜像输出信息:" + outMessage); + } + while ((line = brErr.readLine()) != null) { + errMessage += line; + } + if (StringUtils.isNotEmpty(errMessage)) { + LogUtil.error(LogEnum.BIZ_TRAIN, "shell上传镜像异常信息:" + errMessage); + } + Integer status = process.waitFor(); + LogUtil.info(LogEnum.BIZ_TRAIN, "上传镜像状态:{}", status); + if (status == null) { + if (harborApi.isExistImage(ptImage.getImageUrl())) { + updateImageStatus(ptImage, ImageStateEnum.SUCCESS.getCode()); + } else { + updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); + } + } else if (status == 0) { + updateImageStatus(ptImage, ImageStateEnum.SUCCESS.getCode()); + } else { + updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); + } + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "上传镜像异常:{}", e); + throw new BusinessException("上传镜像异常!"); + } + } + + /** + * 更新镜像上传状态 + * + * @param ptImage + * @param status + * @return java.lang.Integer + **/ + public Integer updateImageStatus(PtImage ptImage, Integer status) { + ptImage.setImageStatus(status); + ptImageMapper.updateById(ptImage); + return ResponseCode.SUCCESS; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/HarborProjectNameSyncTask.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/HarborProjectNameSyncTask.java new file mode 100644 index 0000000..c2d2e9f --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/HarborProjectNameSyncTask.java @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.task; + +import org.dubhe.enums.LogEnum; +import org.dubhe.service.PtImageService; +import org.dubhe.utils.LogUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * @description 从harbor同步projectName + * @date 2020-6-23 + **/ +@Component +public class HarborProjectNameSyncTask { + + @Autowired + private PtImageService ptImageService; + + /** + * 每天晚上11点开始同步 + **/ + @Scheduled(cron = "0 0 23 * * ?") + public void syncProjectName() { + LogUtil.info(LogEnum.BIZ_TRAIN, "开始到harbor同步projectName到harbor_project表。。。。。"); + ptImageService.harborImageNameSync(); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/NoteBookStatusRefreshTask.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/NoteBookStatusRefreshTask.java new file mode 100644 index 0000000..3262d36 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/NoteBookStatusRefreshTask.java @@ -0,0 +1,110 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.task; + +import cn.hutool.core.date.DateUtil; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.domain.entity.NoteBook; +import org.dubhe.domain.dto.NoteBookQueryDTO; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.NoteBookStatusEnum; +import org.dubhe.service.NoteBookService; +import org.dubhe.utils.LogUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * @description: 定时刷新NoteBook状态 回调补偿方案Task 刷新过程中状态 + * @date 2020-04-27 + */ +@Component +public class NoteBookStatusRefreshTask { + + /** + * 设置一次性批处理数量，避免待处理状态过多导致单次处理压力过大 + */ + private static final int ONE_TIME_PROCESSING_QUANTITY = 100; + + @Autowired + private NoteBookService noteBookService; + + /** + * 表示每隔15秒刷新状态 + */ + @Scheduled(cron = "*/15 * * * * ?") + public void process() { + LogUtil.info(LogEnum.NOTE_BOOK, "【回调补偿刷新NoteBook状态Task】 Start at {}", DateUtil.now()); + try { + refreshStatus(ONE_TIME_PROCESSING_QUANTITY); + } catch (Exception e) { + LogUtil.error(LogEnum.NOTE_BOOK, "ERROR!【回调补偿刷新oteBook状态Task】", e); + } finally { + LogUtil.info(LogEnum.NOTE_BOOK, "【回调补偿刷新NoteBook状态Task】 END at {}", DateUtil.now()); + } + } + + /** + * 刷新状态 + * + * @param oneTimeProcessingQuantity + */ + private void refreshStatus(int oneTimeProcessingQuantity) { + List noteBookList = noteBookService.getList(new Page(MagicNumConstant.ZERO, oneTimeProcessingQuantity), NoteBookQueryDTO.getToRefreshCriteria()); + int noteBookSize = noteBookList.size(); + LogUtil.info(LogEnum.NOTE_BOOK, "本次待处理NoteBook数量：{}", noteBookSize); + for (int i = 0; i < noteBookSize; i++) { + if ((i + 1) % MagicNumConstant.FIVE == MagicNumConstant.ZERO) { + LogUtil.info(LogEnum.NOTE_BOOK, "NoteBook处理进度 {}/{} ..", i + 1, noteBookSize); + } + NoteBook noteBook = noteBookList.get(i); + refreshNoteBookStatus(noteBook); + } + } + + /** + * 刷新notebook状态 + * + * @param noteBook + */ + private void refreshNoteBookStatus(NoteBook noteBook) { + NoteBookStatusEnum statusEnum = noteBookService.getStatus(noteBook); + // 刷新处理 + if (noteBookService.refreshNoteBookStatus(statusEnum, noteBook)) { + return; + } + // 超时处理 + timeout(noteBook); + noteBookService.updateById(noteBook); + } + + /** + * 超时处理 + * + * @param noteBook + */ + private void timeout(NoteBook noteBook) { + if (NoteBookStatusEnum.STARTING.getCode().equals(noteBook.getStatus())) { + noteBook.setStatus(NoteBookStatusEnum.STOP.getCode()); + noteBook.setK8sStatusInfo("启动超时！"); + } + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/NoteBookUrlRefreshTask.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/NoteBookUrlRefreshTask.java new file mode 100644 index 0000000..0511ab1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/NoteBookUrlRefreshTask.java @@ -0,0 +1,101 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.task; + +import cn.hutool.core.date.DateUtil; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.domain.entity.NoteBook; +import org.dubhe.enums.LogEnum; +import org.dubhe.service.NoteBookService; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.NotebookUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.List; + +/** + * @description : 定时刷新notebook url + * @date 2020-06-30 + */ +@Component +public class NoteBookUrlRefreshTask { + + /** + * 设置一次性批处理数量，避免待处理状态过多导致单次处理压力过大 + */ + private static final int ONE_TIME_PROCESSING_QUANTITY = 100; + + @Autowired + private NoteBookService noteBookService; + + /** + * 表示每隔2秒刷新URL + */ + @Scheduled(cron = "*/2 * * * * ?") + public void process() { + LogUtil.info(LogEnum.NOTE_BOOK, "【刷新NoteBook URL 任务】 Start at ", DateUtil.now()); + try { + refreshUrl(ONE_TIME_PROCESSING_QUANTITY); + } catch (Exception e) { + LogUtil.error(LogEnum.NOTE_BOOK, "ERROR!【刷新NoteBook URL 任务】", e); + } finally { + LogUtil.info(LogEnum.NOTE_BOOK, "【刷新NoteBook URL 任务】 END at", DateUtil.now()); + } + } + + /** + * 刷新URL + * + * @param oneTimeProcessingQuantity + */ + private void refreshUrl(int oneTimeProcessingQuantity) { + List noteBookList = noteBookService.getRunNotUrlList(new Page(MagicNumConstant.ZERO, oneTimeProcessingQuantity)); + LogUtil.info(LogEnum.NOTE_BOOK, "需要处理的notebook条数为: ", noteBookList.size()); + if (!CollectionUtils.isEmpty(noteBookList)) { + int noteBookSize = noteBookList.size(); + LogUtil.info(LogEnum.NOTE_BOOK, "本次待处理NoteBook数量：{}", noteBookSize); + for (int i = 0; i < noteBookSize; i++) { + if ((i + 1) % MagicNumConstant.FIVE == MagicNumConstant.ZERO) { + LogUtil.info(LogEnum.NOTE_BOOK, "NoteBook处理进度 ", (i + 1), noteBookSize); + } + NoteBook noteBook = noteBookList.get(i); + LogUtil.info(LogEnum.NOTE_BOOK, "NoteBook resource name : {} , ", noteBook.getK8sResourceName(), noteBook.getNoteBookName(), noteBook.getK8sNamespace()); + refreshNoteBookUrl(noteBook); + } + } + } + + /** + * 刷新notebook + * + * @param noteBook + */ + private void refreshNoteBookUrl(NoteBook noteBook) { + LogUtil.info(LogEnum.NOTE_BOOK, "成功进入获取URL接口 : ", noteBook.toString()); + String jupyterUrlWithToken = noteBookService.getJupyterUrl(noteBook); + if (NotebookUtil.checkUrlContainsToken(jupyterUrlWithToken)) { + noteBook.setUrl(jupyterUrlWithToken); + noteBookService.updateById(noteBook); + LogUtil.info(LogEnum.NOTE_BOOK, "成功更新notebook url 的名称为 : ", noteBook.getNoteBookName()); + } + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/TrainJobAsyncTask.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/TrainJobAsyncTask.java new file mode 100644 index 0000000..0ccae1e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/TrainJobAsyncTask.java @@ -0,0 +1,245 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.task; + +import cn.hutool.core.util.StrUtil; +import com.alibaba.fastjson.JSONObject; +import org.dubhe.constant.TrainJobConstant; +import org.dubhe.dao.PtTrainJobMapper; +import org.dubhe.domain.dto.BaseTrainJobDTO; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.PtTrainJob; +import org.dubhe.domain.vo.PtImageAndAlgorithmVO; +import org.dubhe.enums.BizEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.ResourcesPoolTypeEnum; +import org.dubhe.enums.TrainJobStatusEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.k8s.api.NamespaceApi; +import org.dubhe.k8s.api.TrainJobApi; +import org.dubhe.k8s.domain.bo.PtJupyterJobBO; +import org.dubhe.k8s.domain.resource.BizNamespace; +import org.dubhe.k8s.domain.vo.PtJupyterJobVO; +import org.dubhe.k8s.enums.K8sResponseEnum; +import org.dubhe.utils.K8sNameTool; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.NfsUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +/** + * @description 提交训练任务 + * @date 2020-07-17 + */ + +@Component +public class TrainJobAsyncTask { + + @Autowired + private K8sNameTool k8sNameTool; + + @Autowired + private NamespaceApi namespaceApi; + + @Autowired + private TrainJobConstant trainJobConstant; + + @Autowired + private NfsUtil nfsUtil; + + @Autowired + private PtTrainJobMapper ptTrainJobMapper; + + @Autowired + private TrainJobApi trainJobApi; + + /** + * 提交job + * + * @param baseTrainJobDTO 训练任务信息 + * @param currentUser 用户 + * @param ptImageAndAlgorithmVO 镜像和算法信息 + */ + public void doJob(BaseTrainJobDTO baseTrainJobDTO, UserDTO currentUser, PtImageAndAlgorithmVO ptImageAndAlgorithmVO, PtTrainJob ptTrainJob) { + PtJupyterJobBO jobBo = null; + String k8sJobName = ""; + boolean flag = false; + try { + //判断是否存在相应的namespace,如果没有则创建 + String namespace = getNamespace(currentUser); + + //封装PtJupyterJobBO对象,调用创建训练任务接口 + jobBo = pkgPtJupyterJobBo(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, namespace); + if (null == jobBo) { + LogUtil.error(LogEnum.BIZ_TRAIN, "user{}create TrainJob，Encapsulating ptjupyterjobbo object is empty，the received parameters namespace：{}", currentUser.getId(), namespace); + updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, k8sJobName, flag); + } + PtJupyterJobVO ptJupyterJobResult = trainJobApi.create(jobBo); + k8sJobName = ptJupyterJobResult.getName(); + if (null == ptJupyterJobResult || !ptJupyterJobResult.isSuccess()) { + if (null != ptJupyterJobResult && ("" + K8sResponseEnum.LACK_OF_RESOURCES).equals(ptJupyterJobResult.getCode())) { + updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, k8sJobName, flag); + LogUtil.error(LogEnum.BIZ_TRAIN, "user{}create TrainJob, K8s creation failed, the received parameters are{}, the wrong information is{}", currentUser.getUsername(), + jobBo, ptJupyterJobResult.getMessage()); + } + String message = null == ptJupyterJobResult ? "未知的错误" : ptJupyterJobResult.getMessage(); + LogUtil.error(LogEnum.BIZ_TRAIN, "user{}create TrainJob, K8s creation failed, the received parameters are {}, the wrong information is{}", currentUser.getUsername(), jobBo, message); + updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, k8sJobName, flag); + } + flag = true; + //更新训练任务状态 + updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, k8sJobName, flag); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_TRAIN, "user{}create TrainJob, K8s creation failed, the received parameters are {}, the wrong information is{}", currentUser.getUsername(), + jobBo, e); + updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, k8sJobName, flag); + } + } + + + /** + * 获取namespace + * + * @param currentUser 用户 + * @return String 命名空间 + */ + private String getNamespace(UserDTO currentUser) { + String namespaceStr = k8sNameTool.generateNameSpace(currentUser.getId()); + BizNamespace bizNamespace = namespaceApi.get(namespaceStr); + if (null == bizNamespace) { + BizNamespace namespace = namespaceApi.create(namespaceStr, null); + if (null == namespace || !namespace.isSuccess()) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to create namespace during training job..."); + throw new BusinessException("内部错误"); + } + } + return namespaceStr; + } + + + /** + * 封装出创建job所需的BO + * + * @param baseTrainJobDTO 训练任务信息 + * @param ptImageAndAlgorithmVO 镜像和算法信息 + * @param namespace 命名空间 + * @return PtJupyterJobBO jupyter任务BO + */ + private PtJupyterJobBO pkgPtJupyterJobBo(BaseTrainJobDTO baseTrainJobDTO, UserDTO currentUser, + PtImageAndAlgorithmVO ptImageAndAlgorithmVO, String namespace) { + //绝对路径 + String commonPath = nfsUtil.getNfsConfig().getBucket() + trainJobConstant.getManage() + StrUtil.SLASH + + currentUser.getId() + StrUtil.SLASH + baseTrainJobDTO.getJobName(); + //相对路径 + String relativeCommonPath = StrUtil.SLASH + trainJobConstant.getManage() + StrUtil.SLASH + + currentUser.getId() + StrUtil.SLASH + baseTrainJobDTO.getJobName(); + String[] codeDirArray = ptImageAndAlgorithmVO.getCodeDir().split(StrUtil.SLASH); + String workspaceDir = codeDirArray[codeDirArray.length - 1]; + // 算法路径待拷贝的地址 + String sourcePath = nfsUtil.getNfsConfig().getBucket() + ptImageAndAlgorithmVO.getCodeDir().substring(1); + String trainDir = commonPath.substring(1) + StrUtil.SLASH + workspaceDir; + LogUtil.info(LogEnum.BIZ_TRAIN, "Algorithm path copy::sourcePath:{},commonPath:{},trainDir:{}", sourcePath, commonPath, trainDir); + boolean bool = nfsUtil.copyPath(sourcePath.substring(1), trainDir); + if (!bool) { + LogUtil.error(LogEnum.BIZ_TRAIN, "During the process of user {} creating training Job and encapsulating k8s creating job interface parameters, it failed to copy algorithm directory {} to the specified directory {}", currentUser.getUsername(), sourcePath, + commonPath); + return null; + } + + List list = new ArrayList<>(); + JSONObject runParams = baseTrainJobDTO.getRunParams(); + + StringBuilder sb = new StringBuilder(); + sb.append(ptImageAndAlgorithmVO.getRunCommand()); + // 拼接out,log和dataset + String pattern = trainJobConstant.getPythonFormat(); + if (ptImageAndAlgorithmVO.getIsTrainOut()) { + nfsUtil.createDir(commonPath + StrUtil.SLASH + trainJobConstant.getOutPath()); + baseTrainJobDTO.setOutPath(relativeCommonPath + StrUtil.SLASH + trainJobConstant.getOutPath()); + sb.append(pattern).append(trainJobConstant.getDockerOutPath()); + } + if (ptImageAndAlgorithmVO.getIsTrainLog()) { + nfsUtil.createDir(commonPath + StrUtil.SLASH + trainJobConstant.getLogPath()); + baseTrainJobDTO.setLogPath(relativeCommonPath + StrUtil.SLASH + trainJobConstant.getLogPath()); + sb.append(pattern).append(trainJobConstant.getDockerLogPath()); + } + if (ptImageAndAlgorithmVO.getIsVisualizedLog()) { + nfsUtil.createDir(commonPath + StrUtil.SLASH + trainJobConstant.getVisualizedLogPath()); + baseTrainJobDTO.setVisualizedLogPath(relativeCommonPath + StrUtil.SLASH + trainJobConstant.getVisualizedLogPath()); + sb.append(pattern).append(trainJobConstant.getDockerVisualizedLogPath()); + } + + sb.append(pattern).append(trainJobConstant.getDockerDataset()); + + String command = sb.toString(); + if (null != runParams && !runParams.isEmpty()) { + sb.append(pattern); + runParams.entrySet() + .forEach(entry -> sb.append(entry.getKey()).append("=").append(entry.getValue()).append(pattern)); + command = sb.toString().substring(0, sb.toString().length() - pattern.length()); + } + list.add("-c"); + command = "echo 'training mission begins... " + command + "\r\n" + "'&& cd " + trainJobConstant.getDockerTrainPath() + StrUtil.SLASH + + workspaceDir + " && " + command + " && echo 'the training mission is over' "; + list.add(command); + PtJupyterJobBO jobBo = new PtJupyterJobBO(); + jobBo.setNamespace(namespace) + .setName(baseTrainJobDTO.getJobName()) + .setImage(ptImageAndAlgorithmVO.getImageName()) + .putNfsMounts(trainJobConstant.getDockerDatasetPath(), nfsUtil.getNfsConfig().getRootDir() + nfsUtil.getNfsConfig().getBucket().substring(1) + baseTrainJobDTO.getDataSourcePath()) + .setCmdLines(list) + .putNfsMounts(trainJobConstant.getDockerTrainPath(), nfsUtil.getNfsConfig().getRootDir() + commonPath.substring(1)) + .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.ALGORITHM)); + + jobBo.setCpuNum(baseTrainJobDTO.getPtTrainJobSpecs().getSpecsInfo().getInteger("cpuNum")).setMemNum(baseTrainJobDTO.getPtTrainJobSpecs().getSpecsInfo().getInteger("memNum")); + if (ResourcesPoolTypeEnum.GPU.getCode().equals(baseTrainJobDTO.getPtTrainJobSpecs().getResourcesPoolType())) { + jobBo.setUseGpu(true).setGpuNum(baseTrainJobDTO.getPtTrainJobSpecs().getSpecsInfo().getInteger("gpuNum")); + } else { + jobBo.setUseGpu(false); + } + return jobBo; + } + + /** + * 训练任务异步处理更新训练状态 + * + * @param user 用户 + * @param ptTrainJob 训练任务 + * @param baseTrainJobDTO 训练任务信息 + * @param k8sJobName k8s创建的job名称 + * @param flag 创建训练任务是否异常(true：正常，false：失败) + **/ + private void updateTrainStatus(UserDTO user, PtTrainJob ptTrainJob, BaseTrainJobDTO baseTrainJobDTO, String k8sJobName, boolean flag) { + + ptTrainJob.setK8sJobName(k8sJobName) + .setOutPath(baseTrainJobDTO.getOutPath()) + .setLogPath(baseTrainJobDTO.getLogPath()) + .setVisualizedLogPath(baseTrainJobDTO.getVisualizedLogPath()); + LogUtil.info(LogEnum.BIZ_TRAIN, "user {} training tasks are processed asynchronously to update training status，receiving parameters:{}", user.getId(), ptTrainJob); + if (flag) { + ptTrainJobMapper.updateById(ptTrainJob); + } else { + ptTrainJob.setTrainStatus(TrainJobStatusEnum.CREATE_FAILED.getStatus()); + //训练任务创建失败 + ptTrainJobMapper.updateById(ptTrainJob); + throw new BusinessException("内部错误"); + } + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/TransactionAsyncManager.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/TransactionAsyncManager.java new file mode 100644 index 0000000..2a65e55 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/TransactionAsyncManager.java @@ -0,0 +1,64 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.task; + +import org.dubhe.aspect.LogAspect; +import org.dubhe.domain.dto.BaseTrainJobDTO; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.PtTrainJob; +import org.dubhe.domain.vo.PtImageAndAlgorithmVO; +import org.slf4j.MDC; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionSynchronizationAdapter; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import javax.annotation.Resource; +import java.util.concurrent.Executor; + +/** + * @description 事务提交后触发异步操作 + * @date 2020-07-17 + */ +@Component +public class TransactionAsyncManager { + + @Autowired + private TrainJobAsyncTask trainJobAsyncTask; + + @Resource(name = "trainJobAsyncExecutor") + private Executor trainJobAsyncExecutor; + + + public void execute(BaseTrainJobDTO baseTrainJobDTO, UserDTO currentUser, PtImageAndAlgorithmVO ptImageAndAlgorithmVO, PtTrainJob ptTrainJob) { + + String traceId = MDC.get(LogAspect.TRACE_ID); + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() { + + @Override + public void afterCommit() { + trainJobAsyncExecutor.execute( + () -> { + MDC.put(LogAspect.TRACE_ID, traceId); + trainJobAsyncTask.doJob(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); + MDC.remove(LogAspect.TRACE_ID); + } + ); + } + }); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java new file mode 100644 index 0000000..00bb2a4 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java @@ -0,0 +1,80 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import org.apache.commons.collections4.CollectionUtils; +import org.dubhe.base.BaseImageDTO; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.dao.PtImageMapper; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.PtImage; +import org.dubhe.enums.ImageSourceEnum; +import org.dubhe.enums.ImageStateEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.BusinessException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * @description: 镜像 + * @date: 2020-06-22 + */ +@Component +public class ImageUtil { + + @Autowired + private PtImageMapper ptImageMapper; + + /** + * 获取镜像url + * + * @param baseImageDTO 镜像参数 + * @return BaseImageDTO 镜像url + **/ + public String getImages(BaseImageDTO baseImageDTO, UserDTO user) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("image_name", baseImageDTO.getImageName()).eq("image_tag", baseImageDTO.getImageTag()) + .eq("image_status", ImageStateEnum.SUCCESS.getCode()); + List ptImages = ptImageMapper.selectList(queryWrapper); + if (CollectionUtils.isEmpty(ptImages)) { + LogUtil.error(LogEnum.BIZ_TRAIN, " User {} gets image ,the imageName is {}, the imageTag is {}, and the result of query image table (PT_image) is empty", user.getUsername(), baseImageDTO.getImageName(), baseImageDTO.getImageTag()); + throw new BusinessException("镜像不存在"); + } + //获取镜像为用户自定义镜像或预置镜像，且两者自身不能重复 + if (ptImages.size() > MagicNumConstant.TWO) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} got more images than scheduled, the imageName provided is {} and the imageTag is {}. The parameters are illegal", user.getUsername(), baseImageDTO.getImageName(), baseImageDTO.getImageTag()); + throw new BusinessException("镜像不合法"); + } + for (PtImage ptImage : ptImages) { + if (ImageSourceEnum.PRE.getCode().equals(ptImage.getImageResource())) { + baseImageDTO.setImageName(ptImage.getImageUrl()); + } else if (user.getId().equals(ptImage.getCreateUserId())) { + baseImageDTO.setImageName(ptImage.getImageUrl()); + } + } + if (StringUtils.isBlank(baseImageDTO.getImageName())) { + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} gets image, the imageName provided is {} and the imageTag is {}. The parameters are illegal", user.getUsername(), baseImageDTO.getImageName(), baseImageDTO.getImageTag()); + throw new BusinessException("镜像不合法"); + } + return baseImageDTO.getImageName(); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java new file mode 100644 index 0000000..998a597 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java @@ -0,0 +1,245 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +import cn.hutool.core.date.DateField; +import cn.hutool.core.date.DatePattern; +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.util.RandomUtil; +import org.dubhe.constant.SymbolConstant; +import org.dubhe.enums.BizNfsEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.NotebookBizException; +import org.dubhe.k8s.domain.PtBaseResult; +import org.apache.shiro.UnavailableSecurityManagerException; +import org.dubhe.domain.dto.UserDTO; + +import java.util.Date; + +/** + * @description: Notebook 工具类 + * @date 2020-04-27 + */ +public class NotebookUtil { + + private static final char SEPARATOR = '-'; + private static final String NOTEBOOK = "notebook"; + private static final String NAMESPACE = "namespace"; + private static final String RESOURCE_NAME = "resource-name"; + private static final String IMAGE_NAME = "10.5.24.118:5000/notebook-tf-of-pytorch"; + private static final String K8S_MOUNT_PATH = "/tf"; + private static final String K8S_FILE_SEPARATOR = "/"; + private static final int TRUNCATION_INDEX = 250; + private static final int NOTEBOOK_TIMEOUT_SECOND = 180; + private static final String JUPYTER_TOKEN_KEY = "?token="; + + public static final String K8S_REGEX = "[a-z0-9]([-a-z0-9]*[a-z0-9])?"; + public static final String K8S_NOTEBOOK_REGEX = "^[\\u4e00-\\u9fa5_a-zA-Z0-9\\-]+$"; + public static final String NOTEBOOK_NOT_EXISTS = "Notebook不存在"; + public static final Long ANONYMITY_USER_ID = 0L; + + public static final int CPU_MIN_NUMBER = 1; + public static final int CPU_MAX_NUMBER = 8; + public static final int GPU_MAX_NUMBER = 4; + public static final int GPU_MIN_NUMBER = 0; + public static final int MEMORY_MIN_NUMBER = 1; + public static final int MEMORY_MAX_NUMBER = 8; + public static final int DISK_MEMORY_MIN_NUMBER = 1; + public static final int DISK_MEMORY_MAX_NUMBER = 1024; + + public static final String FAILED = "失败!"; + + private NotebookUtil() { + + } + + /** + * 验证是否包含失败关键字 + * + * @param info + * @return boolean + */ + public static boolean validateFailedInfo(String info) { + return info != null && info.contains(FAILED); + } + + /** + * 获取当前登录人ID + * 如果当前没登录人，则返回匿名用户0 + * + * @return long + */ + public static long getCurUserId() { + try { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + return userDTO == null ? ANONYMITY_USER_ID : userDTO.getId(); + } catch (UnavailableSecurityManagerException e) { + LogUtil.error(LogEnum.NOTE_BOOK, "未找到登录用户"); + return ANONYMITY_USER_ID; + } + } + + /** + * @param userId 当前登录人 + * @return namespace + * @deprecated use K8sNameTool.generateNameSpace(..) instead + * 根据当前用户 生成 Notebook的NameSpace + */ + @Deprecated + public static String generateNameSpace(long userId) { + return NOTEBOOK + SEPARATOR + NAMESPACE + SEPARATOR + userId; + } + + /** + * @param notebookName notebook名称 + * @return resourceName + * @deprecated use K8sNameTool.generateResourceName(..) instead + * 根据当前nodebook 生成 ResourceName + */ + @Deprecated + public static String generateResourceName(String notebookName) { + return NOTEBOOK + SEPARATOR + RESOURCE_NAME + SEPARATOR + notebookName; + } + + /** + * @return jupyter镜像名称 + * @deprecated use Harbor image instead + */ + @Deprecated + public static String getImageName() { + return IMAGE_NAME; + } + + /** + * @return String 容器内映射到宿主机的路径 + */ + public static String getK8sMountPath() { + return K8S_MOUNT_PATH; + } + + /** + * @param userId 当前登录人 + * @return 根据规则生成随机PVC路径 + * @deprecated use K8sNameTool.getNfsPath(..) instead + * 根据当前用户生成PVC路径（notebook重新启动得用原先路径） + */ + @Deprecated + public static String generatePvcPath(long userId) { + return K8S_FILE_SEPARATOR + + "nfs" + + K8S_FILE_SEPARATOR + + NOTEBOOK + + K8S_FILE_SEPARATOR + + userId + + K8S_FILE_SEPARATOR + + DateUtil.format(new Date(), DatePattern.PURE_DATETIME_FORMAT) + RandomUtil.randomString(4); + } + + /** + * 获取k8s响应结果的tMessage信息 + * 如果超长(250)，则截断 + * + * @param info + * @return String + */ + public static String getK8sStatusInfo(String info) { + if (info == null) { + return SymbolConstant.BLANK; + } + return StringUtils.truncationString(info, TRUNCATION_INDEX); + } + + + /** + * 获取k8s响应结果的tMessage信息 + * 如果超长(250)，则截断 + * + * @param ptBaseResult + * @return String + */ + public static String getK8sStatusInfo(PtBaseResult ptBaseResult) { + if (ptBaseResult == null) { + return SymbolConstant.BLANK; + } + return getK8sStatusInfo(ptBaseResult.getMessage()); + } + + /** + * 获取k8s异常结果的tMessage信息 + * 如果超长(250)，则截断 + * + * @param e 异常 + * @return String + */ + public static String getK8sStatusInfo(Exception e) { + if (e == null) { + return SymbolConstant.BLANK; + } else if (e instanceof NullPointerException) { + return "K8s NullPointerException"; + } + return getK8sStatusInfo(e.getMessage()); + } + + /** + * 验证 jupyter URL 是否包含token + * + * @param jupyterUrl + * @return boolean + */ + public static boolean checkUrlContainsToken(String jupyterUrl) { + if (StringUtils.isBlank(jupyterUrl)) { + return false; + } + return jupyterUrl.contains(JUPYTER_TOKEN_KEY); + } + + /** + * 判断notebook是否启动超时 + * + * @param lastStartDate + * @return boolean + */ + public static boolean notebookStartTimeout(Date lastStartDate) { + if (lastStartDate == null) { + return true; + } + Date timeOutPoint = DateUtil.offset(new Date(), DateField.SECOND, -NOTEBOOK_TIMEOUT_SECOND); + return timeOutPoint.after(lastStartDate); + } + + /** + * 获取超时时间秒 + * + * @return Long + */ + public static Long getTimeoutSecondLong() { + return K8sCallBackTool.getTimeoutSecondLong(NOTEBOOK_TIMEOUT_SECOND); + } + + + /** + * 生成第三方notebook名称 + * + * @param bizNfsEnum 业务源 + * @param sourceId 第三方源主键 + * @return String + */ + public static String generateName(BizNfsEnum bizNfsEnum, long sourceId) { + return bizNfsEnum.getBizCode() + SEPARATOR + sourceId; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java new file mode 100644 index 0000000..5a6e206 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.utils; + +/** + * @description: 训练任务工具类 + * @date: 2020-07-14 + */ +public class TrainUtil { + public static final String REGEXP = "^[a-zA-Z0-9\\-\\_\\u4e00-\\u9fa5]+$"; + public static final String REGEXP_NAME = "^[a-zA-Z0-9\\-\\_]+$"; + public static final String REGEXP_TAG = "^[a-zA-Z0-9\\-\\_\\.]+$"; + + public static final String RUNTIME = "%02d:%02d:%02d"; + public static final String FOUR_DECIMAL = "%04d"; + + public static final int NUMBER_ZERO = 0; + public static final int NUMBER_ONE = 1; + public static final int NUMBER_TWO = 2; + public static final int NUMBER_SEVEN = 7; + public static final int NUMBER_TWENTY = 20; + public static final int NUMBER_THIRTY_TWO = 32; + public static final int NUMBER_SIXTY_FOUR = 64; + public static final int NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN = 127; + public static final int NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT = 128; + public static final int NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE = 255; + public static final int NUMBER_ONE_THOUSAND = 1000; + public static final int NUMBER_ONE_THOUSAND_AND_TWENTY_FOUR = 1024; + +} diff --git a/dubhe-server/dubhe-admin/src/main/resources/banner.txt b/dubhe-server/dubhe-admin/src/main/resources/banner.txt new file mode 100644 index 0000000..f961ed5 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/resources/banner.txt @@ -0,0 +1,2 @@ + + :: Spring Boot :: (v2.2.5.RELEASE) diff --git a/dubhe-server/dubhe-admin/src/main/resources/generator.properties b/dubhe-server/dubhe-admin/src/main/resources/generator.properties new file mode 100644 index 0000000..78e13bf --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/resources/generator.properties @@ -0,0 +1,20 @@ +#数据库类型转Java类型 +tinyint=Integer +smallint=Integer +mediumint=Integer +int=Integer +integer=Integer +bigint=Long +float=Float +double=Double +decimal=BigDecimal +bit=Boolean +char=String +varchar=String +tinytext=String +text=String +mediumtext=String +longtext=String +date=Timestamp +datetime=Timestamp +timestamp=Timestamp \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/resources/log4jdbc.log4j2.properties b/dubhe-server/dubhe-admin/src/main/resources/log4jdbc.log4j2.properties new file mode 100644 index 0000000..302525f --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/resources/log4jdbc.log4j2.properties @@ -0,0 +1,4 @@ +# If you use SLF4J. First, you need to tell log4jdbc-log4j2 that you want to use the SLF4J logger +log4jdbc.spylogdelegator.name=net.sf.log4jdbc.log.slf4j.Slf4jSpyLogDelegator +log4jdbc.auto.load.popular.drivers=false +log4jdbc.drivers=com.mysql.cj.jdbc.Driver \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/resources/logback-spring.xml b/dubhe-server/dubhe-admin/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..025abc2 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/resources/logback-spring.xml @@ -0,0 +1,193 @@ + + + dubhe + + + + + + + + + ${log.pattern} + ${log.charset} + + + + + + logs/${log.path}/info/dubhe-info.log + + logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + ${log.charset} + + + true + + INFO + ACCEPT + DENY + + + + + + logs/${log.path}/debug/dubhe-debug.log + + logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + ${log.charset} + + + true + + DEBUG + ACCEPT + DENY + + + + + + logs/${log.path}/error/dubhe-error.log + + logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + ${log.charset} + + + true + + ERROR + ACCEPT + DENY + + + + + + logs/${log.path}/warn/dubhe-warn.log + + logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + ${log.charset} + + + true + + + WARN + ACCEPT + DENY + + + + + + logs/${log.path}/trace/dubhe-trace.log + + logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + ${log.charset} + + + true + + TRACE + ACCEPT + DENY + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dubhe-server/dubhe-admin/src/main/resources/mapper/PtTrainJobMapper.xml b/dubhe-server/dubhe-admin/src/main/resources/mapper/PtTrainJobMapper.xml new file mode 100644 index 0000000..cdd4e00 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/resources/mapper/PtTrainJobMapper.xml @@ -0,0 +1,39 @@ + + + + + + + \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java new file mode 100644 index 0000000..4ea8af0 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java @@ -0,0 +1,121 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.subject.Subject; +import org.apache.shiro.web.mgt.DefaultWebSecurityManager; +import org.dubhe.support.login.UsernamePasswordCaptchaToken; +import org.junit.Assert; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.web.WebAppConfiguration; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.ResultMatcher; +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.web.context.WebApplicationContext; + +import java.io.UnsupportedEncodingException; + +/** + * @description 基础测试类 + * @date 2020-04-20 + */ +@ActiveProfiles(value = "dev") +@RunWith(SpringRunner.class) +@SpringBootTest +@EnableTransactionManagement +@WebAppConfiguration +public class BaseTest { + + @Autowired + private WebApplicationContext wac; + protected MockMvc mockMvc; + + @Autowired + private DefaultWebSecurityManager defaultWebSecurityManager; + + public BaseTest() { + } + + @Before + public void setup() { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).build(); + SecurityUtils.setSecurityManager(defaultWebSecurityManager); + Subject subject = SecurityUtils.getSubject(); + UsernamePasswordCaptchaToken token = new UsernamePasswordCaptchaToken("admin", "admin"); + token.setRememberMe(true); + subject.login(token); + } + + /** + * mockMvcTest + * content 返回结果 + * status 返回状态码 + * + * @param mockHttpServletRequestBuilder 模拟HTTP请求 + * @param s 传入参数 + * @param ok 预期结果 + * @param i 预期状态码 + * @throws throws Exception + */ + public void mockMvcTest(MockHttpServletRequestBuilder mockHttpServletRequestBuilder, String s, ResultMatcher ok, int i) throws Exception { + MockHttpServletResponse response = this.mockMvc.perform( + mockHttpServletRequestBuilder + .contentType(MediaType.APPLICATION_JSON) + .content(s) + ).andExpect(ok) + .andReturn() + .getResponse(); + response.setCharacterEncoding("UTF-8"); + //得到返回状态码 + int status = response.getStatus(); + //得到返回结果 + String content = response.getContentAsString(); + //断言，判断返回代码是否正确 + Assert.assertEquals(i, status); + System.out.println(content); + } + + /** + * @param @param response + * @param @param i + * @param @throws UnsupportedEncodingException 入参 + * @return void 返回类型 + * @throws @Title: mockMvcWithNoRequestBody + */ + public void mockMvcWithNoRequestBody(MockHttpServletResponse response, int i) throws UnsupportedEncodingException { + response.setCharacterEncoding("UTF-8"); + // 得到返回代码 + int status = response.getStatus(); + // 得到返回结果 + String content = response.getContentAsString(); + // 断言，判断返回代码是否正确 + Assert.assertEquals(i, status); + System.out.println("返回的参数" + content); + } + +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java new file mode 100644 index 0000000..2cfb705 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; +/** + * @description 实体类 + * @date 2020-03-25 + */ +@RunWith(SpringRunner.class) +@SpringBootTest +public class DictControllerTest { + @Autowired + private WebApplicationContext wac; + private MockMvc mockMvc; + + public DictControllerTest() { + } + + @Before + public void setup() { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).build(); + } + + @Test + public void whenQueryUserAll() throws Exception { + this.mockMvc.perform(MockMvcRequestBuilders.get("/api/dict/all", new Object[0]).contentType(MediaType.APPLICATION_JSON_UTF8)).andExpect(MockMvcResultMatchers.status().isOk()); + } +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java new file mode 100644 index 0000000..d88d4c7 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java @@ -0,0 +1,37 @@ +/** Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import com.alibaba.fastjson.JSON; +import org.dubhe.domain.dto.PtTrainAlgorithmQueryDTO; +import org.junit.Test; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; + +/** + * @description 代理转发单元测试 + * @date 2020-06-23 + */ + +public class ForwardTest extends BaseTest { + @Test + public void ForwardControllerTest() throws Exception { + PtTrainAlgorithmQueryDTO ptTrainAlgorithmQueryDTO = new PtTrainAlgorithmQueryDTO(); + ptTrainAlgorithmQueryDTO.setAlgorithmSource(1).setCurrent(1).setSize(10).setSort("id").setOrder("asc"); + mockMvcTest(MockMvcRequestBuilders.get("/forward/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmQueryDTO), MockMvcResultMatchers.status().isOk(), 200); + } +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java new file mode 100644 index 0000000..eef1b5c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import org.dubhe.enums.BizEnum; +import org.dubhe.utils.K8sCallBackTool; +import org.dubhe.utils.K8sNameTool; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * @description K8sCallBackToolTest测试类 + * @date 2020-05-28 + */ +@ActiveProfiles(value = "notebook") +@RunWith(SpringRunner.class) +@SpringBootTest(classes= AppRun.class,webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) +public class K8sCallBackToolTest{ + + @Autowired + private K8sCallBackTool k8sCallBackTool; + + @Autowired + private K8sNameTool k8sNameTool; + + @Test + public void token(){ + System.out.println(k8sCallBackTool.generateToken()); + Assert.assertTrue(k8sCallBackTool.validateToken(k8sCallBackTool.generateToken())); + } + + @Test + public void getPodCallbackUrl(){ + Assert.assertEquals("http://xxx.xxx.xxx.xxx:xxxx/api/k8s/callback/pod/notebook",k8sCallBackTool.getPodCallbackUrl(k8sNameTool.getPodLabel(BizEnum.NOTEBOOK))); + } + +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java new file mode 100644 index 0000000..33c9db1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java @@ -0,0 +1,122 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + + +import org.dubhe.enums.BizEnum; +import org.dubhe.enums.BizNfsEnum; +import org.dubhe.utils.K8sNameTool; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; + + +/** + * @description K8sNameToolTest测试类 + * @date 2020-05-13 + */ +@ActiveProfiles(value = "notebook") +@RunWith(SpringRunner.class) +@SpringBootTest +public class K8sNameToolTest { + + @Autowired + private K8sNameTool k8sNameTool; + + @Test + public void generateResourceName(){ + Assert.assertEquals("notebook-rn-info", k8sNameTool.generateResourceName(BizEnum.NOTEBOOK,"info")); + } + + @Test + public void generateNameSpace(){ + Assert.assertEquals("namespace-0", k8sNameTool.generateNameSpace(0L)); + } + + @Test + public void getUserIdFromNameSpace(){ + Assert.assertSame(0L, k8sNameTool.getUserIdFromNameSpace("namespace-0")); + } + + + @Test + public void getNfsPath(){ + Assert.assertTrue(k8sNameTool.getNfsPath(BizNfsEnum.NOTEBOOK,0L).startsWith("/notebook/0/")); + } + + @Test + public void getAbsoluteNfsPath(){ + Assert.assertEquals("/nfs/dubhe-dev/notebook/0/20200513162830yody/",k8sNameTool.getAbsoluteNfsPath("/notebook/0/20200513162830yody/")); + } + + @Test + public void appendBucket(){ + Assert.assertEquals("/dubhe-dev/notebook/0/20200513162830yody/",k8sNameTool.appendBucket("/notebook/0/20200513162830yody/")); + Assert.assertEquals("/dubhe-dev/notebook/0/20200513162830yody/",k8sNameTool.appendBucket("notebook/0/20200513162830yody/")); + } + + @Test + public void convertNfsPath(){ + Assert.assertEquals("/algorithm-manage/0/20200513162830yody/",k8sNameTool.convertNfsPath( + "/notebook/0/20200513162830yody/" + ,BizNfsEnum.NOTEBOOK + ,BizNfsEnum.ALGORITHM)); + } + + @Test + public void validateBizNfsPath(){ + Assert.assertTrue(k8sNameTool.validateBizNfsPath( + "/notebook/0/20200513162830yody/" + ,BizNfsEnum.NOTEBOOK)); + Assert.assertFalse(k8sNameTool.validateBizNfsPath( + "/notebook/0/20200513162830yody/" + ,BizNfsEnum.ALGORITHM)); + Assert.assertFalse(k8sNameTool.validateBizNfsPath( + "/dubhe-dev/notebook/0/20200513162830yody/" + ,BizNfsEnum.ALGORITHM)); + Assert.assertFalse(k8sNameTool.validateBizNfsPath( + "/nfs/dubhe-dev/notebook/0/20200513162830yody/" + ,BizNfsEnum.ALGORITHM)); + } + + @Test + public void getByCreateResource(){ + Assert.assertSame(BizNfsEnum.ALGORITHM , BizNfsEnum.getByCreateResource(1)); + Assert.assertNull( BizNfsEnum.getByCreateResource(10)); + } + + @Test + public void removeNfsRootPath(){ + Assert.assertEquals( "/dubhe-dev/algorithm-manage/0/20200519111111test", k8sNameTool.removeNfsRootPath("/nfs/dubhe-dev/algorithm-manage/0/20200519111111test")); + Assert.assertEquals( "/nfs2/dubhe-dev/algorithm-manage/0/20200519111111test", k8sNameTool.removeNfsRootPath("/nfs2/dubhe-dev/algorithm-manage/0/20200519111111test")); + } + + @Test + public void getPodLabel(){ + Assert.assertEquals( "notebook", k8sNameTool.getPodLabel(BizEnum.NOTEBOOK)); + } + + @Test + public void getDatasetPath(){ + Assert.assertEquals( "/dataset", k8sNameTool.getDatasetPath()); + } +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java new file mode 100644 index 0000000..1bb22b5 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java @@ -0,0 +1,78 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import com.alibaba.fastjson.JSON; +import org.dubhe.domain.dto.PtImageUploadDTO; +import org.junit.Test; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +/** + * @description 镜像接口单元测试 + * @date 2020-06-28 + */ +public class PtImageTest extends BaseTest { + + /** + * 查询镜像 + */ + @Test + public void getImageTest() throws Exception { + MultiValueMap params = new LinkedMultiValueMap<>(); + params.add("current", "1"); + params.add("size", "10"); + params.add("sort", "id"); + params.add("order", "desc"); + params.add("imageResource", "1"); + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/ptImage/info").param("imageResource", "1")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + /** + * 通过projectName查询镜像 + */ + @Test + public void getTagsByImageNameTest() throws Exception { + String imageName = "redis"; + mockMvc.perform( + MockMvcRequestBuilders.get("/api/v1/ptImage") + .param("imageName", imageName)) + .andExpect(MockMvcResultMatchers.status().is2xxSuccessful()) + .andReturn().getResponse().getContentAsString(); + } + + /** + * 上传镜像包到harbor + */ + @Test + public void uploadImageTest() throws Exception { + PtImageUploadDTO ptImageUploadDTO = new PtImageUploadDTO(); + ptImageUploadDTO.setImageName("mysql"); + ptImageUploadDTO.setImagePath("F:/mysql.tar"); + ptImageUploadDTO.setImageTag("5.7"); + ptImageUploadDTO.setRemark("测试上传镜像"); + + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/ptImage/uploadImage"), JSON.toJSONString(ptImageUploadDTO), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + } + +} + diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java new file mode 100644 index 0000000..5c6c230 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java @@ -0,0 +1,91 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import com.alibaba.fastjson.JSON; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageCreateDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageUpdateDTO; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; + +/** + * @description 算法用途单元测试 + * @date 2020-06-23 + */ + +@RunWith(SpringRunner.class) +@SpringBootTest +public class PtTrainAlgorithmUsageTest extends BaseTest { + + /** + * 修改任务参数 算法id=2在算法表中runcommand为空 + */ + @Test + public void queryAllTest() throws Exception { + + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithmUsage").param("isContainDefault", "1")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + @Test + public void queryAllTest2() throws Exception { + + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithmUsage").param("isContainDefault", "0")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + @Test + public void createTest() throws Exception { + + PtTrainAlgorithmUsageCreateDTO ptTrainAlgorithmUsageCreateDTO = new PtTrainAlgorithmUsageCreateDTO(); + ptTrainAlgorithmUsageCreateDTO.setAuxInfo("测试"); + + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/algorithmUsage"), + JSON.toJSONString(ptTrainAlgorithmUsageCreateDTO), MockMvcResultMatchers.status().is2xxSuccessful(), + 200); + + } + + + @Test + public void deleteTest() throws Exception { + Long[] longs = new Long[1]; + longs[0] = 13L; + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/algorithmUsage"), JSON.toJSONString(longs), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + + } + + @Test + public void updateTest() throws Exception { + PtTrainAlgorithmUsageUpdateDTO ptTrainAlgorithmUsageUpdateDTO = new PtTrainAlgorithmUsageUpdateDTO(); + + ptTrainAlgorithmUsageUpdateDTO.setId(12L); + ptTrainAlgorithmUsageUpdateDTO.setAuxInfo("更新测试"); + + mockMvcTest(MockMvcRequestBuilders.put("/api/v1/algorithmUsage"), + JSON.toJSONString(ptTrainAlgorithmUsageUpdateDTO), MockMvcResultMatchers.status().is2xxSuccessful(), + 200); + } + + +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java new file mode 100644 index 0000000..74634d3 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +/** + * @description 用户查询训练日志 + * @date 2020-6-20 + **/ +@SpringBootTest +@RunWith(SpringRunner.class) +public class PtTrainLogServiceTest extends BaseTest { + + @Test + public void searchTrainLogInfo() throws Exception { + MultiValueMap params = new LinkedMultiValueMap<>(); + params.add("jobId", "68"); + params.add("startLine", "1"); + params.add("lines", "50"); + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainLog").params(params)) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java new file mode 100644 index 0000000..fad45f0 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java @@ -0,0 +1,208 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import org.dubhe.domain.dto.PtTrainJobCreateDTO; +import org.dubhe.domain.dto.PtTrainJobDeleteDTO; +import org.dubhe.domain.dto.PtTrainJobStopDTO; +import org.dubhe.domain.dto.PtTrainJobUpdateDTO; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.transaction.annotation.Transactional; + +/** + * @description 训练任务管理模块任务参数管理单元测试 + * @date 2020-5-11 + */ + +@RunWith(SpringRunner.class) +@SpringBootTest +public class PtTrainModelOptJobApiTest extends BaseTest { + + + /** + * 修改任务参数 算法id=2在算法表中runcommand为空 + * + */ + @Test + @Transactional(rollbackFor = Exception.class) + public void createTrainJobTest() throws Exception { + PtTrainJobCreateDTO ptTrainJobCreateDTO = new PtTrainJobCreateDTO(); + ptTrainJobCreateDTO.setAlgorithmId(18L); + ptTrainJobCreateDTO.setDataSourceName("dataset/68"); + ptTrainJobCreateDTO.setDataSourcePath("dataset/68"); + ptTrainJobCreateDTO.setDescription("job描述"); + ptTrainJobCreateDTO.setTrainJobSpecsId(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + JSONObject runParams = new JSONObject(); + runParams.put("key1", 33); + runParams.put("key2", 33); + runParams.put("key3", 33); + runParams.put("key4", 33); + ptTrainJobCreateDTO.setRunParams(runParams); + ptTrainJobCreateDTO.setSaveParams(true); + ptTrainJobCreateDTO.setTrainName("trainjobtest"); + ptTrainJobCreateDTO.setTrainParamDesc("job描述"); + ptTrainJobCreateDTO.setTrainParamName("paramname5"); + + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobCreateDTO), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + } + + @Test + public void getTrainJobTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + @Test + public void getTrainJobSpecsTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/trainJobSpecs").param("resourcesPoolType", "0")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + @Test + public void getTrainJobDetailTest() throws Exception { + mockMvcWithNoRequestBody( + mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/jobDetail").param("id", "20")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), + 200); + } + + /** + * 我的训练任务统计 + * + * @throws Exception + */ + @Test + public void statisticsMineTest() throws Exception { + mockMvcWithNoRequestBody(this.mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/mine")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + /** + * @param @throws Exception 入参 + * @return void 返回类型 + * @throws @date 2020年6月16日 上午10:19:12 + * @Title: getTrainJobVersionTest + * @version V1.0 + */ + @Test + public void getTrainJobVersionTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/10")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + + } + + /** + * @param @throws Exception 入参 + * @return void 返回类型 + * @throws @date 2020年6月16日 上午10:31:25 + * @Title: getTrainDataSourceStatusTest + * @version V1.0 + */ + @Test + public void getTrainDataSourceStatusTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc + .perform(MockMvcRequestBuilders.get("/api/v1/trainJob/dataSourceStatus").param("dataSourcePath", + "dataset/68,dataset/20741/versionFile/V0003")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + + } + + /** + * 更新训练任务测试 + * + * @param @throws Exception 入参 + * @return void 返回类型 + * @throws @date 2020年6月16日 上午10:46:55 + * @Title: updateTrainJobTest + * @version V1.0 + */ + @Test + public void updateTrainJobTest() throws Exception { + // TODO Auto-generated method stub + PtTrainJobUpdateDTO ptTrainJobUpdateDTO = new PtTrainJobUpdateDTO(); + ptTrainJobUpdateDTO.setId(39L); + ptTrainJobUpdateDTO.setAlgorithmId(91L); + ptTrainJobUpdateDTO.setDataSourceName("dataset/68"); + ptTrainJobUpdateDTO.setDataSourcePath("dataset/68"); + ptTrainJobUpdateDTO.setTrainJobSpecsId(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobUpdateDTO), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + + } + + /** + * 删除训练任务 + * + * @param + * @return void 返回类型 + * @throws @date 2020年6月16日 上午10:50:31 + * @throws Exception + * @Title: deleteTrainJobTest + * @version V1.0 + */ + @Test + public void deleteTrainJobTest() throws Exception { + // TODO Auto-generated method stub + PtTrainJobDeleteDTO ptTrainJobDeleteDTO = new PtTrainJobDeleteDTO(); + ptTrainJobDeleteDTO.setId(38L); + ptTrainJobDeleteDTO.setTrainId(36L); + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobDeleteDTO), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + + } + + @Test + public void deleteTrainJobWithNoIdTest() throws Exception { + // TODO Auto-generated method stub + PtTrainJobDeleteDTO ptTrainJobDeleteDTO = new PtTrainJobDeleteDTO(); + ptTrainJobDeleteDTO.setTrainId(36L); + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobDeleteDTO), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + + } + + /** + * 停止训练任务测试 + * + * @param @throws Exception 入参 + * @return void 返回类型 + * @throws @date 2020年6月16日 上午10:55:22 + * @Title: testStopTrainJob + * @version V1.0 + */ + @Test + public void testStopTrainJob() throws Exception { + + PtTrainJobStopDTO dto = new PtTrainJobStopDTO(); + dto.setId(46L); + dto.setTrainId(37L); + + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainJob/stop"), JSON.toJSONString(dto), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + } + + +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java new file mode 100644 index 0000000..91178aa --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java @@ -0,0 +1,114 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import org.dubhe.domain.dto.PtTrainAlgorithmCreateDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUpdateDTO; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; + +/** + * @description: 算法管理模块算法管理单元测试 + * @date: 2020-06-18 + */ +public class TrainAlgorithmTest extends BaseTest { + + /** + * 查询算法列表 + */ + @Test + public void ptTrainAlgorithmQueryTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithm").param("algorithmSource", "2")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + /** + * 查询当前用户的算法个数 + */ + @Test + public void getAlgorithmCountTest() throws Exception { + MockHttpServletResponse response = this.mockMvc.perform( + MockMvcRequestBuilders.get("/api/v1/algorithm/myAlgorithmCount") + ).andExpect(MockMvcResultMatchers.status().isOk()) + .andReturn() + .getResponse(); + response.setCharacterEncoding("UTF-8"); + //得到返回状态码 + int status = response.getStatus(); + //得到返回结果 + String content = response.getContentAsString(); + //断言，判断返回代码是否正确 + Assert.assertEquals(200, status); + System.out.println(content); + + } + + /** + * 模型管理查询对应路径的算法 + */ + @Test + public void getTrainAlgorithmFromPathTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithm/fromPath").param("codeDir", "/algorithm-manage/common/resnet50v1.5/")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + /** + * 新增算法 + */ + @Test + public void ptTrainAlgorithmCreateTest() throws Exception { + PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO = new PtTrainAlgorithmCreateDTO(); + JSONObject jsonObject = new JSONObject(); + ptTrainAlgorithmCreateDTO.setAlgorithmName("test") + .setDescription("test") + .setImageName("tensorflow/tensorflow:latest") + .setCodeDir("algorithm-manage/18/20200617073954700eqhl/") + .setRunParams(jsonObject) + .setAlgorithmUsage("test") + .setRunCommand("test"); + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmCreateDTO), MockMvcResultMatchers.status().isOk(), 200); + } + + /** + * 修改算法 + */ + @Test + public void ptTrainAlgorithmUpdateTest() throws Exception { + PtTrainAlgorithmUpdateDTO ptTrainAlgorithmUpdateDTO = new PtTrainAlgorithmUpdateDTO(); + ptTrainAlgorithmUpdateDTO.setId(1L) + .setAlgorithmName("test") + .setDescription("test"); + mockMvcTest(MockMvcRequestBuilders.put("/api/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmUpdateDTO), MockMvcResultMatchers.status().isOk(), 200); + } + + + /** + * 删除算法 + */ + @Test + public void ptTrainAlgorithmDeleteTest() throws Exception { + Long[] ids = {(long) 1, (long) 2}; + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/algorithm"), JSON.toJSONString(ids), MockMvcResultMatchers.status().isOk(), 200); + } + +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainModelOptJobApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainModelOptJobApiTest.java new file mode 100644 index 0000000..492f673 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainModelOptJobApiTest.java @@ -0,0 +1,122 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import org.dubhe.domain.dto.PtTrainJobCreateDTO; +import org.dubhe.domain.dto.PtTrainJobStopDTO; +import org.dubhe.domain.dto.PtTrainQueryDTO; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; + +/** + * @description: + * @date: 2020-04-17 + */ +public class TrainModelOptJobApiTest extends BaseTest { + + /** + * 停止任务测试 + * + * @throws Exception + */ + @Test + public void testStopTrainJob() throws Exception { + + PtTrainJobStopDTO dto = new PtTrainJobStopDTO(); + dto.setId(1L); + dto.setTrainId(10L); + MockHttpServletResponse response = this.mockMvc.perform( + MockMvcRequestBuilders. + post("/api/train_job/1") + .contentType(MediaType.APPLICATION_JSON).content(JSON.toJSONString(dto) + )) + .andExpect(MockMvcResultMatchers.status().isOk()) + .andReturn().getResponse(); + JSONObject jsonObject = JSON.parseObject(response.getContentAsString()); + Assert.assertNotNull(jsonObject.get("code")); + } + + + /** + * 我的训练任务统计 + * + * @throws Exception + */ + @Test + public void statisticsMine() throws Exception { + + MockHttpServletResponse response = this.mockMvc.perform( + MockMvcRequestBuilders. + get("/api/train_job/mine") + ) + .andExpect(MockMvcResultMatchers.status().isOk()) + .andReturn().getResponse(); + JSONObject jsonObject = JSON.parseObject(response.getContentAsString()); + Assert.assertNotNull(jsonObject.get("code")); + } + + /** + * 分页查询训练任务 + */ + @Test + public void testGetTrainJob() throws Exception { + PtTrainQueryDTO dto = new PtTrainQueryDTO(); + MockHttpServletResponse response = this.mockMvc.perform( + MockMvcRequestBuilders. + get("/api/v1/trainJob") + .contentType(MediaType.APPLICATION_JSON) + .content(JSON.toJSONString(dto)) + ) + .andExpect(MockMvcResultMatchers.status().isOk()) + .andReturn().getResponse(); + JSONObject jsonObject = JSON.parseObject(response.getContentAsString()); + Assert.assertNotNull(jsonObject.get("code")); + } + + /** + * 分页查询训练任务 + */ + @Test + public void testCreateTrainJob() throws Exception { + PtTrainJobCreateDTO dto = new PtTrainJobCreateDTO(); + dto.setAlgorithmId(3L); + dto.setRunParams(new JSONObject()); + dto.setDataSourceName("dataset/68"); + dto.setDataSourcePath("dataset/68"); + dto.setTrainName("test-train"+System.currentTimeMillis()); + dto.setTrainParamDesc("test-train"); + dto.setTrainJobSpecsId(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + + MockHttpServletResponse response = this.mockMvc.perform( + MockMvcRequestBuilders. + post("/api/v1/trainJob") + .contentType(MediaType.APPLICATION_JSON) + .content(JSON.toJSONString(dto)) + ) + .andExpect(MockMvcResultMatchers.status().isOk()) + .andReturn().getResponse(); + JSONObject jsonObject = JSON.parseObject(response.getContentAsString()); + Assert.assertNotNull(jsonObject.get("code")); + } +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java new file mode 100644 index 0000000..c2f68d2 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java @@ -0,0 +1,213 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import org.dubhe.domain.dto.PtTrainParamCreateDTO; +import org.dubhe.domain.dto.PtTrainParamUpdateDTO; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.transaction.annotation.Transactional; + +/** + * @description: 训练任务管理模块任务参数管理单元测试 + * @date: 2020-5-11 + */ +@RunWith(SpringRunner.class) +@SpringBootTest +public class TrainParamApiTest extends BaseTest { + + /** + * 查询任务参数列表 + */ + @Test + public void ptTrainParamQueryTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainParams")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + /** + * 查询任务参数列表 + */ + @Test + public void ptTrainParamQueryTest1() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainParams").param("paramName", "lpf").param("resourcesPoolType", "0")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + /** + * 新增任务参数 + */ + @Test + @Transactional(rollbackFor = Exception.class) + @Rollback(false) + public void ptTrainParamCreateTest() throws Exception { + PtTrainParamCreateDTO ptTrainParamCreateDTO = new PtTrainParamCreateDTO(); + JSONObject runParams = new JSONObject(); + runParams.put("data_url", "/nfs/testuser1/mnist/MNIST_data"); + ptTrainParamCreateDTO.setParamName("新增任务参数名称").setAlgorithmId((long) 1).setRunParams(runParams) + .setDescription("描述信息").setDataSourcePath("1") + .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamCreateDTO), MockMvcResultMatchers.status().isOk(), 200); + + } + + /** + * 新增任务参数 + */ + @Test + @Transactional(rollbackFor = Exception.class) + public void ptTrainParamCreateTest1() throws Exception { + PtTrainParamCreateDTO ptTrainParamCreateDTO = new PtTrainParamCreateDTO(); + JSONObject runParams = new JSONObject(); + runParams.put("key1", 2); + runParams.put("key2", 2); + runParams.put("key3", 2); + runParams.put("key4", 2); + ptTrainParamCreateDTO.setParamName("新增任务参数名称") + .setAlgorithmId((long) 1) + .setRunParams(runParams) + .setDescription("描述信息") + .setDataSourcePath("/usr/local/data/out.json") + .setDataSourceName("out.json") + .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamCreateDTO), MockMvcResultMatchers.status().is4xxClientError(), 400); + + } + + /** + * 新增任务参数 + */ + @Test + @Transactional(rollbackFor = Exception.class) + public void ptTrainParamCreateTest2() throws Exception { + PtTrainParamCreateDTO ptTrainParamCreateDTO = new PtTrainParamCreateDTO(); + JSONObject runParams = new JSONObject(); + runParams.put("key1", 3); + runParams.put("key2", 3); + runParams.put("key3", 3); + runParams.put("key4", 3); + ptTrainParamCreateDTO.setParamName("新增任务参数名称" + System.currentTimeMillis()).setAlgorithmId(Long.MAX_VALUE).setRunParams(runParams) + .setDescription("描述信息") + .setDataSourcePath("/usr/local/data/out.json") + .setDataSourceName("out.json") + .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamCreateDTO), MockMvcResultMatchers.status().is4xxClientError(), 400); + + } + + /** + * 修改任务参数 + */ + @Test + @Transactional(rollbackFor = Exception.class) + @Rollback(false) + public void ptTrainParamUpdateTest() throws Exception { + PtTrainParamUpdateDTO ptTrainParamUpdateDTO = new PtTrainParamUpdateDTO(); + JSONObject runParams = new JSONObject(); + runParams.put("key1", 11); + runParams.put("key2", 11); + runParams.put("key3", 11); + runParams.put("key4", 11); + ptTrainParamUpdateDTO.setId((long) 1).setParamName("修改任务参数名称" + System.currentTimeMillis()).setAlgorithmId((long) 1) + .setRunParams(runParams) + .setDescription("描述信息") + .setDataSourcePath("/usr/local/data/out.json") + .setDataSourceName("out.json") + .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("harbor.dubhe.ai/tensorflow/tensorflow:latest"); + mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamUpdateDTO), MockMvcResultMatchers.status().isOk(), 200); + } + + /** + * 修改任务参数 + */ + @Test + @Transactional(rollbackFor = Exception.class) + public void ptTrainParamUpdateTest2() throws Exception { + PtTrainParamUpdateDTO ptTrainParamUpdateDTO = new PtTrainParamUpdateDTO(); + JSONObject runParams = new JSONObject(); + runParams.put("key1", 22); + runParams.put("key2", 22); + runParams.put("key3", 22); + runParams.put("key4", 22); + ptTrainParamUpdateDTO.setId((long) 2).setParamName("修改测试").setAlgorithmId((long) 1).setRunParams(runParams) + .setDescription("描述信息").setDataSourcePath("/usr/local/data/out.json") + .setDataSourceName("out.json") + .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamUpdateDTO), MockMvcResultMatchers.status().is4xxClientError(), 400); + } + + /** + * 修改任务参数 + */ + @Test + @Transactional(rollbackFor = Exception.class) + public void ptTrainParamUpdateTest1() throws Exception { + PtTrainParamUpdateDTO ptTrainParamUpdateDTO = new PtTrainParamUpdateDTO(); + JSONObject runParams = new JSONObject(); + runParams.put("key1", 33); + runParams.put("key2", 33); + runParams.put("key3", 33); + runParams.put("key4", 33); + ptTrainParamUpdateDTO.setId((long) 1).setParamName("修改任务参数名称").setAlgorithmId((long) 10).setRunParams(runParams) + .setDescription("描述信息").setDataSourcePath("/usr/local/data/out.json") + .setDataSourceName("out.json") + .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamUpdateDTO), MockMvcResultMatchers.status().is4xxClientError(), 400); + } + + /** + * 删除任务参数 + */ + @Test + @Transactional(rollbackFor = Exception.class) + @Rollback(false) + public void ptTrainParamDeleteTest() throws Exception { + Long[] ids = {1L}; + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainParams"), JSON.toJSONString(ids), MockMvcResultMatchers.status().isOk(), 200); + } + + /** + * 删除任务参数 + */ + @Test + @Transactional(rollbackFor = Exception.class) + public void ptTrainParamDeleteTest0() throws Exception { + Long[] ids = {}; + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainParams"), JSON.toJSONString(ids), MockMvcResultMatchers.status().is4xxClientError(), 400); + } + + /** + * 删除任务参数 + * 传入id数组在映射表中有id不存在测试 + * id=10在表中不存在 + */ + @Test + @Transactional(rollbackFor = Exception.class) + public void ptTrainParamDeleteTest1() throws Exception { + Long[] ids = {1L, 10L}; + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainParams"), JSON.toJSONString(ids), MockMvcResultMatchers.status().is4xxClientError(), 400); + } + +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java new file mode 100644 index 0000000..e10dedb --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.data.service; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import org.dubhe.BaseTest; +import org.dubhe.data.service.impl.FileServiceImpl; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * @description 文件业务层测试类 + * @date 2020-05-12 + */ +public class FileServiceImplTest extends BaseTest { + + @Autowired + private FileServiceImpl fileService; + + @Test + public void list() { + System.out.println(fileService.listByLimit(3L, 10, new QueryWrapper<>())); + } + +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java new file mode 100644 index 0000000..3f638f7 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java @@ -0,0 +1,52 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.data.service; + +import org.dubhe.BaseTest; +import org.dubhe.data.domain.entity.Label; +import org.dubhe.data.service.impl.LabelServiceImpl; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Arrays; +import java.util.List; + +/** + * @description 标签业务层测试类 + * @date 2020-05-12 + */ +public class LabelBOServiceImplTest extends BaseTest { + + @Autowired + private LabelServiceImpl labelService; + + @Test + public void exist() { + System.out.println(labelService.exist(8L)); + } + + @Test + public void save() { + List