Browse Source

init

tags/v0.3.0
之江实验室 4 years ago
commit
e2743c1873
100 changed files with 7977 additions and 0 deletions
  1. +211
    -0
      LICENSE
  2. +103
    -0
      README.md
  3. BIN
      docs/images/dingtalk.jpg
  4. BIN
      docs/images/dubhe_dashboard.png
  5. +22
    -0
      docs/images/logo.svg
  6. BIN
      docs/images/tech-arc.jpg
  7. +53
    -0
      dubhe-server/.gitignore
  8. +10
    -0
      dubhe-server/.gitmessage
  9. +98
    -0
      dubhe-server/README.md
  10. +121
    -0
      dubhe-server/common/pom.xml
  11. +32
    -0
      dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java
  12. +44
    -0
      dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java
  13. +87
    -0
      dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java
  14. +23
    -0
      dubhe-server/common/src/main/java/org/dubhe/annotation/Log.java
  15. +68
    -0
      dubhe-server/common/src/main/java/org/dubhe/annotation/Query.java
  16. +33
    -0
      dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java
  17. +87
    -0
      dubhe-server/common/src/main/java/org/dubhe/aspect/LogAspect.java
  18. +58
    -0
      dubhe-server/common/src/main/java/org/dubhe/base/BaseConvert.java
  19. +48
    -0
      dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java
  20. +76
    -0
      dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java
  21. +44
    -0
      dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java
  22. +46
    -0
      dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java
  23. +76
    -0
      dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java
  24. +99
    -0
      dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java
  25. +62
    -0
      dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java
  26. +33
    -0
      dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java
  27. +56
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/GlobalCorsConfig.java
  28. +44
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java
  29. +65
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/KaptchaConfig.java
  30. +44
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java
  31. +95
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java
  32. +310
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java
  33. +41
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java
  34. +214
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java
  35. +70
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/Resources.java
  36. +36
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java
  37. +138
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java
  38. +39
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java
  39. +50
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java
  40. +35
    -0
      dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java
  41. +46
    -0
      dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java
  42. +39
    -0
      dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java
  43. +40
    -0
      dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java
  44. +66
    -0
      dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java
  45. +46
    -0
      dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java
  46. +46
    -0
      dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java
  47. +47
    -0
      dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java
  48. +100
    -0
      dubhe-server/common/src/main/java/org/dubhe/constant/TrainJobConstant.java
  49. +36
    -0
      dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java
  50. +65
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/dto/UserDTO.java
  51. +64
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java
  52. +120
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/entity/Menu.java
  53. +87
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/entity/Role.java
  54. +49
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/entity/RoleMenu.java
  55. +85
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java
  56. +58
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java
  57. +126
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/entity/User.java
  58. +58
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserAvatar.java
  59. +47
    -0
      dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserRole.java
  60. +35
    -0
      dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java
  61. +33
    -0
      dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java
  62. +76
    -0
      dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java
  63. +36
    -0
      dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java
  64. +80
    -0
      dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java
  65. +91
    -0
      dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java
  66. +61
    -0
      dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java
  67. +95
    -0
      dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java
  68. +101
    -0
      dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java
  69. +69
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java
  70. +72
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java
  71. +48
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java
  72. +48
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java
  73. +39
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java
  74. +44
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/LoginException.java
  75. +42
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java
  76. +45
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java
  77. +47
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java
  78. +190
    -0
      dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java
  79. +121
    -0
      dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java
  80. +60
    -0
      dubhe-server/common/src/main/java/org/dubhe/filter/FileLogFilter.java
  81. +68
    -0
      dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java
  82. +64
    -0
      dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java
  83. +58
    -0
      dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java
  84. +97
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java
  85. +80
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java
  86. +47
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java
  87. +100
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java
  88. +355
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java
  89. +131
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java
  90. +66
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java
  91. +37
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java
  92. +203
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java
  93. +168
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java
  94. +255
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java
  95. +39
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java
  96. +255
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java
  97. +49
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java
  98. +286
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java
  99. +76
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java
  100. +114
    -0
      dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java

+ 211
- 0
LICENSE View File

@@ -0,0 +1,211 @@

Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/

TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION

1. Definitions.

"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.

"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.

"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.

"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.

"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.

"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.

"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).

"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.

"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."

"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.

2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.

3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.

4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:

(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and

(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and

(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and

(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.

You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.

5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.

6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.

7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.

8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.

9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.

END OF TERMS AND CONDITIONS

APPENDIX: How to apply the Apache License to your work.

To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.

Copyright [yyyy] [name of copyright owner]

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Other dependencies and licenses:
----------------------------------------------------------------------------------------

Open Source Software Licensed Under the Apache License, Version 2.0:
The below software in this distribution may have been modified.
----------------------------------------------------------------------------------------
1. EL-ADMIN
Copyright 2019-2020 Zheng Jie

+ 103
- 0
README.md View File

@@ -0,0 +1,103 @@
<p align="center">
<a href="http://tianshu.org.cn">
<img width="200" src="http://tianshu.org.cn/template/default/assets/img/logo4.png">
</a>
</p>

<h1 align="center">天枢一站式AI模型开发平台</h1>

**天枢人工智能开源开放平台**(简称:**天枢平台**)是天枢平台由之江实验室牵头,联合北京一流科技、中国信通院和浙江大学共同自研的人工智能开源平台。整个平台由一站式AI模型开发平台、高性能深度学习框架和模型炼知框架三大子系统组成。

其中, **一站式AI模型开发平台面**(简称:**一站式开发平台**)面向AI模型生产的生命周期,提供了包括数据处理、模型开发、模型训练和模型管理等功能,方便用户一站式构建AI算法。

## 平台优势

* **一站式开发**:为用户提供一站式深度学习开发功能,通过智能数据处理、便利的模型开发和模型训练,打通深度学习全链路;

* **集成先进算法**:除了囊括常规AI算法外,之江天枢还集成了多领域的独家算法,提供业界领先性能。

* **灵活易用**:除了一站式深度学习开发平台,亦提供可视化和动静结合编码方式,调试灵活,小白亦可快速上手。

* **性能优越**:集成自主研发的分布式训练平台,提供高性能的分布式计算体验,节省训练成本和训练时间。

## 页面预览
![概览](/docs/images/dubhe_dashboard.png "概览")


## 功能列表
<table>
<tbody>
<tr>
<td rowspan="12">一站式开发平台</td>
</tr>
<tr>
<td>数据管理</td>
<td>数据集管理</td>
</tr>
<tr>
<td rowspan="2">模型开发</td>
<td>Notebook</td>
</tr>
<tr>
<td>算法管理</td>
</tr>
<tr>
<td rowspan="2">训练管理</td>
<td>镜像管理</td>
</tr>
<tr>
<td>训练任务</td>
</tr>
<tr>
<td>模型管理</td>
<td>模型列表</td>
</tr>
<tr>
<td rowspan="5">控制台</td>
<td>用户管理</td>
</tr>
<tr>
<td>角色管理</td>
</tr>
<tr>
<td>菜单管理</td>
</tr>
<tr>
<td>字典管理</td>
</tr>
<tr>
<td>集群状态</td>
</tr>
</tbody>
</table>

## 技术架构
![技术架构](./docs/images/tech-arc.jpg "技术架构")

## 技术栈
- 后端: [Spring Boot](https://spring.io/projects/spring-boot)
- 前端: [Vue.js](https://vuejs.org/), [Element](https://element.eleme.cn/)
- 数据处理 [Yolo](https://pjreddie.com/darknet/yolo/) ...
- 可视化: [Django](https://www.djangoproject.com/) ...
- 中间件: [MySQL](https://www.mysql.com/), [MyBatis-Plus](https://mp.baomidou.com/), [Redis](https://redis.io/)
- 基础设施: [Docker](https://www.docker.com/), [Kubernetes](https://kubernetes.io/)

## 目录结构

```
├── dubhe_data_process 数据处理服务
├── dubhe-server 后端服务
├── dubhe-visual-server 可视化服务
├── webapp 前端服务
```

## 反馈问题

- [在线社区](http://www.aiiaos.cn/index.php?s=/forum/index/forum/id/45.html)
- 钉钉交流群

<a href="./docs/images/dingtalk.jpg"><img src="http://cdn.qjycloud.com/dingtalk.jpg" width="320" /></a>


## 许可证书
本项目的发布受[Apache 2.0 license](./LICENSE)许可认证。

BIN
docs/images/dingtalk.jpg View File

Before After
Width: 877  |  Height: 1184  |  Size: 192 kB

BIN
docs/images/dubhe_dashboard.png View File

Before After
Width: 2880  |  Height: 2456  |  Size: 136 kB

+ 22
- 0
docs/images/logo.svg
File diff suppressed because it is too large
View File


BIN
docs/images/tech-arc.jpg View File

Before After
Width: 2077  |  Height: 1165  |  Size: 63 kB

+ 53
- 0
dubhe-server/.gitignore View File

@@ -0,0 +1,53 @@
/target/
.mvn*
mvnw
mvnw.cmd
kubeconfig

### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache

### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr

### NetBeans ###
/nbproject/private/
/build/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/

# system ignore
.DS_Store
Thumbs.db
*.orig

# maven ignore
target/
output/
*.jar
!.mvn/wrapper/*
*.war
*.zip
*.tar
#*.tar.gz

# eclipse ignore
.settings/
.project
.classpath
logs/
/dubhe-k8s/src/main/resources/kubeconfig
/dubhe-k8s/src/main/resources
*.log
/dubhe-admin/kubeconfig

+ 10
- 0
dubhe-server/.gitmessage View File

@@ -0,0 +1,10 @@

# type: subject
# type:feat增加新功能;fix修复错误;docs修改文档;style修改样式;refactor代码重构;test增加测试模块,不涉及生产环境的代码;chore更新核心模块,包配置文件,不涉及生产环境的代码
# subject: 一两句话简述提交原因,与下方详细描述间隔一行


# body:提交详细原因,与下方脚注间隔一行


# footer(可选):1.提供链接关联issue、2.关闭issue

+ 98
- 0
dubhe-server/README.md View File

@@ -0,0 +1,98 @@
# 之江天枢-服务端

**之江天枢一站式人工智能开源平台**(简称:**之江天枢**),包括海量数据处理、交互式模型构建(包含Notebook和模型可视化)、AI模型高效训练。多维度产品形态满足从开发者到大型企业的不同需求,将提升人工智能技术的研发效率、扩大算法模型的应用范围,进一步构建人工智能生态“朋友圈”。

## 源码部署

### 准备环境
安装如下软件环境。
- OpenJDK:1.8+
- Redis: 3.0+
- Maven: 3.0+
- MYSQL: 5.5.0+

### 下载源码
``` bash
git clone https://codeup.teambition.com/zhejianglab/dubhe-server.git
# 进入项目根目录
cd dubhe-server
```

### 创建DB
在MySQL中依次执行如下sql文件
```
sql/v1/00-Dubhe-DB.sql
sql/v1/01-Dubhe-DDL.sql
sql/v1/02-Dubhe-DML.sql
```

### 配置
根据实际情况修改如下配置文件。
```
dubhe-admin/src/main/resources/config/application-prod.yml
```

### 构建
``` bash
# 构建,生成的 jar 包位于 ./dubhe-admin/target/dubhe-admin-1.0.jar
mvn clean compile package
```

### 启动
``` bash
# 指定启动环境为 prod
java -jar ./dubhe-admin/target/dubhe-admin-1.0.jar --spring.profiles.active=prod
```

## 本地开发

### 必要条件:
导入maven项目,下载所需的依赖包
mysql下创建数据库dubhe,初始化数据脚本
安装redis

### 启动:
mvn spring-boot:run

## 代码结构:
```
├── common 公共模块
├── dubhe-admin 开发与训练模块
│   ├── src
│   │   └── main
│   │   ├── java
│   │   │   └── org
│   │   │   └── dubhe
│   │   │   ├── AppRun.java
│   │   │   ├── domain 实体对象
│   │   │   ├── repository 数据库层
│   │   │   ├── rest 控制层
│   │   │   └── service 服务层
│   │   │   ├── dto 数据传输对象
│   │   │   ├── impl 服务实现
│   │   │   └── mapper 对象转化
│   │   └── resources 配置文件
├── dubhe-data 数据处理模块
├── dubhe-model 模型管理模块
├── dubhe-system 系统管理
```

## docker服务器
上传镜像功能依赖docker服务,harbor与dokcer的信任配置如下:
### 1、对外开放端口
vi /lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock -H tcp://0.0.0.0:2375 -H unix://var/run/docker.sock
### 2、信任harbor地址
vi /etc/docker/daemon.json
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"insecure-registries":[harbor地址],
"log-opts": {
"max-size": "100m"
}
}
### 3、重新启动
systemctl daemon-reload
service docker restart
systemctl status docker

+ 121
- 0
dubhe-server/common/pom.xml View File

@@ -0,0 +1,121 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dubheplatform</artifactId>
<groupId>zhejianglab</groupId>
<version>1.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<properties>
<hutool.version>5.0.6</hutool.version>
<minio.version>7.0.2</minio.version>
<aspectjweaver.version>1.8.9</aspectjweaver.version>
</properties>

<artifactId>common</artifactId>
<name>公共模块</name>

<dependencies>
<!--工具包-->
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>${hutool.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>21.0</version>
</dependency>
<dependency>
<groupId>com.github.penggle</groupId>
<artifactId>kaptcha</artifactId>
<version>${kaptcha.version}</version>
</dependency>
<!-- shiro -->
<dependency>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-spring</artifactId>
<version>${shiro.version}</version>
</dependency>
<dependency>
<groupId>com.auth0</groupId>
<artifactId>java-jwt</artifactId>
<version>${jwt.version}</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>${minio.version}</version>
</dependency>

<!-- aspect -->
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
<version>${aspectjweaver.version}</version>
</dependency>

<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-core</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>1.3.2</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>net.lingala.zip4j</groupId>
<artifactId>zip4j</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.20</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<!-- 跳过单元测试 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
</plugins>
<resources>
<resource>
<filtering>true</filtering>
<directory>src/main/resources</directory>
</resource>
</resources>
</build>
</project>

+ 32
- 0
dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java View File

@@ -0,0 +1,32 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.annotation;

import java.lang.annotation.*;

/**
* @description API版本控制注解
* @date 2020-04-06
*/
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface ApiVersion {
//标识版本号
int value() default 1;
}

+ 44
- 0
dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java View File

@@ -0,0 +1,44 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.annotation;

import java.lang.annotation.*;

/**
* 数据权限过滤Mapper拦截
*
* @date 2020-06-22
*/
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface DataPermission {

/**
* 不需要数据权限的方法名
*/
String[] ignores() default {};

/**
* 只在方法的注解上使用,代表方法的数据权限类型,如果不加注解,只会识别带"select"方法名的方法
*
* @return
*/
String[] permission() default {};

}

+ 87
- 0
dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java View File

@@ -0,0 +1,87 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.annotation;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import javax.validation.Constraint;
import javax.validation.ConstraintValidator;
import javax.validation.ConstraintValidatorContext;
import javax.validation.Payload;

/**
* @date: 2020-05-21
*/
@Target({ ElementType.METHOD, ElementType.FIELD, ElementType.ANNOTATION_TYPE })
@Retention(RetentionPolicy.RUNTIME)
@Constraint(validatedBy = EnumValue.Validator.class)
public @interface EnumValue {

String message() default "custom.value.invalid";

Class<?>[] groups() default {};

Class<? extends Payload>[] payload() default {};

Class<? extends Enum<?>> enumClass();

String enumMethod();

class Validator implements ConstraintValidator<EnumValue, Object> {

private Class<? extends Enum<?>> enumClass;
private String enumMethod;

@Override
public void initialize(EnumValue enumValue) {
enumMethod = enumValue.enumMethod();
enumClass = enumValue.enumClass();
}

@Override
public boolean isValid(Object value, ConstraintValidatorContext constraintValidatorContext) {
if (value == null) {
return Boolean.TRUE;
}

if (enumClass == null || enumMethod == null) {
return Boolean.TRUE;
}
Class<?> valueClass = value.getClass();

try {
Method method = enumClass.getMethod(enumMethod, valueClass);
if (!Boolean.TYPE.equals(method.getReturnType()) && !Boolean.class.equals(method.getReturnType())) {
throw new RuntimeException(String.format("%s method return is not boolean type in the %s class", enumMethod, enumClass));
}

Boolean result = (Boolean)method.invoke(null, value);
return result == null ? false : result;
} catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
throw new RuntimeException(e);
} catch (NoSuchMethodException | SecurityException e) {
throw new RuntimeException(String.format("This %s(%s) method does not exist in the %s", enumMethod, valueClass, enumClass), e);
}
}

}
}

+ 23
- 0
dubhe-server/common/src/main/java/org/dubhe/annotation/Log.java View File

@@ -0,0 +1,23 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.annotation;

/**
* @description 日志
* @date 2020-03-15
*/
public @interface Log {
}

+ 68
- 0
dubhe-server/common/src/main/java/org/dubhe/annotation/Query.java View File

@@ -0,0 +1,68 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.annotation;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* @description 构建Wrapper的注解
* @date 2020-03-26
*/
@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
public @interface Query {


String propName() default "";

Type type() default Type.EQ;


String blurry() default "";

enum Type {
// 相等
EQ
// 不等于
, NE
// 大于
, GT
// 大于等于
, GE
// 小于
, LT
// 小于等于
, LE,
BETWEEN,
NOT_BETWEEN,
LIKE,
NOT_LIKE,
LIkE_LEFT,
LIKE_RIGHT,
IS_NULL,
IS_NOT_NULL,
IN,
NOT_IN,
INSQL,
NOT_INSQL,
ORDER_BY
}

}


+ 33
- 0
dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java View File

@@ -0,0 +1,33 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.annotation;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* @description 限流
* @date 2020-03-15
*/
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface RateLimit {
double limitNum() default 20;
}

+ 87
- 0
dubhe-server/common/src/main/java/org/dubhe/aspect/LogAspect.java View File

@@ -0,0 +1,87 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.aspect;

import java.util.UUID;

import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.dubhe.enums.LogEnum;
import org.dubhe.utils.LogUtil;
import org.slf4j.MDC;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;

import lombok.extern.slf4j.Slf4j;

/**
* @date 2020/04/10
*/
@Component
@Aspect
@Slf4j
public class LogAspect {

public static final String TRACE_ID = "traceId";

@Pointcut("execution(* org.dubhe..service..*.*(..))) "
+ "and execution(* org.dubhe..schedule..*.*(..))) "
+ "and execution(* org.dubhe..util*..*.*(..)))")
public void serviceAspect() {
}

@Pointcut("execution(* org.dubhe..rest..*.*(..))) ")
public void restAspect() {
}

@Pointcut(" execution(* org.dubhe..task..*.*(..))) ")
public void taskAspect() {
}

@Pointcut(" serviceAspect() || taskAspect() ")
public void aroundAspect() {
}

@Around("aroundAspect()")
public Object around(JoinPoint joinPoint) throws Throwable {
if (StringUtils.isEmpty(MDC.get(TRACE_ID))) {
MDC.put(TRACE_ID, UUID.randomUUID().toString());
}
return combineLogInfo(joinPoint);
}

@Around("restAspect()")
public Object aroundRest(JoinPoint joinPoint) throws Throwable {
MDC.clear();
MDC.put(TRACE_ID, UUID.randomUUID().toString());
return combineLogInfo(joinPoint);
}
private Object combineLogInfo(JoinPoint joinPoint) throws Throwable {
Object[] param = joinPoint.getArgs();
LogUtil.info(LogEnum.REST_REQ, "uri:{},input:{},==>begin", joinPoint.getSignature(), param);
long start = System.currentTimeMillis();
Object result = ((ProceedingJoinPoint) joinPoint).proceed();
long end = System.currentTimeMillis();
LogUtil.info(LogEnum.REST_REQ, "uri:{},output:{},proc_time:{},<==end", joinPoint.getSignature().toString(),
result, end - start);
return result;
}

}

+ 58
- 0
dubhe-server/common/src/main/java/org/dubhe/base/BaseConvert.java View File

@@ -0,0 +1,58 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.base;

import java.util.List;

/**
* @description DTO Entity 转换
* @date 2020-03-15
*/
public interface BaseConvert<D, E> {

/**
* DTO转Entity
*
* @param dto /
* @return /
*/
E toEntity(D dto);

/**
* Entity转DTO
*
* @param entity /
* @return /
*/
D toDto(E entity);

/**
* DTO集合转Entity集合
*
* @param dtoList /
* @return /
*/
List<E> toEntity(List<D> dtoList);

/**
* Entity集合转DTO集合
*
* @param entityList /
* @return /
*/
List<D> toDto(List<E> entityList);

}

+ 48
- 0
dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java View File

@@ -0,0 +1,48 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.base;

import lombok.Getter;
import lombok.Setter;

import java.io.Serializable;
import java.sql.Timestamp;

/**
* @description DTO基础类
* @date 2020-03-15
*/
@Getter
@Setter
public class BaseDTO implements Serializable {

private Boolean deleted;

private Timestamp createTime;

private Timestamp updateTime;

@Override
public String toString() {
return "BaseDTO{" +
"deleted=" + deleted +
", createTime=" + createTime +
", updateTime=" + updateTime +
'}';
}
}

+ 76
- 0
dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java View File

@@ -0,0 +1,76 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.base;

import com.baomidou.mybatisplus.annotation.FieldFill;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableLogic;
import lombok.Data;
import org.apache.commons.lang3.builder.ToStringBuilder;

import java.io.Serializable;
import java.lang.reflect.Field;
import java.sql.Timestamp;

/**
* @description Entity基础类
* @date 2020-03-15
*/
@Data
public class BaseEntity implements Serializable {

private static final long serialVersionUID = 4936056317364745513L;

/**
* 删除标识
**/
@TableField(value = "deleted",fill = FieldFill.INSERT)
@TableLogic
private Boolean deleted = false;

@TableField(value = "create_user_id",fill = FieldFill.INSERT)
private Long createUserId;

@TableField(value = "update_user_id",fill = FieldFill.INSERT_UPDATE)
private Long updateUserId;

@TableField(value = "create_time",fill = FieldFill.INSERT)
private Timestamp createTime;


@TableField(value = "update_time",fill = FieldFill.INSERT_UPDATE)
private Timestamp updateTime;

@Override
public String toString() {
ToStringBuilder builder = new ToStringBuilder(this);
Field[] fields = this.getClass().getDeclaredFields();
try {
for (Field f : fields) {
f.setAccessible(true);
builder.append(f.getName(), f.get(this)).append("\n");
}
} catch (Exception e) {
builder.append("toString builder encounter an error");
}
return builder.toString();
}

public @interface Update {
}
}

+ 44
- 0
dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java View File

@@ -0,0 +1,44 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/
package org.dubhe.base;

import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.experimental.Accessors;

import javax.validation.constraints.NotBlank;
import java.io.Serializable;

/**
* @description 镜像基础类DTO
* @date: 2020-07-14
*/
@Data
@Accessors(chain = true)
public class BaseImageDTO implements Serializable {

private static final long serialVersionUID = 1L;

@ApiModelProperty(value = "镜像版本", required = true)
@NotBlank(message = "镜像版本不能为空")
private String imageTag;

@ApiModelProperty(value = "镜像名称", required = true)
@NotBlank(message = "镜像名称不能为空")
private String imageName;

}

+ 46
- 0
dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java View File

@@ -0,0 +1,46 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.base;

import io.swagger.annotations.ApiModelProperty;
import lombok.Data;

import java.io.Serializable;
import java.sql.Timestamp;

/**
* @description: VO基础类
* @date: 2020-05-22
*/
@Data
public class BaseVO implements Serializable {

private static final long serialVersionUID = 1L;

@ApiModelProperty("创建人")
private Long createUserId;

@ApiModelProperty("创建时间")
private Timestamp createTime;

@ApiModelProperty("修改人")
private Long updateUserId;

@ApiModelProperty("修改时间")
private Timestamp updateTime;
}

+ 76
- 0
dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java View File

@@ -0,0 +1,76 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.base;


import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.slf4j.MDC;

import java.io.Serializable;

/**
* @description 统一的公共响应体
* @date 2020-03-16
*/
@Data
@ApiModel(description = "统一的公共响应体")
public class DataResponseBody<T> implements Serializable {

/**
* 返回状态码
*/
@ApiModelProperty("返回状态码")
private Integer code;
/**
* 返回信息
*/
@ApiModelProperty("返回信息")
private String msg;
/**
* 泛型数据
*/
@ApiModelProperty("泛型数据")
private T data;
/**
* 链路追踪ID
*/
@ApiModelProperty("链路追踪ID")
private String traceId;

public DataResponseBody() {
this(ResponseCode.SUCCESS, null);
}

public DataResponseBody(T data) {
this(ResponseCode.SUCCESS, null, data);
}

public DataResponseBody(Integer code, String msg) {
this(code, msg, null);
}

public DataResponseBody(Integer code, String msg, T data) {
this.code = code;
this.msg = msg;
this.data = data;
this.traceId = MDC.get("traceId");
}

}

+ 99
- 0
dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java View File

@@ -0,0 +1,99 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.base;

/**
* @date: 2020-05-14
*/
public final class MagicNumConstant {

public static final int NEGATIVE_ONE = -1;
public static final int ZERO = 0;
public static final int ONE = 1;
public static final int TWO = 2;
public static final int THREE = 3;
public static final int FOUR = 4;
public static final int FIVE = 5;
public static final int SIX = 6;
public static final int SEVEN = 7;
public static final int EIGHT = 8;
public static final int NINE = 9;
public static final int TEN = 10;

public static final int ELEVEN = 11;
public static final int SIXTEEN = 16;
public static final int TWENTY = 20;
public static final int FIFTY = 50;
public static final int SIXTY = 60;
public static final int SIXTY_TWO = 62;
public static final int SIXTY_FOUR = 64;
public static final int INTEGER_TWO_HUNDRED_AND_FIFTY_FIVE = 255;
public static final int ONE_HUNDRED = 100;
public static final int ONE_HUNDRED_TWENTY_EIGHT = 128;
public static final int TWO_HUNDRED = 200;
public static final int FIVE_HUNDRED = 500;
public static final int FIVE_HUNDRED_AND_SIXTEEN = 516;
public static final int ONE_THOUSAND = 1000;
public static final int BINARY_TEN_EXP = 1024;
public static final int ONE_THOUSAND_ONE_HUNDRED = 1100;
public static final int ONE_THOUSAND_ONE_HUNDRED_ONE = 1101;
public static final int ONE_THOUSAND_TWO_HUNDRED = 1200;
public static final int ONE_THOUSAND_TWO_HUNDRED_ONE = 1201;
public static final int ONE_THOUSAND_TWENTY_FOUR = 1024;
public static final int ONE_THOUSAND_THREE_HUNDRED = 1300;
public static final int ONE_THOUSAND_THREE_HUNDRED_ONE = 1301;
public static final int ONE_THOUSAND_THREE_HUNDRED_NINE = 1309;
public static final int ONE_THOUSAND_FIVE_HUNDRED = 1500;
public static final int TWO_THOUSAND = 2000;
public static final int TWO_THOUSAND_TWENTY_EIGHT = 2048;
public static final int THREE_THOUSAND = 3000;
public static final int FOUR_THOUSAND = 4000;
public static final int NINE_THOUSAND = 9000;
public static final int NINE_THOUSAND_NINE_HUNDRED_NINTY_NINE = 9999;
public static final int TEN_THOUSAND = 10000;
public static final int FIFTEEN_THOUSAND = 15000;
public static final int HUNDRED_THOUSAND = 100000;
public static final int MILLION = 1000000;
public static final int ONE_MINUTE = 60000;

public static final long NEGATIVE_ONE__LONG = -1L;
public static final long ZERO_LONG = 0L;
public static final long ONE_LONG = 1L;
public static final long TWO_LONG = 2L;
public static final long THREE_LONG = 3L;
public static final long FOUR_LONG = 4L;
public static final long FIVE_LONG = 5L;
public static final long SIX_LONG = 6L;
public static final long SEVEN_LONG = 7L;
public static final long EIGHT_LONG = 8L;
public static final long NINE_LONG = 9L;
public static final long TEN_LONG = 10L;

public static final long TWELVE_LONG = 12L;
public static final long SIXTY_LONG = 60L;
public static final long TEN_THOUSAND_LONG = 10000L;
public static final long ONE_ZERO_ONE_ZERO_ONE_ZERO_LONG = 101010L;
public static final long NINE_ZERO_NINE_ZERO_NINE_ZERO_LONG = 909090L;
public static final long ONE_YEAR_BEFORE_LONG = 1552579200000L;

public static final int SIXITY_0XFF = 0xFF;


private MagicNumConstant() {
}
}

+ 62
- 0
dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java View File

@@ -0,0 +1,62 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.base;

import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.experimental.Accessors;
import org.dubhe.constant.NumberConstant;

import javax.validation.constraints.Min;

/**
* @description: 分页基类
* @date: 2020-05-8
*/
@Data
@Accessors(chain = true)
public class PageQueryBase<T> {

@ApiModelProperty(value = "分页-当前页数")
@Min(value = 1, message = "current不能小于1")
private Integer current;

@ApiModelProperty(value = "分页-每页展示数")
@Min(value = 1, message = "size不能小于1")
private Integer size;


@ApiModelProperty(value = "排序字段")
private String sort;

@ApiModelProperty(value = "排序方式,asc | desc")
private String order;

public Page<T> toPage() {
Page<T> page = new Page();
if (this.current != null) {
page.setCurrent(this.current);
}
if (this.size != null && this.size < NumberConstant.MAX_PAGE_SIZE) {
page.setSize(this.size);
}
return page;
}

}

+ 33
- 0
dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java View File

@@ -0,0 +1,33 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.base;

/**
* @description 返回状态码
* @date 2020-02-23
*/
public class ResponseCode {
public static Integer SUCCESS = 200;
public static Integer UNAUTHORIZED = 401;
public static Integer ERROR = 10000;
public static Integer ENTITY_NOT_EXIST = 10001;
public static Integer BADREQUEST = 10002;
public static Integer SERVICE_ERROR = 10003;
public static Integer DOCKER_ERROR = 10004;

}

+ 56
- 0
dubhe-server/common/src/main/java/org/dubhe/config/GlobalCorsConfig.java View File

@@ -0,0 +1,56 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.config;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;

/**
* @description 允许跨域
* @date 2020-02-23
*/
@Configuration
public class GlobalCorsConfig {

@Bean
public CorsFilter corsFilter() {
//1.添加CORS配置信息
CorsConfiguration config = new CorsConfiguration();
//放行哪些原始域
config.addAllowedOrigin("*");
//是否发送Cookie信息
config.setAllowCredentials(true);
//放行哪些原始域(请求方式)
config.addAllowedMethod("OPTIONS");
config.addAllowedMethod("HEAD");
config.addAllowedMethod("GET");
config.addAllowedMethod("PUT");
config.addAllowedMethod("POST");
config.addAllowedMethod("DELETE");
config.addAllowedMethod("PATCH");
config.addAllowedHeader("*");

//2.添加映射路径
UrlBasedCorsConfigurationSource configSource = new UrlBasedCorsConfigurationSource();
configSource.registerCorsConfiguration("/**", config);

//3.返回新的CorsFilter.
return new CorsFilter(configSource);
}
}

+ 44
- 0
dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java View File

@@ -0,0 +1,44 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;

/**
* @description k8s命名相关配置
*
* @date 2020-05-13
*/
@Getter
@Configuration
public class K8sNameConfig {

@Value("${k8s.nfs-root-path}")
private String nfsRootPath;

@Value("${k8s.namespace}")
private String namespace;

@Value("${minio.bucketName}")
private String fileBucket;

@Value("${train-job.docker-dataset-path}")
private String datasetPath;
}

+ 65
- 0
dubhe-server/common/src/main/java/org/dubhe/config/KaptchaConfig.java View File

@@ -0,0 +1,65 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import com.google.code.kaptcha.impl.DefaultKaptcha;
import com.google.code.kaptcha.util.Config;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;

import java.util.Properties;

/**
* @description 验证码配置
* @date 2020-02-23
*/
@Component
public class KaptchaConfig {
private final static String CODE_LENGTH = "4";
private final static String SESSION_KEY = "verification_session_key";

@Bean
public DefaultKaptcha defaultKaptcha() {
DefaultKaptcha defaultKaptcha = new DefaultKaptcha();
Properties properties = new Properties();

// 设置边框
properties.setProperty("kaptcha.border", "yes");
// 设置边框颜色
properties.setProperty("kaptcha.border.color", "105,179,90");
// 设置字体颜色
properties.setProperty("kaptcha.textproducer.font.color", "blue");
// 设置图片宽度
properties.setProperty("kaptcha.image.width", "108");
// 设置图片高度
properties.setProperty("kaptcha.image.height", "28");
// 设置字体尺寸
properties.setProperty("kaptcha.textproducer.font.size", "26");
// 设置session key
properties.setProperty("kaptcha.session.key", SESSION_KEY);
// 设置验证码长度
properties.setProperty("kaptcha.textproducer.char.length", CODE_LENGTH);
// 设置字体
properties.setProperty("kaptcha.textproducer.font.names", "宋体,楷体,黑体");
//去噪点
properties.setProperty("kaptcha.noise.impl", "com.google.code.kaptcha.impl.NoNoise");
Config config = new Config(properties);
defaultKaptcha.setConfig(config);
return defaultKaptcha;
}
}

+ 44
- 0
dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java View File

@@ -0,0 +1,44 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;

@Configuration
public class LocalDateTimeSerializerConfig {
@Value("${spring.jackson.date-format}")
private String pattern;

@Bean
public LocalDateTimeSerializer localDateTimeDeserializer() {
return new LocalDateTimeSerializer(DateTimeFormatter.ofPattern(pattern));
}

@Bean
public Jackson2ObjectMapperBuilderCustomizer jackson2ObjectMapperBuilderCustomizer() {
return builder -> builder.serializerByType(LocalDateTime.class, localDateTimeDeserializer());
}

}

+ 95
- 0
dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java View File

@@ -0,0 +1,95 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler;
import org.apache.ibatis.reflection.MetaObject;
import org.dubhe.constant.StringConstant;
import org.dubhe.domain.dto.UserDTO;
import org.dubhe.enums.SwitchEnum;
import org.dubhe.utils.DateUtil;
import org.dubhe.utils.JwtUtils;
import org.springframework.stereotype.Component;

import java.util.Objects;

/**
* @description 处理新增和更新的基础数据填充,配合BaseEntity和MyBatisPlusConfig使用
* @date 2020-6-10
*/
@Component
public class MetaHandlerConfig implements MetaObjectHandler {


private final String LOCK_USER_ID = "LOCK_USER_ID";


/**
* 新增数据执行
*
* @param metaObject
*/

@Override
public void insertFill(MetaObject metaObject) {
if (Objects.isNull(getFieldValByName(StringConstant.CREATE_TIME, metaObject))) {
this.setFieldValByName(StringConstant.CREATE_TIME, DateUtil.getCurrentTimestamp(), metaObject);
}
if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_TIME, metaObject))) {
this.setFieldValByName(StringConstant.UPDATE_TIME, DateUtil.getCurrentTimestamp(), metaObject);
}
synchronized (LOCK_USER_ID){
if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_USER_ID, metaObject))) {
this.setFieldValByName(StringConstant.UPDATE_USER_ID, getUserId(), metaObject);
}
if (Objects.isNull(getFieldValByName(StringConstant.CREATE_USER_ID, metaObject))) {
this.setFieldValByName(StringConstant.CREATE_USER_ID, getUserId(), metaObject);
}
}
if (Objects.isNull(getFieldValByName(StringConstant.DELETED, metaObject))) {
this.setFieldValByName(StringConstant.DELETED, SwitchEnum.getBooleanValue(SwitchEnum.OFF.getValue()), metaObject);
}
}

/**
* 更新数据执行
*
* @param metaObject
*/
@Override
public void updateFill(MetaObject metaObject) {
if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_TIME, metaObject))) {
this.setFieldValByName(StringConstant.UPDATE_TIME, DateUtil.getCurrentTimestamp(), metaObject);
}
if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_USER_ID, metaObject))) {
this.setFieldValByName(StringConstant.UPDATE_USER_ID, getUserId(), metaObject);
}

}


/**
* 获取用户ID
*
* @return
*/
private Long getUserId() {
UserDTO userDTO = JwtUtils.getCurrentUserDto();
return Objects.isNull(userDTO) ? null : userDTO.getId();
}
}

+ 310
- 0
dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java View File

@@ -0,0 +1,310 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import com.baomidou.mybatisplus.core.override.MybatisMapperProxy;
import com.baomidou.mybatisplus.core.parser.ISqlParser;
import com.baomidou.mybatisplus.core.parser.SqlParserHelper;
import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor;
import com.baomidou.mybatisplus.extension.plugins.tenant.TenantHandler;
import com.baomidou.mybatisplus.extension.plugins.tenant.TenantSqlParser;
import com.google.common.collect.Sets;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
import net.sf.jsqlparser.expression.operators.relational.InExpression;
import net.sf.jsqlparser.schema.Column;
import org.apache.ibatis.mapping.MappedStatement;
import org.apache.shiro.UnavailableSecurityManagerException;
import org.dubhe.annotation.DataPermission;
import org.dubhe.base.MagicNumConstant;
import org.dubhe.constant.PermissionConstant;
import org.dubhe.domain.dto.UserDTO;
import org.dubhe.domain.entity.Role;
import org.dubhe.enums.LogEnum;
import org.dubhe.utils.JwtUtils;
import org.dubhe.utils.LogUtil;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ApplicationListener;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.core.annotation.AnnotationUtils;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.util.CollectionUtils;

import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.*;
import java.util.stream.Collectors;

/**
* @description MybatisPlus配置类
* @date 2020-06-24
*/
@EnableTransactionManagement
@Configuration
public class MybatisPlusConfig implements ApplicationListener<ContextRefreshedEvent>, ApplicationContextAware {

/**
* 以此字段作为租户实现数据隔离
*/
private static final String TENANT_ID_COLUMN = "create_user_id";
/**
* 以0作为公共数据的标识
*/
private static final long PUBLIC_TENANT_ID = MagicNumConstant.ZERO;
private static final Set<Long> PUBLIC_TENANT_ID_SET = new HashSet<Long>() {{
add(PUBLIC_TENANT_ID);
}};
private static final String PACKAGE_SEPARATOR = ".";
private static final Set<String> SELECT_PERMISSION = new HashSet<String>() {{
add(PermissionConstant.SELECT);
}};
private static final Set<String> UPDATE_DELETE_PERMISSION = new HashSet<String>() {{
add(PermissionConstant.UPDATE);
add(PermissionConstant.DELETE);
}};

private static final String SELECT_STR = "select";
/**
* 优先级高于dataFilters,如果ignore,则不进行sql注入
*/
private Map<String, Set<String>> dataFilters = new HashMap<>();

private ApplicationContext applicationContext;
public Set<Long> tenantId;

/**
* mybatis plus 分页插件
* 其中增加了通过多租户实现了数据权限功能
*
* @return
*/
@Bean
public PaginationInterceptor paginationInterceptor() {
PaginationInterceptor paginationInterceptor = new PaginationInterceptor();
List<ISqlParser> sqlParserList = new ArrayList<>();
TenantSqlParser tenantSqlParser = new TenantSqlParser();
tenantSqlParser.setTenantHandler(new TenantHandler() {
@Override
public Expression getTenantId(boolean where) {
Set<Long> tenants = tenantId;

final boolean multipleTenantIds = tenants.size() > MagicNumConstant.ONE;
if (multipleTenantIds) {
return multipleTenantIdCondition(tenants);
} else {
return singleTenantIdCondition(tenants);
}
}

private Expression singleTenantIdCondition(Set<Long> tenants) {
return new LongValue((Long) tenants.toArray()[0]);
}

private Expression multipleTenantIdCondition(Set<Long> tenants) {
final InExpression inExpression = new InExpression();
inExpression.setLeftExpression(new Column(getTenantIdColumn()));
final ExpressionList itemsList = new ExpressionList();
final List<Expression> inValues = new ArrayList<>(tenants.size());
tenants.forEach(i ->
inValues.add(new LongValue(i))
);
itemsList.setExpressions(inValues);
inExpression.setRightItemsList(itemsList);
return inExpression;
}

@Override
public String getTenantIdColumn() {
return TENANT_ID_COLUMN;
}

@Override
public boolean doTableFilter(String tableName) {
return false;
}
});
sqlParserList.add(tenantSqlParser);
paginationInterceptor.setSqlParserList(sqlParserList);
paginationInterceptor.setSqlParserFilter(metaObject -> {
MappedStatement ms = SqlParserHelper.getMappedStatement(metaObject);
String method = ms.getId();
if (!dataFilters.containsKey(method) || isAdmin()) {
return true;
}
Set<String> permission = dataFilters.get(method);
tenantId = getTenantId(permission);
return false;
});
return paginationInterceptor;
}

/**
* 判断用户是否是管理员
* 如果未登录,无法请求任何接口,所以不会到该层,因此匿名认为是定时任务,给予admin权限。
*
* @return 判断用户是否是管理员
*/
private boolean isAdmin() {
UserDTO user;
try {
user = JwtUtils.getCurrentUserDto();
} catch (UnavailableSecurityManagerException e) {
return true;
}
if (Objects.isNull(user)) {
return true;
}
List<Role> roles;
if ((roles = user.getRoles()) == null) {
return false;
}
Set<String> permissions = roles.stream().map(Role::getPermission).collect(Collectors.toSet());
if (CollectionUtils.isEmpty(permissions)) {
return false;
}
return user.getId() == PermissionConstant.ANONYMOUS_USER || user.getId() == PermissionConstant.ADMIN_USER_ID;
}

/**
* 如果是管理员,在前一步isAdmin已过滤;
* 如果是匿名用户,在shiro层被过滤;
* 因此只会是无角色、权限用户或普通用户
*
* @return Set<Long> 租户ID集合
*/
private Set<Long> getTenantId(Set<String> permission) {
UserDTO user = JwtUtils.getCurrentUserDto();
List<Role> roles;
if (Objects.isNull(user) || (roles = user.getRoles()) == null) {
if (permission.contains(PermissionConstant.SELECT)) {
return PUBLIC_TENANT_ID_SET;
}
return Collections.EMPTY_SET;
}
Set<String> permissions = roles.stream().map(Role::getPermission).collect(Collectors.toSet());
if (CollectionUtils.isEmpty(permissions)) {
if (permission.contains(PermissionConstant.SELECT)) {
return PUBLIC_TENANT_ID_SET;
}
return Collections.EMPTY_SET;
}
if (permission.contains(PermissionConstant.SELECT)) {
return new HashSet<Long>() {{
add(PUBLIC_TENANT_ID);
add(user.getId());
}};
}
return new HashSet<Long>() {{
add(user.getId());
}};
}

/**
* 设置上下文
* #需要通过上下文 获取SpringBean
*
* @param applicationContext spring上下文
* @throws BeansException 找不到bean异常
*/
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}

@Override
public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) {
Class<? extends Annotation> annotationClass = DataPermission.class;
Map<String, Object> beanWithAnnotation = applicationContext.getBeansWithAnnotation(annotationClass);
Set<Map.Entry<String, Object>> entitySet = beanWithAnnotation.entrySet();
for (Map.Entry<String, Object> entry : entitySet) {
Proxy proxy = (Proxy) entry.getValue();
Class clazz = getMapperClass(proxy);
populateDataFilters(clazz);
}
}

/**
* 根据mapper对应代理对象获取Class
*
* @param proxy mapper对应代理对象
* @return
*/
private Class getMapperClass(Proxy proxy) {
try {
Field field = proxy.getClass().getSuperclass().getDeclaredField("h");
field.setAccessible(true);
MybatisMapperProxy mapperProxy = (MybatisMapperProxy) field.get(proxy);
field = mapperProxy.getClass().getDeclaredField("mapperInterface");
field.setAccessible(true);
return (Class) field.get(mapperProxy);
} catch (NoSuchFieldException | IllegalAccessException e) {
LogUtil.error(LogEnum.BIZ_DATASET, "reflect error", e);
}
return null;
}

/**
* 填充数据权限过滤,处理那些需要排除的方法
*
* @param clazz 需要处理的类(mapper)
*/
private void populateDataFilters(Class clazz) {
if (clazz == null) {
return;
}
Method[] methods = clazz.getMethods();
DataPermission dataPermission = AnnotationUtils.findAnnotation((Class<?>) clazz, DataPermission.class);
Set<String> ignores = Sets.newHashSet(dataPermission.ignores());
for (Method method : methods) {
if (ignores.contains(method.getName())) {
continue;
}
Set<String> permission = getDataPermission(method);
dataFilters.put(clazz.getName() + PACKAGE_SEPARATOR + method.getName(), permission);
}
}

/**
* 获取方法上权限注解
* 权限注解包含
* 1.用户拥有指定权限才可以执行该方法:比如 PermissionConstant.SELECT 表示用户必须拥有select权限,才可以使用该方法
* 2.方法权限校验排除:比如 ignores = {"insert"} 表示insert方法不做权限处理
*
* @param method 方法对象
* @return
*/
private Set<String> getDataPermission(Method method) {
DataPermission dataPermission = AnnotationUtils.findAnnotation(method, DataPermission.class);
// 无注解时以方法名判断
if (dataPermission == null) {
if (method.getName().contains(SELECT_STR)) {
return SELECT_PERMISSION;
}
return UPDATE_DELETE_PERMISSION;
}
return Sets.newHashSet(dataPermission.permission());
}

}

+ 41
- 0
dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java View File

@@ -0,0 +1,41 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

/**
* @description NFS config
* @create 2020/5/13
*/
@Data
@Component
public class NfsConfig {

@Value("${k8s.nfs}")
private String nfsIp;

@Value("${k8s.nfs-root-path}")
private String rootDir;

@Value("/${minio.bucketName}/")
private String bucket;

}

+ 214
- 0
dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java View File

@@ -0,0 +1,214 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import cn.hutool.core.lang.Assert;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.parser.ParserConfig;
import com.alibaba.fastjson.serializer.SerializerFeature;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.dubhe.utils.StringUtils;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.data.redis.RedisProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cache.Cache;
import org.springframework.cache.annotation.CachingConfigurerSupport;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cache.interceptor.CacheErrorHandler;
import org.springframework.cache.interceptor.KeyGenerator;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.cache.RedisCacheConfiguration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisOperations;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.RedisSerializationContext;
import org.springframework.data.redis.serializer.RedisSerializer;

import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
/**
* @description redis配置类
* @date 2020-03-25
*/
@Slf4j
@Configuration
@EnableCaching
@ConditionalOnClass(RedisOperations.class)
@EnableConfigurationProperties(RedisProperties.class)
public class RedisConfig extends CachingConfigurerSupport {

/**
* 设置 redis 数据默认过期时间,默认2小时
* 设置@cacheable 序列化方式
*/
@Bean
public RedisCacheConfiguration redisCacheConfiguration() {
FastJsonRedisSerializer<Object> fastJsonRedisSerializer = new FastJsonRedisSerializer<>(Object.class);
RedisCacheConfiguration configuration = RedisCacheConfiguration.defaultCacheConfig();
configuration = configuration.serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(fastJsonRedisSerializer)).entryTtl(Duration.ofHours(2));
return configuration;
}

@SuppressWarnings("all")
@Bean(name = "redisTemplate")
@ConditionalOnMissingBean(name = "redisTemplate")
public RedisTemplate<Object, Object> redisTemplate(RedisConnectionFactory redisConnectionFactory) {
RedisTemplate<Object, Object> template = new RedisTemplate<>();
//序列化
FastJsonRedisSerializer<Object> fastJsonRedisSerializer = new FastJsonRedisSerializer<>(Object.class);
// value值的序列化采用fastJsonRedisSerializer
template.setValueSerializer(fastJsonRedisSerializer);
template.setHashValueSerializer(fastJsonRedisSerializer);
// 全局开启AutoType,这里方便开发,使用全局的方式
ParserConfig.getGlobalInstance().setAutoTypeSupport(true);
// 建议使用这种方式,小范围指定白名单
// ParserConfig.getGlobalInstance().addAccept("org.dubhe.domain");
// key的序列化采用StringRedisSerializer
template.setKeySerializer(new StringRedisSerializer());
template.setHashKeySerializer(new StringRedisSerializer());
template.setConnectionFactory(redisConnectionFactory);
return template;
}

/**
* 自定义缓存key生成策略,默认将使用该策略
*/
@Bean
@Override
public KeyGenerator keyGenerator() {
return (target, method, params) -> {
Map<String, Object> container = new HashMap<>(3);
Class<?> targetClassClass = target.getClass();
// 类地址
container.put("class", targetClassClass.toGenericString());
// 方法名称
container.put("methodName", method.getName());
// 包名称
container.put("package", targetClassClass.getPackage());
// 参数列表
for (int i = 0; i < params.length; i++) {
container.put(String.valueOf(i), params[i]);
}
// 转为JSON字符串
String jsonString = JSON.toJSONString(container);
// 做SHA256 Hash计算,得到一个SHA256摘要作为Key
return DigestUtils.sha256Hex(jsonString);
};
}

@Bean
@Override
public CacheErrorHandler errorHandler() {
// 异常处理,当Redis发生异常时,打印日志,但是程序正常走
log.info("初始化 -> [{}]", "Redis CacheErrorHandler");
return new CacheErrorHandler() {
@Override
public void handleCacheGetError(RuntimeException e, Cache cache, Object key) {
log.error("Redis occur handleCacheGetError:key -> [{}]", key, e);
}

@Override
public void handleCachePutError(RuntimeException e, Cache cache, Object key, Object value) {
log.error("Redis occur handleCachePutError:key -> [{}];value -> [{}]", key, value, e);
}

@Override
public void handleCacheEvictError(RuntimeException e, Cache cache, Object key) {
log.error("Redis occur handleCacheEvictError:key -> [{}]", key, e);
}

@Override
public void handleCacheClearError(RuntimeException e, Cache cache) {
log.error("Redis occur handleCacheClearError:", e);
}
};
}

}

/**
* Value 序列化
*
* @param <T>
*/
class FastJsonRedisSerializer<T> implements RedisSerializer<T> {

private Class<T> clazz;

FastJsonRedisSerializer(Class<T> clazz) {
super();
this.clazz = clazz;
}

@Override
public byte[] serialize(T t) {
if (t == null) {
return new byte[0];
}
return JSON.toJSONString(t, SerializerFeature.WriteClassName).getBytes(StandardCharsets.UTF_8);
}

@Override
public T deserialize(byte[] bytes) {
if (bytes == null || bytes.length <= 0) {
return null;
}
String str = new String(bytes, StandardCharsets.UTF_8);
return JSON.parseObject(str, clazz);
}

}

/**
* 重写序列化器
*
*/
class StringRedisSerializer implements RedisSerializer<Object> {

private final Charset charset;

StringRedisSerializer() {
this(StandardCharsets.UTF_8);
}

private StringRedisSerializer(Charset charset) {
Assert.notNull(charset, "Charset must not be null!");
this.charset = charset;
}

@Override
public String deserialize(byte[] bytes) {
return (bytes == null ? null : new String(bytes, charset));
}

@Override
public byte[] serialize(Object object) {
String string = JSON.toJSONString(object);
if (StringUtils.isBlank(string)) {
return null;
}
string = string.replace("\"", "");
return string.getBytes(charset);
}
}

+ 70
- 0
dubhe-server/common/src/main/java/org/dubhe/config/Resources.java View File

@@ -0,0 +1,70 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import org.springframework.context.annotation.PropertySource;
import org.springframework.context.i18n.LocaleContextHolder;

import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.ResourceBundle;

/**
* @description 加载配置
* @date 2020-03-15
*/
@PropertySource(value = {"classpath:config/thirdParty.properties", "classpath:i18n/messages*.properties"})
public final class Resources {
/**
* 第三方登录配置
*/
public static final ResourceBundle THIRDPARTY = ResourceBundle.getBundle("config/thirdParty");
/**
* 国际化信息
*/
private static final Map<String, ResourceBundle> MESSAGES = new HashMap<String, ResourceBundle>();

/**
* 国际化信息
*/
public static String getMessage(String key, Object... params) {
Locale locale = LocaleContextHolder.getLocale();
ResourceBundle message = MESSAGES.get(locale.getLanguage());
if (message == null) {
synchronized (MESSAGES) {
message = MESSAGES.get(locale.getLanguage());
if (message == null) {
message = ResourceBundle.getBundle("i18n/messages", locale);
MESSAGES.put(locale.getLanguage(), message);
}
}
}
if (params != null && params.length > 0) {
return String.format(message.getString(key), params);
}
return message.getString(key);
}

/**
* 清除国际化信息
*/
public static void flushMessage() {
MESSAGES.clear();
}
}

+ 36
- 0
dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java View File

@@ -0,0 +1,36 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import org.springframework.context.annotation.Configuration;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;

/**
* @description 关闭Security
* @date 2020-03-25
*/
@Configuration
public class SecurityConfig extends WebSecurityConfigurerAdapter {

@Override
protected void configure(HttpSecurity http) throws Exception {
http.authorizeRequests().antMatchers("/**").permitAll().and().csrf().disable();
}

}

+ 138
- 0
dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java View File

@@ -0,0 +1,138 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import com.fasterxml.classmate.TypeResolver;
import com.google.common.base.Predicates;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.Ordered;
import org.springframework.data.domain.Pageable;
import springfox.documentation.builders.ApiInfoBuilder;
import springfox.documentation.builders.ParameterBuilder;
import springfox.documentation.builders.PathSelectors;
import springfox.documentation.schema.AlternateTypeRule;
import springfox.documentation.schema.AlternateTypeRuleConvention;
import springfox.documentation.schema.ModelRef;
import springfox.documentation.service.ApiInfo;
import springfox.documentation.service.Parameter;
import springfox.documentation.spi.DocumentationType;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;

import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import static com.google.common.collect.Lists.newArrayList;
import static springfox.documentation.schema.AlternateTypeRules.newRule;

/**
* @description api页面 /swagger-ui.html
* @date 2020-03-25
*/

@Configuration
@EnableSwagger2
public class SwaggerConfig {

@Value("${jwt.header}")
private String tokenHeader;

@Value("${jwt.token-start-with}")
private String tokenStartWith;

@Value("${swagger.enabled}")
private Boolean enabled;

@Value("${server.rest-version}")
private String restVersion;

@Bean
@SuppressWarnings("all")
public Docket createRestApi() {
List<Parameter> pars = new ArrayList<>();
pars.add(new ParameterBuilder().name(tokenHeader).description("token")
.modelRef(new ModelRef("string"))
.parameterType("header")
.defaultValue(tokenStartWith + " ")
.required(true)
.build());
pars.add(new ParameterBuilder().name("version").description("版本")
.modelRef(new ModelRef("string"))
.parameterType("path")
.defaultValue(restVersion)
.required(false)
.build());
return new Docket(DocumentationType.SWAGGER_2)
.enable(enabled)
.apiInfo(apiInfo())
.select()
.paths(Predicates.not(PathSelectors.regex("/error.*")))
.build().directModelSubstitute(Timestamp.class, Date.class)
.globalOperationParameters(pars);
}

private ApiInfo apiInfo() {
return new ApiInfoBuilder()
.title("之江天枢一站式人工智能开源平台")
.version("1.0")
.build();
}

}

/**
* 将Pageable转换展示在swagger中
*/
@Configuration
class SwaggerDataConfig {

@Bean
public AlternateTypeRuleConvention pageableConvention(final TypeResolver resolver) {
return new AlternateTypeRuleConvention() {
@Override
public int getOrder() {
return Ordered.HIGHEST_PRECEDENCE;
}

@Override
public List<AlternateTypeRule> rules() {
return newArrayList(newRule(resolver.resolve(Pageable.class), resolver.resolve(Page.class)));
}
};
}

@ApiModel
@Data
private static class Page {
@ApiModelProperty("页码 (0..N)")
private Integer page;

@ApiModelProperty("每页显示的数目")
private Integer size;

@ApiModelProperty("以下列格式排序标准:property[,asc | desc]。 默认排序顺序为升序。 支持多种排序条件:如:id,asc")
private List<String> sort;
}
}

+ 39
- 0
dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java View File

@@ -0,0 +1,39 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/
package org.dubhe.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

/**
* @description harbor相关配置
* @date 2020-07-17
*/
@Data
@Component
@ConfigurationProperties(prefix = "harbor")
public class TrainHarborConfig {

private String address;

private String username;

private String password;

private String modelName;
}

+ 50
- 0
dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java View File

@@ -0,0 +1,50 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import org.dubhe.interceptor.K8sCallBackPodInterceptor;
import org.dubhe.utils.K8sCallBackTool;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.InterceptorRegistration;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;

import javax.annotation.Resource;

/**
* @description Web Mvc Config
*
* @date 2020-05-28
*/
@Configuration
public class WebMvcConfig implements WebMvcConfigurer {

@Resource
private K8sCallBackPodInterceptor k8sCallBackPodInterceptor;

@Override
public void addInterceptors(InterceptorRegistry registry) {
InterceptorRegistration registration = registry.addInterceptor(k8sCallBackPodInterceptor);
// 拦截配置
registration.addPathPatterns(K8sCallBackTool.getK8sCallbackPaths());

}



}

+ 35
- 0
dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java View File

@@ -0,0 +1,35 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.config;

import org.dubhe.handle.CustomRequestMappingHandlerMapping;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.autoconfigure.web.servlet.WebMvcRegistrations;
import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping;

/**
* @description API版本控制
* @date 2020-04-06
*/
@SpringBootConfiguration
public class WebMvcRegistrationsConfig implements WebMvcRegistrations {
@Override
public RequestMappingHandlerMapping getRequestMappingHandlerMapping() {
return new CustomRequestMappingHandlerMapping();
}
}

+ 46
- 0
dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java View File

@@ -0,0 +1,46 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.constant;

import lombok.Getter;

/**
* @description 算法枚举类
* @date 2020-05-12
*/
@Getter
public enum AlgorithmSourceEnum {

/**
* MINE 算法来源 我的算法
*/
MINE(1, "MINE"),
/**
* PRE 算法来源 预置算法
*/
PRE(2,"PRE");

private Integer status;

private String message;

AlgorithmSourceEnum(Integer status, String message) {
this.status = status;
this.message = message;
}
}

+ 39
- 0
dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java View File

@@ -0,0 +1,39 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.constant;

/**
* @Description 数字常量
* @Date 2020-6-9
*/
public class NumberConstant {

public final static int NUMBER_0 = 0;
public final static int NUMBER_1 = 1;
public final static int NUMBER_2 = 2;
public final static int NUMBER_3 = 3;
public final static int NUMBER_5 = 5;
public final static int NUMBER_10 = 10;
public final static int NUMBER_30 = 30;
public final static int NUMBER_50 = 50;
public final static int NUMBER_60 = 60;
public final static int HOUR_SECOND = 60 * 60;
public final static int DAY_SECOND = 60 * 60 * 24;
public final static int WEEK_SECOND = 60 * 60 * 24 * 7;
public final static int MAX_PAGE_SIZE = 2000;
}

+ 40
- 0
dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java View File

@@ -0,0 +1,40 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.constant;

import lombok.Data;
import org.springframework.stereotype.Component;

/**
* @description: 权限常量
* @since: 2020-05-25 14:39
*/
@Component
@Data
public class PermissionConstant {

/**
* 超级用户
*/
public static final long ADMIN_USER_ID = 1L;
public static final long ANONYMOUS_USER = -1L;
public static final String SELECT = "select";
public static final String UPDATE = "update";
public static final String DELETE = "delete";

}

+ 66
- 0
dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java View File

@@ -0,0 +1,66 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.constant;

/**
* @description 权限标识,对应 menu 表中的 permission 字段
* @date 2020-05-14
*/
public final class Permissions {

/**
* 数据管理
*/
public static final String DATA = "data";
public static final String DATA_DATASET = "data:dataset";

/**
* 模型开发
*/
public static final String DEVELOPMENT = "development";
public static final String DEVELOPMENT_NOTEBOOK = "development:notebook";
public static final String DEVELOPMENT_ALGORITHM = "development:algorithm";
/**
* 训练管理
*/
public static final String TRAINING = "training";
public static final String TRAINING_IMAGE = "training:image";
public static final String TRAINING_JOB = "training:job";

/**
* 模型管理
*/
public static final String MODEL = "model";
public static final String MODEL_MODEL = "model:model";

/**
* 控制台
*/
public static final String SYSTEM = "system";
public static final String SYSTEM_USER = "system:user";
public static final String SYSTEM_ROLE = "system:role";
public static final String SYSTEM_MENU = "system:menu";
public static final String SYSTEM_DICT = "system:dict";
public static final String SYSTEM_NODE = "system:node";
public static final String SYSTEM_LOG = "system:log";
public static final String SYSTEM_TEAM = "system:team";

private Permissions() {
}
}

+ 46
- 0
dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java View File

@@ -0,0 +1,46 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.constant;

/**
* @description 字符串constant
* @date 2020-05-14
*/
public final class StringConstant {

public static final String MSIE = "MSIE";
public static final String MOZILLA = "Mozilla";

/**
* 公共字段
*/
public static final String CREATE_TIME = "createTime";
public static final String UPDATE_TIME = "updateTime";
public static final String UPDATE_USER_ID = "updateUserId";
public static final String CREATE_USER_ID = "createUserId";
public static final String DELETED = "deleted";
public static final String UTF8 = "utf-8";

/**
* 测试环境
*/
public static final String PROFILE_ACTIVE_TEST = "test";

private StringConstant() {
}
}

+ 46
- 0
dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java View File

@@ -0,0 +1,46 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.constant;

/**
* @description 符号常量
* @Date 2020-5-29
*/
public class SymbolConstant {
public static final String SLASH = "/";
public static final String COMMA = ",";
public static final String COLON = ":";
public static final String LINEBREAK = "\n";
public static final String BLANK = "";
public static final String QUESTION = "?";
public static final String ZERO = "0";
public static final String DOT = ".";
public static final String TOKEN = "token";
public static final String GET = "get";
public static final String SET = "set";

public static final String BRACKETS = "{}";
public static final String BACKSLASH = "\\";
public static final String BACKSLASH_MARK= "\\\"";
public static final String DOUBLE_MARK= "\"\"";
public static final String MARK= "\"";

private SymbolConstant() {
}

}

+ 47
- 0
dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java View File

@@ -0,0 +1,47 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.constant;

import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

/**
* @description 算法常量
* @date 2020-06-02
*/
@Data
@Component
public class TrainAlgorithmConstant {

@Value("${train-algorithm.is-train-out}")
private Boolean isTrainOut;

@Value("${train-algorithm.is-train-log}")
private Boolean isTrainLog;

@Value("${train-algorithm.is-visualized-log}")
private Boolean isVisualizedLog;

@Value("${train-algorithm.algorithm-source}")
private Integer algorithmSource;

@Value("${train-algorithm.fork}")
private Boolean fork;

}

+ 100
- 0
dubhe-server/common/src/main/java/org/dubhe/constant/TrainJobConstant.java View File

@@ -0,0 +1,100 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.constant;

import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

/**
* @description 训练常量
* @create: 2020-05-12
*/
@Component
@Data
public class TrainJobConstant {

@Value("${train-job.namespace}")
private String namespace;

@Value("${train-job.version-label}")
private String versionLabel;

@Value("${train-job.separator}")
private String separator;

@Value("${train-job.pod-name}")
private String podName;

@Value("${train-job.python-format}")
private String pythonFormat;

@Value("${train-job.manage}")
private String manage;

@Value("${train-job.out-path}")
private String outPath;

@Value("${train-job.log-path}")
private String logPath;

@Value("${train-job.visualized-log-path}")
private String visualizedLogPath;

@Value("${train-job.docker-dataset-path}")
private String dockerDatasetPath;

@Value("${train-job.docker-train-path}")
private String dockerTrainPath;

@Value("${train-job.docker-out-path}")
private String dockerOutPath;

@Value("${train-job.docker-log-path}")
private String dockerLogPath;

@Value("${train-job.docker-dataset}")
private String dockerDataset;

@Value("${train-job.docker-visualized-log-path}")
private String dockerVisualizedLogPath;

@Value("${train-job.load-path}")
private String loadPath;

@Value("${train-job.load-key}")
private String loadKey;

@Value("${train-job.eight}")
private String eight;

@Value("${train-job.plus-eight}")
private String plusEight;

public static final String TRAIN_ID = "trainId";

public static final String TRAIN_VERSION = "trainVersion";

public static final String RUN_TIME = "runtime";

public static final String TRAIN_STATUS = "trainStatus";

public static final String CREATE_TIME = "createTime";

public static final String ALGORITHM_NAME = "algorithmName";
}

+ 36
- 0
dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java View File

@@ -0,0 +1,36 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.constant;

import org.springframework.stereotype.Component;

import lombok.Data;

/**
* @description 算法用途
* @date: 2020-06-23
*/
@Component
@Data
public class UserAuxiliaryInfoConstant {


public static final String ALGORITHM_USAGE ="algorithem_usage";

}

+ 65
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/dto/UserDTO.java View File

@@ -0,0 +1,65 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.domain.dto;

import com.fasterxml.jackson.annotation.JsonIgnore;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.dubhe.domain.entity.Role;
import org.dubhe.domain.entity.UserAvatar;

import java.io.Serializable;
import java.sql.Timestamp;
import java.util.Date;
import java.util.List;

/**
* @description 用户信息
* @date 2020-06-29
*/
@Data
public class UserDTO implements Serializable {

@ApiModelProperty(hidden = true)
private Long id;

private String username;

private String nickName;

private String sex;

private String email;

private String phone;

private Boolean enabled;

private String remark;

@JsonIgnore
private String password;

private Date lastPasswordResetTime;

private Timestamp createTime;

private List<Role> roles;

private UserAvatar userAvatar;


}

+ 64
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java View File

@@ -0,0 +1,64 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.domain.entity;
import java.io.Serializable;

import org.dubhe.base.MagicNumConstant;

import com.alibaba.fastjson.annotation.JSONField;

import cn.hutool.core.date.DateUtil;
import lombok.Data;
import lombok.experimental.Accessors;

/**
* @description 日志对象封装类
* @date 2020-06-29
*/
@Data
@Accessors(chain = true)
public class LogInfo implements Serializable {

@JSONField(ordinal = MagicNumConstant.ONE)
private String traceId;

@JSONField(ordinal = MagicNumConstant.TWO)
private String type;

@JSONField(ordinal = MagicNumConstant.THREE)
private String level;

@JSONField(ordinal = MagicNumConstant.FOUR)
private String cName;

@JSONField(ordinal = MagicNumConstant.FIVE)
private String mName;
@JSONField(ordinal = MagicNumConstant.SIX)
private String line;
@JSONField(ordinal = MagicNumConstant.SEVEN)
private String time = DateUtil.now();

@JSONField(ordinal = MagicNumConstant.EIGHT)
private Object info;

public void setInfo(Object info) {
this.info = info;
}
}

+ 120
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/entity/Menu.java View File

@@ -0,0 +1,120 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.domain.entity;

import com.baomidou.mybatisplus.annotation.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.dubhe.base.BaseEntity;
import org.hibernate.validator.constraints.Length;

import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import java.io.Serializable;
import java.sql.Timestamp;
import java.util.Objects;

/**
* @description 菜单实体
* @date 2020-06-29
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@TableName("menu")
public class Menu extends BaseEntity implements Serializable {

private static final long serialVersionUID = 3100515433018008777L;

@TableId(value = "id", type = IdType.AUTO)
private Long id;

@NotBlank
private String name;

@TableField(value = "sort")
private Long sort = 999L;

@TableField(value = "path")
private String path;

@TableField(value = "component")
private String component;

/**
* 类型,目录、菜单、按钮
*/
@TableField(value = "type")
private Integer type;

/**
* 权限
*/
@TableField(value = "permission")
private String permission;

@TableField(value = "component_name")
private String componentName;

@TableField(value = "icon")
private String icon;

/**
* 布局类型
*/
@TableField(value = "layout")
private String layout;

@TableField(value = "cache")
private Boolean cache;

@TableField(value = "hidden")
private Boolean hidden;

/**
* 上级菜单ID
*/
@TableField(value = "pid")
private Long pid;

@TableField(value = "deleted",fill = FieldFill.INSERT)
private Boolean deleted;


@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Menu menu = (Menu) o;
return Objects.equals(id, menu.id);
}

@Override
public int hashCode() {
return Objects.hash(id);
}

public @interface Update {
}
}

+ 87
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/entity/Role.java View File

@@ -0,0 +1,87 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.dubhe.domain.entity;

import com.baomidou.mybatisplus.annotation.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.dubhe.base.BaseEntity;
import org.hibernate.validator.constraints.Length;

import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotEmpty;
import java.io.Serializable;
import java.sql.Timestamp;
import java.util.Objects;
import java.util.Set;

/**
* @description 角色实体
* @date 2020-06-29
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@TableName("role")
public class Role extends BaseEntity implements Serializable {

private static final long serialVersionUID = -812009584744832371L;

@TableId(value = "id", type = IdType.AUTO)
private Long id;

@TableField(value = "name")
private String name;

/**
* 权限
*/
@TableField(value = "permission")
private String permission;

@TableField(value = "remark")
private String remark;

@TableField(exist = false)
private Set<Menu> menus;

@TableField(value = "deleted",fill = FieldFill.INSERT)
private Boolean deleted = false;

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Role role = (Role) o;
return Objects.equals(id, role.id);
}

@Override
public int hashCode() {
return Objects.hash(id);
}

public @interface Update {
}
}

+ 49
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/entity/RoleMenu.java View File

@@ -0,0 +1,49 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.dubhe.domain.entity;

import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

import java.io.Serializable;


/**
* @description 角色菜单关系实体
* @date 2020-06-29
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@TableName("roles_menus")
public class RoleMenu implements Serializable {

private static final long serialVersionUID = -6296866205797727963L;

@TableField(value = "menu_id")
private Long menuId;

@TableField(value = "role_id")
private Long roleId;


}

+ 85
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java View File

@@ -0,0 +1,85 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.domain.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import java.sql.Timestamp;
import java.util.List;
import java.util.Objects;

/**
* @description 团队实体
* @date 2020-06-29
*/
@TableName("team")
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class Team {
@TableId(value = "id", type = IdType.AUTO)
@NotNull(groups = Update.class)
private Long id;

@TableField(value = "name")
@NotBlank
private String name;

@TableField(value = "enabled")
@NotNull
private Boolean enabled;

/**
* 团队成员
*/
@TableField(exist = false)
private List<User> teamUserList;

@TableField(value = "create_time")
private Timestamp createTime;

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Team dept = (Team) o;
return Objects.equals(id, dept.id) &&
Objects.equals(name, dept.name);
}

@Override
public int hashCode() {
return Objects.hash(id, name);
}

public @interface Update {
}
}

+ 58
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java View File

@@ -0,0 +1,58 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.domain.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;

import javax.validation.constraints.NotNull;
import java.io.Serializable;

/**
* @description 团队用户关系实体
* @date 2020-06-29
*/
@Data
@TableName("teams_users_roles")
public class TeamUserRole implements Serializable {

@TableId(value = "id", type = IdType.AUTO)
@NotNull()
private Long id;

/**
* 团队
*/
@TableField(exist = false)
private Team team;

/**
* 用户
*/
@TableField(exist = false)
private User user;

/**
* 角色
*/
@TableField(exist = false)
private Role role;

}

+ 126
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/entity/User.java View File

@@ -0,0 +1,126 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.domain.entity;

import com.baomidou.mybatisplus.annotation.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.dubhe.base.BaseEntity;

import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
import java.util.Objects;


/**
* @description 用户实体
* @date 2020-06-29
*/
@Data
@TableName("user")
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class User extends BaseEntity implements Serializable {

private static final long serialVersionUID = -3836401769559845765L;

@TableId(value = "id", type = IdType.AUTO)
@NotNull(groups = Update.class)
private Long id;

@NotBlank
@TableField(value = "username")
private String username;

/**
* 用户昵称
*/
@TableField(value = "nick_name")
@NotBlank
private String nickName;

/**
* 性别
*/
@TableField(value = "sex")
private String sex;

@NotBlank
@TableField(value = "email")
private String email;

@NotBlank
@TableField(value = "phone")
private String phone;

@NotNull
@TableField(value = "enabled")
private Boolean enabled;

@TableField(value = "password")
private String password;

@TableField(value = "last_password_reset_time")
private Date lastPasswordResetTime;

@TableField(value = "remark")
private String remark;

@TableField(value = "avatar_id")
private Long avatarId;


@TableField(exist = false)
private UserAvatar userAvatar;


@NotEmpty
@TableField(exist = false)
private List<Role> roles;

@TableField(value = "deleted",fill = FieldFill.INSERT)
private Boolean deleted = false;



@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
User user = (User) o;
return Objects.equals(id, user.id) &&
Objects.equals(username, user.username);
}

@Override
public int hashCode() {
return Objects.hash(id, username);
}

public @interface Update {
}
}

+ 58
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserAvatar.java View File

@@ -0,0 +1,58 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.domain.entity;

import cn.hutool.core.util.ObjectUtil;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.dubhe.base.BaseEntity;

import java.io.Serializable;


/**
* @description 用户头像实体
* @date 2020-06-29
*/
@Data
@NoArgsConstructor
@TableName("user_avatar")
public class UserAvatar extends BaseEntity implements Serializable {

@TableId(value = "id", type = IdType.AUTO)
private Long id;

@TableField(value = "real_name")
private String realName;

@TableField(value = "path")
private String path;

@TableField(value = "size")
private String size;


public UserAvatar(UserAvatar userAvatar, String realName, String path, String size) {
this.id = ObjectUtil.isNotEmpty(userAvatar) ? userAvatar.getId() : null;
this.realName = realName;
this.path = path;
this.size = size;
}
}

+ 47
- 0
dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserRole.java View File

@@ -0,0 +1,47 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.domain.entity;

import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

import java.io.Serializable;

/**
* @description 用户角色关系实体
* @date 2020-06-29
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@TableName("users_roles")
public class UserRole implements Serializable {

private static final long serialVersionUID = -6296866205797727963L;

@TableField(value = "user_id")
private Long userId;

@TableField(value = "role_id")
private Long roleId;


}

+ 35
- 0
dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java View File

@@ -0,0 +1,35 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.dto.callback;

import io.swagger.annotations.ApiModel;
import lombok.Data;

/**
* @descripton 个性化参数实现 与 个性化参数个性化校验
*
* @date 2020-05-28
*/
@ApiModel(description = "k8s pod异步回调Algorithm")
@Data
public class AlgorithmK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO {
@Override
public String toString() {
return super.toString();
}
}

+ 33
- 0
dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java View File

@@ -0,0 +1,33 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.dto.callback;

import io.swagger.annotations.ApiModel;
import lombok.Data;

/**
* @descripton k8s方异步回调统一汇总类,即不管什么业务所有请求参数都放在这个类中
*
* @date 2020-05-28
*/
@ApiModel(description = "k8s方 pod异步回调汇总类")
@Data
public class AllK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO{


}

+ 76
- 0
dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java View File

@@ -0,0 +1,76 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.dto.callback;

import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;

import javax.validation.constraints.NotEmpty;

/**
* @descripton 统一通用参数实现与校验
*
* @date 2020-05-28
*/
@ApiModel(description = "k8s pod异步回调基类")
@Data
public class BaseK8sPodCallbackCreateDTO {

@ApiModelProperty(required = true,value = "k8s namespace")
@NotEmpty(message = "namespace 不能为空!")
private String namespace;

@ApiModelProperty(required = true,value = "k8s resource name")
@NotEmpty(message = "resourceName 不能为空!")
private String resourceName;

@ApiModelProperty(required = true,value = "k8s pod name")
@NotEmpty(message = "podName 不能为空!")
private String podName;

@ApiModelProperty(value = "k8s pod phase",notes = "对应PodPhaseEnum")
@NotEmpty(message = "phase 不能为空!")
private String phase;

@ApiModelProperty(value = "k8s pod containerStatuses state")
private String messages;

public BaseK8sPodCallbackCreateDTO(){

}

public BaseK8sPodCallbackCreateDTO(String namespace,String resourceName,String podName,String phase,String messages){
this.namespace = namespace;
this.resourceName = resourceName;
this.podName = podName;
this.phase = phase;
this.messages = messages;
}

@Override
public String toString() {
return "BaseK8sPodCallbackReq{" +
"namespace='" + namespace + '\'' +
", resourceName='" + resourceName + '\'' +
", podName='" + podName + '\'' +
", phase='" + phase + '\'' +
", messages=" + messages +
'}';
}
}

+ 36
- 0
dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java View File

@@ -0,0 +1,36 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.dto.callback;

import io.swagger.annotations.ApiModel;
import lombok.Data;

/**
* @descripton k8s pod异步回调Notebook
*
* @date 2020-05-28
*/
@ApiModel(description = "k8s pod异步回调Notebook")
@Data
public class NotebookK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO {

@Override
public String toString() {
return super.toString();
}
}

+ 80
- 0
dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java View File

@@ -0,0 +1,80 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.enums;

import lombok.Getter;

import java.util.HashMap;
import java.util.Map;

/**
* @desc: 业务模块
*
* @date 2020.05.25
*/
@Getter
public enum BizEnum {

/**
* 模型开发
*/
NOTEBOOK("模型开发","notebook",0),
/**
* 算法管理
*/
ALGORITHM("算法管理","algorithm",1),
;

/**
* 业务模块名称
*/
private String bizName;
/**
* 业务模块名称
*/
private String bizCode;
/**
* 业务源代号
*/
private Integer createResource;

BizEnum(String bizName,String bizCode, Integer createResource) {
this.createResource = createResource;
this.bizName = bizName;
this.bizCode = bizCode;
}

private static final Map<Integer,BizEnum> RESOURCE_ENUM_MAP = new HashMap<Integer,BizEnum>(){
{
for (BizEnum enums:BizEnum.values()){
put(enums.getCreateResource(),enums);
}
}
};

/**
* 根据createResource获取BizEnum
* @param createResource
* @return
*/
public static BizEnum getByCreateResource(int createResource){
return RESOURCE_ENUM_MAP.get(createResource);
}


}

+ 91
- 0
dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java View File

@@ -0,0 +1,91 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.enums;

import java.util.HashMap;
import java.util.Map;

/**
* @desc: 业务NFS路径枚举
* @date 2020.05.13
*/
public enum BizNfsEnum {
/**
* 模型开发 NFS 路径命名
*/
NOTEBOOK(BizEnum.NOTEBOOK, "notebook"),
/**
* 算法管理 NFS 路径命名
*/
ALGORITHM(BizEnum.ALGORITHM, "algorithm-manage"),
;

BizNfsEnum(BizEnum bizEnum, String bizNfsPath) {
this.bizEnum = bizEnum;
this.bizNfsPath = bizNfsPath;
}

/**
* 业务模块
*/
private BizEnum bizEnum;
/**
* 业务模块nfs路径
*/
private String bizNfsPath;


private static final Map<Integer, BizNfsEnum> RESOURCE_ENUM_MAP = new HashMap<Integer, BizNfsEnum>() {
{
for (BizNfsEnum enums : BizNfsEnum.values()) {
put(enums.getCreateResource(), enums);
}
}
};

/**
* 根据createResource获取BizNfsEnum
*
* @param createResource
* @return
*/
public static BizNfsEnum getByCreateResource(int createResource) {
return RESOURCE_ENUM_MAP.get(createResource);
}


public String getBizName() {
return bizEnum == null ? null : bizEnum.getBizName();
}

public Integer getCreateResource() {
return bizEnum == null ? null : bizEnum.getCreateResource();
}

public String getBizNfsPath() {
return bizNfsPath;
}

public BizEnum getBizEnum(){
return bizEnum;
}

public String getBizCode() {
return bizEnum == null ? null :bizEnum.getBizCode();
}
}

+ 61
- 0
dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java View File

@@ -0,0 +1,61 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.enums;

import lombok.Getter;

/**
* @description 日志类型枚举类
* @date 2020-06-23
*/
@Getter
public enum LogEnum {

// 系统报错日志
SYS_ERR,
// 用户请求日志
REST_REQ,
// 训练模块
BIZ_TRAIN,
// 系统模块
BIZ_SYS,
// 模型模块
BIZ_MODEL,
// 数据集模块
BIZ_DATASET,
// k8s模块
BIZ_K8S,
//note book
NOTE_BOOK,
//NFS UTILS
NFS_UTIL;

/**
* 判断日志类型不能为空
*
* @param logType 日志类型
* @return boolean 返回类型
*/
public static boolean isLogType(LogEnum logType) {

if (logType != null) {
return true;
}
return false;
}
}

+ 95
- 0
dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java View File

@@ -0,0 +1,95 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.enums;

/**
* @Description 是否开关枚举
* @Date 2020-06-01
*/
public enum SwitchEnum {
/**
* OFF 否
*/
OFF(0, "否"),

/**
* ON 否
*/
ON(1, "是"),

;

private Integer value;

private String desc;

SwitchEnum(Integer value, String desc) {
this.value = value;
this.desc = desc;
}

public Integer getValue() {
return this.value;
}

public String getDesc() {
return desc;
}

public void setDesc(String desc) {
this.desc = desc;
}

public static SwitchEnum getEnumValue(Integer value) {
switch (value) {
case 0:
return OFF;
case 1:
return ON;
default:
return OFF;
}
}

public static Boolean getBooleanValue(Integer value) {
switch (value) {
case 1:
return true;
case 0:
return false;
default:
return false;
}
}

public static boolean isExist(Integer value) {
for (SwitchEnum itm : SwitchEnum.values()) {
if (value.compareTo(itm.getValue()) == 0) {
return true;
}
}
return false;
}


@Override
public String toString() {
return "[" + this.value + "]" + this.desc;
}

}

+ 101
- 0
dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java View File

@@ -0,0 +1,101 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.enums;

import lombok.Getter;

import java.util.Arrays;
import java.util.List;

/**
* @description 训练任务枚举类
* @date 2020-04-27
*/
@Getter
public enum TrainJobStatusEnum {

/**
* PENDING
*/
PENDING(0, "PENDING"),
/**
* RUNNING
*/
RUNNING(1, "RUNNING"),
/**
* SUCCEEDED
*/
SUCCEEDED(2, "SUCCEEDED"),
/**
* FAILED
*/
FAILED(3, "FAILED"),
/**
* STOP
*/
STOP(4, "STOP"),
/**
* UNKNOWN
*/
UNKNOWN(5, "UNKNOWN"),
/**
* DELETED
*/
DELETED(6, "DELETED"),

/**
* CREATE_FAILED
*/
CREATE_FAILED(7, "CREATE_FAILED");

private Integer status;

private String message;

TrainJobStatusEnum(Integer status, String message) {
this.status = status;
this.message = message;
}

public static TrainJobStatusEnum get(String msg) {
for (TrainJobStatusEnum statusEnum : values()) {
if (statusEnum.message.equalsIgnoreCase(msg)) {
return statusEnum;
}
}
return UNKNOWN;
}

public static boolean isEnd(String msg) {
List<String> endList = Arrays.asList("SUCCEEDED", "FAILED", "STOP", "CREATE_FAILED");
return endList.stream().anyMatch(s -> s.equalsIgnoreCase(msg));
}

public static boolean isEnd(Integer num) {
List<Integer> endList = Arrays.asList(2, 3, 4, 7);
return endList.stream().anyMatch(s -> s.equals(num));
}

public static boolean checkStopStatus(Integer num) {
return SUCCEEDED.getStatus().equals(num) ||
FAILED.getStatus().equals(num) ||
STOP.getStatus().equals(num) ||
CREATE_FAILED.getStatus().equals(num) ||
DELETED.getStatus().equals(num);
}
}

+ 69
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java View File

@@ -0,0 +1,69 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.exception;

import lombok.Getter;

/**
* @description 通用异常code
* @date 2020-03-26
*/
@Getter
public enum BaseErrorCode implements ErrorCode {

/**
* undefined error
*/
UNDEFINED(10000, "操作成功!"),
ERROR(10001, "操作失败!"),
ERROR_SYSTEM(10002, "系统繁忙!"),
UNAUTHORIZED(401, "无权访问!"),
/**
* system 模块异常码
*/
SYSTEM_USERNAME_ALREADY_EXISTS(20000, "账号已存在!"),
SYSTEM_CODE_ALREADY_EXISTS(20001, "Code already exists!"),
SYSTEM_USER_IS_NOT_EXISTS(20002, "用户不存在!"),
SYSTEM_USER_ALREADY_REGISTER(20003, "账号已注册!"),
SYSTEM_USER_REGISTER_EMAIL_INFO_EXPIRED(20004, "邮箱验证码已过期!"),
SYSTEM_USER_EMAIL_ALREADY_EXISTS(20004, "该邮箱已被注册!"),
SYSTEM_USER_EMAIL_PASSWORD_ERROR(20005, "邮件密码错误!"),
SYSTEM_USER_EMAIL_CODE_CANNOT_EXCEED_TIMES(20006, "邮件发送不能超过三次!"),
SYSTEM_USER_EMAIL_OR_CODE_ERROR(20007, "邮箱地址或验证码错误、请重新输入!"),
SYSTEM_USER_IS_LOCKED(20008, "用户已锁定!"),
SYSTEM_USER_USERNAME_OR_PASSWORD_ERROR(20009, "账号或密码不正确!"),
SYSTEM_USER_USERNAME_IS_LOCKED(20010, "账号已锁定,6小时后解锁!"),
SYSTEM_USER_CANNOT_UPDATE_ADMIN(20011, "仅超级管理员可操作!"),
SYSTEM_USER_TOKEN_INFO_IS_NULL(20012, "登录信息不存在!"),
SYSTEM_USER_EMAIL_NOT_EXISTS(20013, "该邮箱未注册!"),
SYSTEM_USER_CANNOT_DELETE(20014, "系统默认用户不可删除!"),
SYSTEM_ROLE_CANNOT_DELETE(20015, "系统默认角色不可删除!"),


;


Integer code;
String msg;

BaseErrorCode(Integer code, String msg) {
this.code = code;
this.msg = msg;
}

}

+ 72
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java View File

@@ -0,0 +1,72 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.exception;

import lombok.Getter;
import org.dubhe.base.DataResponseBody;
import org.dubhe.base.ResponseCode;

/**
* @description 业务异常
* @date 2020-03-13
*/
@Getter
public class BusinessException extends RuntimeException {

private DataResponseBody responseBody;

public BusinessException(String msg) {
super(msg);
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
}

public BusinessException(String msg, Throwable cause) {
super(msg,cause);
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
}

public BusinessException(Throwable cause) {
super(cause);
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST);
}

public BusinessException(Integer code, String msg, String info, Throwable cause) {
super(msg,cause);
if (info == null) {
this.responseBody = new DataResponseBody(code, msg);
} else {
this.responseBody = new DataResponseBody(code, msg + ":" + info);
}
}

public BusinessException(ErrorCode errorCode, Throwable cause) {
this(errorCode.getCode(), errorCode.getMsg(), null, cause);
}

public BusinessException(ErrorCode errorCode, String info, Throwable cause) {
this(errorCode.getCode(), errorCode.getMsg(), info, cause);
}

public BusinessException(ErrorCode errorCode) {
this(errorCode, null);
}

public BusinessException(Integer code,String msg) {
this.responseBody = new DataResponseBody(code, msg);
}
}

+ 48
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java View File

@@ -0,0 +1,48 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.exception;

import lombok.Getter;
import org.dubhe.base.DataResponseBody;
import org.dubhe.base.ResponseCode;

/**
* @description 验证码异常
* @date 2020-02-23
*/
@Getter
public class CaptchaException extends RuntimeException {
private static final long serialVersionUID = 1L;

private DataResponseBody responseBody;
private Throwable cause;

public CaptchaException(String msg) {
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
}

public CaptchaException(String msg, Throwable cause) {
this.cause = cause;
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
}

public CaptchaException(Throwable cause) {
this.cause = cause;
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST);
}
}

+ 48
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java View File

@@ -0,0 +1,48 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.exception;

import lombok.Getter;
import org.dubhe.base.DataResponseBody;
import org.dubhe.base.ResponseCode;

/**
* @Description docker 异常
* @Date 2020-6-6
**/
@Getter
public class DockerBizException extends RuntimeException {
private static final long serialVersionUID = 1L;

private DataResponseBody responseBody;
private Throwable cause;

public DockerBizException(String msg) {
this.responseBody = new DataResponseBody(ResponseCode.DOCKER_ERROR, msg);
}

public DockerBizException(String msg, Throwable cause) {
this.cause = cause;
this.responseBody = new DataResponseBody(ResponseCode.DOCKER_ERROR, msg);
}

public DockerBizException(Throwable cause) {
this.cause = cause;
this.responseBody = new DataResponseBody(ResponseCode.DOCKER_ERROR);
}
}

+ 39
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java View File

@@ -0,0 +1,39 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.exception;


/**
* @description 异常code
* @date 2020-03-26
*/
public interface ErrorCode {

/**
* 错误码
* @return code
*/
Integer getCode();

/**
* error info
* @return
*/
String getMsg();

}

+ 44
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/LoginException.java View File

@@ -0,0 +1,44 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.exception;

import lombok.Getter;
import org.dubhe.base.DataResponseBody;
import org.dubhe.base.ResponseCode;

/**
* @description 登录异常
* @date 2020-02-23
*/
@Getter
public class LoginException extends RuntimeException {
private DataResponseBody responseBody;
private Throwable cause;

public LoginException(String msg) {
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
}

public LoginException(String msg, Throwable cause) {
this.cause = cause;
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
}

public LoginException(Throwable cause) {
this.cause = cause;
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST);
}
}

+ 42
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java View File

@@ -0,0 +1,42 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.exception;

import lombok.Getter;

/**
* Description : NFS utils 工具异常
*
* @date 15.06.2020
*/
@Getter
public class NfsBizException extends BusinessException {

private static final long serialVersionUID = 1L;


public NfsBizException(Throwable cause){
super(cause);
}

public NfsBizException(String msg){
super(msg);
}


}

+ 45
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java View File

@@ -0,0 +1,45 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.exception;

import lombok.Getter;

/**
* @description: Notebook 业务处理异常
*
* @date 2020.04.27
*/
@Getter
public class NotebookBizException extends BusinessException{

private static final long serialVersionUID = 1L;

public NotebookBizException(String msg) {
super(msg);
}

public NotebookBizException(String msg, Throwable cause) {
super(msg,cause);
}

public NotebookBizException(Throwable cause) {
super(cause);
}


}

+ 47
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java View File

@@ -0,0 +1,47 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.exception;

import lombok.Getter;
import org.dubhe.base.DataResponseBody;
import org.dubhe.base.ResponseCode;

/**
* @description 权限异常
* @date 2020-02-23
*/
@Getter
public class UnauthorizedException extends RuntimeException {

private DataResponseBody responseBody;
private Throwable cause;

public UnauthorizedException(String msg) {
this.responseBody = new DataResponseBody(ResponseCode.UNAUTHORIZED, msg);
}

public UnauthorizedException(String msg, Throwable cause) {
this.cause = cause;
this.responseBody = new DataResponseBody(ResponseCode.UNAUTHORIZED, msg);
}

public UnauthorizedException(Throwable cause) {
this.cause = cause;
this.responseBody = new DataResponseBody(ResponseCode.UNAUTHORIZED);
}
}

+ 190
- 0
dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java View File

@@ -0,0 +1,190 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.exception.handler;

import java.util.Objects;

import org.apache.shiro.ShiroException;
import org.apache.shiro.authc.AuthenticationException;
import org.apache.shiro.authc.IncorrectCredentialsException;
import org.apache.shiro.authc.LockedAccountException;
import org.apache.shiro.authc.UnknownAccountException;
import org.dubhe.base.DataResponseBody;
import org.dubhe.base.ResponseCode;
import org.dubhe.enums.LogEnum;
import org.dubhe.exception.BusinessException;
import org.dubhe.exception.CaptchaException;
import org.dubhe.exception.LoginException;
import org.dubhe.exception.NotebookBizException;
import org.dubhe.exception.UnauthorizedException;
import org.dubhe.utils.LogUtil;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.validation.BindException;
import org.springframework.validation.ObjectError;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;

import lombok.extern.slf4j.Slf4j;

/**
* @description 处理异常
* @date 2020-02-23
*/
@Slf4j
@RestControllerAdvice
public class GlobalExceptionHandler {

/**
* 处理所有不可知的异常
*/
@ExceptionHandler(Throwable.class)
public ResponseEntity<DataResponseBody> handleException(Throwable e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
return buildResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR,
new DataResponseBody(ResponseCode.ERROR, e.getMessage()));
}

/**
* UnauthorizedException
*/
@ExceptionHandler(UnauthorizedException.class)
public ResponseEntity<DataResponseBody> badCredentialsException(UnauthorizedException e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
String message = "坏的凭证".equals(e.getMessage()) ? "用户名或密码不正确" : e.getMessage();
return buildResponseEntity(HttpStatus.UNAUTHORIZED, new DataResponseBody(ResponseCode.ERROR, message));
}

/**
* 处理自定义异常
*/
@ExceptionHandler(value = BusinessException.class)
public ResponseEntity<DataResponseBody> badRequestException(BusinessException e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
return buildResponseEntity(HttpStatus.OK, e.getResponseBody());
}

/**
* 处理自定义异常
*/
@ExceptionHandler(value = AuthenticationException.class)
public ResponseEntity<DataResponseBody> badRequestException(AuthenticationException e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
return buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.UNAUTHORIZED, "无权访问"));
}

/**
* shiro 异常捕捉
*/
@ExceptionHandler(value = ShiroException.class)
public ResponseEntity<DataResponseBody> accountException(ShiroException e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
ResponseEntity<DataResponseBody> responseEntity;
if (e instanceof IncorrectCredentialsException) {
responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "密码不正确"));
} else if (e instanceof UnknownAccountException) {
responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "此账户不存在"));
}

else if (e instanceof LockedAccountException) {
responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "未知的账号"));
}

else if (e instanceof UnknownAccountException) {
responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "账户已被禁用"));
}

else {
responseEntity = buildResponseEntity(HttpStatus.OK,
new DataResponseBody(ResponseCode.UNAUTHORIZED, "无权访问"));
}
return responseEntity;
}

/**
* 处理自定义异常
*/
@ExceptionHandler(value = LoginException.class)
public ResponseEntity<DataResponseBody> loginException(LoginException e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
return buildResponseEntity(HttpStatus.UNAUTHORIZED, e.getResponseBody());
}

/**
* 处理自定义异常
*/
@ExceptionHandler(value = CaptchaException.class)
public ResponseEntity<DataResponseBody> captchaException(CaptchaException e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
return buildResponseEntity(HttpStatus.OK, e.getResponseBody());
}

/**
* 处理自定义异常
*/
@ExceptionHandler(value = NotebookBizException.class)
public ResponseEntity<DataResponseBody> captchaException(NotebookBizException e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
return buildResponseEntity(HttpStatus.OK, e.getResponseBody());
}

/**
* 处理所有接口数据验证异常
*/
@ExceptionHandler(MethodArgumentNotValidException.class)
public ResponseEntity<DataResponseBody> handleMethodArgumentNotValidException(MethodArgumentNotValidException e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
String[] str = Objects.requireNonNull(e.getBindingResult().getAllErrors().get(0).getCodes())[1].split("\\.");
String message = e.getBindingResult().getAllErrors().get(0).getDefaultMessage();
String msg = "不能为空";
if (msg.equals(message)) {
message = str[1] + ":" + message;
}
return buildResponseEntity(HttpStatus.BAD_REQUEST, new DataResponseBody(ResponseCode.ERROR, message));
}

@ExceptionHandler(BindException.class)
public ResponseEntity<DataResponseBody> bindException(BindException e) {
// 打印堆栈信息
LogUtil.error(LogEnum.SYS_ERR, e);
ObjectError error = e.getAllErrors().get(0);
return buildResponseEntity(HttpStatus.BAD_REQUEST,
new DataResponseBody(ResponseCode.ERROR, error.getDefaultMessage()));
}

/**
* 统一返回
*
* @param httpStatus
* @param responseBody
* @return
*/
private ResponseEntity<DataResponseBody> buildResponseEntity(HttpStatus httpStatus, DataResponseBody responseBody) {
return new ResponseEntity<>(responseBody, httpStatus);
}
}

+ 121
- 0
dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java View File

@@ -0,0 +1,121 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.factory;

import org.dubhe.base.DataResponseBody;
import org.dubhe.base.ResponseCode;

/**
* @desc DataResponseBody 工厂类
*
* @date 2020.05.28
*/
public class DataResponseFactory {

private DataResponseFactory(){
}

/**
* 成功响应
*
* @param <T>
* @return
*/
public static <T> DataResponseBody success(){
return success(null,null);
}

/**
* 成功响应
*
* @param data
* @param <T>
* @return
*/
public static <T> DataResponseBody success(T data){
return success(null,data);
}

/**
* 成功响应
*
* @param msg
* @return
*/
public static DataResponseBody successWithMsg(String msg){
return success(msg,null);
}

/**
* 成功响应
*
* @param msg
* @param data
* @param <T>
* @return
*/
public static <T> DataResponseBody success(String msg, T data){
return new DataResponseBody(ResponseCode.SUCCESS,msg,data);
}

/**
* 失败响应 msg
*
* @param msg
* @return
*/
public static DataResponseBody failed(String msg){
return failed(ResponseCode.ERROR,msg,null);
}

/**
* 失败响应
*
* @param failedCode
* @param msg
* @return
*/
public static DataResponseBody failed(Integer failedCode,String msg){
return failed(failedCode,msg,null);
}

/**
* 失败响应
*
* @param failedCode
* @return
*/
public static DataResponseBody failed(Integer failedCode){
return failed(failedCode,null,null);
}

/**
* 失败响应
*
* @param failedCode
* @param msg
* @param data
* @param <T>
* @return
*/
public static <T> DataResponseBody failed(Integer failedCode, String msg, T data){
return new DataResponseBody(failedCode,msg,data);
}



}

+ 60
- 0
dubhe-server/common/src/main/java/org/dubhe/filter/FileLogFilter.java View File

@@ -0,0 +1,60 @@
/**
* Copyright 2019-2020 Zheng Jie
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.dubhe.filter;

import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.filter.AbstractMatcherFilter;
import ch.qos.logback.core.spi.FilterReply;

/**
* @description 自定义日志过滤器
* @date 2020-07-21
*/
public class FileLogFilter extends AbstractMatcherFilter<ILoggingEvent> {

Level level;

/**
* 重写decide方法
*
* @param iLoggingEvent event to decide upon.
* @return FilterReply
*/
@Override
public FilterReply decide(ILoggingEvent iLoggingEvent) {
if (!isStarted()) {
return FilterReply.NEUTRAL;
}
if (iLoggingEvent.getLevel().equals(level) && iLoggingEvent.getMessage() != null
&& iLoggingEvent.getMessage().startsWith("{") && iLoggingEvent.getMessage().endsWith("}")) {
return onMatch;
}
return onMismatch;

}

public void setLevel(Level level) {
this.level = level;
}

@Override
public void start() {
if (this.level != null) {
super.start();
}
}
}

+ 68
- 0
dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java View File

@@ -0,0 +1,68 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.handle;

import org.springframework.web.servlet.mvc.condition.RequestCondition;

import javax.servlet.http.HttpServletRequest;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* @description API版本控制
* @date 2020-04-06
*/
public class ApiVersionCondition implements RequestCondition<ApiVersionCondition> {
/**
* 路径中版本的前缀, 这里用 /v[1-9]/的形式
*/
private final static Pattern VERSION_PREFIX_PATTERN = Pattern.compile("v(\\d+)/");
private int apiVersion;

public ApiVersionCondition(int apiVersion) {
this.apiVersion = apiVersion;
}

@Override
public ApiVersionCondition combine(ApiVersionCondition other) {
// 采用最后定义优先原则,则方法上的定义覆盖类上面的定义
return new ApiVersionCondition(other.getApiVersion());
}

@Override
public ApiVersionCondition getMatchingCondition(HttpServletRequest request) {
Matcher m = VERSION_PREFIX_PATTERN.matcher(request.getRequestURI());
if (m.find()) {
Integer version = Integer.valueOf(m.group(1));
if (version >= this.apiVersion) {
return this;
}
}
return null;
}


@Override
public int compareTo(ApiVersionCondition other, HttpServletRequest request) {
// 优先匹配最新的版本号
return other.getApiVersion() - this.apiVersion;
}
public int getApiVersion() {
return apiVersion;
}
}

+ 64
- 0
dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java View File

@@ -0,0 +1,64 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.handle;

import lombok.Data;
import org.apache.shiro.authz.annotation.RequiresPermissions;
import org.dubhe.annotation.ApiVersion;
import org.springframework.core.annotation.AnnotationUtils;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.mvc.condition.RequestCondition;
import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping;

import java.lang.reflect.Method;
import java.util.HashSet;
import java.util.Set;
/**
* @description 权限注解收集
* @date 2020-04-06
*/
@Component
@Data
public class CustomRequestMappingHandlerMapping extends RequestMappingHandlerMapping {
Set<String> permissionsSet =new HashSet<String>();
@Override
protected RequestCondition<ApiVersionCondition> getCustomTypeCondition(Class<?> handlerType) {
ApiVersion apiVersion = AnnotationUtils.findAnnotation(handlerType, ApiVersion.class);
return createCondition(apiVersion);
}

@Override
protected RequestCondition<ApiVersionCondition> getCustomMethodCondition(Method method) {
RequiresPermissions requiresPermissions = AnnotationUtils.findAnnotation(method, RequiresPermissions.class);
if(requiresPermissions != null){
if(requiresPermissions.value()!=null){
for(int i=0;i<requiresPermissions.value().length;i++){
permissionsSet.add(requiresPermissions.value()[i]);
}
}

}

ApiVersion apiVersion = AnnotationUtils.findAnnotation(method, ApiVersion.class);
return createCondition(apiVersion);
}
private RequestCondition<ApiVersionCondition> createCondition(ApiVersion apiVersion) {
return apiVersion == null ? null : new ApiVersionCondition(apiVersion.value());
}

}

+ 58
- 0
dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java View File

@@ -0,0 +1,58 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.interceptor;

import org.dubhe.enums.LogEnum;
import org.dubhe.utils.K8sCallBackTool;
import org.dubhe.utils.LogUtil;
import org.dubhe.utils.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.handler.HandlerInterceptorAdapter;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

/**
* @description k8s pod 异步回调拦截器
*
* @date 2020-05-28
*/
@Component
public class K8sCallBackPodInterceptor extends HandlerInterceptorAdapter {

@Autowired
private K8sCallBackTool k8sCallBackTool;

@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
String uri = request.getRequestURI();
LogUtil.debug(LogEnum.BIZ_K8S,"接收到k8s异步请求,URI:{}",uri);
String k8sCallbackToken = request.getHeader(K8sCallBackTool.K8S_CALLBACK_TOKEN);
if (StringUtils.isBlank(k8sCallbackToken)){
LogUtil.warn(LogEnum.BIZ_K8S,"k8s异步回调没有配置【{}】,URI:{}",K8sCallBackTool.K8S_CALLBACK_TOKEN,uri);
return false;
}
boolean pass = k8sCallBackTool.validateToken(k8sCallbackToken);
if (!pass){
LogUtil.warn(LogEnum.BIZ_K8S,"k8s异步回调token:【{}】 验证不通过,URI:{}",k8sCallbackToken,uri);
}
return pass;
}

}

+ 97
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java View File

@@ -0,0 +1,97 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import cn.hutool.core.util.HexUtil;
import org.dubhe.enums.LogEnum;

import javax.crypto.Cipher;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.InvalidKeyException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

/**
* @description AES加解密工具
* @date 2020-06-01
*/
public class AesUtil {

private static final String AES = "AES";


private AesUtil(){

}


/**
*
* @param mode Cipher mode
* @param key 秘钥
* @return Cipher
* @throws NoSuchAlgorithmException
* @throws NoSuchPaddingException
* @throws InvalidKeyException
*/
private static Cipher getCipher(int mode,String key) throws NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException {
MessageDigest md5Digest = MessageDigest.getInstance("MD5");
SecretKeySpec secretKeySpec = new SecretKeySpec(md5Digest.digest(key.getBytes(StandardCharsets.UTF_8)), AES);
Cipher cipher = Cipher.getInstance(AES);
cipher.init(mode, secretKeySpec);
return cipher;
}

/**
* 加密
*
* @param data 原文
* @param key 秘钥
* @return String 密文
*/
public static String encrypt(String data, String key) {
try {
Cipher cipher = getCipher(Cipher.ENCRYPT_MODE,key);
byte[] content = data.getBytes(StandardCharsets.UTF_8);
return new String(HexUtil.encodeHex(cipher.doFinal(content), false));
} catch (Exception e) {
LogUtil.error(LogEnum.BIZ_SYS,"Md5Util 加密失败 ",e);
return null;
}
}

/**
* 解密
* @param hexData 十六进制密文
* @param key 秘钥
* @return String 密文
*/
public static String decrypt(String hexData, String key) {
try {
Cipher cipher = getCipher(Cipher.DECRYPT_MODE,key);
byte[] content = HexUtil.decodeHex(hexData);
return new String(cipher.doFinal(content), StandardCharsets.UTF_8);
} catch (Exception e) {
LogUtil.error(LogEnum.BIZ_SYS,"Md5Util 解密失败 ",e);
return null;
}
}

}

+ 80
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java View File

@@ -0,0 +1,80 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import java.sql.Timestamp;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;

/**
* @description 日期工具类
* @date 2020-6-10
*/
public class DateUtil {
/**
* 获取当前时间戳
*
* @return
*/
public static Timestamp getCurrentTimestamp() {
return Timestamp.valueOf(LocalDateTime.now());
}



/**
* 获取六小时后时间
* @return
*/
public static long getAfterSixHourTime(){
long l1 = getTimestampOfDateTime(LocalDateTime.now());
long milli = getTimestampOfDateTime(LocalDateTime.now().plusHours(6));
return (milli-l1);
}


/**
* LocalDateTime -> long
* @param localDateTime
* @return
*/
public static long getTimestampOfDateTime(LocalDateTime localDateTime) {
ZoneId zone = ZoneId.systemDefault();
Instant instant = localDateTime.atZone(zone).toInstant();
return instant.toEpochMilli();
}

/**
* 获取第二天凌晨时间
* @return
*/
public static long getSecondTime(){
LocalDateTime localDateTime = LocalDateTime.now();
long l1 = localDateTime.atZone(ZoneId.of("Asia/Shanghai")).toInstant().toEpochMilli();

LocalDate localDate = LocalDate.now();
LocalDate localDate1 = localDate.plusDays(1);
LocalDateTime localDateTime1 = localDate1.atStartOfDay();
long milli = localDateTime1.atZone(ZoneId.of("Asia/Shanghai")).toInstant().toEpochMilli();
System.out.println("当前时间到第二天凌晨的毫秒数"+(milli-l1));
return (milli-l1);
}

}

+ 47
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java View File

@@ -0,0 +1,47 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

/**
* @description 日期工具类
* @date 2020-5-29
**/
public class DubheDateUtil {

private static final int SIXTY = 60;

/**
* 将时间 秒 转为 小时
*
* @param second 秒数
* @return
*/
public static String convert2Str(Long second) {
if (null == second || second < 1) {
return "";
}
Long hh = second / (SIXTY * SIXTY);
second = second % (SIXTY * SIXTY);
Long mm = second / SIXTY;
Long ss = second % SIXTY;

StringBuilder sb = new StringBuilder();
sb.append(String.format("%02d", hh)).append(":").append(String.format("%02d", mm)).append(":").append(String.format("%02d", ss));
return sb.toString();
}
}

+ 100
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java View File

@@ -0,0 +1,100 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.DESKeySpec;
import javax.crypto.spec.IvParameterSpec;
import java.nio.charset.StandardCharsets;

/**
* @description 加密
* @date 2020-03-13
*/
public class EncryptUtils {

private static String strParam = "Passw0rd";

private static Cipher cipher;

private static IvParameterSpec iv = new IvParameterSpec(strParam.getBytes(StandardCharsets.UTF_8));

private static DESKeySpec getDesKeySpec(String source) throws Exception {
if (source == null || source.length() == 0) {
return null;
}
cipher = Cipher.getInstance("DES/CBC/PKCS5Padding");
String strKey = "Passw0rd";
return new DESKeySpec(strKey.getBytes(StandardCharsets.UTF_8));
}

/**
* 对称加密
*/
public static String desEncrypt(String source) throws Exception {
DESKeySpec desKeySpec = getDesKeySpec(source);
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES");
SecretKey secretKey = keyFactory.generateSecret(desKeySpec);
cipher.init(Cipher.ENCRYPT_MODE, secretKey, iv);
return byte2hex(
cipher.doFinal(source.getBytes(StandardCharsets.UTF_8))).toUpperCase();
}

/**
* 对称解密
*/
public static String desDecrypt(String source) throws Exception {
byte[] src = hex2byte(source.getBytes());
DESKeySpec desKeySpec = getDesKeySpec(source);
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES");
SecretKey secretKey = keyFactory.generateSecret(desKeySpec);
cipher.init(Cipher.DECRYPT_MODE, secretKey, iv);
byte[] retByte = cipher.doFinal(src);
return new String(retByte);
}

private static String byte2hex(byte[] inStr) {
String stmp;
StringBuilder out = new StringBuilder(inStr.length * 2);
for (byte b : inStr) {
stmp = Integer.toHexString(b & 0xFF);
if (stmp.length() == 1) {
// 如果是0至F的单位字符串,则添加0
out.append("0").append(stmp);
} else {
out.append(stmp);
}
}
return out.toString();
}

private static byte[] hex2byte(byte[] b) {
int size = 2;
if ((b.length % size) != 0) {
throw new IllegalArgumentException("长度不是偶数");
}
byte[] b2 = new byte[b.length / 2];
for (int n = 0; n < b.length; n += size) {
String item = new String(b, n, 2);
b2[n / 2] = (byte) Integer.parseInt(item, 16);
}
return b2;
}
}

+ 355
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java View File

@@ -0,0 +1,355 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import cn.hutool.core.codec.Base64;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.poi.excel.BigExcelWriter;
import cn.hutool.poi.excel.ExcelUtil;
import org.apache.poi.util.IOUtils;
import org.dubhe.exception.BusinessException;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.security.MessageDigest;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;

/**
* @description File工具类,扩展 hutool 工具包
* @date 2020-03-14
*/
public class FileUtil extends cn.hutool.core.io.FileUtil {

/**
* 定义GB的计算常量
*/
private static final int GB = 1024 * 1024 * 1024;
/**
* 定义MB的计算常量
*/
private static final int MB = 1024 * 1024;
/**
* 定义KB的计算常量
*/
private static final int KB = 1024;

/**
* 格式化小数
*/
private static final DecimalFormat DF = new DecimalFormat("0.00");

/**
* MultipartFile转File
*/
public static File toFile(MultipartFile multipartFile) {
// 获取文件名
String fileName = multipartFile.getOriginalFilename();
// 获取文件后缀
String prefix = "." + getExtensionName(fileName);
File file = null;
try {
// 用uuid作为文件名,防止生成的临时文件重复
file = File.createTempFile(IdUtil.simpleUUID(), prefix);
// MultipartFile to File
multipartFile.transferTo(file);
} catch (IOException e) {
e.printStackTrace();
}
return file;
}

/**
* 获取文件扩展名,不带 .
*/
public static String getExtensionName(String filename) {
if ((filename != null) && (filename.length() > 0)) {
int dot = filename.lastIndexOf('.');
if ((dot > -1) && (dot < (filename.length() - 1))) {
return filename.substring(dot + 1);
}
}
return filename;
}

/**
* Java文件操作 获取不带扩展名的文件名
*/
public static String getFileNameNoEx(String filename) {
if ((filename != null) && (filename.length() > 0)) {
int dot = filename.lastIndexOf('.');
if ((dot > -1) && (dot < (filename.length()))) {
return filename.substring(0, dot);
}
}
return filename;
}

/**
* 文件大小转换
*/
public static String getSize(long size) {
String resultSize;
if (size / GB >= 1) {
//如果当前Byte的值大于等于1GB
resultSize = DF.format(size / (float) GB) + "GB ";
} else if (size / MB >= 1) {
//如果当前Byte的值大于等于1MB
resultSize = DF.format(size / (float) MB) + "MB ";
} else if (size / KB >= 1) {
//如果当前Byte的值大于等于1KB
resultSize = DF.format(size / (float) KB) + "KB ";
} else {
resultSize = size + "B ";
}
return resultSize;
}

/**
* inputStream 转 File
*/
static File inputStreamToFile(InputStream ins, String name) throws Exception {
File file = new File(System.getProperty("java.io.tmpdir") + File.separator + name);
if (file.exists()) {
return file;
}
OutputStream os = new FileOutputStream(file);
int bytesRead;
int len = 8192;
byte[] buffer = new byte[len];
while ((bytesRead = ins.read(buffer, 0, len)) != -1) {
os.write(buffer, 0, bytesRead);
}
os.close();
ins.close();
return file;
}

/**
* 将文件名解析成文件的上传路径
*/
public static File upload(MultipartFile file, String filePath) {
Date date = new Date();
SimpleDateFormat format = new SimpleDateFormat("yyyyMMddhhmmssS");
String name = getFileNameNoEx(file.getOriginalFilename());
String suffix = getExtensionName(file.getOriginalFilename());
String nowStr = "-" + format.format(date);
try {
String fileName = name + nowStr + "." + suffix;
String path = filePath + fileName;
// getCanonicalFile 可解析正确各种路径
File dest = new File(path).getCanonicalFile();
// 检测是否存在目录
if (!dest.getParentFile().exists()) {
dest.getParentFile().mkdirs();
}
// 文件写入
file.transferTo(dest);
return dest;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}

public static String fileToBase64(File file) throws Exception {
FileInputStream inputFile = new FileInputStream(file);
String base64;
byte[] buffer = new byte[(int) file.length()];
inputFile.read(buffer);
inputFile.close();
base64 = Base64.encode(buffer);
return base64.replaceAll("[\\s*\t\n\r]", "");
}

/**
* 导出excel
*/
public static void downloadExcel(List<Map<String, Object>> list, HttpServletResponse response) throws IOException {
String tempPath = System.getProperty("java.io.tmpdir") + IdUtil.fastSimpleUUID() + ".xlsx";
File file = new File(tempPath);
BigExcelWriter writer = ExcelUtil.getBigWriter(file);
// 一次性写出内容,使用默认样式,强制输出标题
writer.write(list, true);
//response为HttpServletResponse对象
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;charset=utf-8");
//test.xls是弹出下载对话框的文件名,不能为中文,中文请自行编码
response.setHeader("Content-Disposition", "attachment;filename=file.xlsx");
ServletOutputStream out = response.getOutputStream();
// 终止后删除临时文件
file.deleteOnExit();
writer.flush(out, true);
//此处记得关闭输出Servlet流
IoUtil.close(out);
}

/**
* 下载文件
*/
public static void download(String path, HttpServletResponse response) {
if (path == null) {
return;
}
FileInputStream fis = null;
ServletOutputStream out = null;
try {
File file = new File(path);
fis = new FileInputStream(file);
out = response.getOutputStream();
IOUtils.copy(fis, out);
response.flushBuffer();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (fis != null) {
try {
fis.close();
} catch (IOException e) {
e.printStackTrace();
}
}
//此处记得关闭输出Servlet流
IoUtil.close(out);
}
}

public static String getFileType(String type) {
String documents = "txt doc pdf ppt pps xlsx xls docx";
String music = "mp3 wav wma mpa ram ra aac aif m4a";
String video = "avi mpg mpe mpeg asf wmv mov qt rm mp4 flv m4v webm ogv ogg";
String image = "bmp dib pcp dif wmf gif jpg tif eps psd cdr iff tga pcd mpt png jpeg";
if (image.contains(type)) {
return "图片";
} else if (documents.contains(type)) {
return "文档";
} else if (music.contains(type)) {
return "音乐";
} else if (video.contains(type)) {
return "视频";
} else {
return "其他";
}
}

public static void checkSize(long maxSize, long size) {
// 1M
int len = 1024 * 1024;
if (size > (maxSize * len)) {
throw new BusinessException("文件超出规定大小");
}
}

/**
* 判断两个文件是否相同
*/
public static boolean check(File file1, File file2) {
String img1Md5 = getMd5(file1);
String img2Md5 = getMd5(file2);
return img1Md5.equals(img2Md5);
}

/**
* 判断两个文件是否相同
*/
public static boolean check(String file1Md5, String file2Md5) {
return file1Md5.equals(file2Md5);
}

private static byte[] getByte(File file) {
// 得到文件长度
byte[] b = new byte[(int) file.length()];
try {
InputStream in = new FileInputStream(file);
try {
in.read(b);
} catch (IOException e) {
e.printStackTrace();
}
} catch (FileNotFoundException e) {
e.printStackTrace();
return null;
}
return b;
}

private static String getMd5(byte[] bytes) {
// 16进制字符
char[] hexDigits = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
try {
MessageDigest mdTemp = MessageDigest.getInstance("MD5");
mdTemp.update(bytes);
byte[] md = mdTemp.digest();
int j = md.length;
char[] str = new char[j * 2];
int k = 0;
// 移位 输出字符串
for (byte byte0 : md) {
str[k++] = hexDigits[byte0 >>> 4 & 0xf];
str[k++] = hexDigits[byte0 & 0xf];
}
return new String(str);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}

/**
* 下载文件
*
* @param request /
* @param response /
* @param file /
*/
public static void downloadFile(HttpServletRequest request, HttpServletResponse response, File file, boolean deleteOnExit) {
response.setCharacterEncoding(request.getCharacterEncoding());
response.setContentType("application/octet-stream");
FileInputStream fis = null;
try {
fis = new FileInputStream(file);
response.setHeader("Content-Disposition", "attachment; filename=" + file.getName());
IOUtils.copy(fis, response.getOutputStream());
response.flushBuffer();
} catch (Exception e) {
e.printStackTrace();
} finally {
if (fis != null) {
try {
fis.close();
if (deleteOnExit) {
file.deleteOnExit();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}

public static String getMd5(File file) {
return getMd5(getByte(file));
}

}

+ 131
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java View File

@@ -0,0 +1,131 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;
import org.apache.commons.io.IOUtils;
import org.dubhe.enums.LogEnum;


import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;

import static org.dubhe.constant.StringConstant.UTF8;
import static org.dubhe.constant.SymbolConstant.BLANK;

/**
* @description: httpClient工具类,不校验SSL证书
* @date: 2020-5-21
*/
public class HttpClientUtils {

public static String sendHttps(String path) {
InputStream inputStream = null;
BufferedReader bufferedReader = null;
InputStreamReader inputStreamReader = null;
StringBuilder stringBuider = new StringBuilder();
String result = BLANK;
HttpsURLConnection con = null;
try {
con = getConnection(path);
con.connect();

/**将返回的输入流转换成字符串**/
inputStream = con.getInputStream();
inputStreamReader = new InputStreamReader(inputStream, UTF8);
bufferedReader = new BufferedReader(inputStreamReader);

String str = null;
while ((str = bufferedReader.readLine()) != null) {
stringBuider.append(str);
}

result = stringBuider.toString();
LogUtil.info(LogEnum.BIZ_SYS,"Request path:{}, SUCCESS, result:{}", path, result);

} catch (Exception e) {
LogUtil.error(LogEnum.BIZ_SYS,"Request path:{}, ERROR, exception:{}", path, e);
return result;
} finally {
closeResource(bufferedReader,inputStreamReader,inputStream,con);
}

return result;
}

private static void closeResource(BufferedReader bufferedReader,InputStreamReader inputStreamReader,InputStream inputStream,HttpsURLConnection con) {

IOUtils.closeQuietly(bufferedReader);

if (inputStreamReader != null) {
IOUtils.closeQuietly(inputStreamReader);
}
if (inputStream != null) {
IOUtils.closeQuietly(inputStream);
}
if (con != null) {
con.disconnect();
}
}


private static HttpsURLConnection getConnection(String path){
HttpsURLConnection con = null;
try {
/**创建并初始化SSLContext对象**/
TrustManager[] trustManagers = {new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
}

@Override
public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
}

@Override
public X509Certificate[] getAcceptedIssuers() {
return null;
}
}};
SSLContext sslContext = SSLContext.getInstance("SSL", "SunJSSE");
sslContext.init(null, trustManagers, new java.security.SecureRandom());

/**得到SSLSocketFactory对象**/
SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory();

URL url = new URL(path);
con = (HttpsURLConnection) url.openConnection();
con.setSSLSocketFactory(sslSocketFactory);
con.setUseCaches(false);
}catch (Exception e){
LogUtil.error(LogEnum.BIZ_SYS,"Request path:{}, error, exception:{}", path, e);
}
return con;

}

}



+ 66
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java View File

@@ -0,0 +1,66 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import lombok.extern.slf4j.Slf4j;

/**
* @description: HttpUtil
* @date 2020.04.30
*/
@Slf4j
public class HttpUtils {

private HttpUtils() {

}

/**
* 判断http请求是否成功
*
* @param httpCode
* 1XX Informational(信息性状态码)
* 2XX Success(成功状态码)
* 3XX Redirection(重定向状态码)
* 4XX Client Error(客户端错误状态码)
* 5XX Server Error(服务器错误状态码)
* @return
*/
public static boolean isSuccess(String httpCode){
if (StringUtils.isBlank(httpCode)){
return false;
}
return httpCode.length() == 3 && httpCode.startsWith("2");
}

/**
* 判断http请求是否成功
*
* @param httpCode
* 1XX Informational(信息性状态码)
* 2XX Success(成功状态码)
* 3XX Redirection(重定向状态码)
* 4XX Client Error(客户端错误状态码)
* 5XX Server Error(服务器错误状态码)
* @return
*/
public static boolean isSuccess(int httpCode) {
return isSuccess(String.valueOf(httpCode));
}

}

+ 37
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java View File

@@ -0,0 +1,37 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
/**
* @description JWT配置类
* @date 2020-04-15
*/
@Configuration
public class JwcConfig {
@Value("${jwt.online-key}")
public String onlineKey;
@Autowired
public RedisUtils redisUtils;
@Value("${jwt.token-validity-in-seconds}")
public Long jwtExpiration;
@Value("${jwt.base64-secret}")
public String secret;
}

+ 203
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java View File

@@ -0,0 +1,203 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import com.auth0.jwt.JWT;
import com.auth0.jwt.JWTCreator.Builder;
import com.auth0.jwt.JWTVerifier;
import com.auth0.jwt.algorithms.Algorithm;
import com.auth0.jwt.exceptions.JWTDecodeException;
import com.auth0.jwt.interfaces.Claim;
import com.auth0.jwt.interfaces.DecodedJWT;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.crypto.SecureRandomNumberGenerator;
import org.apache.shiro.mgt.SecurityManager;
import org.apache.shiro.util.ThreadContext;
import org.dubhe.domain.dto.UserDTO;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.io.UnsupportedEncodingException;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.concurrent.TimeUnit;

/**
* @description JWT
* @date 2020-03-14
*/
@Component
public class JwtUtils {


/**
* 请求头
*/
public static final String AUTH_HEADER = "Authorization";
/**
* 过期时间20分钟
*/
private static final long EXPIRE_TIME = 20 * 60 * 1000;
private static final String FIELD_NAME = "USER_NAME";
private static JwcConfig jwcConfig;
private static RedisUtils redisUtils;


@Autowired
public void setJwcConfig(JwcConfig mtJwcConfig) {
jwcConfig = mtJwcConfig;
}

@Autowired
public void setRedisUtils(RedisUtils myredisUtils) {
redisUtils = myredisUtils;
}

/**
* 验证token是否正确
*/
public static boolean verify(String token, String username) {
try {
Algorithm algorithm = null;
algorithm = Algorithm.HMAC256(jwcConfig.secret);
JWTVerifier verifier = JWT.require(algorithm).withClaim(FIELD_NAME, username).build();
verifier.verify(token);
return true;
} catch (UnsupportedEncodingException exception) {
return false;
}
}

/**
* 获得token中的自定义信息,无需secret解密也能获得
*/
public static String getClaimFiled(String token, String filed) {
try {
DecodedJWT jwt = JWT.decode(token);
return jwt.getClaim(filed).asString();
} catch (JWTDecodeException e) {
return null;
}
}

/**
* 生成签名
*/
public static String sign(String userName) {
try {
Date date = new Date(System.currentTimeMillis() + EXPIRE_TIME);
Algorithm algorithm = Algorithm.HMAC256(jwcConfig.secret);
// 附带username,
String token = JWT.create().withClaim(FIELD_NAME, userName).withExpiresAt(date).sign(algorithm);
saveToken(userName, token);
return token;
} catch (UnsupportedEncodingException e) {
return null;
}
}

/**
* 获取用户名
*/
public static String getUserName(String token) {
return getClaimFiled(token, FIELD_NAME);
}

/**
* 获取 token的签发时间
*/
public static Date getIssuedAt(String token) {
try {
DecodedJWT jwt = JWT.decode(token);
return jwt.getIssuedAt();
} catch (JWTDecodeException e) {
return null;
}
}

/**
* 验证 token是否过期
*/
public static boolean isTokenExpired(String token) {
Date now = Calendar.getInstance().getTime();
DecodedJWT jwt = JWT.decode(token);
return jwt.getExpiresAt().before(now);
}

/**
* 刷新 token的过期时间
*/
public static String refreshTokenExpired(String token) {
DecodedJWT jwt = JWT.decode(token);
Map<String, Claim> claims = jwt.getClaims();
try {
Date date = new Date(System.currentTimeMillis() + EXPIRE_TIME);
Algorithm algorithm = Algorithm.HMAC256(jwcConfig.secret);
Builder builer = JWT.create().withExpiresAt(date);
for (Entry<String, Claim> entry : claims.entrySet()) {
builer.withClaim(entry.getKey(), entry.getValue().asString());
}
String newToken = builer.sign(algorithm);
String userName = claims.get(FIELD_NAME).asString();
String tokenKey = getUserTokenKey(userName);
if (redisUtils.get(tokenKey) == null) {
return null;
}
saveToken(userName, newToken);
return newToken;
} catch (UnsupportedEncodingException e) {
return null;
}
}

/**
* 生成16位随机盐
*/
public static String generateSalt() {
SecureRandomNumberGenerator secureRandom = new SecureRandomNumberGenerator();
String hex = secureRandom.nextBytes(16).toHex();
return hex;
}

/**
* 获取当前用户
*/
public static UserDTO getCurrentUserDto() {
Object principal = null;
SecurityManager manager = ThreadContext.getSecurityManager();
if (!Objects.isNull(manager)) {
principal = SecurityUtils.getSubject().getPrincipal();
}
return Objects.isNull(principal) ? null : (UserDTO) principal;
}

/**
* 获取用户token key
*/
private static String getUserTokenKey(String userName) {
return jwcConfig.onlineKey + userName;
}

private static void saveToken(String userName, String token) {
String tokenKey = getUserTokenKey(userName);
redisUtils.set(tokenKey, token, jwcConfig.jwtExpiration, TimeUnit.MINUTES);
}
}

+ 168
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java View File

@@ -0,0 +1,168 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateUtil;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;

/**
* @description k8s命名相关工具类 token使用MD5加密,并设置超时时间
* @date 2020-05-28
*/
@Component
public class K8sCallBackTool {

/**
* k8s 回调 token秘钥
*/
@Value("${k8s.callback.token.secret-key}")
private String secretKey;
/**
* k8s 回调token超时时间
*/
@Value("${k8s.callback.token.expire-seconds}")
private Integer expireSeconds;
/**
* k8s 回调域名或IP:Port
*/
@Value("${k8s.callback.url}")
private String url;

/**
* k8s 回调token key
*/
public static final String K8S_CALLBACK_TOKEN = "k8sCallbackToken";
/**
* 失败重试次数
*/
private static final int RETRY_COUNT = 3;

/**
* k8s 回调匹配地址
*/
private static final List<String> K8S_CALLBACK_PATH;
/**
* k8s 回调路径
*/
private static final String K8S_CALLBACK_PATH_POD = "/api/k8s/callback/pod/";

static {
K8S_CALLBACK_PATH = new ArrayList<>();
// 添加需要token权限校验的地址(Shiro匿名访问的地址)
K8S_CALLBACK_PATH.add(K8S_CALLBACK_PATH_POD + "**");
}

/**
* 获取 k8s 回调匹配地址
*
* @return List<String>
*/
public static List<String> getK8sCallbackPaths() {
return new ArrayList<>(K8S_CALLBACK_PATH);
}


/**
* 生成k8s回调
*
* @return String
*/
public String generateToken() {
String expireTime = DateUtil.format(
DateUtil.offset(new Date(), DateField.SECOND, expireSeconds),
DatePattern.PURE_DATETIME_PATTERN
);
return AesUtil.encrypt(expireTime, secretKey);
}

/**
* 验证token
*
* @param token
* @return boolean
*/
public boolean validateToken(String token) {
String expireTime = AesUtil.decrypt(token, secretKey);
String nowTime = DateUtil.format(
new Date(),
DatePattern.PURE_DATETIME_PATTERN
);
return expireTime.compareTo(nowTime) > 0;
}


/**
* 判断当前是否可以再次重试
*
* @param retryTimes 第n次试图重试
* @return boolean
*/
public boolean continueRetry(int retryTimes) {
return retryTimes <= RETRY_COUNT;
}

/**
* 获取回调地址
*
* @param podLabel
* @return String
*/
public String getPodCallbackUrl(String podLabel) {
return url + K8S_CALLBACK_PATH_POD + podLabel;
}


/**
* 获取超时时间秒
*
* @param timeoutSecond 超时秒数
* @return Long
*/
public static Long getTimeoutSecondLong(int timeoutSecond) {
return Long.valueOf(
DateUtil.format(
DateUtil.offset(
new Date(), DateField.SECOND, timeoutSecond
),
DatePattern.PURE_DATETIME_PATTERN
)
);
}

/**
* 获取当前秒数
*
* @return Long
*/
public static Long getCurrentSecondLong() {
return Long.valueOf(
DateUtil.format(
new Date(),
DatePattern.PURE_DATETIME_PATTERN
)
);
}

}

+ 255
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java View File

@@ -0,0 +1,255 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.RandomUtil;
import org.dubhe.config.K8sNameConfig;
import org.dubhe.constant.SymbolConstant;
import org.dubhe.enums.BizEnum;
import org.dubhe.enums.BizNfsEnum;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.apache.commons.lang3.StringUtils;

import java.util.Date;

/**
* @description k8s命名相关工具类
* @date 2020-05-13
*/
@Component
public class K8sNameTool {
@Autowired
private K8sNameConfig k8sNameConfig;
/**
* 命名分隔符
*/
private static final char SEPARATOR = '-';
/**
* 文件分隔符
*/
private static final String K8S_FILE_SEPARATOR = "/";
/**
* 资源名称前缀
*/
private static final String RESOURCE_NAME = "rn";
/**
* 随机长度值
*/
private static final int RANDOM_LENGTH = 4;

/**
* 生成 ResourceName
*
* @param bizEnum 业务枚举
* @param resourceInfo 资源备注信息(保证同业务下唯一并且命名规范)
* @return String
*/
public String generateResourceName(BizEnum bizEnum, String resourceInfo) {
return bizEnum.getBizCode() + SEPARATOR + RESOURCE_NAME + SEPARATOR + resourceInfo;
}

/**
* 生成 Notebook的NameSpace
*
* @param userId
* @return namespace
*/
public String generateNameSpace(long userId) {
return this.k8sNameConfig.getNamespace() + SEPARATOR + userId;
}

/**
* 从resourceName中获取资源信息
*
* @param bizEnum 业务枚举
* @param resourceName
* @return resourceInfo
*/
public String getResourceInfoFromResourceName(BizEnum bizEnum, String resourceName) {
if (StringUtils.isEmpty(resourceName) || !resourceName.contains(bizEnum.getBizCode() + SEPARATOR)) {
return null;
}
return resourceName.replace(bizEnum.getBizCode() + SEPARATOR + RESOURCE_NAME + SEPARATOR, SymbolConstant.BLANK);
}

/**
* 从namespace 获取使用者ID
*
* @param namespace
* @return Long
*/
public Long getUserIdFromNameSpace(String namespace) {
if (StringUtils.isEmpty(namespace) || !namespace.contains(this.k8sNameConfig.getNamespace() + SEPARATOR)) {
return null;
}
return Long.valueOf(namespace.replace(this.k8sNameConfig.getNamespace() + SEPARATOR, ""));
}


/**
* 生成业务模块相对路径
*
* @param bizNfsEnum 业务NFS路径枚举
* @return String 例如: /{biz}/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/
*/
public String getNfsPath(BizNfsEnum bizNfsEnum, long userId) {
if (bizNfsEnum == null) {
return null;
}
return optimizationPath(K8S_FILE_SEPARATOR
+ bizNfsEnum.getBizNfsPath()
+ K8S_FILE_SEPARATOR
+ userId
+ K8S_FILE_SEPARATOR
+ DateUtil.format(new Date(), DatePattern.PURE_DATETIME_MS_FORMAT) + RandomUtil.randomString(RANDOM_LENGTH)
+ K8S_FILE_SEPARATOR);
}

/**
* 去除NFS根路径
*
* @param nfsPath
* @return String
*/
public String removeNfsRootPath(String nfsPath) {
if (StringUtils.isBlank(nfsPath) || !nfsPath.startsWith(k8sNameConfig.getNfsRootPath())) {
return nfsPath;
}
return optimizationPath(K8S_FILE_SEPARATOR + nfsPath.replace(k8sNameConfig.getNfsRootPath(), ""));
}

/**
* 路径添加bucket
*
* @param nfsPath
* @return String
*/
public String appendBucket(String nfsPath) {
return optimizationPath(K8S_FILE_SEPARATOR
+ k8sNameConfig.getFileBucket()
+ K8S_FILE_SEPARATOR
+ nfsPath);
}

/**
* 路径添加时间戳随机数
*
* @param nfsPath
* @return String
*/
public String appendTimeStampAndRandomNum(String nfsPath) {
return optimizationPath(nfsPath
+ DateUtil.format(new Date(), DatePattern.PURE_DATETIME_MS_FORMAT) + RandomUtil.randomString(RANDOM_LENGTH)
+ K8S_FILE_SEPARATOR);
}

/**
* nfs路径根据业务转换
*
* @param nfsPath
* @param sourceBizNfsEnum 源业务 NFS Path
* @param targetBizNfsEnum 目标业务 NFS Path
* @return String
*/
public String convertNfsPath(String nfsPath, BizNfsEnum sourceBizNfsEnum, BizNfsEnum targetBizNfsEnum) {
if (!validateBizNfsPath(nfsPath, sourceBizNfsEnum) || targetBizNfsEnum == null) {
return nfsPath;
}
return optimizationPath(nfsPath.replace(K8S_FILE_SEPARATOR + sourceBizNfsEnum.getBizNfsPath() + K8S_FILE_SEPARATOR
, K8S_FILE_SEPARATOR + targetBizNfsEnum.getBizNfsPath() + K8S_FILE_SEPARATOR));
}

/**
* 获取NFS绝对路径
*
* @param nfsPath
* @return String
*/
public String getAbsoluteNfsPath(String nfsPath) {
if (StringUtils.isBlank(nfsPath)) {
return nfsPath;
}
return optimizationPath(k8sNameConfig.getNfsRootPath()
+ K8S_FILE_SEPARATOR
+ k8sNameConfig.getFileBucket()
+ K8S_FILE_SEPARATOR
+ nfsPath);
}

/**
* 验证 nfsPath 是否是所属业务路径
*
* @param nfsPath
* @param bizNfsEnum
* @return boolean
*/
public boolean validateBizNfsPath(String nfsPath, BizNfsEnum bizNfsEnum) {
return org.apache.commons.lang3.StringUtils.isNotBlank(nfsPath)
&& bizNfsEnum != null
&& nfsPath.contains(bizNfsEnum.getBizNfsPath())
&& !nfsPath.contains(K8S_FILE_SEPARATOR + k8sNameConfig.getFileBucket() + K8S_FILE_SEPARATOR)
&& !nfsPath.contains(K8S_FILE_SEPARATOR + k8sNameConfig.getNfsRootPath() + K8S_FILE_SEPARATOR);
}

/**
* 路径优化
*
* @param path
* @return String
*/
private String optimizationPath(String path) {
if (StringUtils.isBlank(path)) {
return path;
}
return path.replaceAll("///*", K8S_FILE_SEPARATOR);
}

/**
* 获取k8s pod标签
*
* @param bizEnum
* @return String
*/
public String getPodLabel(BizEnum bizEnum) {
return bizEnum == null ? null : bizEnum.getBizCode();
}

/**
* 获取数据集在镜像中路径
*
* @return String
*/
public String getDatasetPath() {
return k8sNameConfig.getDatasetPath();
}


/**
* 自送生成K8S名称,供K8S使用
*
* @return String
*/
public String getK8sName() {
return DateUtil.format(new Date(), DatePattern.PURE_DATETIME_MS_FORMAT)
+ RandomUtil.randomString(RANDOM_LENGTH);
}

}

+ 39
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java View File

@@ -0,0 +1,39 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import java.text.SimpleDateFormat;
import java.util.Calendar;

/**
* @description Key生成工具类
* @date 2020-06-08
*/
public class KeyUtil {

/**
* 生成训练任务的key,目前规则为:train-{user_id}-{time}
* @param userId 用户id
* @return 训练任务的key
*/
public static String generateTrainKey(long userId) {
Calendar calendar = Calendar.getInstance();
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
return "train-" + userId + '-' + dateFormat.format(calendar.getTime());
}
}

+ 255
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java View File

@@ -0,0 +1,255 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import ch.qos.logback.classic.Level;
import com.alibaba.fastjson.JSON;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.dubhe.aspect.LogAspect;
import org.dubhe.base.MagicNumConstant;
import org.dubhe.constant.SymbolConstant;
import org.dubhe.domain.entity.LogInfo;
import org.dubhe.enums.LogEnum;
import org.slf4j.MDC;
import org.slf4j.helpers.MessageFormatter;

import java.util.Arrays;
import java.util.UUID;

/**
* @description 日志工具类
* @date 2020-06-29
*/
@Slf4j
public class LogUtil {
/**
* info级别的日志
*
* @param logType 日志类型
* @param object 打印的日志参数
*/
public static void info(LogEnum logType, Object... object) {

logHandle(logType, Level.INFO, object);
}

/**
* debug级别的日志
*
* @param logType 日志类型
* @param object 打印的日志参数
*/
public static void debug(LogEnum logType, Object... object) {
logHandle(logType, Level.DEBUG, object);
}

/**
* error级别的日志
*
* @param logType 日志类型
* @param object 打印的日志参数
*/
public static void error(LogEnum logType, Object... object) {
errorObjectHandle(object);
logHandle(logType, Level.ERROR, object);
}

/**
* warn级别的日志
*
* @param logType 日志类型
* @param object 打印的日志参数
*/
public static void warn(LogEnum logType, Object... object) {
logHandle(logType, Level.WARN, object);
}

/**
* trace级别的日志
*
* @param logType 日志类型
* @param object 打印的日志参数
*/
public static void trace(LogEnum logType, Object... object) {
logHandle(logType, Level.TRACE, object);
}

/**
* 日志处理
*
* @param logType 日志类型
* @param level 日志级别
* @param object 打印的日志参数
*/
private static void logHandle(LogEnum logType, Level level, Object[] object) {

LogInfo logInfo = generateLogInfo(logType, level, object);
String logInfoJsonStr = logJsonStringLengthLimit(logInfo);
switch (Level.toLevel(logInfo.getLevel()).levelInt) {
case Level.TRACE_INT:
log.trace(logInfoJsonStr);
break;
case Level.DEBUG_INT:
log.debug(logInfoJsonStr);
break;
case Level.INFO_INT:
log.info(logInfoJsonStr);
break;
case Level.WARN_INT:
log.warn(logInfoJsonStr);
break;
case Level.ERROR_INT:
log.error(logInfoJsonStr);
break;
default:
}

}

/**
* 日志信息组装的内部方法
*
* @param logType 日志类型
* @param level 日志级别
* @param object 打印的日志参数
* @return LogInfo日志对象信息
*/
private static LogInfo generateLogInfo(LogEnum logType, Level level, Object[] object) {
LogInfo logInfo = new LogInfo();
// 日志类型检测
if (!LogEnum.isLogType(logType)) {
level = Level.ERROR;
object = new Object[MagicNumConstant.ONE];
object[MagicNumConstant.ZERO] = String.valueOf("logType【").concat(String.valueOf(logType))
.concat("】is error !");
logType = LogEnum.SYS_ERR;
}

// 获取trace_id
if (StringUtils.isEmpty(MDC.get(LogAspect.TRACE_ID))) {
MDC.put(LogAspect.TRACE_ID, UUID.randomUUID().toString());
}
// 设置logInfo的level,type,traceId属性
logInfo.setLevel(level.levelStr).setType(logType.toString()).setTraceId(MDC.get(LogAspect.TRACE_ID));
// 设置logInfo的堆栈信息
setLogStackInfo(logInfo);
// 设置logInfo的info信息
setLogInfo(logInfo, object);
// 截取loginfo的长度并转换成json字符串
return logInfo;

}

/**
* 设置loginfo的堆栈信息
*
* @param logInfo 日志对象
*/
private static void setLogStackInfo(LogInfo logInfo) {
StackTraceElement[] elements = Thread.currentThread().getStackTrace();
if (elements.length >= MagicNumConstant.SIX) {
logInfo.setCName(elements[MagicNumConstant.FIVE].getClassName())
.setMName(elements[MagicNumConstant.FIVE].getMethodName())
.setLine(String.valueOf(elements[MagicNumConstant.FIVE].getLineNumber()));
}
}

/**
* 限制log日志的长度并转换成json
*
* @param logInfo 日志对象
* @return String 日志对象Json字符串
*/
private static String logJsonStringLengthLimit(LogInfo logInfo) {
try {
String jsonString = JSON.toJSONString(logInfo.getInfo());
if (jsonString.length() > MagicNumConstant.TEN_THOUSAND) {
jsonString = jsonString.substring(MagicNumConstant.ZERO, MagicNumConstant.TEN_THOUSAND);
}

logInfo.setInfo(jsonString);
jsonString = JSON.toJSONString(logInfo);
jsonString = jsonString.replace(SymbolConstant.BACKSLASH_MARK, SymbolConstant.MARK)
.replace(SymbolConstant.DOUBLE_MARK, SymbolConstant.MARK)
.replace(SymbolConstant.BRACKETS, SymbolConstant.BLANK);

return jsonString;

} catch (Exception e) {
logInfo.setLevel(Level.ERROR.levelStr).setType(LogEnum.SYS_ERR.toString())
.setInfo("cannot serialize exception: " + ExceptionUtils.getStackTrace(e));
return JSON.toJSONString(logInfo);
}
}

/**
* 设置日志对象的info信息
*
* @param logInfo 日志对象
* @param object 打印的日志参数
*/
private static void setLogInfo(LogInfo logInfo, Object[] object) {
for (Object obj : object) {
if (obj instanceof Exception) {
log.error((ExceptionUtils.getStackTrace((Throwable) obj)));
}
}

if (object.length > MagicNumConstant.ONE) {
logInfo.setInfo(MessageFormatter.arrayFormat(object[MagicNumConstant.ZERO].toString(),
Arrays.copyOfRange(object, MagicNumConstant.ONE, object.length)).getMessage());

} else if (object.length == MagicNumConstant.ONE && object[MagicNumConstant.ZERO] instanceof Exception) {
logInfo.setInfo((ExceptionUtils.getStackTrace((Exception) object[MagicNumConstant.ZERO])));
} else if (object.length == MagicNumConstant.ONE) {
logInfo.setInfo(
object[MagicNumConstant.ZERO] == null ? SymbolConstant.BLANK : object[MagicNumConstant.ZERO]);
} else {
logInfo.setInfo(SymbolConstant.BLANK);
}

}

/**
* 处理Exception的情况
*
* @param object 打印的日志参数
*/
private static void errorObjectHandle(Object[] object) {
if (object.length >= MagicNumConstant.TWO) {
object[MagicNumConstant.ZERO] = String.valueOf(object[MagicNumConstant.ZERO])
.concat(SymbolConstant.BRACKETS);
}

if (object.length == MagicNumConstant.TWO && object[MagicNumConstant.ONE] instanceof Exception) {
log.error((ExceptionUtils.getStackTrace((Throwable) object[MagicNumConstant.ONE])));
object[MagicNumConstant.ONE] = ExceptionUtils.getStackTrace((Exception) object[MagicNumConstant.ONE]);

} else if (object.length >= MagicNumConstant.THREE) {
for (int i = 0; i < object.length; i++) {
if (object[i] instanceof Exception) {
log.error((ExceptionUtils.getStackTrace((Throwable) object[i])));
object[i] = ExceptionUtils.getStackTrace((Exception) object[i]);
}

}
}
}

}

+ 49
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java View File

@@ -0,0 +1,49 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import org.dubhe.base.MagicNumConstant;

/**
* @description: 计算工具类
* @create: 2020/6/4 14:53
*/
public class MathUtils {


/**
* 字符串整数加法
* num1+num2
* @param num1
* @param num2
* @return
*/
public static String add(String num1,String num2){
return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO:Integer.valueOf(num1)) + (!RegexUtil.isDigits(num2)?MagicNumConstant.ZERO:Integer.valueOf(num2)));
}
/**
* 字符串整数减法
* num1 - num2
* @param num1
* @param num2
* @return
*/
public static String reduce(String num1,String num2){
return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO:Integer.valueOf(num1)) - (!RegexUtil.isDigits(num2)?MagicNumConstant.ZERO:Integer.valueOf(num2)));
}
}

+ 286
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java View File

@@ -0,0 +1,286 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import cn.hutool.core.io.IoUtil;
import cn.hutool.http.HttpRequest;
import com.alibaba.fastjson.JSONObject;
import io.minio.MinioClient;
import io.minio.PutObjectOptions;
import io.minio.Result;
import io.minio.errors.*;
import io.minio.messages.DeleteError;
import io.minio.messages.Item;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.commons.lang.StringUtils;
import org.dubhe.base.MagicNumConstant;
import org.dubhe.enums.LogEnum;
import org.dubhe.exception.BusinessException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* @description Minio工具类
* @date 2020-05-09
*/
@Service
public class MinioUtil {

@Value("${minio.url}")
private String url;
@Value("${minio.accessKey}")
private String accessKey;
@Value("${minio.secretKey}")
private String secretKey;

private MinioClient client;

@PostConstruct
public void init() {
try {
client = new MinioClient(url, accessKey, secretKey);
} catch (InvalidEndpointException e) {
LogUtil.warn(LogEnum.BIZ_DATASET, "MinIO endpoint invalid. e:", e);
} catch (InvalidPortException e) {
LogUtil.warn(LogEnum.BIZ_DATASET, "MinIO endpoint port invalid. e:", e);
}
}

/**
* 写文件
*
* @param bucket 桶名称
* @param fullFilePath 文件存储的全路径,包括文件名,非'/'开头. e.g. dataset/12/annotation/test.txt
* @param content file content. can not be null
*/
public void writeString(String bucket, String fullFilePath, String content) throws Exception {
boolean isExist = client.bucketExists(bucket);
if (!isExist) {
client.makeBucket(bucket);
}
InputStream inputStream = IoUtil.toUtf8Stream(content);
PutObjectOptions options = new PutObjectOptions(inputStream.available(), MagicNumConstant.NEGATIVE_ONE);
client.putObject(bucket, fullFilePath, inputStream, options);
}

/**
* 读取文件
*
* @param bucket
* @param fullFilePath 文件存储的全路径,包括文件名,非'/'开头. e.g. dataset/12/annotation/test.txt
* @return String
*/
public String readString(String bucket, String fullFilePath) throws Exception {
try (InputStream is = client.getObject(bucket, fullFilePath)) {
return IoUtil.read(is, Charset.defaultCharset());
}
}

/**
* 文件删除
*
* @param bucket
* @param fullFilePath 文件存储的全路径,包括文件名,非'/'开头. e.g. dataset/12/annotation/test.txt
*/
public void del(String bucket, String fullFilePath) throws Exception {
Iterable<Result<Item>> items = client.listObjects(bucket, fullFilePath);
Set<String> files = new HashSet<>();
for (Result<Item> item : items) {
files.add(item.get().objectName());
}
Iterable<Result<DeleteError>> results = client.removeObjects(bucket, files);
for (Result<DeleteError> result : results) {
result.get();
}
}

/**
* 批量删除文件
*
* @param bucket
* @param objectNames
*/
public void delFiles(String bucket,List<String> objectNames) throws Exception{
Iterable<Result<DeleteError>> results = client.removeObjects(bucket, objectNames);
for (Result<DeleteError> result : results) {
result.get();
}
}

/**
* 获取对象名称
*
*/
public List<String> getObjects(String bucketName, String prefix)throws Exception{
List<String> fileNames = new ArrayList<>();
Iterable<Result<Item>> results = client.listObjects(bucketName, prefix);
for(Result<Item> result:results){
Item item = result.get();
fileNames.add(item.objectName());
}
return fileNames;
}

/**
* 获取文件流
*
*/
public InputStream getObjectInputStream(String bucket,String objectName)throws Exception{
return client.getObject(bucket, objectName);
}

/**
* 文件夹复制
*
* @param bucket
* @param sourceFiles 源文件
* @param targetDir 目标文件夹
*/
public void copyDir(String bucket, List<String> sourceFiles, String targetDir) {
sourceFiles.forEach(sourceFile -> {
InputStream inputStream = null;
try {
String sourceObjectName = sourceFile;
String targetObjectName = targetDir + "/" + StringUtils.substringAfterLast(sourceObjectName, "/");
inputStream = client.getObject(bucket, sourceObjectName);
byte[] buf = new byte[512];
int bytesRead;
int count = MagicNumConstant.ZERO;
while ((bytesRead = inputStream.read(buf, MagicNumConstant.ZERO, buf.length)) >= MagicNumConstant.ZERO) {
count += bytesRead;
}
PutObjectOptions options = new PutObjectOptions(count, MagicNumConstant.ZERO);
client.putObject(bucket, targetObjectName, client.getObject(bucket, sourceObjectName), options);
} catch (Exception e) {
LogUtil.error(LogEnum.BIZ_DATASET, "MinIO file copy exception, {}", e);
} finally {
try {
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
LogUtil.error(LogEnum.BIZ_DATASET, "MinIO file read stream closed failed, {}", e);
}
}
});
}

@Data
@Service
public class MinioWebTokenBody {

@Value("${minioweb.GetToken.url}")
private String tokenUrl;
@Value("${minioweb.GetToken.param.id}")
private int id;
@Value("${minioweb.GetToken.param.jsonrpc}")
private String jsonrpc;
@Value("${minioweb.GetToken.param.method}")
private String method;
@Value("${minioweb.zip.url}")
private String zipUrl;

/**
* 生成文件下载请求参数方法
*
* @param bucketName
* @param prefix
* @param objects
* @return MinioDownloadDto
*/
public MinioDownloadDto getDownloadParam(String bucketName, String prefix, List<String> objects, String zipName) {
String paramTemplate = "{\"id\":%d,\"jsonrpc\":\"%s\",\"params\":{\"username\":\"%s\",\"password\":\"%s\"},\"method\":\"%s\"}";
String downloadBodyTemplate = "{\"bucketName\":\"%s\",\"prefix\":\"%s\",\"objects\":[%s]}";
String param = String.format(paramTemplate, id, jsonrpc, accessKey, secretKey, method);
String result = HttpRequest.post(url + tokenUrl).contentType("application/json").body(param).execute().body();
String token = JSONObject.parseObject(result).getJSONObject("result").getString("token");
return new MinioDownloadDto(token, String.format(downloadBodyTemplate, bucketName, prefix, getStrFromList(objects)), zipName);
}

public String getStrFromList(List<String> objects) {
List<String> result = new ArrayList<>();
objects.stream().forEach(s -> {
result.add("\"" + s + "\"");
});
return StringUtils.join(result, ",");
}

}

@ApiModel
@Data
public class MinioDownloadDto {
@ApiModelProperty("下载压缩包请求token")
private String token;
@ApiModelProperty("下载压缩包请求参数")
private String body;
@ApiModelProperty("下载压缩包请求需要的header")
private Map<String, Object> headers;
@ApiModelProperty("下载压缩包文件名称")
private String zipName;

public MinioDownloadDto() {
}

public MinioDownloadDto(String token, String body, String zipName) {
this.token = token;
this.body = body;
this.zipName = zipName;
Map<String, Object> headers = new HashMap<>();
headers.put("Content-Type", "text/plain;charset=UTF-8");
this.headers = headers;
}

}


/**
* 生成一个给HTTP PUT请求用的presigned URL。浏览器/移动端的客户端可以用这个URL进行上传,
* 即使其所在的存储桶是私有的。这个presigned URL可以设置一个失效时间,默认值是7天
*
* @param bucketName 存储桶名称
* @param objectName 存储桶里的对象名称
* @param expires 失效时间(以秒为单位),默认是7天,不得大于七天
* @return String
*/
public String getEncryptedPutUrl(String bucketName, String objectName, Integer expires) {
if (StringUtils.isEmpty(objectName)) {
throw new BusinessException("object name cannot be empty");
}
try {
return client.presignedPutObject(bucketName, objectName, expires);
} catch (Exception e) {
LogUtil.error(LogEnum.BIZ_DATASET, e.getMessage());
throw new BusinessException("MinIO an error occurred, please contact the administrator");
}
}

}

+ 76
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java View File

@@ -0,0 +1,76 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import com.emc.ecs.nfsclient.nfs.nfs3.Nfs3;
import com.emc.ecs.nfsclient.rpc.CredentialUnix;
import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.dubhe.config.NfsConfig;
import org.dubhe.enums.LogEnum;
import org.springframework.stereotype.Component;

import java.io.IOException;

/**
* @description NFS工厂类
* @date 2020-05-13
*/
@Component
public class NfsFactory implements PooledObjectFactory<Nfs3> {

private final NfsConfig nfsConfig;

public NfsFactory(NfsConfig nfsConfig) {
this.nfsConfig = nfsConfig;
}

@Override
public PooledObject<Nfs3> makeObject() {
Nfs3 nfs3 = null;
try {
nfs3 = new Nfs3(nfsConfig.getNfsIp(), nfsConfig.getRootDir(), new CredentialUnix(0, 0, null), 3);
} catch (IOException e) {
LogUtil.error(LogEnum.NFS_UTIL, "创建NFS对象失败: ", e);
}
return new DefaultPooledObject<>(nfs3);
}

@Override
public void destroyObject(PooledObject<Nfs3> pooledObject) {
LogUtil.info(LogEnum.NFS_UTIL, "销毁NFS对象: ", pooledObject.getObject());
}

@Override
public boolean validateObject(PooledObject<Nfs3> pooledObject) {
LogUtil.info(LogEnum.NFS_UTIL, "验证NFS对象: ", pooledObject.getObject());
return true;
}

@Override
public void activateObject(PooledObject<Nfs3> pooledObject) {
LogUtil.info(LogEnum.NFS_UTIL, "激活NFS对象: ", pooledObject.getObject());

}

@Override
public void passivateObject(PooledObject<Nfs3> pooledObject) {
LogUtil.info(LogEnum.NFS_UTIL, "钝化NFS对象: ", pooledObject.getObject());
}
}

+ 114
- 0
dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java View File

@@ -0,0 +1,114 @@
/**
* Copyright 2020 Zhejiang Lab. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* =============================================================
*/

package org.dubhe.utils;

import com.emc.ecs.nfsclient.nfs.nfs3.Nfs3;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.dubhe.enums.LogEnum;
import org.springframework.stereotype.Component;

/**
* @description NFS3 连接池
* @date 2020-05-18
*/
@Component
public class NfsPool {
/**
* NFS工厂对象
*/
private NfsFactory nfsFactory;
/**
* GenericObjectPool对象
*/
private final GenericObjectPool<Nfs3> genericObjectPool;
/**
* 最大总共连接数
*/
public static final int MAX_TOTAL = 300;
/**
* 最小连接数
*/
public static final int MIN = 20;
/**
* 最大连接数
*/
public static final int MAX = 300;
/**
* 最大等待时间 单位毫秒
*/
public static final int MAX_WAIT_TIME = 3000;

/**
* 初始化连接池
*
* @param nfsFactory
*/
public NfsPool(NfsFactory nfsFactory) {
this.nfsFactory = nfsFactory;
GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig();
poolConfig.setMaxTotal(MAX_TOTAL);
poolConfig.setMinIdle(MIN);
poolConfig.setMaxIdle(MAX);
poolConfig.setMaxWaitMillis(MAX_WAIT_TIME);
this.genericObjectPool = new GenericObjectPool<>(nfsFactory, poolConfig);
}

/**
* 从连接池中取连接
*
* @return nfs3
*/
public Nfs3 getNfs() {
try {
LogUtil.info(LogEnum.NFS_UTIL,"NFS线程 活跃数量:{} ,空闲数量: {} , 等待队列数量 : {}",genericObjectPool.getNumActive(),genericObjectPool.getNumIdle(),genericObjectPool.getNumWaiters());
return genericObjectPool.borrowObject();
} catch (Exception e) {
LogUtil.error(LogEnum.NFS_UTIL, "获取NFS连接失败: {} ", e);
return null;
}
}

/**
* 释放连接到连接池
*
* @param nfs3
*/
public void revertNfs(Nfs3 nfs3) {
try {
if(nfs3 != null){
LogUtil.info(LogEnum.NFS_UTIL,"成功释放对象 : {} ", nfs3);
genericObjectPool.returnObject(nfs3);
}
} catch (Exception e) {
LogUtil.error(LogEnum.NFS_UTIL, " 释放NFS连接失败: ", e);
}
}

/**
* 销毁公共池
*/
public void destroyPool() {
try {
genericObjectPool.close();
} catch (Exception e) {
LogUtil.error(LogEnum.NFS_UTIL, "销毁NFS连接失败: ", e);
}
}

}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save