diff --git a/dubhe-server/README.md b/dubhe-server/README.md index 333111d..a2b4755 100644 --- a/dubhe-server/README.md +++ b/dubhe-server/README.md @@ -7,7 +7,7 @@ ### 准备环境 安装如下软件环境。 - OpenJDK：1.8+ -- Redis: 3.0+ +- Redis: 5.0+ - Maven: 3.0+ - MYSQL: 5.7.0+ @@ -46,6 +46,9 @@ java -jar ./dubhe-admin/target/dubhe-admin-1.0-exec.jar --spring.profiles.active ## task模块 java -jar ./dubhe-task/target/dubhe-task-1.0.jar --spring.profiles.active=prod + +## serving gateway模块 +java -jar ./dubhe-serving-gateway/target/dubhe-serving-gateway.jar --spring.profiles.active=prod ``` ## 本地开发 @@ -80,6 +83,8 @@ java -jar ./dubhe-task/target/dubhe-task-1.0.jar --spring.profiles.active=prod ├── dubhe-model 模型管理模块 ├── dubhe-system 系统管理 ├── dubhe-task 定时任务模块 +├── dubhe-serving 云端serving模块 +├── dubhe-serving-gateway 云端serving gateway网关模块 ``` ## docker服务器 diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java index 6cc1bb5..f4193f9 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java index 34de34a..0ed9b97 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermissionMethod.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermissionMethod.java index 0d4e087..a7ba413 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermissionMethod.java +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermissionMethod.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java index d5f15b8..ddff6b1 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/FlagValidator.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/FlagValidator.java index d47071b..8821c91 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/FlagValidator.java +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/FlagValidator.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java index f7bc427..87001dc 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java +++ b/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/aspect/PermissionAspect.java b/dubhe-server/common/src/main/java/org/dubhe/aspect/PermissionAspect.java index cac6f48..c6c8ef6 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/aspect/PermissionAspect.java +++ b/dubhe-server/common/src/main/java/org/dubhe/aspect/PermissionAspect.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java index c868f28..e91a107 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java index 95d527c..6c949b4 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java index a726f2d..57da55c 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseService.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseService.java index 2a43bb9..310b673 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseService.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java index 4cf377e..c347515 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/DataContext.java b/dubhe-server/common/src/main/java/org/dubhe/base/DataContext.java index 20c8c57..3223c90 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/DataContext.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/DataContext.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java b/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java index 97ff300..6699252 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java b/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java index 0454bd0..84e7c4a 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java b/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java index 77752ed..ce56dd3 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java b/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java index 5ee446f..247a50a 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/ScheduleTaskHandler.java b/dubhe-server/common/src/main/java/org/dubhe/base/ScheduleTaskHandler.java index 52da2a1..a4ade6c 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/base/ScheduleTaskHandler.java +++ b/dubhe-server/common/src/main/java/org/dubhe/base/ScheduleTaskHandler.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java index 9c03757..de702d5 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java index ffb7788..8fba81e 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java index f9d17fa..7202208 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java index 8235596..0740c65 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java index c399147..2881aa9 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/RecycleConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/RecycleConfig.java index 418b97e..1a54aef 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/RecycleConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/RecycleConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java index dad8e7c..f76f43d 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java b/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java index 143b7f2..e926b95 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java index 8880a09..5f8c817 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java index 14a4bd7..e4a4ebd 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java index bc4341c..cd4c9ea 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/TrainJobConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/TrainJobConfig.java index bdb8a47..0565cb1 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/TrainJobConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/TrainJobConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -60,6 +60,18 @@ public class TrainJobConfig { private String dockerModelPath; + private String dockerTeacherModelPath; + + private String dockerTeacherModelKey; + + private String dockerStudentModelKey; + + private String dockerStudentModelPath; + + private String atlasAnaconda; + + private String atlasPythonioencoding; + private String dockerValDatasetPath; private String loadValDatasetKey; diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/TrainPoolConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/TrainPoolConfig.java index 049aa1e..93a377e 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/TrainPoolConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/TrainPoolConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java index 85a83be..d8bb2a5 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java index 60c2962..90f6a9e 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java b/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java index 1f8e234..bd95bce 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/constant/ErrorMessageConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ErrorMessageConstant.java similarity index 91% rename from dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/constant/ErrorMessageConstant.java rename to dubhe-server/common/src/main/java/org/dubhe/constant/ErrorMessageConstant.java index 03c3b7d..3b6209e 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/constant/ErrorMessageConstant.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/ErrorMessageConstant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. * ============================================================= */ -package org.dubhe.data.machine.constant; +package org.dubhe.constant; /** * @description 状态机异常量类 diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java new file mode 100644 index 0000000..8fc6f9d --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java @@ -0,0 +1,128 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +import lombok.Data; +import org.springframework.stereotype.Component; + +/** + * @description 模型优化常量 + * @date 2020-05-25 + */ +@Component +@Data +public class ModelOptConstant { + + /** + * 模型优化任务结果字典 + */ + public static final String TASK_RESULT_DICT_NAME = "opt_result"; + /** + * 数据集挂载路径 + */ + public static final String DATASET_MOUNT_PATH = "/usr/local/dataset"; + /** + *算法挂载路径 + */ + public static final String ALGORITHM_MOUNT_PATH = "/usr/local/algorithm"; + /** + *输入模型挂载路径 + */ + public static final String INPUT_MODEL_BEFORE_MOUNT_PATH = "/usr/local/input/model"; + /** + *输出日志挂载路径 + */ + public static final String OUTPUT_LOG_MOUNT_PATH = "/usr/local/output/log"; + /** + *压缩前评估结果挂载路径 + */ + public static final String OUTPUT_RESULT_BEFORE_MOUNT_PATH = "/usr/local/output/result/before"; + /** + *输出模型挂载路径 + */ + public static final String OUTPUT_MODEL_MOUNT_PATH = "/usr/local/output/model"; + /** + *压缩后评估结果挂载路径 + */ + public static final String OUTPUT_RESULT_AFTER_MOUNT_PATH = "/usr/local/output/result/after"; + /** + * 使用的CPU配额 + */ + public static final int CPU_NUM = 2048; + /** + * 使用的GPU配额 + */ + public static final int GPU_NUM = 1; + /** + * 使用的内存配额 + */ + public static final int MEMORY_NUM = 8000; + /** + * CNN剪枝算法启动命令 + */ + public static final String OPT_START_SLIMMING_COMMAND = "cd /usr/local/algorithm && python run.py --prune_method=%s --model=%s --data_type=%s --dataset_dir=%s --model_save_dir=%s --log_dir=%s --before_result_dir=%s --after_result_dir=%s"; + /** + * 量化算法启动命令 + */ + public static final String OPT_START_QUANTIFY_COMMAND = "cd /usr/local/algorithm && python3 of_cnn_evaluate.py --model=%s --model_load_dir=%s --val_data_dir=%s --log_dir=%s --result_dir=%s --use_tensorrt=False --use_int8_online=False && python3 of_cnn_evaluate.py --model=%s --model_load_dir=%s --val_data_dir=%s --log_dir=%s --result_dir=%s --use_tensorrt=True --use_int8_online=True"; + /** + * 非内置运行命令 + */ + public static final String MY_OPT_COMMAND = "cd %s && %s"; + /** + * 模型优化结果日志路径 + */ + public static final String OPTIMIZE_LOG = "/log"; + /** + * 模型优化结果保存模型路径 + */ + public static final String OPTIMIZE_MODEL = "/model"; + /** + * 模型优化结果保存优化前模型评估参数 + */ + public static final String OPTIMIZE_JSON_BEFORE = "/before"; + /** + * 模型优化结果保存优化后模型评估参数 + */ + public static final String OPTIMIZE_JSON_AFTER = "/after"; + /** + * 模型优化结果JSON文件名称 + */ + public static final String OPTIMIZE_JSON_NAME = "/results_eval.json"; + /** + * 我的压缩算法根路径 + */ + public static final String MY_OPT_ALGORITHM_ROOT_PATH = "/model-opt/myAlgorithms/"; + /** + * 我的数据集 + */ + public static final String MY_OPT_DATASET_ROOT_PATH = "/model-opt/myDataset/"; + /** + * 算法准确度名称 + */ + public static final String ACCURACY = "accuracy"; + /** + * 复制模型后缀文件夹名称 + */ + public static final String COPY_MODEL_POSTFIX = "/model"; + /** + * 基于神经元权重剪枝 + */ + public static final String NEURONAL_PRUNING = "基于神经元权重剪枝"; + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java new file mode 100644 index 0000000..7bfba57 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java @@ -0,0 +1,127 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +import lombok.Getter; +import org.dubhe.exception.ErrorCode; + +/** + * @description 模型优化错误码 + * @date 2020-05-22 + */ +@Getter +public enum ModelOptErrorEnum implements ErrorCode { + + /** + * 该模型优化任务不存在 + */ + MODEL_OPT_TASK_ABSENT(1400, "该模型优化任务不存在"), + /** + * 该模型优化任务实例不存在 + */ + MODEL_OPT_TASK_INSTANCE_ABSENT(1401, "该模型优化任务实例不存在"), + /** + * 模型优化任务实例创建失败 + */ + MODEL_OPT_TASK_INSTANCE_CREATE_FAILED(1402, "模型优化任务实例创建失败"), + /** + * 模型优化任务名称已存在 + */ + MODEL_OPT_TASK_NAME_EXIST(1403, "模型优化任务名称已存在"), + /** + * 该任务已存在等待或进行中的实例 + */ + MODEL_OPT_TASK_INSTANCE_EXIST(1404, "该任务已存在等待或进行中的实例"), + /** + * 请先取消等待或进行中的实例 + */ + MODEL_OPT_TASK_DELETE_ERROR(1405, "请先取消等待或进行中的实例"), + + /** + * 请先添加模型优化结果字典 + */ + MODEL_OPT_TASK_RESULT_DICT_EMPTY(1406, "请先添加模型优化结果字典"), + + /** + * 模型压缩算法不存在 + */ + MODEL_OPT_ALGORITHM_NOT_EXIST(1407, "模型压缩算法不存在"), + + /** + * 任务未完成，不能下载 + */ + MODEL_OPT_TASK_UNFINISHED(1408, "任务未完成，不能下载"), + + /** + * 数据集不存在，请选择正确的数据集 + */ + MODEL_OPT_DATASET_ABSENT(1409, "数据集不存在，请选择正确的数据集"), + + /** + * 当前状态不能进行此操作 + */ + MODEL_OPT_TASK_INSTANCE_STATUS_ERROR(1410, "当前状态不能进行此操作"), + + /** + * 模型不存在 + */ + MODEL_OPT_MODEL_NOT_EXIST(1411, "模型不存在"), + + /** + * 请先登录 + */ + NO_USER_LOGGED_IN(1412, "请先登录"), + + /** + * 当前用户无此操作权限 + */ + NO_PERMISSION(1413, "当前用户无此操作权限"), + + /** + * 内部错误 + */ + INTERNAL_SERVER_ERROR(1414, "内部错误"), + + /** + * 字典参数不存在，请先添加字典 + */ + PARAM_DICT_NOT_EXIST(1415, "字典参数不存在，请先添加字典"), + + /** + * 无可用镜像，请先在字典中配置镜像 + */ + IMAGES_DICT_NOT_EXIST(1416, "无可用镜像，请先在字典中配置镜像"), + + /** + * 容器启动失败 + */ + CONTAINER_START_FAILED(1417, "容器启动失败"), + + /** + * 数据集名称已存在 + */ + DATASET_NAME_EXIST(1418, "数据集名称已存在"); + + private Integer code; + private String msg; + + ModelOptErrorEnum(Integer code, String msg) { + this.code = code; + this.msg = msg; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java new file mode 100644 index 0000000..99aa735 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + +import lombok.Getter; + +/** + * @description 模型优化常量 + * @date 2020-05-22 + */ +@Getter +public enum ModelOptInstanceStatusEnum { + + /** + * 模型优化任务状态 + */ + WAITING("-1", "等待中"), + RUNNING("0", "进行中"), + COMPLETED("1", "已完成"), + CANCELED("2", "已取消"), + EXEC_FAILED("3", "执行失败"), + + ; + private String value; + private String msg; + + ModelOptInstanceStatusEnum(String value, String msg) { + this.value = value; + this.msg = msg; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java index 220fe22..e7ee3d7 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,15 +27,22 @@ public class NumberConstant { public final static int NUMBER_1 = 1; public final static int NUMBER_2 = 2; public final static int NUMBER_3 = 3; + public final static int NUMBER_4 = 4; public final static int NUMBER_5 = 5; + public final static int NUMBER_6 = 6; + public final static int NUMBER_8 = 8; public final static int NUMBER_10 = 10; + public final static int NUMBER_25 = 25; public final static int NUMBER_30 = 30; public final static int NUMBER_50 = 50; public final static int NUMBER_60 = 60; public final static int NUMBER_1024 = 1024; public final static int NUMBER_1000 = 1000; + public final static int NUMBER_100 = 100; public final static int HOUR_SECOND = 60 * 60; public final static int DAY_SECOND = 60 * 60 * 24; public final static int WEEK_SECOND = 60 * 60 * 24 * 7; public final static int MAX_PAGE_SIZE = 2000; + public final static int MAX_MESSAGE_LENGTH = 1024 * 1024 * 1024; + } diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java index 0d30a7d..82a28e3 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java b/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java index d9d4f7a..62b0e88 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,6 +48,7 @@ public final class Permissions { */ public static final String MODEL = "model"; public static final String MODEL_MODEL = "model:model"; + public static final String MODEL_OPTIMIZE = "model:optimize"; /** * 控制台 @@ -61,6 +62,13 @@ public final class Permissions { public static final String SYSTEM_LOG = "system:log"; public static final String SYSTEM_TEAM = "system:team"; + /** + * 云端Serving + */ + public static final String SERVING = "serving"; + public static final String SERVING_DEPLOYMENT = "serving:deployment"; + public static final String SERVING_BATCH = "serving:batch"; + private Permissions() { } } diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/RedisConstants.java b/dubhe-server/common/src/main/java/org/dubhe/constant/RedisConstants.java new file mode 100644 index 0000000..b9715eb --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/RedisConstants.java @@ -0,0 +1,48 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.constant; + + +import org.dubhe.base.MagicNumConstant; + +/** + * @description redis 常量 + * @date 2020-12-17 + */ +public class RedisConstants { + /** + * 分布式锁过期时间 + */ + public static final Long DELAY_CUD_RESOURCE_EXPIRE_TIME = MagicNumConstant.TEN_LONG; + /** + * 分布式锁 + */ + public static final String DELAY_CUD_RESOURCE_KEY = "k8sclient:task:k8s_delay_cud_resource"; + /** + * 延时启动队列 + */ + public static final String DELAY_APPLY_ZSET_KEY = "k8sclient:task:delay_apply_zset"; + /** + * 延时停止队列 + */ + public static final String DELAY_STOP_ZSET_KEY = "k8sclient:task:delay_stop_zset"; + /** + * 延时队列值 + */ + public static final String DELAY_ZSET_VALUE = "%s__%s"; +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java index 414e5fe..b03bfb6 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,8 @@ package org.dubhe.constant; +import java.util.regex.Pattern; + /** * @description 字符串constant * @date 2020-05-14 @@ -27,6 +29,16 @@ public final class StringConstant { public static final String MOZILLA = "Mozilla"; public static final String REQUEST_METHOD_GET = "GET"; + /** + * 字母、数字、英文横杠和下划线匹配 + */ + public static final String REGEXP = "^[a-zA-Z0-9\\-\\_\\u4e00-\\u9fa5]+$"; + /** + * 整数匹配 + */ + public static final Pattern PATTERN_NUM = Pattern.compile("^[-\\+]?[\\d]*$"); + + /** * 公共字段 */ @@ -39,7 +51,27 @@ public final class StringConstant { public static final String UTF8 = "utf-8"; public static final String JSON_REQUEST = "application/json"; public static final String K8S_CALLBACK_URI = "/api/k8s/callback/pod"; + public static final String K8S_CALLBACK_PATH_DEPLOYMENT = "/api/k8s/callback/deployment"; public static final String MULTIPART = "multipart/form-data"; + /** + * 分页内容 + */ + public static final String RESULT = "result"; + /** + * 排序规则 + */ + public static final String SORT_ASC = "asc"; + + public static final String SORT_DESC = "desc"; + + public static final String QUERY = "query"; + + public static final String NGINX_LOWERCASE = "nginx"; + + public static final String TRUE_LOWERCASE = "true"; + + public static final String GRPC_CAPITALIZE = "GRPC"; + /** * 测试环境 diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java index 6cb1be9..de5650a 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,14 +33,20 @@ public class SymbolConstant { public static final String TOKEN = "token"; public static final String GET = "get"; public static final String SET = "set"; - + public static final String HTTP = "http"; + public static final String GRPC = "grpc"; public static final String BRACKETS = "{}"; public static final String BACKSLASH = "\\"; public static final String BACKSLASH_MARK= "\\\""; public static final String DOUBLE_MARK= "\"\""; public static final String MARK= "\""; - public static final String FLAG_EQUAL = "="; + public static final String LEFT_PARENTHESIS = "["; + public static final String RIGHT_PARENTHESIS = "]"; + public static final String APOSTROPHE = "'"; + public static final String HYPHEN = "-"; + public static final String EVENT_SEPARATOR = "&&"; + public static final String POST = "POST"; private SymbolConstant() { } diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java index 71477d0..1a85787 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java index 8975b02..5364203 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java +++ b/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/dto/CommonPermissionDataDTO.java b/dubhe-server/common/src/main/java/org/dubhe/domain/dto/CommonPermissionDataDTO.java index c1cfcc4..a0d6dc5 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/dto/CommonPermissionDataDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/dto/CommonPermissionDataDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java index f01468d..1f42e5c 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java index d0eb560..71ad1ad 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java index c6ec6c3..59146f4 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java +++ b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/GlobalRequestRecordDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/GlobalRequestRecordDTO.java index 6937ae2..e350450 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/GlobalRequestRecordDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/GlobalRequestRecordDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/dto/StateChangeDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/StateChangeDTO.java similarity index 92% rename from dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/dto/StateChangeDTO.java rename to dubhe-server/common/src/main/java/org/dubhe/dto/StateChangeDTO.java index 8486e8c..9db46cd 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/machine/dto/StateChangeDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/StateChangeDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. * ============================================================= */ -package org.dubhe.data.machine.dto; +package org.dubhe.dto; import lombok.*; import org.springframework.stereotype.Component; diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java index e291579..2daf37d 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java index 5fd0740..7d9601c 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sDeploymentCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sDeploymentCallbackCreateDTO.java new file mode 100644 index 0000000..6c9b13b --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sDeploymentCallbackCreateDTO.java @@ -0,0 +1,82 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dto.callback; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +/** + * @descripton 统一通用参数实现与校验 + * @date 2020-11-26 + */ +@ApiModel(description = "k8s deployment异步回调基类") +@Data +public class BaseK8sDeploymentCallbackCreateDTO { + @ApiModelProperty(required = true, value = "k8s namespace") + @NotBlank(message = "namespace 不能为空！") + private String namespace; + + @ApiModelProperty(required = true, value = "k8s resource name") + @NotBlank(message = "resourceName 不能为空！") + private String resourceName; + + @ApiModelProperty(required = true, value = "k8s deployment name") + @NotBlank(message = "deployment 不能为空！") + private String deploymentName; + + /** + * deployment已 Running的pod数 + */ + @ApiModelProperty(required = true, value = "k8s deployment readyReplicas") + @NotNull(message = "readyReplicas 不能为空！") + private Integer readyReplicas; + + /** + * deployment总pod数 + */ + @ApiModelProperty(required = true, value = "k8s deployment replicas") + @NotNull(message = "replicas 不能为空！") + private Integer replicas; + + public BaseK8sDeploymentCallbackCreateDTO() { + + } + + public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas) { + this.namespace = namespace; + this.resourceName = resourceName; + this.deploymentName = deploymentName; + this.readyReplicas = readyReplicas; + this.replicas = replicas; + } + + @Override + public String toString() { + return "BaseK8sDeploymentCallbackCreateDTO{" + + "namespace='" + namespace + '\'' + + ", resourceName='" + resourceName + '\'' + + ", deploymentName='" + deploymentName + '\'' + + ", readyReplicas='" + readyReplicas + '\'' + + ", replicas='" + replicas + '\'' + + '}'; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java index ffc4a9f..82897de 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BatchServingK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BatchServingK8sPodCallbackCreateDTO.java new file mode 100644 index 0000000..8d1e305 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BatchServingK8sPodCallbackCreateDTO.java @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dto.callback; + +import io.swagger.annotations.ApiModel; +import lombok.Data; + +/** + * @description k8s pod异步回调云端serving + * @date 2020-11-27 + */ +@ApiModel(description = "k8s pod异步回调云端serving") +@Data +public class BatchServingK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO { + + @Override + public String toString() { + return super.toString(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java new file mode 100644 index 0000000..12aa34f --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java @@ -0,0 +1,30 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dto.callback; + +import io.swagger.annotations.ApiModel; +import lombok.Data; + +/** + * @description 模型优化自定义回调参数类 + * @date 2020-06-03 + */ +@ApiModel(description = "k8s pod异步回调Algorithm") +@Data +public class ModelOptK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO { +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java index 57b41eb..1378cdd 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ServingK8sDeploymentCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ServingK8sDeploymentCallbackCreateDTO.java new file mode 100644 index 0000000..8611f62 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ServingK8sDeploymentCallbackCreateDTO.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.dto.callback; + +import io.swagger.annotations.ApiModel; +import lombok.Data; + +/** + * @description k8s deployment异步回调云端serving + * @date 2020-09-16 + */ +@ApiModel(description = "k8s deployment异步回调云端serving") +@Data +public class ServingK8sDeploymentCallbackCreateDTO extends BaseK8sDeploymentCallbackCreateDTO { + + @Override + public String toString() { + return super.toString(); + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java index 691c0d2..4ce8332 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,15 +32,32 @@ public enum BizEnum { /** * 模型开发 */ - NOTEBOOK("模型开发","notebook",0), + NOTEBOOK("模型开发", "notebook", 0), /** * 算法管理 */ - ALGORITHM("算法管理","algorithm",1), + ALGORITHM("算法管理", "algorithm", 1), /** * 模型管理 */ - MODEL("模型管理","model",2), + MODEL("模型管理", "model", 2), + + /** + * 训练管理 + */ + TRAIN("训练管理", "train", 3), + /** + * 云端Serving-在线服务 + */ + SERVING("云端Serving", "serving", 4), + /** + * 批量服务 + */ + BATCH_SERVING("批量服务", "batchserving", 5), + /** + * 模型优化 + */ + MODEL_OPT("模型优化管理","modelopt",6), ; /** @@ -56,28 +73,42 @@ public enum BizEnum { */ private Integer createResource; - BizEnum(String bizName,String bizCode, Integer createResource) { + BizEnum(String bizName, String bizCode, Integer createResource) { this.createResource = createResource; this.bizName = bizName; this.bizCode = bizCode; } - private static final Map RESOURCE_ENUM_MAP = new HashMap(){ + private static final Map RESOURCE_ENUM_MAP = new HashMap() { { - for (BizEnum enums:BizEnum.values()){ - put(enums.getCreateResource(),enums); + for (BizEnum enums : BizEnum.values()) { + put(enums.getCreateResource(), enums); } } }; /** * 根据createResource获取BizEnum - * @param createResource - * @return + * @param createResource 业务源代号 + * @return 枚举对象 */ - public static BizEnum getByCreateResource(int createResource){ + public static BizEnum getByCreateResource(int createResource) { return RESOURCE_ENUM_MAP.get(createResource); } + /** + * 根据createResource获取code + * + * @param createResource 业务源代号 + * @return String 业务模块名称 + */ + public static String getCodeByResource(Integer createResource) { + for (BizEnum enums : BizEnum.values()) { + if (enums.getCreateResource().equals(createResource)) { + return enums.getBizCode(); + } + } + return null; + } } diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java index 796f5b7..9c0af4f 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,6 +37,10 @@ public enum BizNfsEnum { * 模型管理 NFS 路径命名 */ MODEL(BizEnum.MODEL, "model"), + /** + * 模型优化 NFS 路径命名 + */ + MODEL_OPT(BizEnum.MODEL_OPT, "model-opt"), ; BizNfsEnum(BizEnum bizEnum, String bizNfsPath) { diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/DatasetTypeEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/DatasetTypeEnum.java index 5f72aaa..5c5953b 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/DatasetTypeEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/DatasetTypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageSourceEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/ImageSourceEnum.java similarity index 95% rename from dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageSourceEnum.java rename to dubhe-server/common/src/main/java/org/dubhe/enums/ImageSourceEnum.java index 471bf71..7a1912b 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageSourceEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/ImageSourceEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageStateEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/ImageStateEnum.java similarity index 95% rename from dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageStateEnum.java rename to dubhe-server/common/src/main/java/org/dubhe/enums/ImageStateEnum.java index 127e747..fdc1414 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ImageStateEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/ImageStateEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. * ============================================================= */ - + package org.dubhe.enums; /** diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/ImageTypeEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/ImageTypeEnum.java new file mode 100644 index 0000000..18b881e --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/ImageTypeEnum.java @@ -0,0 +1,92 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.enums; + + +import lombok.Getter; + +import java.util.HashMap; +import java.util.Map; + +/** + * @description 镜像项目枚举类 + * @date 2020-12-11 + */ +@Getter +public enum ImageTypeEnum { + + /** + * notebook镜像 + */ + NOTEBOOK("notebook镜像", "notebook", 0), + + /** + * 训练镜像 + */ + TRAIN("训练镜像", "train", 1); + + /** + * 镜像项目名称 + */ + private String name; + /** + * 镜像项目代码 + */ + private String code; + /** + * 镜像项目类型 + */ + private Integer type; + + ImageTypeEnum(String name, String code, Integer type) { + this.name = name; + this.code = code; + this.type = type; + } + + private static final Map RESOURCE_ENUM_MAP = new HashMap() { + { + for (ImageTypeEnum enums : ImageTypeEnum.values()) { + put(enums.getType(), enums); + } + } + }; + + /** + * 根据type获取ImageTypeEnum + * @param type + * @return 镜像项目枚举对象 + */ + public static ImageTypeEnum getType(int type) { + return RESOURCE_ENUM_MAP.get(type); + } + + /** + * 根据type获取code + * + * @param type 镜像项目类型 + * @return String 镜像项目代码 + */ + public static String getType(Integer type) { + for (ImageTypeEnum enums : ImageTypeEnum.values()) { + if (enums.getType().equals(type)) { + return enums.getCode(); + } + } + return null; + } +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java index 603f590..a266177 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,8 @@ public enum LogEnum { BIZ_SYS, // 模型模块 BIZ_MODEL, + // 模型优化 + MODEL_OPT, // 数据集模块 BIZ_DATASET, // k8s模块 @@ -50,19 +52,26 @@ public enum LogEnum { LOCAL_FILE_UTIL, //FILE UTILS FILE_UTIL, + // 云端Serving模块 + SERVING, //FILE UTILS UPLOAD_TEMP, //STATE MACHINE STATE_MACHINE, //全局垃圾回收 GARBAGE_RECYCLE, - //DATA_SEQUENCE - DATA_SEQUENCE, //IO UTIL IO_UTIL, + //DATA_SEQUENCE + DATA_SEQUENCE, // 日志切面 - LOG_ASPECT + LOG_ASPECT, + //镜像 + IMAGE, + //度量 + MEASURE; ; + /** * 判断日志类型不能为空 * diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/ModelResourceEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/ModelResourceEnum.java new file mode 100644 index 0000000..a4f0c55 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/ModelResourceEnum.java @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + +import lombok.Getter; + +/** + * @description 模型资源枚举类 + * @date 2020-11-19 + */ +@Getter +public enum ModelResourceEnum { + + /** + * 我的模型 + */ + MINE(0, "我的模型"), + /** + * 预置模型 + */ + PRESET(1, "预置模型"), + /** + * 炼知模型 + */ + ATLAS(2, "炼知模型"); + + private Integer type; + + private String description; + + ModelResourceEnum(Integer type, String description) { + this.type = type; + this.description = description; + } + + /** + * 根据类型获取枚举类对象 + * + * @param type 类型 + * @return 枚举类对象 + */ + public static ModelResourceEnum getType(Integer type) { + for (ModelResourceEnum modelResourceEnum : values()) { + if (modelResourceEnum.getType().compareTo(type) == 0) { + //获取指定的枚举 + return modelResourceEnum; + } + } + return null; + } +} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/OperationTypeEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/OperationTypeEnum.java index 59b284f..7d911ad 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/OperationTypeEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/OperationTypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/RecycleResourceEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/RecycleResourceEnum.java index 1d6b257..3639d35 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/RecycleResourceEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/RecycleResourceEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,10 +33,20 @@ public enum RecycleResourceEnum { * 数据集文件回收 */ DATASET_RECYCLE_FILE("datasetRecycleFile", "数据集文件回收"), + + /** + * 医学数据集文件回收 + */ + DATAMEDICINE_RECYCLE_FILE("dataMedicineRecycleFile", "数据集文件回收"), + /** * 数据集版本文件回收 */ DATASET_RECYCLE_VERSION_FILE("datasetRecycleVersionFile", "数据集版本文件回收"), + /** + * 云端Serving输入文件回收 + */ + SERVING_RECYCLE_FILE("servingRecycleFile", "云端Serving文件回收"), ; diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java index 2c4fcaa..35d5c4b 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java index 58a91d4..024da9c 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java +++ b/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java b/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java index 233f35c..5bf674c 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java index 9cb5a8b..cee727d 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java index fc8e9aa..f2ec3e6 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/DataSequenceException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/DataSequenceException.java index ca6869b..b4790dd 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/DataSequenceException.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/DataSequenceException.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java index 8f80555..94de0f3 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java b/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java index 2bbc60e..b8d367c 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java index e1124f3..2fa03d0 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java index cd9c067..3f23953 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/exception/StateMachineException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/StateMachineException.java similarity index 89% rename from dubhe-server/dubhe-data/src/main/java/org/dubhe/data/exception/StateMachineException.java rename to dubhe-server/common/src/main/java/org/dubhe/exception/StateMachineException.java index 08d6387..774c9e2 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/exception/StateMachineException.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/StateMachineException.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,9 @@ * limitations under the License. * ============================================================= */ -package org.dubhe.data.exception; +package org.dubhe.exception; import lombok.Getter; -import org.dubhe.exception.BusinessException; /** * @description 状态机异常类 diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java index fed0939..54b4929 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java b/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java index b94613f..41367b7 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java +++ b/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java b/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java index 86a20a4..c0b717f 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java +++ b/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java @@ -18,10 +18,10 @@ package org.dubhe.factory; import org.dubhe.base.DataResponseBody; import org.dubhe.base.ResponseCode; + /** - * @desc DataResponseBody 工厂类 - * - * @date 2020.05.28 + * @description DataResponseBody 工厂类 + * @date 2020-03-16 */ public class DataResponseFactory { diff --git a/dubhe-server/common/src/main/java/org/dubhe/filter/BaseLogFilter.java b/dubhe-server/common/src/main/java/org/dubhe/filter/BaseLogFilter.java index 50c7bc5..08dc43b 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/filter/BaseLogFilter.java +++ b/dubhe-server/common/src/main/java/org/dubhe/filter/BaseLogFilter.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/filter/ConsoleLogFilter.java b/dubhe-server/common/src/main/java/org/dubhe/filter/ConsoleLogFilter.java index 69b1f78..c442842 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/filter/ConsoleLogFilter.java +++ b/dubhe-server/common/src/main/java/org/dubhe/filter/ConsoleLogFilter.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/filter/GlobalRequestLogFilter.java b/dubhe-server/common/src/main/java/org/dubhe/filter/GlobalRequestLogFilter.java index ae8929b..4e3f09b 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/filter/GlobalRequestLogFilter.java +++ b/dubhe-server/common/src/main/java/org/dubhe/filter/GlobalRequestLogFilter.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java b/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java index 029457f..6d928b1 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java +++ b/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java b/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java index de9d09f..64be203 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java +++ b/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java b/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java index 90beef9..7b8d398 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java +++ b/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/interceptor/MySqlInterceptor.java b/dubhe-server/common/src/main/java/org/dubhe/interceptor/MySqlInterceptor.java index 534a650..3654574 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/interceptor/MySqlInterceptor.java +++ b/dubhe-server/common/src/main/java/org/dubhe/interceptor/MySqlInterceptor.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/interceptor/PaginationInterceptor.java b/dubhe-server/common/src/main/java/org/dubhe/interceptor/PaginationInterceptor.java index 0eeb8f4..5176942 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/interceptor/PaginationInterceptor.java +++ b/dubhe-server/common/src/main/java/org/dubhe/interceptor/PaginationInterceptor.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java index e4ecd90..5399f7c 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java index 150d09f..e55822e 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,7 @@ package org.dubhe.utils; import java.sql.Timestamp; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneId; +import java.time.*; import java.util.Date; /** @@ -95,4 +92,25 @@ public class DateUtil { return dateFormat.format(date); } + /** + * + * @return 当前字符串时间yyyyMMddHHmmss + */ + public static String getTimestampStr() { + SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmssSSS"); + return df.format(System.currentTimeMillis()); + } + + /** + * 获取当前时间与目标时间相差秒数 + * @param timestamp 目标时间 + * @return + */ + public static long getSeconds(Timestamp timestamp) { + LocalDateTime time = timestamp.toLocalDateTime(); + LocalDateTime now = LocalDateTime.now(); + Duration duration = Duration.between(time, now); + return duration.abs().getSeconds(); + } + } diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java index 768f527..b9f77f0 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java index 33ad123..f05cbb1 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java index 7dde12f..fe3a3b1 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java index e16145d..9e09cb1 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java index 70a72db..cb31df0 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/IOUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/IOUtil.java index dd6a94b..e82e0be 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/IOUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/IOUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java b/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java index d0d433b..a69eb78 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java index fedd799..032f70a 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java index 7e2cfbd..7dc31b9 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,11 +67,13 @@ public class K8sCallBackTool { * k8s 回调路径 */ private static final String K8S_CALLBACK_PATH_POD = "/api/k8s/callback/pod/"; + private static final String K8S_CALLBACK_PATH_DEPLOYMENT = "/api/k8s/callback/deployment/"; static { K8S_CALLBACK_PATH = new ArrayList<>(); // 添加需要token权限校验的地址（Shiro匿名访问的地址） K8S_CALLBACK_PATH.add(K8S_CALLBACK_PATH_POD + "**"); + K8S_CALLBACK_PATH.add(K8S_CALLBACK_PATH_DEPLOYMENT + "**"); } /** @@ -133,6 +135,16 @@ public class K8sCallBackTool { return url + K8S_CALLBACK_PATH_POD + podLabel; } + /** + * 获取回调地址 + * + * @param businessLabel + * @return String + */ + public String getDeploymentCallbackUrl(String businessLabel) { + return url + K8S_CALLBACK_PATH_DEPLOYMENT + businessLabel; + } + /** * 获取超时时间秒 diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java index 1f8846d..de3b916 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java index 13470d4..f3dcb62 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/LocalFileUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/LocalFileUtil.java index e5bad3d..5e77bc5 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/LocalFileUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/LocalFileUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ import org.springframework.util.FileCopyUtils; import java.io.*; import java.util.Enumeration; +import java.util.Objects; import java.util.zip.ZipEntry; /** @@ -54,6 +55,27 @@ public class LocalFileUtil { private static final String WINDOWS = "Windows"; + /** + * nfs服务暴露的IP地址 + */ + @Value("${k8s.nfs}") + private String nfsIp; + + /** + * 文件存储服务器用户名 + */ + @Value("${data.server.userName}") + private String userName; + + /** + * 拷贝文件命令 + */ + public static final String COPY_COMMAND = "ssh %s@%s \"mkdir -p %s && cp -r %s %s && echo success\""; + /** + * 拷贝文件夹下文件命令 + */ + public static final String COPY_DIR_COMMAND = "ssh %s@%s \"mkdir -p %s && cp -rf %s* %s && echo success\""; + @Value("${k8s.nfs-root-path}") private String nfsRootPath; @@ -198,53 +220,53 @@ public class LocalFileUtil { } } + /** - * NFS 复制目录到指定目录下 多个文件 包含目录与文件并存情况 - * - * 通过本地文件复制方式 + * 复制单个文件到指定目录下 单个文件 * - * @param sourcePath 需要复制的文件目录 例如：/abc/def + * @param sourcePath 需要复制的文件 例如：/abc/def/cc.txt * @param targetPath 需要放置的目标目录 例如：/abc/dd * @return boolean */ - public boolean copyPath(String sourcePath, String targetPath) { + private boolean copyLocalFile(String sourcePath, String targetPath) { if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { return false; } sourcePath = formatPath(sourcePath); targetPath = formatPath(targetPath); - try { - return copyLocalPath(nfsConfig.getRootDir() + sourcePath, nfsConfig.getRootDir() + targetPath); - } catch (Exception e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, " failed to Copy file original path: {} ,target path： {} ,copyPath: {}", sourcePath, targetPath, e); + try (InputStream input = new FileInputStream(sourcePath); + FileOutputStream output = new FileOutputStream(targetPath)) { + FileCopyUtils.copy(input, output); + return true; + } catch (IOException e) { + LogUtil.error(LogEnum.LOCAL_FILE_UTIL, " failed to copy file original path: {} ,target path： {} ,copyLocalFile:{} ", sourcePath, targetPath, e); return false; } } /** - * 复制文件到指定目录下 单个文件 + * NFS 复制目录到指定目录下 多个文件 包含目录与文件并存情况 * - * @param sourcePath 需要复制的文件 例如：/abc/def/cc.txt + * 通过本地文件复制方式 + * + * @param sourcePath 需要复制的文件目录 例如：/abc/def * @param targetPath 需要放置的目标目录 例如：/abc/dd * @return boolean */ - private boolean copyLocalFile(String sourcePath, String targetPath) { + public boolean copyPath(String sourcePath, String targetPath) { if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { return false; } sourcePath = formatPath(sourcePath); targetPath = formatPath(targetPath); - try (InputStream input = new FileInputStream(sourcePath); - FileOutputStream output = new FileOutputStream(targetPath)) { - FileCopyUtils.copy(input, output); - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, " failed to copy file original path: {} ,target path： {} ,copyLocalFile:{} ", sourcePath, targetPath, e); + try { + return copyLocalPath(nfsConfig.getRootDir() + sourcePath, nfsConfig.getRootDir() + targetPath); + } catch (Exception e) { + LogUtil.error(LogEnum.LOCAL_FILE_UTIL, " failed to Copy file original path: {} ,target path： {} ,copyPath: {}", sourcePath, targetPath, e); return false; } } - /** * 复制文件 到指定目录下 多个文件 包含目录与文件并存情况 * @@ -304,4 +326,77 @@ public class LocalFileUtil { return path; } + /** + * 拷贝文件 + * + * @param sourcePath 需要复制的文件 例如：/abc/def/cc.txt + * @param targetPath 需要放置的目标目录 例如：/abc/dd + * @return + */ + public boolean copyFile(String sourcePath, String targetPath) { + //绝对路径 + String sourceAbsolutePath = formatPath(nfsConfig.getRootDir() + sourcePath); + String targetPathAbsolutePath = formatPath(nfsConfig.getRootDir() + targetPath); + String[] command = new String[]{"/bin/sh", "-c", String.format(COPY_COMMAND, userName, nfsIp, targetPathAbsolutePath, sourceAbsolutePath, targetPathAbsolutePath)}; + boolean flag = false; + Process process; + try { + process = Runtime.getRuntime().exec(command); + if (isCopySuccess(process)) { + flag = true; + } + } catch (IOException e) { + LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "copy file failed, filePath:{}, targetPath:{}", sourcePath, targetPath, e); + } + return flag; + } + + /** + * 拷贝文件夹下内容 + * @param sourcePath 需要复制的文件目录 例如：/abc/dd + * @param targetPath 需要放置的目标目录 例如：/abc/dd + * @return + */ + public boolean copyDir(String sourcePath, String targetPath) { + //绝对路径 + String sourceAbsolutePath = formatPath(nfsConfig.getRootDir() + sourcePath); + String targetPathAbsolutePath = formatPath(nfsConfig.getRootDir() + targetPath); + String[] command = new String[]{"/bin/sh", "-c", String.format(COPY_DIR_COMMAND, userName, nfsIp, targetPathAbsolutePath, sourceAbsolutePath, targetPathAbsolutePath)}; + boolean flag = false; + Process process; + try { + process = Runtime.getRuntime().exec(command); + if (isCopySuccess(process)) { + flag = true; + } + } catch (IOException e) { + LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "copy file failed, filePath:{}, targetPath:{}", sourcePath, targetPath, e); + } + return flag; + } + + /** + * 判断拷贝结果 + * + * @param process + * @return + */ + public boolean isCopySuccess(Process process) { + boolean flag = false; + try (InputStream stream = process.getInputStream(); + InputStreamReader iReader = new InputStreamReader(stream); + BufferedReader bReader = new BufferedReader(iReader)) { + String line; + while (Objects.nonNull(line = bReader.readLine())) { + boolean temp = line.contains("success"); + if (temp) { + flag = true; + } + } + } catch (Exception e) { + LogUtil.error(LogEnum.SERVING, "Read stream failed : {}", e); + } + return flag; + } + } \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java index 25b273a..cc336cc 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java index d0f9bf3..cf97b0f 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package org.dubhe.utils; import org.dubhe.base.MagicNumConstant; +import org.dubhe.constant.NumberConstant; /** * @description 计算工具类 @@ -25,6 +26,7 @@ import org.dubhe.base.MagicNumConstant; */ public class MathUtils { + private static String FLOAT_DIVISION_FORMATE = "%1$0"; /** * 字符串整数加法 @@ -33,9 +35,10 @@ public class MathUtils { * @param num2 * @return */ - public static String add(String num1,String num2){ - return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO:Integer.valueOf(num1)) + (!RegexUtil.isDigits(num2)?MagicNumConstant.ZERO:Integer.valueOf(num2))); + public static String add(String num1, String num2) { + return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO : Integer.valueOf(num1)) + (!RegexUtil.isDigits(num2) ? MagicNumConstant.ZERO : Integer.valueOf(num2))); } + /** * 字符串整数减法 * num1 - num2 @@ -43,7 +46,30 @@ public class MathUtils { * @param num2 * @return */ - public static String reduce(String num1,String num2){ - return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO:Integer.valueOf(num1)) - (!RegexUtil.isDigits(num2)?MagicNumConstant.ZERO:Integer.valueOf(num2))); + public static String reduce(String num1, String num2) { + return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO : Integer.valueOf(num1)) - (!RegexUtil.isDigits(num2) ? MagicNumConstant.ZERO : Integer.valueOf(num2))); + } + + /** + * + * 浮点数除法 num1/num2 + * @param num1 + * @param num2 + * @param decimal 结果小数位数 + * @return + */ + public static Float floatDivision(String num1, String num2, Integer decimal) { + if (!RegexUtil.isFloat(num1) || !RegexUtil.isFloat(num2)) { + return null; + } + if (Float.valueOf(num2).equals(0f)) { + return null; + } + if (decimal != null && decimal > NumberConstant.NUMBER_0) { + Integer d = Integer.valueOf(MagicNumConstant.ONE + String.format(FLOAT_DIVISION_FORMATE + decimal + "d", NumberConstant.NUMBER_0)); + return (float) (Math.round((Float.valueOf(num1) / Float.valueOf(num2)) * d)) / d; + } else { + return Float.valueOf(num1) / Float.valueOf(num2); + } } } diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java index 42c934f..4461288 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package org.dubhe.utils; import cn.hutool.core.io.IoUtil; import cn.hutool.http.HttpRequest; import com.alibaba.fastjson.JSONObject; +import io.minio.CopyConditions; import io.minio.MinioClient; import io.minio.PutObjectOptions; import io.minio.Result; @@ -31,6 +32,7 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; import org.apache.commons.lang.StringUtils; import org.dubhe.base.MagicNumConstant; +import org.dubhe.constant.NumberConstant; import org.dubhe.enums.LogEnum; import org.dubhe.exception.BusinessException; import org.springframework.beans.factory.annotation.Value; @@ -40,12 +42,7 @@ import javax.annotation.PostConstruct; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * @description Minio工具类 @@ -91,6 +88,22 @@ public class MinioUtil { client.putObject(bucket, fullFilePath, inputStream, options); } + /** + * 写文件 + * + * @param bucket 桶名称 + * @param fullFilePath 文件存储的全路径，包括文件名，非'/'开头. e.g. dataset/12/annotation/test.txt + * @param inputStream file inputStream. can not be null + */ + public void writeInputStream(String bucket, String fullFilePath, InputStream inputStream) throws Exception { + boolean isExist = client.bucketExists(bucket); + if (!isExist) { + client.makeBucket(bucket); + } + PutObjectOptions options = new PutObjectOptions(inputStream.available(), MagicNumConstant.NEGATIVE_ONE); + client.putObject(bucket, fullFilePath, inputStream, options); + } + /** * 读取文件 * @@ -125,10 +138,10 @@ public class MinioUtil { /** * 批量删除文件 * - * @param bucket 桶 - * @param objectNames 对象名称 + * @param bucket 桶 + * @param objectNames 对象名称 */ - public void delFiles(String bucket,List objectNames) throws Exception{ + public void delFiles(String bucket, List objectNames) throws Exception { Iterable> results = client.removeObjects(bucket, objectNames); for (Result result : results) { result.get(); @@ -138,30 +151,47 @@ public class MinioUtil { /** * 获取对象名称 * - * @param bucketName 桶名称 - * @param prefix 前缀 - * @return + * @param bucketName 桶名称 + * @param prefix 前缀 + * @return List 对象名称列表 * @throws Exception */ - public List getObjects(String bucketName, String prefix)throws Exception{ + public List getObjects(String bucketName, String prefix) throws Exception { List fileNames = new ArrayList<>(); Iterable> results = client.listObjects(bucketName, prefix); - for(Result result:results){ + for (Result result : results) { Item item = result.get(); fileNames.add(item.objectName()); } return fileNames; } + /** + * 获取路径下文件数量 + * + * @param bucketName 桶名称 + * @param prefix 前缀 + * @return InputStream 文件流 + * @throws Exception + */ + public int getCount(String bucketName, String prefix) throws Exception { + int count = NumberConstant.NUMBER_0; + Iterable> results = client.listObjects(bucketName, prefix); + for (Result result : results) { + count++; + } + return count; + } + /** * 获取文件流 * * @param bucket 桶 * @param objectName 对象名称 - * @return + * @return InputStream 文件流 * @throws Exception */ - public InputStream getObjectInputStream(String bucket,String objectName)throws Exception{ + public InputStream getObjectInputStream(String bucket, String objectName) throws Exception { return client.getObject(bucket, objectName); } @@ -201,6 +231,25 @@ public class MinioUtil { }); } + /** + * minio拷贝操作 + * + * @Param bucket 桶名 + * @Param sourceFiles 需要复制的标注文件名 + * @Param targetDir 目标文件夹路径 + */ + public void copyObject(String bucket, List sourceFiles, String targetDir) { + CopyConditions copyConditions = new CopyConditions(); + sourceFiles.forEach(sourceFile -> { + try { + String targetName = targetDir + "/" + org.dubhe.utils.StringUtils.substringAfterLast(sourceFile, "/"); + client.copyObject(bucket, targetName, null, null, bucket, sourceFile, null, copyConditions); + } catch (Exception e) { + LogUtil.error(LogEnum.BIZ_DATASET, "MinIO file copy failed, {}", e); + } + }); + } + @Data @Service public class MinioWebTokenBody { @@ -219,9 +268,9 @@ public class MinioUtil { /** * 生成文件下载请求参数方法 * - * @param bucketName 桶名称 - * @param prefix 前缀 - * @param objects 对象名称 + * @param bucketName 桶名称 + * @param prefix 前缀 + * @param objects 对象名称 * @return MinioDownloadDto 下载请求参数 */ public MinioDownloadDto getDownloadParam(String bucketName, String prefix, List objects, String zipName) { diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java index 197d15c..f47f304 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java index 862a97a..d6e031d 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java index 0856532..459b11b 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -111,8 +111,11 @@ public class NfsUtil { stream = new BufferedInputStream(new NfsFileInputStream(nfs3File)); } catch (IOException e) { throw new NfsBizException("nfs获取对象输出流失败！"); + } finally { + nfsPool.revertNfs(nfs3File.getNfs()); } return stream; + } /** diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java index 86c2977..00365d6 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java index 8f791ab..7a2cb7c 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java b/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java index fb04133..fe29ab3 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java index 170a030..d7f59d5 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -172,6 +172,42 @@ public class RedisUtils { } } + /** + * + * @param script 脚本字符串 + * @param key 键 + * @param args 脚本其他参数 + * @return + */ + public Object executeRedisScript(String script, String key, Object... args) { + try { + RedisScript redisScript = new DefaultRedisScript<>(script, Long.class); + redisTemplate.setValueSerializer(new Jackson2JsonRedisSerializer<>(Object.class)); + return redisTemplate.execute(redisScript, Collections.singletonList(key), args); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR, "executeRedisScript script {} key {} expireTime {} args {} error:{}", script, key, args, e); + return MagicNumConstant.ZERO_LONG; + } + } + + /** + * + * @param script 脚本字符串 + * @param key 键 + * @param args 脚本其他参数 + * @return + */ + public Object executeRedisObjectScript(String script, String key, Object... args) { + try { + RedisScript redisScript = new DefaultRedisScript<>(script, Object.class); + redisTemplate.setValueSerializer(new Jackson2JsonRedisSerializer<>(Object.class)); + return redisTemplate.execute(redisScript, Collections.singletonList(key), args); + } catch (Exception e) { + LogUtil.error(LogEnum.SYS_ERR, "executeRedisObjectScript script {} key {} expireTime {} args {} error:{}", script, key, args, e); + return null; + } + } + // ============================String============================= /** @@ -285,24 +321,6 @@ public class RedisUtils { return result != null && result.equals(MagicNumConstant.ONE_LONG); } - /** - * - * @param script 脚本字符串 - * @param key 键 - * @param args 脚本其他参数 - * @return - */ - public Object executeRedisScript(String script, String key, Object... args) { - try { - RedisScript redisScript = new DefaultRedisScript<>(script, Long.class); - redisTemplate.setValueSerializer(new Jackson2JsonRedisSerializer<>(Object.class)); - return redisTemplate.execute(redisScript, Collections.singletonList(key), args); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "executeRedisScript script {} key {} expireTime {} args {} error:{}", script, key, args, e); - return MagicNumConstant.ZERO_LONG; - } - } - // ================================Map================================= /** @@ -575,6 +593,25 @@ public class RedisUtils { } } + /** + * 将zSet数据放入缓存 + * @param key 健 + * @param score 分数 + * @param value 值 + * @return + */ + public Boolean zAdd(String key,Long score,Object value){ + try{ + if (StringUtils.isEmpty(key) || score == null || value == null){ + return false; + } + return redisTemplate.opsForZSet().add(key, value, score); + }catch (Exception e){ + LogUtil.error(LogEnum.SYS_ERR, "RedisUtils zAdd key {} score {} value {} error:{}", key, score, value, e.getMessage(), e); + return false; + } + } + /** * 返回有序集合所有成员，从大到小排序 * @@ -590,6 +627,47 @@ public class RedisUtils { } } + /** + * 弹出有序集合 score 在 [min,max] 内由小到大从 offset 取 count 个 + * @param key 健 + * @param min score最小值 + * @param max score最大值 + * @param offset 起始下标 + * @param count 偏移量 + * @return + */ + public List zRangeByScorePop(String key,double min,double max,long offset,long count){ + try{ + String script = "local elementSet = redis.call('ZRANGEBYSCORE',KEYS[1],ARGV[1],ARGV[2],'LIMIT',ARGV[3],ARGV[4]) if elementSet ~= false and #elementSet ~= 0 then redis.call('ZREM' , KEYS[1] , elementSet[1]) end return elementSet"; + Object result = executeRedisObjectScript(script, key, min, max,offset,count); + return (List) result; + }catch (Exception e){ + LogUtil.error(LogEnum.SYS_ERR, "RedisUtils zRangeByScorePop key {} min {} max {} offset {} count {} error:{}", key,min, max, offset, count, e.getMessage(), e); + return new ArrayList<>(); + } + } + + /** + * 弹出有序集合 score 在 [min,max] 内由小到大从 0 取 1 个 + * @param key 健 + * @param min score最小值 + * @param max score最大值 + * @return + */ + public List zRangeByScorePop(String key,double min, double max){ + return zRangeByScorePop( key,min, max,0,1); + } + + /** + * 弹出有序集合 score 在 [0,max] 内由小到大从 offset 取 count 个 + * @param key 健 + * @param max score最大值 + * @return + */ + public List zRangeByScorePop(String key,double max){ + return zRangeByScorePop( key,0, max,0,1); + } + // ===============================list================================= diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java index 0859db5..3f0aeae 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java index d320c47..a85034e 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,22 +28,26 @@ import java.util.regex.Pattern; */ @Slf4j public class RegexUtil { - private static final String DIGIT = "^[0-9]*$"; + private static final String DIGIT = "^[0-9]*$"; + private static final String FLOAT = "^[-+]?[0-9]*\\.?[0-9]+$"; + /** - * str待匹配文本 - * regex 正则表达式 - *返回str中匹配regex的第一个子串 + * 字符串匹配 + * + * @param str 待匹配文本 + * @param regex 正则表达式 + * @return 返回str中匹配regex的第一个子串 */ - public static String getMatcher(String str,String regex) { - try{ - if (StringUtils.isEmpty(str) || StringUtils.isEmpty(regex)){ + public static String getMatcher(String str, String regex) { + try { + if (StringUtils.isEmpty(str) || StringUtils.isEmpty(regex)) { return ""; } Pattern p = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); Matcher matcher = p.matcher(str); matcher.find(); return matcher.group(); - }catch (IllegalStateException e){ + } catch (IllegalStateException e) { log.error(e.getMessage(), e); return ""; } @@ -51,13 +55,27 @@ public class RegexUtil { /** * 数字匹配 + * * @param str * @return */ - public static boolean isDigits(String str){ - if (StringUtils.isEmpty(str)){ + public static boolean isDigits(String str) { + if (StringUtils.isEmpty(str)) { return false; } return str.matches(DIGIT); } + + /** + * 浮点数匹配 + * + * @param str + * @return + */ + public static boolean isFloat(String str) { + if (StringUtils.isEmpty(str)) { + return false; + } + return str.matches(FLOAT); + } } diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java b/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java index 8bf7444..501f555 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java index d21756d..cbee763 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/StateMachineProxyUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/StateMachineProxyUtil.java new file mode 100644 index 0000000..cd04585 --- /dev/null +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/StateMachineProxyUtil.java @@ -0,0 +1,122 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.utils; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.poi.ss.formula.functions.T; +import org.dubhe.dto.StateChangeDTO; +import org.dubhe.enums.LogEnum; +import org.dubhe.exception.StateMachineException; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import org.springframework.util.ReflectionUtils; + +/** + * @description 代理执行状态机 + * @date 2020-08-27 + */ +@Component +public class StateMachineProxyUtil { + + /** + * 代理执行单个状态机的状态切换 + * + * @param stateChangeDTO 数据集状态切换信息 + * @param objectService 服务类 + */ + public static void proxyExecutionSingleState(StateChangeDTO stateChangeDTO,Object objectService) { + checkSingleParam(stateChangeDTO); + //获取全局状态机中的指定状态机 + Field field = ReflectionUtils.findField(objectService.getClass(), stateChangeDTO.getStateMachineType()); + if (field == null) { + throw new StateMachineException("在全局状态机中未找到指定状态机"); + } + //获取需要执行的状态机对象 + Object stateMachineObject = SpringContextHolder.getBean(field.getName()); + try { + //获取目标执行方法的参数类型 + List> paramTypesList = getMethodParamTypes(stateMachineObject, stateChangeDTO.getEventMethodName()); + //构造目标执行方法 + Method method = ReflectionUtils.findMethod(stateMachineObject.getClass(), stateChangeDTO.getEventMethodName(), paramTypesList.toArray(new Class[paramTypesList.size()])); + if (stateChangeDTO.getObjectParam().length != paramTypesList.size()) { + LogUtil.error(LogEnum.STATE_MACHINE, " 目标执行方法参数 {} 与传入的数量不一致 {} ", paramTypesList.size(), stateChangeDTO.getObjectParam().length); + } else { + ReflectionUtils.invokeMethod(method, stateMachineObject, stateChangeDTO.getObjectParam()); + } + } catch (ClassNotFoundException e) { + LogUtil.error(LogEnum.STATE_MACHINE, "未找到指定类： {} ", e); + } + } + + /** + * 代理执行多个状态机的状态切换 + * + * @param stateChangeDTOList 多个状态机切换信息 + * @param objectService 服务类 + */ + public static void proxyExecutionRelationState(List stateChangeDTOList,Object objectService) { + if (!CollectionUtils.isEmpty(stateChangeDTOList)) { + for (StateChangeDTO stateChangeDTO : stateChangeDTOList) { + proxyExecutionSingleState(stateChangeDTO,objectService); + } + } + } + + /** + * 校验参数是否正常 + * + * @param stateChangeDTO 数据集状态切换信息 + */ + public static void checkSingleParam(StateChangeDTO stateChangeDTO) { + if (StringUtils.isEmpty(stateChangeDTO.getStateMachineType())) { + throw new StateMachineException("未指定状态机类"); + } + if (StringUtils.isEmpty(stateChangeDTO.getEventMethodName())) { + throw new StateMachineException("未指定状态机需要执行事件"); + } + } + + /** + * 根据方法名获取所有参数的类型 + * + * @param classInstance 类实例 + * @param methodName 方法名 + * @return List> 对象集合 + * @throws ClassNotFoundException + */ + public static List> getMethodParamTypes(Object classInstance, String methodName) throws ClassNotFoundException { + List> paramTypes = new ArrayList<>(); + Method[] methods = classInstance.getClass().getMethods(); + for (Method method : methods) { + if (method.getName().equals(methodName)) { + Class[] params = method.getParameterTypes(); + for (Class classParamType : params) { + paramTypes.addAll(Collections.singleton((Class) Class.forName(classParamType.getName()))); + } + break; + } + } + return paramTypes; + } + +} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java index 1253e4f..28c8b87 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,19 +19,23 @@ package org.dubhe.utils; import eu.bitwalker.useragentutils.Browser; import eu.bitwalker.useragentutils.UserAgent; +import org.apache.commons.lang.RandomStringUtils; import org.dubhe.base.MagicNumConstant; +import org.dubhe.constant.NumberConstant; import org.dubhe.constant.SymbolConstant; import javax.servlet.http.HttpServletRequest; import java.net.InetAddress; import java.net.UnknownHostException; +import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; /** - * @description 字符串工具类, 继承org.apache.commons.lang3.StringUtils类 + * @description 字符串工具类, 继承org.apache.commons.lang3.StringUtils类 * @date 2020-03-25 */ public class StringUtils extends org.apache.commons.lang3.StringUtils { @@ -277,10 +281,11 @@ public class StringUtils extends org.apache.commons.lang3.StringUtils { /** * 字符串截取前 + * * @param str * @return */ - public static String substringBefore(String str, String separator){ + public static String substringBefore(String str, String separator) { if (!isEmpty(str) && separator != null) { if (separator.isEmpty()) { @@ -296,10 +301,11 @@ public class StringUtils extends org.apache.commons.lang3.StringUtils { /** * 字符串截取后 + * * @param str * @return */ - public static String substringAfter(String str, String separator){ + public static String substringAfter(String str, String separator) { if (isEmpty(str)) { return str; @@ -310,4 +316,33 @@ public class StringUtils extends org.apache.commons.lang3.StringUtils { return pos == -1 ? "" : str.substring(pos + separator.length()); } } + + /** + * 获取UUID字符串 + * + * @return + */ + public static String getUUID() { + return UUID.randomUUID().toString().replace(SymbolConstant.HYPHEN, SymbolConstant.BLANK); + } + + /** + * 生成4位随机[a-z]字符串 + * + * @return + */ + public static String getRandomString() { + return RandomStringUtils.randomAlphabetic(NumberConstant.NUMBER_4).toLowerCase(); + } + + /** + * 生成时间戳 + 4位随机数 + * + * @return + */ + public static String getTimestamp() { + SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmss"); + String dateStr = df.format(System.currentTimeMillis()); + return dateStr + getRandomString(); + } } diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java index 481f977..6cb0344 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java b/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java index 8ddfd6c..465fe04 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java b/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java index c8b46e7..c6fa6f2 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java index a958000..2a31df0 100644 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java +++ b/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java b/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java index 6a5f963..c6f5c9f 100644 --- a/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java +++ b/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java b/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java index da5488a..61119be 100644 --- a/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java +++ b/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java b/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java index 55823a2..40be6b7 100644 --- a/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java +++ b/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java b/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java index 2e2d785..5264e93 100644 --- a/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java +++ b/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java index b45f320..b308dcc 100644 --- a/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java +++ b/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java index 27bef50..b667dd5 100644 --- a/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java +++ b/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java index ab0ffec..a7369e2 100644 --- a/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java +++ b/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java index 87dae06..9ff3c93 100644 --- a/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java +++ b/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/deploy.sh b/dubhe-server/deploy.sh index 4622654..89d6d9b 100644 --- a/dubhe-server/deploy.sh +++ b/dubhe-server/deploy.sh @@ -22,8 +22,8 @@ usage() { start_application() { echo "starting java process" - echo "nohup java -jar ${JAR_NAME} > ${JAVA_OUT} --spring.profiles.active=${ENV} 2>&1 &" - nohup java -jar ${JAR_NAME} > ${JAVA_OUT} --spring.profiles.active=${ENV} 2>&1 & + echo "nohup java -Dfile.encoding=utf-8 -jar ${JAR_NAME} > ${JAVA_OUT} --spring.profiles.active=${ENV} 2>&1 &" + nohup java -Dfile.encoding=utf-8 -jar ${JAR_NAME} > ${JAVA_OUT} --spring.profiles.active=${ENV} 2>&1 & echo "started java process" } diff --git a/dubhe-server/dubhe-admin/pom.xml b/dubhe-server/dubhe-admin/pom.xml index d563dfe..8a4c76e 100644 --- a/dubhe-server/dubhe-admin/pom.xml +++ b/dubhe-server/dubhe-admin/pom.xml @@ -34,7 +34,7 @@ zhejianglab - dubhe-data + dubhe-model 1.0 @@ -45,7 +45,7 @@ zhejianglab - dubhe-model + dubhe-k8s 1.0 @@ -56,7 +56,18 @@ zhejianglab - dubhe-k8s + dubhe-dcm + 1.0 + + + zhejianglab + common + + + + + zhejianglab + dubhe-serving 1.0 diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java index cdbe26a..58e3188 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/HarborImagePushAsync.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/HarborImagePushAsync.java index 0de73ce..eae4676 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/HarborImagePushAsync.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/HarborImagePushAsync.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -60,7 +60,7 @@ public class HarborImagePushAsync { @Async public void execShell(String imagePath, String imageNameandTag, PtImage ptImage) { try { - String imageResource = trainHarborConfig.getAddress() + StrUtil.SLASH + trainHarborConfig.getModelName() + String imageResource = trainHarborConfig.getAddress() + StrUtil.SLASH + ptImage.getProjectName() + StrUtil.SLASH + imageNameandTag; String cmdStr = "docker login --username=" + trainHarborConfig.getUsername() + " " + trainHarborConfig.getAddress() + " --password=" + trainHarborConfig.getPassword() + " ; docker " + "load < " + imagePath + " |awk '{print $3}' |xargs -I str docker tag str " + imageResource + " ; docker push " + imageResource + "; docker rmi " + imageResource; diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/StopTrainJobAsync.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/StopTrainJobAsync.java index 2c908ed..3bd6783 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/StopTrainJobAsync.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/StopTrainJobAsync.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainAlgorithmUploadAsync.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainAlgorithmUploadAsync.java index e52f688..843ad9a 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainAlgorithmUploadAsync.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainAlgorithmUploadAsync.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainJobAsync.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainJobAsync.java index efca62e..c7952e2 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainJobAsync.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainJobAsync.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,8 +19,8 @@ package org.dubhe.async; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson.JSONObject; import org.dubhe.base.MagicNumConstant; -import org.dubhe.constant.SymbolConstant; import org.dubhe.config.TrainJobConfig; +import org.dubhe.constant.SymbolConstant; import org.dubhe.dao.PtTrainJobMapper; import org.dubhe.domain.dto.BaseTrainJobDTO; import org.dubhe.domain.dto.UserDTO; @@ -28,6 +28,7 @@ import org.dubhe.domain.entity.PtTrainJob; import org.dubhe.domain.vo.PtImageAndAlgorithmVO; import org.dubhe.enums.BizEnum; import org.dubhe.enums.LogEnum; +import org.dubhe.enums.ModelResourceEnum; import org.dubhe.enums.ResourcesPoolTypeEnum; import org.dubhe.enums.TrainJobStatusEnum; import org.dubhe.exception.BusinessException; @@ -39,9 +40,15 @@ import org.dubhe.k8s.domain.bo.PtJupyterJobBO; import org.dubhe.k8s.domain.resource.BizDistributeTrain; import org.dubhe.k8s.domain.resource.BizNamespace; import org.dubhe.k8s.domain.vo.PtJupyterJobVO; -import org.dubhe.utils.*; +import org.dubhe.utils.K8sNameTool; +import org.dubhe.utils.LocalFileUtil; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.NfsUtil; +import org.dubhe.utils.StringUtils; +import org.dubhe.utils.TrainUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; + import java.util.ArrayList; import java.util.List; @@ -76,6 +83,12 @@ public class TrainJobAsync { @Autowired private DistributeTrainApi distributeTrainApi; + public static final String DATASET_VOLUME_MOUNTS = "/dataset"; + + public static final String WORKSPACE_VOLUME_MOUNTS = "/workspace"; + + public static final String MODEL_VOLUME_MOUNTS = "/model"; + /** * 提交分布式训练 @@ -113,7 +126,7 @@ public class TrainJobAsync { } /** - * 构造分布式训练DistributeTrainBO + * 构造分布式训练DistributeTrainBO(炼知模型暂不支持) * * @param baseTrainJobDTO 训练任务信息 * @param currentUser 用户 @@ -171,6 +184,11 @@ public class TrainJobAsync { } // 拼接python固定参数 数据集 sb.append(paramPrefix).append(trainJobConfig.getDockerDataset()); + + // 模型路径挂载及其参数拼接 + DistributeTrainBO distributeTrainBO = new DistributeTrainBO(); + buildBoAboutModel(baseTrainJobDTO, distributeTrainBO, sb); + JSONObject runParams = baseTrainJobDTO.getRunParams(); if (null != runParams && !runParams.isEmpty()) { // 拼接用户自定义参数 @@ -190,7 +208,7 @@ public class TrainJobAsync { + " ' && " + mainCommand + " && echo 'Distribute training mission is over' "; - DistributeTrainBO distributeTrainBO = new DistributeTrainBO() + distributeTrainBO .setNamespace(namespace) .setName(baseTrainJobDTO.getJobName()) .setSize(ptTrainJob.getResourcesPoolNode()) @@ -198,9 +216,9 @@ public class TrainJobAsync { .setMasterCmd(wholeCommand) .setMemNum(baseTrainJobDTO.getMenNum()) .setCpuNum(baseTrainJobDTO.getCpuNum()) - .setDatasetStoragePath(k8sNameTool.getAbsoluteNfsPath(baseTrainJobDTO.getDataSourcePath())) - .setWorkspaceStoragePath(localFileUtil.formatPath(nfsUtil.getNfsConfig().getRootDir() + basePath)) - .setModelStoragePath(k8sNameTool.getAbsoluteNfsPath(relativePath + StrUtil.SLASH + trainJobConfig.getOutPath())) + .putNfsMounts(DATASET_VOLUME_MOUNTS,k8sNameTool.getAbsoluteNfsPath(baseTrainJobDTO.getDataSourcePath())) + .putNfsMounts(WORKSPACE_VOLUME_MOUNTS,localFileUtil.formatPath(nfsUtil.getNfsConfig().getRootDir() + basePath)) + .putNfsMounts(MODEL_VOLUME_MOUNTS,k8sNameTool.getAbsoluteNfsPath(relativePath + StrUtil.SLASH + trainJobConfig.getOutPath())) .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.ALGORITHM)); //延时启动，单位为分钟 if (baseTrainJobDTO.getDelayCreateTime() != null && baseTrainJobDTO.getDelayCreateTime() > 0) { @@ -309,6 +327,7 @@ public class TrainJobAsync { } List list = new ArrayList<>(); + PtJupyterJobBO jobBo = new PtJupyterJobBO(); JSONObject runParams = baseTrainJobDTO.getRunParams(); StringBuilder sb = new StringBuilder(); @@ -336,12 +355,8 @@ public class TrainJobAsync { if (StringUtils.isNotBlank(valDataSourcePath)) { sb.append(pattern).append(trainJobConfig.getLoadValDatasetKey()).append(SymbolConstant.FLAG_EQUAL).append(trainJobConfig.getDockerValDatasetPath()); } - //将模型加载路径拼接到 - String modelLoadPathDir = baseTrainJobDTO.getModelLoadPathDir(); - if (StringUtils.isNotBlank(modelLoadPathDir)) { - //将模型路径model_load_dir路径 - sb.append(pattern).append(trainJobConfig.getLoadKey()).append(SymbolConstant.FLAG_EQUAL).append(trainJobConfig.getDockerModelPath()); - } + //模型路径挂载及其参数拼接 + buildBoAboutModel(baseTrainJobDTO, jobBo, sb); if (null != runParams && !runParams.isEmpty()) { runParams.forEach((k, v) -> @@ -357,20 +372,25 @@ public class TrainJobAsync { list.add("-c"); String workPath = trainJobConfig.getDockerTrainPath() + StrUtil.SLASH + workspaceDir; - String command = "echo 'training mission begins... " + executeCmd + "\r\n '" + - " && cd " + workPath + - " && " + executeCmd + - " && echo 'the training mission is over' "; + String command; + Integer modelResource = baseTrainJobDTO.getModelResource(); + if(null != modelResource && modelResource.intValue() == ModelResourceEnum.ATLAS.getType().intValue()) { + command = "&& " + trainJobConfig.getAtlasAnaconda() + + " && cd " + workPath + + " && " + trainJobConfig.getAtlasPythonioencoding() + executeCmd; + } else { + command = " && cd " + workPath + " && " + executeCmd; + } + command = "echo 'training mission begins... " + executeCmd + "\r\n '" + command + " && echo 'the training mission is over' "; + list.add(command); - PtJupyterJobBO jobBo = new PtJupyterJobBO(); jobBo.setNamespace(namespace) .setName(baseTrainJobDTO.getJobName()) .setImage(ptImageAndAlgorithmVO.getImageName()) .putNfsMounts(trainJobConfig.getDockerDatasetPath(), nfsUtil.getNfsConfig().getRootDir() + nfsUtil.getNfsConfig().getBucket().substring(1) + baseTrainJobDTO.getDataSourcePath()) .setCmdLines(list) .putNfsMounts(trainJobConfig.getDockerTrainPath(), nfsUtil.getNfsConfig().getRootDir() + commonPath.substring(1)) - .putNfsMounts(trainJobConfig.getDockerModelPath(), nfsUtil.formatPath(nfsUtil.getAbsolutePath(modelLoadPathDir))) .putNfsMounts(trainJobConfig.getDockerValDatasetPath(), nfsUtil.formatPath(nfsUtil.getAbsolutePath(valDataSourcePath))) .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.ALGORITHM)); //延时启动，单位为分钟 @@ -390,6 +410,80 @@ public class TrainJobAsync { return jobBo; } + /** + * 模型路径挂载及其参数拼接 + * + * @param baseTrainJobDTO 训练任务基本信息 + * @param jobBo 训练任务实体 + * @param sb 训练命令参数 + */ + private void buildBoAboutModel(BaseTrainJobDTO baseTrainJobDTO, Object jobBo, StringBuilder sb) { + if(null == baseTrainJobDTO.getModelResource()) { + return; + } + String modelLoadPathDir = baseTrainJobDTO.getModelLoadPathDir(); + //非炼知模型 + if (StringUtils.isNotBlank(modelLoadPathDir)) { + //将模型路径model_load_dir路径 + sb.append(trainJobConfig.getPythonFormat()).append(trainJobConfig.getLoadKey()).append(SymbolConstant.FLAG_EQUAL).append(trainJobConfig.getDockerModelPath()); + if(jobBo instanceof PtJupyterJobBO) { + PtJupyterJobBO ptJupyterJobBO = (PtJupyterJobBO)jobBo; + ptJupyterJobBO.putNfsMounts(trainJobConfig.getDockerModelPath(), nfsUtil.formatPath(nfsUtil.getAbsolutePath(modelLoadPathDir))); + } else if(jobBo instanceof DistributeTrainBO) { + DistributeTrainBO distributeTrainBO = (DistributeTrainBO)jobBo; + distributeTrainBO.putNfsMounts(trainJobConfig.getDockerModelPath(), nfsUtil.formatPath(nfsUtil.getAbsolutePath(modelLoadPathDir))); + } + return; + } + //炼知模型中的教师模型 + appendAtlasModelPath(baseTrainJobDTO.getTeacherModelPathList(), jobBo, sb, true); + //炼知模型中的学生模型 + appendAtlasModelPath(baseTrainJobDTO.getStudentModelPathList(), jobBo, sb, false); + } + + /** + * 炼知模型路径挂载及其参数拼接 + * + * @param modelPathList 模型路径集合 + * @param jobBo 训练任务实体 + * @param sb 训练命令参数 + * @param isTeacher 是否教师模型 + */ + private void appendAtlasModelPath(List modelPathList, Object jobBo, StringBuilder sb, boolean isTeacher) { + if(null == modelPathList || modelPathList.isEmpty()) { + return; + } + StringBuilder appendModelPath = new StringBuilder(); + String preModelKey; + String preModelPath; + if(isTeacher){ + preModelKey = trainJobConfig.getDockerTeacherModelKey(); + preModelPath = trainJobConfig.getDockerTeacherModelPath(); + } else { + preModelKey = trainJobConfig.getDockerStudentModelKey(); + preModelPath = trainJobConfig.getDockerStudentModelPath(); + } + modelPathList.stream() + .forEach(modelPath -> { + String[] urlArray = modelPath.split(SymbolConstant.SLASH); + String dockerModelPath = urlArray[urlArray.length - TrainUtil.NUMBER_ONE]; + String mountPath = preModelPath + SymbolConstant.SLASH + dockerModelPath; + appendModelPath.append(mountPath).append(SymbolConstant.COMMA); + if(jobBo instanceof PtJupyterJobBO) { + PtJupyterJobBO ptJupyterJobBO = (PtJupyterJobBO)jobBo; + ptJupyterJobBO.putNfsMounts(mountPath, nfsUtil.formatPath(nfsUtil.getAbsolutePath(modelPath))); + } else if(jobBo instanceof DistributeTrainBO) { + DistributeTrainBO distributeTrainBO = (DistributeTrainBO)jobBo; + distributeTrainBO.putNfsMounts(mountPath, nfsUtil.formatPath(nfsUtil.getAbsolutePath(modelPath))); + } + }); + String resultPath = SymbolConstant.MARK + + appendModelPath.toString().substring(TrainUtil.NUMBER_ZERO, appendModelPath.toString().length() - TrainUtil.NUMBER_ONE) + + SymbolConstant.MARK; + + sb.append(trainJobConfig.getPythonFormat()).append(preModelKey).append(SymbolConstant.FLAG_EQUAL).append(resultPath); + } + /** * 训练任务异步处理更新训练状态 * diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TransactionAsyncManager.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TransactionAsyncManager.java index 053df6b..b7e4fa5 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TransactionAsyncManager.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TransactionAsyncManager.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/GlobalFilter.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/GlobalFilter.java index 21342f2..9d14108 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/GlobalFilter.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/GlobalFilter.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/RequestBodyWrapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/RequestBodyWrapper.java index f041be7..954f620 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/RequestBodyWrapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/RequestBodyWrapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/ResponseBodyWrapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/ResponseBodyWrapper.java index dc0bc3e..6dc8eba 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/ResponseBodyWrapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/ResponseBodyWrapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java index baaf923..679a027 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java new file mode 100644 index 0000000..13c2997 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.dubhe.domain.entity.ModelOptBuiltIn; +import org.dubhe.domain.vo.ModelOptAlgorithmQueryVO; +import org.dubhe.domain.vo.ModelOptDatasetQueryVO; +import org.dubhe.domain.vo.ModelOptModelQueryVO; + +import java.util.List; + +/** + * @description 内置模型 + * @date 2020-05-22 + */ +public interface ModelOptBuiltInMapper extends BaseMapper { + /** + * 获取模型优化数据集 + * @param type 算法类型 + * @return 模型优化数据集列表 + */ + List getDataset(@Param("type") Integer type, @Param("model") String model, @Param("algorithm") String algorithm); + + /** + * 获取模型优化教师模型 + * @param type + * @return + */ + List getModel(@Param("type") Integer type, @Param("dataset") String dataset, @Param("algorithm") String algorithm); + + /** + * 获取模型优化算法 + * @param type + * @return + */ + List getAlgorithm(@Param("type") Integer type, @Param("model") String model, @Param("dataset") String dataset); + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java new file mode 100644 index 0000000..31d7799 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Select; +import org.dubhe.domain.entity.ModelOptDataset; + +import java.util.List; + +/** + * @description 模型优化数据集 + * @date 2021-01-06 + */ +public interface ModelOptDatasetMapper extends BaseMapper { + + @Select("select id, name, path from model_opt_dataset where deleted = 0 order by id desc") + List getAllDataset(); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java new file mode 100644 index 0000000..c49c1c7 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; +import org.dubhe.domain.entity.ModelOptTaskInstance; + +import java.util.List; + +/** + * @description 模型优化实例 + * @date 2020-05-22 + */ +public interface ModelOptTaskInstanceMapper extends BaseMapper { + + /** + * 将实例状态标记为执行失败 + * + * @param instId 实例id + * @return 更新行数 + */ + @Update("update model_opt_task_instance set status = '3' where id = #{instId} and status in ('-1','0')") + Integer markInstanceExecFailed(Long instId); + + /** + * 获取5分钟以前状态仍为等待中的实例 + * + * @return 实例列表 + */ + @Select("select * from model_opt_task_instance where status = '-1' and create_time < date_sub(now(), interval 5 minute)") + List selectWaitingFor5MinutesInstances(); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java new file mode 100644 index 0000000..9ed43a4 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java @@ -0,0 +1,28 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.dubhe.domain.entity.ModelOptTask; + +/** + * @description 模型优化任务 + * @date 2020-05-22 + */ +public interface ModelOptTaskMapper extends BaseMapper { +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelQueryMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelQueryMapper.java deleted file mode 100644 index 0e1003f..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelQueryMapper.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.dao; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.dubhe.domain.dto.ModelQueryDTO; -import org.dubhe.domain.entity.ModelQuery; -import org.dubhe.domain.entity.ModelQueryBrance; - -/** - * @description model mapper - * @date 2020-10-09 - */ -public interface ModelQueryMapper extends BaseMapper { - /** - * 根据modelId查询模型信息 - * - * @param modelId 模型id - * @return modelQuery返回查询的模型对象 - */ - @Select("select name,url from pt_model_info where id=#{modelId}") - ModelQuery findModelNameById(@Param("modelId") Integer modelId); - - /** - * 根据模型路径查询模型版本信息 - * - * @param modelLoadPathDir 模型路径 - * @return ModelQueryBrance 模型版本信息 - */ - @Select("select version from pt_model_branch where url=#{modelLoadPathDir}") - ModelQueryBrance findModelVersionByUrl(@Param("modelLoadPathDir") String modelLoadPathDir); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java index 1a95d68..5c1d1a6 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java index 4b0281f..280f68b 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java index 974a03f..0136d8e 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,9 +19,12 @@ package org.dubhe.dao; import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Select; import org.dubhe.annotation.DataPermission; import org.dubhe.domain.entity.PtImage; +import java.util.List; + /** * @description 镜像 Mapper 接口 * @date 2020-04-27 diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java index ea05002..094b95e 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtMeasureMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtMeasureMapper.java new file mode 100644 index 0000000..b716125 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtMeasureMapper.java @@ -0,0 +1,32 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.dao; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.dubhe.annotation.DataPermission; +import org.dubhe.domain.entity.PtMeasure; + +/** + * @description 度量 Mapper 接口 + * @date 2020-11-16 + */ +@DataPermission(ignoresMethod = {"insert"}) +public interface PtMeasureMapper extends BaseMapper { + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java index 8a4d571..d22ce96 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java index de786d3..fa7d656 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java index 08c992c..290db66 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java index 886b966..a5dd5ec 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java index 3e8f025..7e043d4 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java index ff08d99..2a9efd1 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java index 21ee36b..354c8e9 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ import lombok.experimental.Accessors; import org.dubhe.domain.entity.PtTrainJobSpecs; import java.io.Serializable; +import java.util.List; /** * @description 创建训练任务的数据包 @@ -42,6 +43,14 @@ public class BaseTrainJobDTO implements Serializable { private String visualizedLogPath; private Integer delayCreateTime; private Integer delayDeleteTime; + private String modelPath; + private List teacherModelPathList; + private List studentModelPathList; + private Long modelId; + private Long modelBranchId; + private Integer modelResource; + private String teacherModelIds; + private String studentModelIds; /** * @return 每个节点的GPU数量 diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainParamDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainParamDTO.java new file mode 100644 index 0000000..a3fb87e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainParamDTO.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; + +/** + * @description 创建训练模板的数据包 + * @date 2020-07-15 + */ +@Data +@Accessors(chain = true) +public class BaseTrainParamDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + private Long modelId; + private Long modelBranchId; + private Integer modelResource; + private String teacherModelIds; + private String studentModelIds; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptDatasetCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptDatasetCreateDTO.java new file mode 100644 index 0000000..1a5e6e8 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptDatasetCreateDTO.java @@ -0,0 +1,25 @@ +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotEmpty; +import java.io.Serializable; + +/** + * @description 模型优化模块我的数据集创建参数 + * @date 2021-01-06 + */ +@Data +public class ModelOptDatasetCreateDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "名称", required = true) + @NotEmpty(message = "数据集名称不能为空") + private String name; + + @ApiModelProperty(value = "路径", required = true) + @NotEmpty(message = "数据集路径不能为空") + private String path; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java new file mode 100644 index 0000000..17532f4 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java @@ -0,0 +1,47 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + * @description 模型优化日志查询参数 + * @date 2020-05-29 + */ +@Data +public class ModelOptLogQueryDTO { + + @ApiModelProperty("模型优化实例ID") + @NotNull(message = "模型优化实例ID") + private Long instId; + + @ApiModelProperty(value = "起始行") + @Min(value = 0, message = "startLine不能小于0") + private Integer startLine; + + @ApiModelProperty(value = "行数") + @Min(value = 1, message = "lines不能小于1") + @Max(value = 1000, message = "lines不能大于1000") + private Integer lines; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java new file mode 100644 index 0000000..d4e8751 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java @@ -0,0 +1,97 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; +import javax.validation.constraints.Size; +import java.io.Serializable; + +/** + * @description 创建模型优化任务 + * @date 2020-05-22 + */ +@Data +@Accessors(chain = true) +public class ModelOptTaskCreateDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "任务名称", required = true) + @NotEmpty(message = "任务名称不能为空") + @Size(max = 32, message = "任务名称长度不能超过32") + @Pattern(regexp = TrainUtil.REGEXP, message = "任务名称仅支持字母、数字、汉字、英文横杠和下划线") + private String name; + + @ApiModelProperty(value = "任务描述") + @Size(max = 500, message = "任务描述长度不能超过500") + private String description; + + @ApiModelProperty(value = "是否内置", required = true) + @NotNull(message = "请选择是否内置") + private Boolean isBuiltIn; + + @ApiModelProperty(value = "数据集id") + private Long datasetId; + + @ApiModelProperty(value = "数据集名称", required = true) + @NotEmpty(message = "数据集不能为空") + private String datasetName; + + @ApiModelProperty(value = "数据集路径", required = true) + @NotEmpty(message = "数据集路径为空") + private String datasetPath; + + @ApiModelProperty(value = "模型id") + private Long modelId; + + @ApiModelProperty(value = "模型名称") + private String modelName; + + @ApiModelProperty(value = "模型路径") + private String modelAddress; + + @ApiModelProperty(value = "优化算法类别") + private Integer algorithmType; + + @ApiModelProperty(value = "优化算法id") + private Long algorithmId; + + @ApiModelProperty(value = "是否编辑") + private Boolean editAlgorithm ; + + @ApiModelProperty(value = "优化算法", required = true) + @NotEmpty(message = "请选择优化算法") + private String algorithmName; + + @ApiModelProperty(value = "算法路径", required = true) + @NotEmpty(message = "算法路径不能为空") + private String algorithmPath; + + @ApiModelProperty(value = "运行命令") + private String command; + + @ApiModelProperty(value = "运行参数") + private JSONObject params; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java new file mode 100644 index 0000000..f25cdc0 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 删除模型优化任务参数 + * @date 2020-05-22 + */ +@Data +public class ModelOptTaskDeleteDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为空") + private Long id; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java new file mode 100644 index 0000000..34599be --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 取消模型优化任务参数 + * @date 2020-06-29 + */ +@Data +public class ModelOptTaskInstanceCancelDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "实例id不能为空") + private Long id; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java new file mode 100644 index 0000000..528c288 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 删除任务实例对象 + * @date 2021-01-07 + */ +@Data +public class ModelOptTaskInstanceDeleteDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "实例ID") + @NotNull(message = "实例ID不能为空") + private Long id; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java new file mode 100644 index 0000000..4e7b570 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 查看任务实例详情参数 + * @date 2020-06-29 + */ +@Data +public class ModelOptTaskInstanceDetailDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "实例id不能为空") + private Long id; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java new file mode 100644 index 0000000..ce556ee --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java @@ -0,0 +1,62 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.dubhe.annotation.Query; +import org.dubhe.base.PageQueryBase; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.List; + +/** + * @description 模型优化任务实例查询条件 + * @date 2020-05-22 + */ +@Data +public class ModelOptTaskInstanceQueryDTO extends PageQueryBase implements Serializable { + @ApiModelProperty("任务ID") + @Query(propName = "task_id") + private Long taskId; + /** + * 任务状态 + */ + @ApiModelProperty("任务状态") + @Query + private String status; + /** + * 优化算法 + */ + @ApiModelProperty("优化算法") + @Query(propName = "algorithm_name") + private String algorithmName; + /** + * 提交时间 + */ + @ApiModelProperty("提交时间") + @Query(propName = "create_time", type = Query.Type.BETWEEN) + private List createTime; + /** + * 完成时间 + */ + @ApiModelProperty("完成时间") + @Query(propName = "end_time", type = Query.Type.BETWEEN) + private List endTime; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java new file mode 100644 index 0000000..1bbe465 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 重新提交任务实例参数 + * @date 2020-05-22 + */ +@Data +public class ModelOptTaskInstanceResubmitDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "实例id不能为空") + private Long id; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java new file mode 100644 index 0000000..85fe54a --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java @@ -0,0 +1,48 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.annotation.Query; +import org.dubhe.base.PageQueryBase; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.List; + +/** + * @description 模型优化任务查询条件 + * @date 2020-05-22 + */ +@Data +@Accessors(chain = true) +public class ModelOptTaskQueryDTO extends PageQueryBase implements Serializable { + @ApiModelProperty("任务id") + @Query(propName = "id") + private Long id; + + @Query(type = Query.Type.LIKE) + @ApiModelProperty("任务名称或id") + private String name; + + @ApiModelProperty("更新时间") + @Query(type = Query.Type.BETWEEN, propName = "update_time") + private List updateTime; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java new file mode 100644 index 0000000..e0bce93 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import lombok.Data; +import lombok.experimental.Accessors; + +/** + * @description 模型优化创建资源运行的参数类 + * @date 2020-05-25 + */ +@Data +@Accessors(chain = true) +public class ModelOptTaskRunParamDTO { + + /** + * 实例id + **/ + private Long instId; + /** + * 使用的镜像 + **/ + private String imageUrl; + /** + * 输入目录 + **/ + private String inputDir; + /** + * 输出目录 + **/ + private String outputDir; + /** + * 算法目录 + **/ + private String algorithmDir; + /** + * 日志目录 + **/ + private String logDir; + /** + * 数据集路径 + **/ + private String datasetDir; + /** + * 压缩前结果json目录 + **/ + private String resultJsonBeforeDir; + /** + * 压缩后结果json目录 + **/ + private String resultJsonAfterDir; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java new file mode 100644 index 0000000..2d20c22 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 模型优化任务提交参数 + * @date 2020-05-22 + */ +@Data +public class ModelOptTaskSubmitDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为空") + private Long id; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java new file mode 100644 index 0000000..84058d8 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java @@ -0,0 +1,100 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; +import javax.validation.constraints.Size; +import java.io.Serializable; + +/** + * @description 修改模型优化任务 + * @date 2020-05-22 + */ +@Data +@Accessors(chain = true) +public class ModelOptTaskUpdateDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "id", required = true) + @NotNull(message = "id不能为空") + private Long id; + + @ApiModelProperty(value = "任务名称", required = true) + @NotEmpty(message = "任务名称不能为空") + @Size(max = 32, message = "任务名称长度不能超过32") + @Pattern(regexp = "^[a-zA-Z0-9\\-\\_\\u4e00-\\u9fa5]+$", message = "任务名称仅支持字母、数字、汉字、英文横杠和下划线") + private String name; + + @ApiModelProperty(value = "任务描述") + @Size(max = 500, message = "任务描述长度不能超过500") + private String description; + + @ApiModelProperty(value = "是否内置", required = true) + @NotNull(message = "请选择是否内置") + private Boolean isBuiltIn; + + @ApiModelProperty(value = "数据集id") + private Long datasetId; + + @ApiModelProperty(value = "数据集名称", required = true) + @NotEmpty(message = "数据集不能为空") + private String datasetName; + + @ApiModelProperty(value = "数据集路径", required = true) + @NotEmpty(message = "数据集路径为空") + private String datasetPath; + + @ApiModelProperty(value = "模型id") + private Long modelId; + + @ApiModelProperty(value = "模型名称") + private String modelName; + + @ApiModelProperty(value = "模型路径") + private String modelAddress; + + @ApiModelProperty(value = "优化算法类型") + private Integer algorithmType; + + @ApiModelProperty(value = "优化算法id") + private Long algorithmId; + + @ApiModelProperty(value = "优化算法", required = true) + @NotEmpty(message = "请选择优化算法") + private String algorithmName; + + @ApiModelProperty(value = "算法路径", required = true) + @NotEmpty(message = "算法路径不能为空") + private String algorithmPath; + + @ApiModelProperty(value = "是否编辑") + private Boolean editAlgorithm ; + + @ApiModelProperty(value = "运行命令") + private String command; + + @ApiModelProperty(value = "运行参数") + private JSONObject params; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelQueryDTO.java index a10256c..c03e128 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java index 29d4b94..0f38743 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java index f388fd8..05457b5 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java index faa1a38..2543fb8 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java index 196a973..228a01e 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java index 76b06fa..36fe478 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java index 3f5b30b..397579c 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java index ea6cf40..3309b18 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java index 46973ec..3d7f1bf 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java index 94adff4..3856980 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDeleteDTO.java index d332df4..6faa8d8 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDeleteDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDeleteDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java index 17cc7ce..a413e01 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package org.dubhe.domain.dto; import io.swagger.annotations.ApiModelProperty; import lombok.Data; import lombok.experimental.Accessors; +import org.dubhe.annotation.FlagValidator; import org.dubhe.base.PageQueryBase; import java.io.Serializable; @@ -34,6 +35,7 @@ public class PtImageQueryDTO extends PageQueryBase implements Serializable { private static final long serialVersionUID = 1L; + @ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") private Integer imageResource; @@ -43,4 +45,8 @@ public class PtImageQueryDTO extends PageQueryBase implements Serializable { @ApiModelProperty(value = "镜像名称或id") private String imageNameOrId; + @ApiModelProperty(value = "镜像项目类型(0:notebook , 1:train)") + @FlagValidator(value = {"0", "1"}, message = "镜像项目类型不支持") + private Integer projectType; + } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryUrlDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryUrlDTO.java new file mode 100644 index 0000000..6e46601 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryUrlDTO.java @@ -0,0 +1,47 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.dubhe.annotation.FlagValidator; + +/** + * @description 查询镜像路径 + * @date 2020-12-14 + */ +@Data +@Accessors(chain = true) +@EqualsAndHashCode +public class PtImageQueryUrlDTO { + + @ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") + private Integer imageResource; + + @ApiModelProperty(value = "镜像名称") + private String imageName; + + @ApiModelProperty(value = "镜像标签") + private String iamgeTag; + + @ApiModelProperty(value = "镜像项目类型(0:notebook , 1:train)") + @FlagValidator(value = {"0", "1"}, message = "镜像项目类型不支持") + private Integer projectType; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java index 8d3c7a1..7aaa3d9 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUpdateDTO.java index 6f08648..047168e 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUpdateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUpdateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java index 970186b..f7ccb0f 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import org.dubhe.utils.TrainUtil; import org.hibernate.validator.constraints.Length; import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; import javax.validation.constraints.Pattern; import java.io.Serializable; @@ -38,6 +39,9 @@ public class PtImageUploadDTO implements Serializable { private static final long serialVersionUID = 1L; + @ApiModelProperty(value = "镜像项目类型", required = true) + @NotNull(message = "镜像项目类型不能为空") + private Integer projectType; @ApiModelProperty(value = "镜像文件路径", required = true) @NotBlank(message = "镜像文件路径不能为空") diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDTO.java new file mode 100644 index 0000000..83d1e57 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDTO.java @@ -0,0 +1,52 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.NotBlank; +import java.io.Serializable; + +/** + * @description 度量管理入参DTO + * @date 2020-11-16 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PtMeasureDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "度量名称", required = true) + @NotBlank(message = "度量名称不能为空") + private String name; + + @ApiModelProperty("度量描述(选填)") + private String description; + + @ApiModelProperty(value = "度量文件路径", required = true) + @NotBlank(message = "度量文件路径不能为空") + private String url; + + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDeleteDTO.java new file mode 100644 index 0000000..843891c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDeleteDTO.java @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; +import java.util.Set; + +/** + * @description 删除度量信息入参DTO + * @date 2020-11-16 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PtMeasureDeleteDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "参数id,参数id不能小于1", required = true) + @NotNull(message = "度量id不能为空") + private Set ids; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureQueryDTO.java new file mode 100644 index 0000000..0ba9e35 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureQueryDTO.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.dubhe.base.PageQueryBase; + +import java.io.Serializable; + +/** + * @description 度量管理查询接口入参DTO + * @date 2020-11-16 + */ +@Data +public class PtMeasureQueryDTO extends PageQueryBase implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "度量名称", required = true) + private String nameOrId; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureUpdateDTO.java new file mode 100644 index 0000000..6b41b2e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureUpdateDTO.java @@ -0,0 +1,60 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.dubhe.utils.TrainUtil; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import java.io.Serializable; + +/** + * @description 修改度量信息入参DTO + * @date 2020-11-16 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PtMeasureUpdateDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "参数id,参数id不能小于1", required = true) + @NotNull(message = "度量id不能为空") + @Min(value = TrainUtil.NUMBER_ONE, message = "度量参数id不能小于1") + private Long id; + + @ApiModelProperty(value = "度量名称", required = true) + @NotBlank(message = "度量名称不能为空") + private String name; + + @ApiModelProperty("度量描述(选填)") + private String description; + + @ApiModelProperty(value = "度量文件路径", required = true) + @NotBlank(message = "度量文件路径不能为空") + private String url; + + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java new file mode 100644 index 0000000..040ab1a --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; +import java.io.Serializable; + +/** + * @description 模型优化上传算法入参 + * @date 2021-01-06 + */ +@Data +@Accessors(chain = true) +public class PtModelAlgorithmCreateDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "算法名称，输入长度不能超过32个字符", required = true) + @NotBlank(message = "算法名称不能为空") + @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "算法名称-输入长度不能超过32个字符") + @Pattern(regexp = TrainUtil.REGEXP, message = "算法名称支持字母、数字、汉字、英文横杠和下划线") + private String name; + + @ApiModelProperty(value = "代码目录（路径规则：/algorithm-manage/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/用户上传的算法具体文件(zip文件）名称或从notebook跳转时为/notebook/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/）", required = true) + @NotBlank(message = "代码目录不能为空") + @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "代码目录-输入长度不能超过128个字符") + private String path; + +} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java index 4544556..d6f4b66 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java index 2370ef2..d9f6eea 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java index 08daa8d..8768150 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java index 21bec33..8f00024 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java index 9c83be2..06785dc 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java index 42e6a84..aff4a97 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java index e2e3abd..3d96b15 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java index f3612f1..f85cd22 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java index 6c12f7e..2b8b19a 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java index 5d43e8e..d586e04 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java index cabc883..f7c85a4 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,10 +23,15 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; import org.dubhe.base.BaseImageDTO; +import org.dubhe.utils.PtModelUtil; import org.dubhe.utils.TrainUtil; import org.hibernate.validator.constraints.Length; -import javax.validation.constraints.*; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; /** * @description 创建训练任务 @@ -132,18 +137,27 @@ public class PtTrainJobCreateDTO extends BaseImageDTO { @ApiModelProperty(value = "训练信息(失败信息)") private String trainMsg; - @ApiModelProperty(value = "是否打开模型选择") - private Integer modelType; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型)") + @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)" + + "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") + @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") + @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") private Integer modelResource; @ApiModelProperty(value = "模型id") - private Integer modelId; - - @ApiModelProperty(value = "模型路径") - private String modelLoadPathDir; - - @ApiModelProperty(value = "模型名称") - private String modelName; + @Min(value = TrainUtil.NUMBER_ONE, message = "模型id必须大于1") + private Long modelId; + + @ApiModelProperty(value = "我的模型版本对应的id") + @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") + private Long modelBranchId; + + @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") + private String teacherModelIds; + + @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") + private String studentModelIds; } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java index 064ef8a..5ce65f6 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java index 6bfdc09..371060b 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java index ba044de..fba7474 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java index ab6e87c..a494745 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java index f217fbb..d041954 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java index c953cdf..4b52592 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java index 6b5e606..f7570dc 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; import org.dubhe.base.BaseImageDTO; +import org.dubhe.utils.PtModelUtil; import org.dubhe.utils.TrainUtil; import org.hibernate.validator.constraints.Length; @@ -30,6 +31,7 @@ import javax.validation.constraints.Max; import javax.validation.constraints.Min; import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; /** * @description 训练任务修改 @@ -117,19 +119,27 @@ public class PtTrainJobUpdateDTO extends BaseImageDTO { @Max(value = TrainUtil.NUMBER_ONE_HUNDRED_AND_SIXTY_EIGHT, message = "训练自动停止时长不能大于168小时即时长不能超过一周（7*24小时）") private Integer delayDeleteTime; - @ApiModelProperty(value = "是否打开模型选择") - private Integer modelType; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型)") + @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)" + + "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") + @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") + @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") private Integer modelResource; - @ApiModelProperty(value = "模型名称") - private String modelName; + @ApiModelProperty(value = "模型id") + @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") + private Long modelId; - @ApiModelProperty(value = "模型加载路径") - private String modelLoadPathDir; + @ApiModelProperty(value = "我的模型版本对应的id") + @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") + private Long modelBranchId; - @ApiModelProperty(value = "模型id") - private Integer modelId; + @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") + private String teacherModelIds; + @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") + private String studentModelIds; } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java index cbe33b1..d3e93b1 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java index 3be8dec..776a75d 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainModelDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainModelDTO.java new file mode 100644 index 0000000..56e1b31 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainModelDTO.java @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.utils.PtModelUtil; +import org.dubhe.utils.TrainUtil; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; +import java.io.Serializable; + +/** + * @description 训练使用模型 + * @date 2020-11-23 + */ +@Data +@Accessors(chain = true) +public class PtTrainModelDTO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("非炼知模型id") + @Min(value = TrainUtil.NUMBER_ONE, message = "模型id必须大于1") + private Long modelId; + + @ApiModelProperty(value = "我的模型版本对应的id") + @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") + private Long modelBranchId; + + @ApiModelProperty("模型是否为预置模型（0默认模型，1预置模型, 2炼知模型" + + "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") + @NotNull(message = "模型类型不能为空") + @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") + @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") + private Integer modelResource; + + @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") + private String teacherModelIds; + + @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") + private String studentModelIds; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java index 1b399f7..0169e00 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,10 +23,15 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; import org.dubhe.base.BaseImageDTO; +import org.dubhe.utils.PtModelUtil; import org.dubhe.utils.TrainUtil; import org.hibernate.validator.constraints.Length; -import javax.validation.constraints.*; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; /** * @description 任务参数创建条件 @@ -89,21 +94,6 @@ public class PtTrainParamCreateDTO extends BaseImageDTO { @ApiModelProperty("是否验证数据集") private Integer valType; - @ApiModelProperty(value = "是否打开模型选择") - private Integer modelType; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型)") - private Integer modelResource; - - @ApiModelProperty(value = "模型名称") - private String modelName; - - @ApiModelProperty(value = "模型加载路径") - private String modelLoadPathDir; - - @ApiModelProperty(value = "模型id") - private Integer modelId; - @ApiModelProperty(value = "训练类型 0：普通训练，1：分布式训练", required = true) @Min(value = TrainUtil.NUMBER_ZERO, message = "训练类型错误") @Max(value = TrainUtil.NUMBER_ONE, message = "训练类型错误") @@ -116,4 +106,28 @@ public class PtTrainParamCreateDTO extends BaseImageDTO { @NotNull(message = "节点个数") private Integer resourcesPoolNode; + @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)" + + "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") + @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") + @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") + private Integer modelResource; + + @ApiModelProperty(value = "模型id") + @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") + private Long modelId; + + @ApiModelProperty(value = "我的模型版本对应的id") + @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") + private Long modelBranchId; + + @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") + private String teacherModelIds; + + @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") + private String studentModelIds; + } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java index b0635f2..edb82e7 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java index c470f37..4a3cf22 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java index 96506fe..87a56ca 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,10 +23,15 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.Accessors; import org.dubhe.base.BaseImageDTO; +import org.dubhe.utils.PtModelUtil; import org.dubhe.utils.TrainUtil; import org.hibernate.validator.constraints.Length; -import javax.validation.constraints.*; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; /** * @description 任务参数修改条件 @@ -94,21 +99,6 @@ public class PtTrainParamUpdateDTO extends BaseImageDTO { @ApiModelProperty("是否验证数据集") private Integer valType; - @ApiModelProperty(value = "是否打开模型选择") - private Integer modelType; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型)") - private Integer modelResource; - - @ApiModelProperty(value = "模型名称") - private String modelName; - - @ApiModelProperty(value = "模型加载路径") - private String modelLoadPathDir; - - @ApiModelProperty(value = "模型id") - private Integer modelId; - @ApiModelProperty(value = "训练类型 0：普通训练，1：分布式训练") @Min(value = TrainUtil.NUMBER_ZERO, message = "训练类型错误") @Max(value = TrainUtil.NUMBER_ONE, message = "训练类型错误") @@ -119,4 +109,27 @@ public class PtTrainParamUpdateDTO extends BaseImageDTO { @Max(value = TrainUtil.NUMBER_EIGHT, message = "节点个数在1~8之间") private Integer resourcesPoolNode; + @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)" + + "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") + @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") + @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") + private Integer modelResource; + + @ApiModelProperty(value = "模型id") + @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") + private Long modelId; + + @ApiModelProperty(value = "我的模型版本对应的id") + @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") + private Long modelBranchId; + + @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") + private String teacherModelIds; + + @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") + @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") + @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") + private String studentModelIds; } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java index f50746d..f1016ad 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java index eff7890..a966041 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java new file mode 100644 index 0000000..d581346 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import org.dubhe.base.BaseEntity; + +/** + * @description 内置模型 + * @date 2020-05-22 + */ +@Data +@TableName("model_opt_built_in") +public class ModelOptBuiltIn extends BaseEntity { + /** + * 主键 + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + /** + * 算法类型：0-剪枝，1-蒸馏，2-量化 + */ + @TableField("type") + private Integer type; + /** + * 算法名称 + */ + @TableField("algorithm") + private String algorithm; + /** + * 算法路径 + */ + @TableField("algorithm_path") + private String algorithmPath; + /** + * 数据集名称 + */ + @TableField("dataset") + private String dataset; + /** + * 数据集路径 + */ + @TableField("dataset_path") + private String datasetPath; + /** + * 模型名称 + */ + @TableField("model") + private String model; + /** + * 模型路径 + */ + @TableField(value = "model_address") + private String modelAddress; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java new file mode 100644 index 0000000..3ad3024 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java @@ -0,0 +1,52 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import org.dubhe.base.BaseEntity; + +/** + * @description 模型优化数据集 + * @date 2021-01-06 + */ +@Data +@TableName("model_opt_dataset") +public class ModelOptDataset extends BaseEntity { + + /** + * 主键 + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 数据集名称 + */ + @TableField(value = "name") + private String name; + + /** + * 数据集路径 + */ + @TableField(value = "path") + private String path; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java new file mode 100644 index 0000000..6a95bf4 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java @@ -0,0 +1,123 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.bean.copier.CopyOptions; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; +import lombok.Data; +import org.dubhe.base.BaseEntity; + +/** + * @description 模型优化任务 + * @date 2020-05-22 + */ +@Data +@TableName(value = "model_opt_task", autoResultMap = true) +public class ModelOptTask extends BaseEntity { + + /** + * 主键 + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + /** + * 任务名称 + */ + @TableField(value = "name") + private String name; + /** + * 任务描述 + */ + @TableField(value = "description") + private String description; + /** + * 是否内置 + */ + @TableField(value = "is_built_in") + private Boolean isBuiltIn; + /** + * 模型名称 + */ + @TableField(value = "model_name") + private String modelName; + /** + * 模型id + */ + @TableField(value = "model_id") + private Long modelId; + /** + * 模型路径 + */ + @TableField(value = "model_address") + private String modelAddress; + /** + * 算法选择类型 + */ + @TableField(value = "algorithm_type") + private Integer algorithmType; + /** + * 优化算法id + */ + @TableField(value = "algorithm_id") + private Long algorithmId; + /** + * 优化算法 + */ + @TableField(value = "algorithm_name") + private String algorithmName; + /** + * 算法路径 + */ + @TableField(value = "algorithm_path") + private String algorithmPath; + /** + * 数据集id + */ + @TableField(value = "dataset_id") + private Long datasetId; + /** + * 数据集名称 + */ + @TableField(value = "dataset_name") + private String datasetName; + /** + * 数据集路径 + */ + @TableField(value = "dataset_path") + private String datasetPath; + /** + * 运行命令 + */ + @TableField(value = "command") + private String command; + /** + * 运行参数 + */ + @TableField(value = "params", typeHandler = FastjsonTypeHandler.class) + private JSONObject params; + + public void copy(ModelOptTask source) { + BeanUtil.copyProperties(source, this, CopyOptions.create().setIgnoreNullValue(true)); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java new file mode 100644 index 0000000..fa66abb --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java @@ -0,0 +1,173 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.entity; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.bean.copier.CopyOptions; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; +import lombok.Data; +import lombok.experimental.Accessors; +import org.dubhe.base.BaseEntity; + +import java.sql.Timestamp; + +/** + * @description 模型优化任务实例 + * @date 2020-05-22 + */ +@Data +@TableName(value = "model_opt_task_instance", autoResultMap = true) +@Accessors(chain = true) +public class ModelOptTaskInstance extends BaseEntity { + /** + * 主键 + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + /** + * 任务id + */ + @TableField(value = "task_id") + private Long taskId; + /** + * 任务名称 + */ + @TableField(value = "task_name") + private String taskName; + /** + * 是否内置 + */ + @TableField(value = "is_built_in") + private Boolean isBuiltIn; + /** + * 模型id + */ + @TableField(value = "model_id") + private Long modelId; + /** + * 模型名称 + */ + @TableField(value = "model_name") + private String modelName; + /** + * 模型路径 + */ + @TableField(value = "model_address") + private String modelAddress; + + /** + * 算法选择类型 + */ + @TableField(value = "algorithm_type") + private Integer algorithmType; + /** + * 优化算法id + */ + @TableField(value = "algorithm_id") + private Long algorithmId; + /** + * 优化算法 + */ + @TableField(value = "algorithm_name") + private String algorithmName; + /** + * 算法路径 + */ + @TableField(value = "algorithm_path") + private String algorithmPath; + /** + * 提交时间 + */ + @TableField(value = "start_time") + private Timestamp startTime; + /** + * 完成时间 + */ + @TableField(value = "end_time") + private Timestamp endTime; + /** + * 输出模型路径 + */ + @TableField(value = "output_model_dir") + private String outputModelDir; + /** + * 日志路径 + */ + @TableField(value = "log_path") + private String logPath; + /** + * 任务状态 + */ + @TableField(value = "status") + private String status; + /** + * 数据集id + */ + @TableField(value = "dataset_id") + private Long datasetId; + /** + * 数据集名称 + */ + @TableField(value = "dataset_name") + private String datasetName; + /** + * 数据集路径 + */ + @TableField(value = "dataset_path") + private String datasetPath; + /** + * 运行命令 + */ + @TableField(value = "command") + private String command; + /** + * 运行参数 + */ + @TableField(value = "params", typeHandler = FastjsonTypeHandler.class) + private JSONObject params; + /** + * 优化前性能参数 + */ + @TableField(value = "opt_result_before") + private String optResultBefore; + /** + * 优化后性能参数 + */ + @TableField(value = "opt_result_after") + private String optResultAfter; + + /** + * 模型优化前性能参数json文件路径 + */ + @TableField(value = "opt_result_json_path_before") + private String optResultJsonPathBefore; + /** + * 模型优化后性能参数json文件路径 + */ + @TableField(value = "opt_result_json_path_after") + private String optResultJsonPathAfter; + + public void copy(ModelOptTaskInstance source) { + BeanUtil.copyProperties(source, this, CopyOptions.create().setIgnoreNullValue(true)); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQuery.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQuery.java index b6366ba..ea52708 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQuery.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQuery.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQueryBrance.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQueryBrance.java index c4c62cb..dadbb1e 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQueryBrance.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQueryBrance.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java index 21196a9..b32d5d9 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java index 41fb51c..46a6f08 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java index c99fde3..460aa1f 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ import org.dubhe.base.BaseEntity; @EqualsAndHashCode(callSuper = true) @Data @Accessors(chain = true) -@TableName(value = "pt_image") +@TableName(value = "pt_image", autoResultMap = true) public class PtImage extends BaseEntity { /** @@ -85,6 +85,6 @@ public class PtImage extends BaseEntity { /** * 资源拥有者ID */ - @TableField(value = "origin_user_id",fill = FieldFill.INSERT) + @TableField(value = "origin_user_id", fill = FieldFill.INSERT) private Long originUserId; } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java index 615c8ad..769b730 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtMeasure.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtMeasure.java new file mode 100644 index 0000000..192ed80 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtMeasure.java @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.entity; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.*; +import org.dubhe.base.BaseEntity; + +/** + * @description 度量管理实体类 + * @date 2020-11-16 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Builder +@TableName("pt_measure") +@NoArgsConstructor +@AllArgsConstructor +public class PtMeasure extends BaseEntity { + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 度量名称 + */ + @TableField("name") + private String name; + + /** + * 度量文件路径 + */ + @TableField("url") + private String url; + + /** + * 资源拥有者ID + */ + @TableField(value = "origin_user_id", fill = FieldFill.INSERT) + private Long originUserId; + + /** + * 度量描述信息 + */ + @TableField("description") + private String description; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java index b227523..ee675ac 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java index ecea1dd..82bf3eb 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java index 722098e..f7cc715 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java index f78c1af..6a1b6a9 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -172,31 +172,23 @@ public class PtTrainJob extends BaseEntity { @TableField(value = "origin_user_id",fill = FieldFill.INSERT) private Long originUserId; - /** - * 是否打开模型原则 - */ - @TableField(value = "model_type") - private Integer modelType; /** * 模型来源 */ @TableField(value = "model_resource") private Integer modelResource; - /** - * 模型加载路径 - */ - @TableField(value = "model_load_dir") - private String modelLoadPathDir; - /** - * 模型名称 - */ - @TableField(value = "model_name") - private String modelName; + /** * 模型id */ @TableField(value = "model_id") - private Integer modelId; + private Long modelId; + + /** + * 模型对应版本id + */ + @TableField(value = "model_branch_id") + private Long modelBranchId; /** * 训练信息(失败信息) @@ -204,4 +196,10 @@ public class PtTrainJob extends BaseEntity { @TableField(value = "train_msg") private String trainMsg; + @TableField(value = "teacher_model_ids") + private String teacherModelIds; + + @TableField(value = "student_model_ids") + private String studentModelIds; + } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java index 9ca2eae..c282bb9 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java index f1ac63e..9024159 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -142,31 +142,24 @@ public class PtTrainParam extends BaseEntity { */ @TableField(value = "val_type") private Integer valType; - /** - * 模型名称 - */ - @TableField(value = "model_name") - private String modelName; + /** * 模型id */ @TableField(value = "model_id") - private Integer modelId; + private Long modelId; + + /** + * 模型版本对应id + */ + @TableField(value = "model_branch_id") + private Long modelBranchId; + /** * 模型来源 */ @TableField(value = "model_resource") private Integer modelResource; - /** - * 模型类型 - */ - @TableField(value = "model_type") - private Integer modelType; - /** - * 模型路径 - */ - @TableField(value = "model_load_dir") - private String modelLoadPathDir; /** * 训练类型,0:普通训练，1：分布式训练 @@ -180,5 +173,11 @@ public class PtTrainParam extends BaseEntity { @TableField(value = "origin_user_id",fill = FieldFill.INSERT) private Long originUserId; + @TableField(value = "teacher_model_ids") + private String teacherModelIds; + + @TableField(value = "student_model_ids") + private String studentModelIds; + } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java new file mode 100644 index 0000000..71815ae --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description 模型优化算法返回 + * @date 2021-01-05 + */ +@Data +public class ModelOptAlgorithmQueryVO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty("算法类型") + private Integer type; + + @ApiModelProperty("算法名称") + private String algorithm; + + @ApiModelProperty("算法路径") + private String algorithmPath; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java new file mode 100644 index 0000000..83aa2d6 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description 模型优化任务创建返回 + * @date 2021-01-07 + */ +@Data +@AllArgsConstructor +public class ModelOptCreateVO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty("算法id") + private Long algorithmId; + + @ApiModelProperty("算法路径") + private String algorithmPath; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java new file mode 100644 index 0000000..1e90ea5 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description 模型优化数据集查询返回 + * @date 2021-01-05 + */ +@Data +public class ModelOptDatasetQueryVO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty("数据集名称") + private String dataset; + @ApiModelProperty("数据集路径") + private String datasetPath; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java new file mode 100644 index 0000000..fb0af79 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.dubhe.domain.entity.ModelOptDataset; + +import java.io.Serializable; + +/** + * @description 模型优化模块我的数据集查询 + * @date 2021-01-06 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ModelOptDatasetVO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty("主键") + private Long id; + + @ApiModelProperty("名称") + private String name; + + @ApiModelProperty("路径") + private String path; + + public ModelOptDatasetVO(ModelOptDataset modelOptDataset) { + this.id = modelOptDataset.getId(); + this.name = modelOptDataset.getName(); + this.path = modelOptDataset.getPath(); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java new file mode 100644 index 0000000..79dcc4e --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description 模型优化内置模型查询 + * @date 2021-01-06 + */ +@Data +public class ModelOptModelQueryVO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty("模型名称") + private String model; + + @ApiModelProperty("模型路径") + private String modelPath; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java new file mode 100644 index 0000000..e1b31c2 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import lombok.Data; + +import java.io.Serializable; + +/** + * @description 模型优化结果对象 + * @date 2020-05-22 + */ +@Data +public class ModelOptResultQueryVO implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * 参数名称 + */ + private String name; + + /** + * 结果对比 + */ + private String positive; + + /** + * 优化后值 + */ + private Float after; + + /** + * 优化前值 + */ + private Float before; + + /** + * 参数单位 + */ + private String unit; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java new file mode 100644 index 0000000..47bc2e5 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java @@ -0,0 +1,128 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import com.alibaba.fastjson.JSONObject; +import lombok.Data; +import org.dubhe.base.BaseVO; + +import java.sql.Timestamp; +import java.util.List; + +/** + * @description 模型优化任务实例 + * @date 2020-05-22 + */ +@Data +public class ModelOptTaskInstanceQueryVO extends BaseVO { + /** + * 主键 + */ + private Long id; + /** + * 任务id + */ + private Long taskId; + /** + * 任务名称 + */ + private String taskName; + + /** + * 是否内置 + */ + private Boolean isBuiltIn; + + /** + * 模型id + */ + private Long modelId; + /** + * 模型名称 + */ + private String modelName; + /** + * 模型路径 + */ + private String modelAddress; + /** + * 算法选择类型 + */ + private Integer algorithmType; + /** + * 优化算法 + */ + private String algorithmName; + /** + * 优化算法id + */ + private Long algorithmId; + /** + * 优化算法路径 + */ + private String algorithmPath; + /** + * 提交时间 + */ + private Timestamp startTime; + /** + * 完成时间 + */ + private Timestamp endTime; + /** + * 日志路径 + */ + private String logPath; + /** + * 任务状态 + */ + private String status; + /** + * 数据集id + */ + private Long datasetId; + /** + * 数据集名称 + */ + private String datasetName; + /** + * 数据集路径 + */ + private String datasetPath; + /** + * 运行命令 + */ + private String command; + /** + * 运行参数 + */ + private JSONObject params; + /** + * 容器名称 + */ + private String podName; + /** + * 模型输出路径 + */ + private String outputModelDir; + /** + * 优化结果 + */ + private List optResult; + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java new file mode 100644 index 0000000..b40783c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java @@ -0,0 +1,79 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import com.alibaba.fastjson.JSONObject; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.dubhe.base.BaseVO; + +/** + * @description 查询模型优化任务返回结果 + * @date 2020-05-22 + */ +@Data +public class ModelOptTaskQueryVO extends BaseVO { + + @ApiModelProperty("ID") + private Long id; + + @ApiModelProperty("任务名称") + private String name; + + @ApiModelProperty("任务描述") + private String description; + + @ApiModelProperty(value = "数据集id") + private Long datasetId; + + @ApiModelProperty("数据集名称") + private String datasetName; + + @ApiModelProperty("数据集路径") + private String datasetPath; + + @ApiModelProperty("是否内置") + private Boolean isBuiltIn; + + @ApiModelProperty("模型id") + private Long modelId; + + @ApiModelProperty("模型名称") + private String modelName; + + @ApiModelProperty("模型路径") + private String modelAddress; + + @ApiModelProperty("优化算法类型") + private Integer algorithmType; + + @ApiModelProperty("优化算法") + private String algorithmName; + + @ApiModelProperty("优化算法id") + private Long algorithmId; + + @ApiModelProperty("优化算法路径") + private String algorithmPath; + + @ApiModelProperty(value = "运行命令") + private String command; + + @ApiModelProperty(value = "运行参数") + private JSONObject params; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java new file mode 100644 index 0000000..c7144ee --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.io.Serializable; + +/** + * @description 模型优化任务修改返回 + * @date 2021-01-08 + */ +@Data +@AllArgsConstructor +public class ModelOptUpdateVO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty("算法id") + private Long algorithmId; + + @ApiModelProperty("算法路径") + private String algorithmPath; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelVO.java new file mode 100644 index 0000000..c69b72c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelVO.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * @description 模型返回结果 + * @date 2021-01-14 + */ +@Data +@AllArgsConstructor +public class ModelVO { + + @ApiModelProperty("模型名称") + private String name; + + @ApiModelProperty("模型版本") + private String modelVersion; + + @ApiModelProperty("模型地址") + private String modelUrl; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java index d4328b6..d67ae3b 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java index 97e6775..55e94da 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java index 260d761..0800505 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java index 9251bd2..31c3ed7 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,4 +54,7 @@ public class PtImageQueryVO implements Serializable { @ApiModelProperty("资源拥有者ID") private Long originUserId; + + @ApiModelProperty("镜像来源") + private Integer imageResource; } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java index 3532c00..dff8f4b 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtMeasureQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtMeasureQueryVO.java new file mode 100644 index 0000000..00ad5ca --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtMeasureQueryVO.java @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description 返回度量查询结果 + * @date 2020-11-16 + */ +@Data +public class PtMeasureQueryVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("id") + private Long id; + + @ApiModelProperty("度量名称") + private String name; + + @ApiModelProperty("度量文件url") + private String url; + + @ApiModelProperty("度量描述") + private String description; + + @ApiModelProperty("创建时间") + private Timestamp createTime; + + @ApiModelProperty("资源拥有者ID") + private Long originUserId; +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java index 28221ea..aed416f 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java index 10b54a9..ef7e38d 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java index 3d318d1..8c62602 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java index af0d1a8..9156b58 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java index efd7962..48c8a78 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -161,6 +161,18 @@ public class PtTrainJobDetailQueryVO implements Serializable { @ApiModelProperty("训练信息(失败信息)") private String trainMsg; - @ApiModelProperty("模型名称") - private String modelName; + @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)") + private Integer modelResource; + + @ApiModelProperty(value = "非炼知模型id") + private Long modelId; + + @ApiModelProperty(value = "我的模型版本对应的id") + private Long modelBranchId; + + @ApiModelProperty(value = "炼知教师模型ids,多个id之前用','隔开") + private String teacherModelIds; + + @ApiModelProperty(value = "炼知学生模型ids,多个id之前用','隔开") + private String studentModelIds; } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java index a0ca8ee..c023b51 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -155,21 +155,21 @@ public class PtTrainJobDetailVO implements Serializable { @ApiModelProperty("训练自动停止倒计时，单位：分钟") private Integer delayDeleteCountDown; - @ApiModelProperty("模型路径") - private String modelLoadPathDir; - - @ApiModelProperty(value = "是否打开模型选择") - private Integer modelType; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型)") + @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)") private Integer modelResource; - @ApiModelProperty(value = "模型名称") - private String modelName; + @ApiModelProperty(value = "非炼知模型id") + private Long modelId; - @ApiModelProperty(value = "模型id") - private Integer modelId; + @ApiModelProperty(value = "我的模型版本对应的id") + private Long modelBranchId; @ApiModelProperty(value = "训练信息(失败信息)") private String trainMsg; + + @ApiModelProperty(value = "炼知教师模型ids,多个id之前用','隔开") + private String teacherModelIds; + + @ApiModelProperty(value = "炼知学生模型ids,多个id之前用','隔开") + private String studentModelIds; } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobModelVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobModelVO.java new file mode 100644 index 0000000..d92b1e1 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobModelVO.java @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.domain.vo; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * @description 返回停止训练任务结果 + * @date 2020-04-28 + */ +@Data +public class PtTrainJobModelVO implements Serializable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty("非炼知模型信息") + private List modelList; + + @ApiModelProperty("炼知教师模型信息") + private List teacherModelList; + + @ApiModelProperty("炼知学生模型信息") + private List studentModelList; +} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java index cf4fcd1..d7237ef 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java index 98f2d1e..8ef03be 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java index b2573b7..6a30cde 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java index a3294ee..1435eb0 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java index 6ee35f3..bbf3226 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -95,19 +95,19 @@ public class PtTrainParamQueryVO extends BaseVO implements Serializable { @ApiModelProperty("是否验证数据集") private Integer valType; - @ApiModelProperty(value = "是否打开模型选择") - private Integer modelType; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型)") + @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)") private Integer modelResource; - @ApiModelProperty(value = "模型名称") - private String modelName; + @ApiModelProperty(value = "模型id") + private Long modelId; + + @ApiModelProperty(value = "我的模型版本对应的id") + private Long modelBranchId; - @ApiModelProperty(value = "模型加载路径") - private String modelLoadPathDir; + @ApiModelProperty(value = "炼知教师模型ids,多个id之前用','隔开") + private String teacherModelIds; - @ApiModelProperty(value = "模型id") - private Integer modelId; + @ApiModelProperty(value = "炼知学生模型ids,多个id之前用','隔开") + private String studentModelIds; } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java index efda628..8281c15 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/AlgorithmStatusEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/AlgorithmStatusEnum.java index b27c8a2..11ac292 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/AlgorithmStatusEnum.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/AlgorithmStatusEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java new file mode 100644 index 0000000..d56b6be --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java @@ -0,0 +1,92 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + +/** + * @description 蒸馏算法命令行枚举 + * @date 2021-01-05 + */ +public enum DistillCommandEnum { + + /** + * 软标签蒸馏 + */ + KD("KD", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_student_kd.sh %s %s %s %s %s"), + /** + * 软标签蒸馏，将BERT蒸馏到BiLSTM + */ + BILSTM("Distilled-BiLSTM", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_student_distilled_lstm.sh %s %s %s %s %s"), + /** + * 软标签蒸馏+层与层蒸馏 + */ + PKD("BERT-PKD", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_student_bert_pkd.sh %s %s %s %s %s"), + /** + * 软标签蒸馏+层与层蒸馏+注意力蒸馏 + */ + TINY("TinyBERT", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_student_tinybert.sh %s %s %s %s %s"), + /** + * 依照概率替换原有的BERT模块和Theseus的模块组成新的模型来训练 + */ + THESEUS("BERT-Theseus", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_theseus.sh %s %s %s %s %s"); + + + /** + * 算法名 + */ + private String name; + + /** + * 命令行 + */ + private String command; + + DistillCommandEnum(String name, String command) { + this.name = name; + this.command = command; + } + + /** + * 根据名称获取命令行 + * @param name 蒸馏算法名称 + * @return 返回命令行 + */ + public static String getCommandByName(String name) { + for (DistillCommandEnum commandEnum : DistillCommandEnum.values()) { + if (commandEnum.getName().equals(name)) { + return commandEnum.getCommand(); + } + } + return ""; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java index 9ccc3f7..e3a80a8 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java new file mode 100644 index 0000000..6f0d749 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.enums; + +import lombok.Getter; + +/** + * @description 优化算法类型 + * @date 2021-01-06 + */ +@Getter +public enum OptimizeTypeEnum { + + SLIMMING(0, "剪枝", "/ChannelSlimming"), + + DISTILL(1, "蒸馏", "/distil"), + + QUANTIFY(2, "量化", "/quantization"), + ; + + /** + * 类型值 + */ + private Integer type; + + /** + * 类型名称 + */ + private String name; + + /** + * 代码目录 + */ + private String codeDir; + + OptimizeTypeEnum(Integer type, String name, String codeDir) { + this.type = type; + this.name = name; + this.codeDir = codeDir; + } + + + public static String getCodeDirByType(Integer type) { + for (OptimizeTypeEnum optimizeTypeEnum : OptimizeTypeEnum.values()) { + if (optimizeTypeEnum.getType().equals(type)) { + return optimizeTypeEnum.getCodeDir(); + } + } + return ""; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java index 53d1e2f..3282f7d 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/TrainTypeEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/TrainTypeEnum.java index 8d50bff..5769737 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/TrainTypeEnum.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/TrainTypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/BatchServingController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/BatchServingController.java new file mode 100644 index 0000000..a9b7991 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/BatchServingController.java @@ -0,0 +1,120 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.base.DataResponseBody; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.dto.BatchServingCreateDTO; +import org.dubhe.domain.dto.BatchServingDeleteDTO; +import org.dubhe.domain.dto.BatchServingDetailDTO; +import org.dubhe.domain.dto.BatchServingQueryDTO; +import org.dubhe.domain.dto.BatchServingStartDTO; +import org.dubhe.domain.dto.BatchServingStopDTO; +import org.dubhe.domain.dto.BatchServingUpdateDTO; +import org.dubhe.service.BatchServingService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; + +/** + * @description 批量服务管理 + * @date 2020-08-27 + */ +@Api(tags = "云端Serving：批量服务管理") +@RestController +@RequestMapping("/api/batchServing") +public class BatchServingController { + + @Resource + private BatchServingService batchServingService; + + @ApiOperation("查询批量服务") + @GetMapping + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody query(BatchServingQueryDTO batchServingQueryDTO) { + return new DataResponseBody(batchServingService.query(batchServingQueryDTO)); + } + + @ApiOperation("创建批量服务") + @PostMapping + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody create(@Validated @RequestBody BatchServingCreateDTO batchServingCreateDTO) { + return new DataResponseBody(batchServingService.create(batchServingCreateDTO)); + } + + @ApiOperation("修改批量服务") + @PutMapping + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody update(@Validated @RequestBody BatchServingUpdateDTO batchServingUpdateDTO) { + return new DataResponseBody(batchServingService.update(batchServingUpdateDTO)); + } + + @ApiOperation("删除批量服务") + @DeleteMapping + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody delete(@Validated @RequestBody BatchServingDeleteDTO batchServingDeleteDTO) { + return new DataResponseBody(batchServingService.delete(batchServingDeleteDTO)); + } + + @ApiOperation("启动批量服务") + @PostMapping("/start") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody start(@Validated @RequestBody BatchServingStartDTO batchServingStartDTO) { + return new DataResponseBody(batchServingService.start(batchServingStartDTO)); + } + + @ApiOperation("停止批量服务") + @PostMapping("/stop") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody stop(@Validated @RequestBody BatchServingStopDTO batchServingStopDTO) { + return new DataResponseBody(batchServingService.stop(batchServingStopDTO)); + } + + @ApiOperation("获取批量服务详情") + @GetMapping("/detail") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody getDetail(@Validated BatchServingDetailDTO batchServingDetailDTO) { + return new DataResponseBody(batchServingService.getDetail(batchServingDetailDTO)); + } + + @GetMapping("/pod/{id}") + @ApiOperation("获取批量服务下pod信息") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody getServingLog(@PathVariable Long id) { + return new DataResponseBody(batchServingService.getPods(id)); + } + + + @GetMapping("/queryById/{id}") + @ApiOperation("获取批量服务状态及进度") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody queryStatusAndProgress(@PathVariable Long id) { + return new DataResponseBody(batchServingService.queryStatusAndProgress(id)); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java index 0043e36..99865f4 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java @@ -1,4 +1,4 @@ -/** Copyright 2020 Zhejiang Lab. All Rights Reserved. +/** Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackDeploymentController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackDeploymentController.java new file mode 100644 index 0000000..d8538c4 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackDeploymentController.java @@ -0,0 +1,62 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.dubhe.base.DataResponseBody; +import org.dubhe.dto.callback.ServingK8sDeploymentCallbackCreateDTO; +import org.dubhe.factory.DataResponseFactory; +import org.dubhe.service.DeploymentCallbackAsyncService; +import org.dubhe.utils.K8sCallBackTool; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; + +import static org.dubhe.constant.StringConstant.K8S_CALLBACK_PATH_DEPLOYMENT; + +/** + * @description k8s deployment回调 + * @date 2020-11-27 + */ +@Api(tags = "k8s回调：deployment") +@RestController +@RequestMapping(K8S_CALLBACK_PATH_DEPLOYMENT) +public class K8sCallbackDeploymentController { + + @Resource(name = "servingAsyncService") + private DeploymentCallbackAsyncService servingAsyncService; + + /** + * 云端serving在线服务异步回调 + * + * @param k8sToken + * @param k8sDeploymentCallbackReq + * @return + */ + @PostMapping(value = "/serving") + @ApiOperation("云端serving deployment 回调") + public DataResponseBody servingPodCallBack(@ApiParam(type = "head") @RequestHeader(name = K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken, + @Validated @RequestBody ServingK8sDeploymentCallbackCreateDTO k8sDeploymentCallbackReq) { + servingAsyncService.deploymentCallBack(k8sDeploymentCallbackReq); + return DataResponseFactory.success("云端serving在线服务异步回调中"); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java index de537b4..4489d6d 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,12 +22,18 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.dubhe.base.DataResponseBody; import org.dubhe.dto.callback.AlgorithmK8sPodCallbackCreateDTO; +import org.dubhe.dto.callback.BatchServingK8sPodCallbackCreateDTO; +import org.dubhe.dto.callback.ModelOptK8sPodCallbackCreateDTO; import org.dubhe.dto.callback.NotebookK8sPodCallbackCreateDTO; import org.dubhe.factory.DataResponseFactory; import org.dubhe.service.PodCallbackAsyncService; import org.dubhe.utils.K8sCallBackTool; import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; @@ -35,7 +41,6 @@ import static org.dubhe.constant.StringConstant.K8S_CALLBACK_URI; /** * @description k8s Pod 异步回调处理类 - * * @date 2020-05-28 */ @Api(tags = "k8s回调：Pod") @@ -47,6 +52,10 @@ public class K8sCallbackPodController { private PodCallbackAsyncService noteBookAsyncService; @Resource(name = "algorithmAsyncServiceImpl") private PodCallbackAsyncService algorithmAsyncServiceImpl; + @Resource(name = "modelOptAsyncServiceImpl") + private PodCallbackAsyncService modelOptAsyncService; + @Resource(name = "batchServingAsyncService") + private PodCallbackAsyncService batchServingAsyncService; /** * notebook异步回调 @@ -57,13 +66,12 @@ public class K8sCallbackPodController { */ @PostMapping(value = "/notebook") @ApiOperation("模型管理 pod 回调") - public DataResponseBody notebookPodCallBack(@ApiParam(type = "head") @RequestHeader(name= K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken - ,@Validated @RequestBody NotebookK8sPodCallbackCreateDTO k8sPodCallbackReq) { + public DataResponseBody notebookPodCallBack(@ApiParam(type = "head") @RequestHeader(name = K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken + , @Validated @RequestBody NotebookK8sPodCallbackCreateDTO k8sPodCallbackReq) { noteBookAsyncService.podCallBack(k8sPodCallbackReq); return DataResponseFactory.success("notebook正在异步处理pod中。"); } - /** * algorithm异步回调 * @@ -73,11 +81,39 @@ public class K8sCallbackPodController { */ @PostMapping(value = "/algorithm") @ApiOperation("算法管理 pod 回调") - public DataResponseBody algorithmPodCallBack(@ApiParam(type = "head") @RequestHeader(name= K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken - ,@Validated @RequestBody AlgorithmK8sPodCallbackCreateDTO k8sPodCallbackReq) { + public DataResponseBody algorithmPodCallBack(@ApiParam(type = "head") @RequestHeader(name = K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken + , @Validated @RequestBody AlgorithmK8sPodCallbackCreateDTO k8sPodCallbackReq) { algorithmAsyncServiceImpl.podCallBack(k8sPodCallbackReq); - return DataResponseFactory.success("算法管理异步回调处理方法中"); + return DataResponseFactory.success("算法管理异步回调处理中"); } + /** + * modelOpt异步回调 + * + * @param k8sToken + * @param k8sPodCallbackReq + * @return + */ + @PostMapping(value = "/modelopt") + @ApiOperation("模型优化管理 pod 回调") + public DataResponseBody modelOptPodCallBack(@ApiParam(type = "head") @RequestHeader(name= K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken, + @Validated @RequestBody ModelOptK8sPodCallbackCreateDTO k8sPodCallbackReq) { + modelOptAsyncService.podCallBack(k8sPodCallbackReq); + return DataResponseFactory.success("model-opt 正在异步处理pod中。"); + } + /** + * 云端serving批量服务异步回调 + * + * @param k8sToken + * @param k8sPodCallbackReq + * @return + */ + @PostMapping(value = "/batchserving") + @ApiOperation("云端serving pod 回调") + public DataResponseBody servingPodCallBack(@ApiParam(type = "head") @RequestHeader(name = K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken, + @Validated @RequestBody BatchServingK8sPodCallbackCreateDTO k8sPodCallbackReq) { + batchServingAsyncService.podCallBack(k8sPodCallbackReq); + return DataResponseFactory.success("云端serving批量服务异步回调处理中"); + } } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java new file mode 100644 index 0000000..25203da --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java @@ -0,0 +1,127 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.base.DataResponseBody; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.dto.ModelOptDatasetCreateDTO; +import org.dubhe.domain.dto.ModelOptTaskCreateDTO; +import org.dubhe.domain.dto.ModelOptTaskDeleteDTO; +import org.dubhe.domain.dto.ModelOptTaskQueryDTO; +import org.dubhe.domain.dto.ModelOptTaskSubmitDTO; +import org.dubhe.domain.dto.ModelOptTaskUpdateDTO; +import org.dubhe.factory.DataResponseFactory; +import org.dubhe.service.ModelOptTaskService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; + +/** + * @description 模型优化任务 + * @date 2020-05-22 + */ +@Api(tags = "模型优化：任务") +@RestController +@RequestMapping("/api/modelOpt/task") +public class ModelOptTaskController { + + @Resource + private ModelOptTaskService modelOptTaskService; + + @GetMapping + @ApiOperation("任务列表分页查询") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody getModelOptList(ModelOptTaskQueryDTO modelOptTaskQueryDTO) { + return DataResponseFactory.success(modelOptTaskService.queryAll(modelOptTaskQueryDTO)); + } + + @PostMapping + @ApiOperation("创建任务") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody create(@Validated @RequestBody ModelOptTaskCreateDTO modelOptTaskCreateDTO) { + return DataResponseFactory.success(modelOptTaskService.create(modelOptTaskCreateDTO)); + } + + @PostMapping("/submit") + @ApiOperation("提交任务，创建实例") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody submit(@Validated @RequestBody ModelOptTaskSubmitDTO submitDTO) { + modelOptTaskService.submit(submitDTO); + return DataResponseFactory.success(); + } + + @PutMapping + @ApiOperation("编辑模型优化任务") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody update(@Validated @RequestBody ModelOptTaskUpdateDTO modelOptTaskUpdateDTO) { + return DataResponseFactory.success(modelOptTaskService.update(modelOptTaskUpdateDTO)); + } + + @DeleteMapping + @ApiOperation("删除模型优化任务") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody delete(@Validated @RequestBody ModelOptTaskDeleteDTO modelOptTaskDeleteDTO) { + modelOptTaskService.delete(modelOptTaskDeleteDTO); + return DataResponseFactory.success(); + } + + @GetMapping(value = "/getBuiltInModel") + @ApiOperation("获取全部内置模型") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody getBuiltInModel(Integer type, String dataset, String algorithm) { + return DataResponseFactory.success(modelOptTaskService.getBuiltInModel(type, dataset, algorithm)); + } + + @GetMapping(value = "/getAlgorithm") + @ApiOperation("获取模型优化算法") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody getAlgorithm(Integer type, String model, String dataset) { + return DataResponseFactory.success(modelOptTaskService.getAlgorithm(type, model, dataset)); + } + + @GetMapping(value = "/getDataset") + @ApiOperation("获取内置模型优化数据集") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody getDataset(Integer type, String model, String algorithm) { + return DataResponseFactory.success(modelOptTaskService.getDataset(type, model, algorithm)); + } + + @GetMapping(value = "/MyDataset") + @ApiOperation("获取我的模型优化数据集") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody getMyDataset() { + return DataResponseFactory.success(modelOptTaskService.getMyDataset()); + } + + @PostMapping(value = "/MyDataset") + @ApiOperation("创建我的模型优化数据集") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody createMyDataset(@Validated @RequestBody ModelOptDatasetCreateDTO modelOptDatasetCreateDTO) { + return DataResponseFactory.success(modelOptTaskService.createMyDataset(modelOptDatasetCreateDTO)); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java new file mode 100644 index 0000000..0b0df29 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java @@ -0,0 +1,93 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.base.DataResponseBody; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.dto.ModelOptTaskInstanceDeleteDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceCancelDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceDetailDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceQueryDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceResubmitDTO; +import org.dubhe.factory.DataResponseFactory; +import org.dubhe.service.ModelOptTaskInstanceService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; + +/** + * @description 模型优化任务实例 + * @date 2020-05-22 + */ +@Api(tags = "模型优化：任务实例") +@RestController +@RequestMapping("/api/modelOpt/taskInstance") +public class ModelOptTaskInstanceController { + + @Resource + private ModelOptTaskInstanceService modelOptTaskInstanceService; + + @ApiOperation("分页查询任务执行记录实例列表") + @GetMapping + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody query(ModelOptTaskInstanceQueryDTO instanceQueryDTO) { + return DataResponseFactory.success(modelOptTaskInstanceService.queryAll(instanceQueryDTO)); + } + + @ApiOperation("重新提交任务实例") + @PostMapping(value = "/resubmit") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody resubmit(@Validated @RequestBody ModelOptTaskInstanceResubmitDTO resubmitDTO) { + modelOptTaskInstanceService.resubmit(resubmitDTO); + return DataResponseFactory.success(); + } + + @ApiOperation("取消模型优化任务实例") + @PutMapping(value = "/cancel") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody cancel(@Validated @RequestBody ModelOptTaskInstanceCancelDTO cancelDTO) { + modelOptTaskInstanceService.cancel(cancelDTO); + return DataResponseFactory.success(); + } + + @ApiOperation("查看任务实例详情") + @GetMapping(value = "/detail") + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody getInstDetail(@Validated ModelOptTaskInstanceDetailDTO detailDTO) { + return DataResponseFactory.success(modelOptTaskInstanceService.getInstDetail(detailDTO)); + } + + @ApiOperation("删除任务实例") + @DeleteMapping + @RequiresPermissions(Permissions.MODEL_OPTIMIZE) + public DataResponseBody delete(@Validated @RequestBody ModelOptTaskInstanceDeleteDTO modelOptTaskInstanceDeleteDTO) { + modelOptTaskInstanceService.delete(modelOptTaskInstanceDeleteDTO); + return DataResponseFactory.success(); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java index 5ac67d6..12d7202 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java index d4a3460..08c289d 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,10 +23,7 @@ import org.apache.shiro.authz.annotation.RequiresPermissions; import org.dubhe.annotation.ApiVersion; import org.dubhe.base.DataResponseBody; import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.PtImageDeleteDTO; -import org.dubhe.domain.dto.PtImageQueryDTO; -import org.dubhe.domain.dto.PtImageUpdateDTO; -import org.dubhe.domain.dto.PtImageUploadDTO; +import org.dubhe.domain.dto.*; import org.dubhe.service.PtImageService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.validation.annotation.Validated; @@ -55,8 +52,8 @@ public class PtImageController { @ApiOperation("通过projectName查询镜像") @GetMapping @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody getTagsByImageName(@RequestParam String imageName) { - return new DataResponseBody(ptImageService.searchImages(imageName)); + public DataResponseBody getTagsByImageName(@RequestParam Integer projectType, @RequestParam String imageName) { + return new DataResponseBody(ptImageService.searchImages(projectType, imageName)); } @PostMapping("uploadImage") @@ -67,18 +64,10 @@ public class PtImageController { return new DataResponseBody(); } - @GetMapping("/project") - @ApiOperation("查询镜像project列表") - @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody getHarborProjectList() { - return new DataResponseBody(ptImageService.getHarborProjectList()); - } - - @DeleteMapping @ApiOperation("删除镜像") @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody deleteTrainImage(@RequestBody PtImageDeleteDTO ptImageDeleteDTO) { + public DataResponseBody deleteTrainImage(@Validated @RequestBody PtImageDeleteDTO ptImageDeleteDTO) { ptImageService.deleteTrainImage(ptImageDeleteDTO); return new DataResponseBody(); } @@ -94,7 +83,22 @@ public class PtImageController { @GetMapping("/imageNameList") @ApiOperation("获取镜像名称列表") @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody getImageNameList() { - return new DataResponseBody(ptImageService.getImageNameList()); + public DataResponseBody getImageNameList(@RequestParam Integer projectType) { + return new DataResponseBody(ptImageService.getImageNameList(projectType)); + } + + @PutMapping("/imageResource") + @ApiOperation("修改镜像来源(notebook定制)") + @RequiresPermissions(Permissions.TRAINING_IMAGE) + public DataResponseBody updateImageResource(@RequestParam Long id) { + ptImageService.updImageResource(id); + return new DataResponseBody(); + } + + @GetMapping("/imageUrl") + @ApiOperation("查询镜像url") + @RequiresPermissions(Permissions.TRAINING_IMAGE) + public DataResponseBody getImageUrl(PtImageQueryUrlDTO ptImageQueryUrlDTO) { + return new DataResponseBody(ptImageService.getImageUrl(ptImageQueryUrlDTO)); } } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtMeasureController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtMeasureController.java new file mode 100644 index 0000000..81c1b24 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtMeasureController.java @@ -0,0 +1,78 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.dubhe.annotation.ApiVersion; +import org.dubhe.base.DataResponseBody; +import org.dubhe.domain.dto.PtMeasureDTO; +import org.dubhe.domain.dto.PtMeasureDeleteDTO; +import org.dubhe.domain.dto.PtMeasureQueryDTO; +import org.dubhe.domain.dto.PtMeasureUpdateDTO; +import org.dubhe.service.PtMeasureService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +/** + * @description 度量管理 + * @date 2020-11-16 + */ +@Api(tags = "度量：度量管理") +@RestController +@ApiVersion(1) +@RequestMapping("/api/{version}/ptMeasure") +public class PtMeasureController { + + @Autowired + private PtMeasureService ptMeasureService; + + @GetMapping + @ApiOperation("查询度量") + public DataResponseBody getMeasure(PtMeasureQueryDTO ptMeasureQueryDTO) { + return new DataResponseBody(ptMeasureService.getMeasure(ptMeasureQueryDTO)); + } + + @PostMapping + @ApiOperation("新建度量") + public DataResponseBody addMeasure(@Validated @RequestBody PtMeasureDTO ptMeasureCreateDTO) { + ptMeasureService.createMeasure(ptMeasureCreateDTO); + return new DataResponseBody(); + } + + @PutMapping + @ApiOperation("修改度量") + public DataResponseBody updateMeasure(@Validated @RequestBody PtMeasureUpdateDTO ptMeasureUpdateDTO) { + ptMeasureService.updateMeasure(ptMeasureUpdateDTO); + return new DataResponseBody(); + } + + @DeleteMapping + @ApiOperation("删除度量") + public DataResponseBody deleteMeasure(@Validated @RequestBody PtMeasureDeleteDTO PtMeasureUpdateDTO) { + ptMeasureService.deleteMeasure(PtMeasureUpdateDTO); + return new DataResponseBody(); + } + + @GetMapping("/byName") + @ApiOperation("通过度量名称查询") + public DataResponseBody getMeasureByName(@RequestParam String name) { + return new DataResponseBody(ptMeasureService.getMeasureByName(name)); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java index 8f17cbd..5cc6fec 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,10 +23,7 @@ import org.apache.shiro.authz.annotation.RequiresPermissions; import org.dubhe.annotation.ApiVersion; import org.dubhe.base.DataResponseBody; import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.PtTrainAlgorithmCreateDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmDeleteDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmQueryDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUpdateDTO; +import org.dubhe.domain.dto.*; import org.dubhe.service.PtTrainAlgorithmService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.validation.annotation.Validated; @@ -81,4 +78,11 @@ public class PtTrainAlgorithmController { return new DataResponseBody(); } + @PostMapping("/uploadAlgorithm") + @ApiOperation("模型优化上传算法") + @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) + public DataResponseBody modelOptimizationUploadAlgorithm(@Validated @RequestBody PtModelAlgorithmCreateDTO ptModelAlgorithmCreateDTO) { + return new DataResponseBody(ptTrainAlgorithmService.modelOptimizationUploadAlgorithm(ptModelAlgorithmCreateDTO)); + } + } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java index 928dc31..299f3c4 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java index a6cb3c7..5a0dec7 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -139,4 +139,11 @@ public class PtTrainJobController { public DataResponseBody getGrafanaUrl( @PathVariable Long jobId) { return new DataResponseBody(ptTrainJobService.getGrafanaUrl(jobId)); } + + @GetMapping("/model") + @ApiOperation("获取job使用的模型") + @RequiresPermissions(Permissions.TRAINING_JOB) + public DataResponseBody getTrainJobModel(@Validated PtTrainModelDTO ptTrainModelDTO) { + return new DataResponseBody(ptTrainJobService.getTrainJobModel(ptTrainModelDTO)); + } } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java index ab82378..f2b28c0 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java index 7b75ae9..05790fb 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ServingController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ServingController.java new file mode 100644 index 0000000..09baeae --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ServingController.java @@ -0,0 +1,142 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.dubhe.base.DataResponseBody; +import org.dubhe.constant.Permissions; +import org.dubhe.domain.dto.PredictParamDTO; +import org.dubhe.domain.dto.ServingInfoCreateDTO; +import org.dubhe.domain.dto.ServingInfoDeleteDTO; +import org.dubhe.domain.dto.ServingInfoDetailDTO; +import org.dubhe.domain.dto.ServingInfoQueryDTO; +import org.dubhe.domain.dto.ServingInfoUpdateDTO; +import org.dubhe.domain.dto.ServingStartDTO; +import org.dubhe.domain.dto.ServingStopDTO; +import org.dubhe.service.ServingService; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; + +/** + * @description 在线服务管理 + * @date 2020-08-24 + */ +@Api(tags = "云端Serving：在线服务管理") +@RestController +@RequestMapping("/api/serving") +public class ServingController { + + @Resource + private ServingService servingService; + + @ApiOperation("查询服务") + @GetMapping + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody query(ServingInfoQueryDTO servingInfoQueryDTO) { + return new DataResponseBody(servingService.query(servingInfoQueryDTO)); + } + + @ApiOperation("创建服务") + @PostMapping + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody create(@Validated @RequestBody ServingInfoCreateDTO servingInfoCreateDTO) { + return new DataResponseBody(servingService.create(servingInfoCreateDTO)); + } + + @ApiOperation("修改服务") + @PutMapping + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody update(@Validated @RequestBody ServingInfoUpdateDTO servingInfoUpdateDTO) { + return new DataResponseBody(servingService.update(servingInfoUpdateDTO)); + } + + @ApiOperation("删除服务") + @DeleteMapping + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody delete(@Validated @RequestBody ServingInfoDeleteDTO servingInfoDeleteDTO) { + return new DataResponseBody(servingService.delete(servingInfoDeleteDTO)); + } + + @ApiOperation("获取服务详情") + @GetMapping("/detail") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody getDetail(@Validated ServingInfoDetailDTO servingInfoDetailDTO) { + return new DataResponseBody(servingService.getDetail(servingInfoDetailDTO)); + } + + @ApiOperation("启动服务") + @PostMapping("/start") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody start(@Validated @RequestBody ServingStartDTO servingStartDTO) { + return new DataResponseBody(servingService.start(servingStartDTO)); + } + + @ApiOperation("停止服务") + @PostMapping("/stop") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody stop(@Validated @RequestBody ServingStopDTO servingStopDTO) { + return new DataResponseBody(servingService.stop(servingStopDTO)); + } + + @ApiOperation("获取接口参数") + @GetMapping("/predictParam") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody getPredictParam(@Validated PredictParamDTO predictParamDTO) { + return new DataResponseBody(servingService.getPredictParam(predictParamDTO)); + } + + @GetMapping("/servingConfig/pod/{modelConfigId}") + @ApiOperation("获取modelConfigId下pod信息") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody getServingLog(@PathVariable Long modelConfigId) { + return new DataResponseBody(servingService.getPods(modelConfigId)); + } + + @PostMapping("/predict") + @ApiOperation("预测") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody predict(MultipartFile[] files, Long id, String url) { + return new DataResponseBody(servingService.predict(id, url, files)); + } + + @GetMapping("/metrics/{servingId}") + @ApiOperation("获取服务的监控信息") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody getServingMetrics(@PathVariable Long servingId) { + return new DataResponseBody(servingService.getMetricsDetail(servingId)); + } + + @GetMapping("/rollback/{servingId}") + @ApiOperation("获取回滚列表") + @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) + public DataResponseBody getRollbackList(@PathVariable Long servingId) { + return new DataResponseBody(servingService.getRollbackList(servingId)); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/BatchServingService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/BatchServingService.java new file mode 100644 index 0000000..cef17c0 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/BatchServingService.java @@ -0,0 +1,113 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import org.dubhe.domain.dto.*; +import org.dubhe.domain.entity.BatchServing; +import org.dubhe.domain.vo.*; +import org.dubhe.dto.callback.BatchServingK8sPodCallbackCreateDTO; +import org.dubhe.k8s.domain.vo.PodVO; + +import java.util.List; +import java.util.Map; + +/** + * @description 批量服务管理 + * @date 2020-08-26 + */ +public interface BatchServingService extends IService { + + /** + * 批量服务查询 + * + * @param batchServingQueryDTO 批量服务查询参数 + * @return Map 批量服务查询返回分页对象 + */ + Map query(BatchServingQueryDTO batchServingQueryDTO); + + /** + * 创建批量服务 + * + * @param batchServingCreateDTO 批量服务创建参数 + * @return BatchServingCreateVO 批量服务创建返回结果 + */ + BatchServingCreateVO create(BatchServingCreateDTO batchServingCreateDTO); + + /** + * 修改批量服务 + * + * @param batchServingUpdateDTO 批量服务修改参数 + * @return BatchServingUpdateVO 批量服务修改返回结果 + */ + BatchServingUpdateVO update(BatchServingUpdateDTO batchServingUpdateDTO); + + /** + * 删除批量服务 + * + * @param batchServingDeleteDTO 批量服务删除参数 + * @return BatchServingDeleteVO 批量服务删除返回结果 + */ + BatchServingDeleteVO delete(BatchServingDeleteDTO batchServingDeleteDTO); + + /** + * 启动批量服务 + * + * @param batchServingStartDTO 批量服务启动参数 + * @return BatchServingStartVO 批量服务启动返回结果 + */ + BatchServingStartVO start(BatchServingStartDTO batchServingStartDTO); + + /** + * 停止批量服务 + * + * @param batchServingStopDTO 批量服务停止参数 + * @return BatchServingStopVO 批量服务停止返回结果 + */ + BatchServingStopVO stop(BatchServingStopDTO batchServingStopDTO); + + /** + * 获取批量服务详情 + * + * @param batchServingDetailDTO 批量服务详情参数 + * @return BatchServingDetailVO 批量服务详情返回结果 + */ + BatchServingDetailVO getDetail(BatchServingDetailDTO batchServingDetailDTO); + + /** + * k8s回调批量服务状态 + * + * @param times 回调请求次数 + * @param req 回调请求对象 + * @return boolean 返回回调结果 + */ + boolean batchServingCallback(int times, BatchServingK8sPodCallbackCreateDTO req); + + /** + * @param id 服务配置id + * @return List 服务配下的pod信息 + */ + List getPods(Long id); + + /** + * 轮询返回状态及进度 + * @param id 批量服务id + * @return BatchServingQueryVO 返回查询结果 + */ + BatchServingQueryVO queryStatusAndProgress(Long id); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/DeploymentCallbackAsyncService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/DeploymentCallbackAsyncService.java new file mode 100644 index 0000000..d302512 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/DeploymentCallbackAsyncService.java @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.dto.callback.BaseK8sDeploymentCallbackCreateDTO; +import org.springframework.scheduling.annotation.Async; + +/** + * @description deployment 异步回调 + * @date 2020-11-27 + */ +public interface DeploymentCallbackAsyncService { + + /** + * deployment 异步回调 + * @param k8sDeploymentCallbackCreateDTO + */ + @Async + void deploymentCallBack (R k8sDeploymentCallbackCreateDTO); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java new file mode 100644 index 0000000..56e61d5 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java @@ -0,0 +1,109 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.domain.dto.ModelOptTaskInstanceCancelDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceDeleteDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceDetailDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceQueryDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceResubmitDTO; +import org.dubhe.domain.entity.ModelOptTaskInstance; +import org.dubhe.domain.vo.ModelOptTaskInstanceQueryVO; +import org.dubhe.dto.callback.ModelOptK8sPodCallbackCreateDTO; + +import java.util.Map; + +/** + * @description 模型优化任务实例 + * @date 2020-05-22 + */ +public interface ModelOptTaskInstanceService { + /** + * 分页查询任务执行记录实例列表 + * + * @param instanceQueryDTO 模型优化实例查询参数包装类 + * @return Map 分页对象 + */ + Map queryAll(ModelOptTaskInstanceQueryDTO instanceQueryDTO); + + /** + * 新增任务实例 + * + * @param modelOptTaskInstance 模型优化实例对象 + */ + void create(ModelOptTaskInstance modelOptTaskInstance); + + /** + * 重新提交任务实例 + * + * @param resubmitDTO 重新提交任务实例参数 + */ + void resubmit(ModelOptTaskInstanceResubmitDTO resubmitDTO); + + /** + * 取消模型优化任务 + * + * @param cancelDTO 取消模型优化任务参数 + */ + void cancel(ModelOptTaskInstanceCancelDTO cancelDTO); + + /** + * 查看单个任务实例详情 + * + * @param detailDTO 查看任务实例详情参数 + * @return ModelOptTaskInstanceVO 任务实例对象 + */ + ModelOptTaskInstanceQueryVO getInstDetail(ModelOptTaskInstanceDetailDTO detailDTO); + + /** + * k8s回调模型优化方法 + * + * @param req 模型优化自定义回调参数类 + * @return 返回回调状态 + */ + boolean modelOptCallBack(ModelOptK8sPodCallbackCreateDTO req); + + /** + * 查询K8S并同步相关实例状态 + */ + void syncInstanceStatus(); + + /** + * 校验该任务是否存在进行中和等待中的实例 + * + * @param taskId 任务ID + * @return Boolean 是否存在进行中和等待中的实例 + */ + Boolean checkUnfinishedInst(Long taskId); + + /** + * 根据任务id删除实例 + * + * @param taskId 任务id + * @return Integer 删除实例数量 + */ + int deleteByTaskId(Long taskId); + + /** + * 删除任务实例 + * + * @param modelOptTaskInstanceDeleteDTO + */ + void delete(ModelOptTaskInstanceDeleteDTO modelOptTaskInstanceDeleteDTO); + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java new file mode 100644 index 0000000..8f1d4f9 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java @@ -0,0 +1,122 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.domain.dto.ModelOptDatasetCreateDTO; +import org.dubhe.domain.dto.ModelOptTaskCreateDTO; +import org.dubhe.domain.dto.ModelOptTaskDeleteDTO; +import org.dubhe.domain.dto.ModelOptTaskQueryDTO; +import org.dubhe.domain.dto.ModelOptTaskSubmitDTO; +import org.dubhe.domain.dto.ModelOptTaskUpdateDTO; +import org.dubhe.domain.vo.ModelOptAlgorithmQueryVO; +import org.dubhe.domain.vo.ModelOptCreateVO; +import org.dubhe.domain.vo.ModelOptDatasetQueryVO; +import org.dubhe.domain.vo.ModelOptDatasetVO; +import org.dubhe.domain.vo.ModelOptModelQueryVO; +import org.dubhe.domain.vo.ModelOptUpdateVO; + +import java.util.List; +import java.util.Map; + +/** + * @description 模型优化任务 + * @date 2020-05-22 + */ +public interface ModelOptTaskService { + + /** + * 查询数据分页 + * + * @param modelOptTaskQueryDTO 模型优化任务查询参数 + * @return Map 模型优化任务分页对象 + */ + Map queryAll(ModelOptTaskQueryDTO modelOptTaskQueryDTO); + + /** + * 创建模型优化任务 + * + * @param modelOptTaskCreateDTO 模型优化任务创建对象 + */ + ModelOptCreateVO create(ModelOptTaskCreateDTO modelOptTaskCreateDTO); + + /** + * 提交模型优化任务，创建任务实例 + * + * @param submitDTO 任务提交参数 + */ + void submit(ModelOptTaskSubmitDTO submitDTO); + + /** + * 修改模型优化任务 + * + * @param modelOptTaskUpdateDTO 模型优化任务修改对象 + */ + ModelOptUpdateVO update(ModelOptTaskUpdateDTO modelOptTaskUpdateDTO); + + /** + * 删除模型优化任务 + * + * @param modelOptTaskDeleteDTO 模型优化任务删除参数 + */ + void delete(ModelOptTaskDeleteDTO modelOptTaskDeleteDTO); + + /** + * 获取内置模型 + * + * @param type 算法类型 + * @param dataset 数据集 + * @param algorithm 训练算法 + * @return List 内置模型列表 + */ + List getBuiltInModel(Integer type, String dataset, String algorithm); + + /** + * 获取优化算法 + * + * @param type 算法类型 + * @param model 模型 + * @param dataset 数据集 + * @return List 获取模型优化算法列表 + */ + List getAlgorithm(Integer type, String model, String dataset); + + /** + * 获取模型优化数据集 + * + * @param type 算法类型 + * @param model 模型 + * @param algorithm 训练算法 + * @return List 模型优化数据集列表 + */ + List getDataset(Integer type, String model, String algorithm); + + /** + * 获取我的模型优化数据集 + * + * @return List 我的模型优化数据集列表 + */ + List getMyDataset(); + + /** + * 创建我的模型优化数据集 + * + * @param modelOptDatasetCreateDTO 数据集创建参数 + * @return ModelOptDatasetVO 模型优化模块我的数据集查询 + */ + ModelOptDatasetVO createMyDataset(ModelOptDatasetCreateDTO modelOptDatasetCreateDTO); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java index 92f33cc..b876bb7 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java index 6fd353f..62e8866 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java index 8fb06ea..5e75cfe 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,11 +17,8 @@ package org.dubhe.service; -import org.dubhe.domain.dto.PtImageDeleteDTO; -import org.dubhe.domain.dto.PtImageQueryDTO; -import org.dubhe.domain.dto.PtImageUpdateDTO; -import org.dubhe.domain.dto.PtImageUploadDTO; -import org.dubhe.domain.entity.HarborProject; +import org.dubhe.domain.dto.*; +import org.dubhe.domain.entity.PtImage; import java.util.List; import java.util.Map; @@ -51,21 +48,12 @@ public interface PtImageService { /** - * 通过imageName查询所含镜像版本信息 + * 根据镜像获取信息 * * @param imageName 镜像名 * @return List 镜像集合 */ - List searchImages(String imageName); - - - /** - * 查询harbor镜像列表 - * - * @return List harbor镜像集合 - **/ - List getHarborProjectList(); - + List searchImages(Integer projectType, String imageName); /** * 删除镜像 @@ -84,8 +72,24 @@ public interface PtImageService { /** * 获取镜像名称列表 - * + * @param projectType 镜像项目类型 * @return Set 镜像列表 */ - Set getImageNameList(); + Set getImageNameList(Integer projectType); + + /** + * 修改镜像来源(notebook定制) + * + * @param id 镜像id + */ + void updImageResource(Long id); + + /** + * 获取镜像URL + * + * @param imageQueryUrlDTO 查询镜像路径DTO + * @return String 镜像完整路径 + */ + String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO); + } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtMeasureService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtMeasureService.java new file mode 100644 index 0000000..41ca656 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtMeasureService.java @@ -0,0 +1,70 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.service; + +import org.dubhe.domain.dto.PtMeasureDTO; +import org.dubhe.domain.dto.PtMeasureDeleteDTO; +import org.dubhe.domain.dto.PtMeasureQueryDTO; +import org.dubhe.domain.dto.PtMeasureUpdateDTO; + +import java.util.Map; + +/** + * @description 度量管理service + * @date 2020-11-16 + */ +public interface PtMeasureService { + + + /** + * 查询度量信息 + * + * @param ptMeasureQueryDTO 查询条件 + * @return Map 度量列表分页信息 + */ + Map getMeasure(PtMeasureQueryDTO ptMeasureQueryDTO); + + /** + * 新建度量 + * + * @param ptMeasureCreateDTO 新建度量入参DTO + */ + void createMeasure(PtMeasureDTO ptMeasureCreateDTO); + + /** + * 修改度量 + * + * @param ptMeasureUpdateDTO 修改度量入参DTO + */ + void updateMeasure(PtMeasureUpdateDTO ptMeasureUpdateDTO); + + /** + * 根据id删除度量 + * + * @param ptMeasureDeleteDTO 删除度量的条件DTO + */ + void deleteMeasure(PtMeasureDeleteDTO ptMeasureDeleteDTO); + + /** + * 根据度量名称返回度量文件信息 + * + * @param name 度量名称 + * @return String 度量文件json字符串 + */ + String getMeasureByName(String name); + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java index 62e4d07..46ea712 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,9 @@ package org.dubhe.service; -import org.dubhe.domain.dto.PtTrainAlgorithmCreateDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmDeleteDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmQueryDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUpdateDTO; +import org.dubhe.domain.dto.*; +import org.dubhe.domain.entity.PtTrainAlgorithm; -import java.util.List; import java.util.Map; /** @@ -45,7 +42,7 @@ public interface PtTrainAlgorithmService { * @param resources 新增算法条件 * @return PtTrainAlgorithmCreateVO 新建训练算法 */ - List create(PtTrainAlgorithmCreateDTO resources); + Long create(PtTrainAlgorithmCreateDTO resources); /** * 修改算法 @@ -53,7 +50,7 @@ public interface PtTrainAlgorithmService { * @param resources 修改算法条件 * @return PtTrainAlgorithmUpdateVO 修改训练算法 */ - List update(PtTrainAlgorithmUpdateDTO resources); + Long update(PtTrainAlgorithmUpdateDTO resources); /** * 删除算法 @@ -67,5 +64,12 @@ public interface PtTrainAlgorithmService { */ Map getAlgorithmCount(); + /** + * 模型优化上传算法 + * + * @param ptModelAlgorithmCreateDTO 模型优化上传算法入参 + * @return PtTrainAlgorithm 新增算法信息 + */ + PtTrainAlgorithm modelOptimizationUploadAlgorithm(PtModelAlgorithmCreateDTO ptModelAlgorithmCreateDTO); } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java index ac8c2d3..a28d0df 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java index 3714da8..5c85338 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -122,4 +122,12 @@ public interface PtTrainJobService { * @return List Pod Metrics Grafana url */ List getGrafanaUrl(Long jobId); + + /** + * 获取训练使用的模型信息 + * + * @param ptTrainModelDTO + * @return PtTrainJobModelVO + */ + PtTrainJobModelVO getTrainJobModel(PtTrainModelDTO ptTrainModelDTO); } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java index ca069b1..960567c 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java index e1d5c02..023e14f 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java index 232b22a..1ba625f 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ServingService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ServingService.java new file mode 100644 index 0000000..d093fa9 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ServingService.java @@ -0,0 +1,164 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service; + +import org.dubhe.domain.dto.PredictParamDTO; +import org.dubhe.domain.dto.ServingInfoCreateDTO; +import org.dubhe.domain.dto.ServingInfoDeleteDTO; +import org.dubhe.domain.dto.ServingInfoDetailDTO; +import org.dubhe.domain.dto.ServingInfoQueryDTO; +import org.dubhe.domain.dto.ServingInfoUpdateDTO; +import org.dubhe.domain.dto.ServingStartDTO; +import org.dubhe.domain.dto.ServingStopDTO; +import org.dubhe.domain.entity.ServingInfo; +import org.dubhe.domain.vo.PredictParamVO; +import org.dubhe.domain.vo.ServingInfoCreateVO; +import org.dubhe.domain.vo.ServingInfoDeleteVO; +import org.dubhe.domain.vo.ServingInfoDetailVO; +import org.dubhe.domain.vo.ServingInfoUpdateVO; +import org.dubhe.domain.vo.ServingMetricsVO; +import org.dubhe.domain.vo.ServingModelConfigVO; +import org.dubhe.domain.vo.ServingStartVO; +import org.dubhe.domain.vo.ServingStopVO; +import org.dubhe.dto.callback.ServingK8sDeploymentCallbackCreateDTO; +import org.dubhe.k8s.domain.vo.PodVO; +import org.springframework.web.multipart.MultipartFile; + +import java.util.List; +import java.util.Map; + +/** + * @description 云端服务管理 + * @date 2020-08-25 + */ +public interface ServingService { + /** + * 查询分页数据 + * + * @param servingInfoQueryDTO 服务查询参数 + * @return Map 云端服务分页对象 + */ + Map query(ServingInfoQueryDTO servingInfoQueryDTO); + + /** + * 创建服务 + * + * @param servingInfoCreateDTO 服务创建参数 + * @return ServingInfoCreateVO 服务创建返回对象 + */ + ServingInfoCreateVO create(ServingInfoCreateDTO servingInfoCreateDTO); + + /** + * 修改服务 + * + * @param servingInfoUpdateDTO 服务对象修改 + * @return ServingInfoUpdateVO 服务修改返回对象 + */ + ServingInfoUpdateVO update(ServingInfoUpdateDTO servingInfoUpdateDTO); + + /** + * 删除服务 + * + * @param servingInfoDeleteDTO 服务对象删除 + * @return ServingInfoDeleteVO 服务删除返回对象 + */ + ServingInfoDeleteVO delete(ServingInfoDeleteDTO servingInfoDeleteDTO); + + /** + * 获取服务详情 + * + * @param servingInfoDetailDTO 获取服务详情参数 + * @return ServingInfoDetailVO 服务详情返回对象 + */ + ServingInfoDetailVO getDetail(ServingInfoDetailDTO servingInfoDetailDTO); + + /** + * 启动服务 + * + * @param servingStartDTO 启动服务参数 + * @return ServingStartVO 启动服务返回对象 + */ + ServingStartVO start(ServingStartDTO servingStartDTO); + + /** + * 预测 + * + * @param id 预测服务ID + * @param url 预测地址 + * @param files 需要预测的图片文件 + * @return ServingPredictVO 预测返回对象 + */ + String predict(Long id, String url, MultipartFile[] files); + + /** + * 停止服务 + * + * @param servingStopDTO 停止服务参数 + * @return ServingStopVO 停止服务返回对象 + */ + ServingStopVO stop(ServingStopDTO servingStopDTO); + + /** + * 获取预测参数 + * + * @param predictParamDTO 获取预测参数服务 + * @return PredictParamVO 预测参数返回对象 + */ + PredictParamVO getPredictParam(PredictParamDTO predictParamDTO); + + /** + * 发送路由更新消息 + * + * @param saveIdList 新增的路由ID列表 + * @param deleteIdList 删除的路由ID列表 + */ + void notifyUpdateServingRoute(List saveIdList, List deleteIdList); + + /** + * 获取modelConfigId下pod信息 + * + * @param id 服务配置id + * @return 服务配下的pod信息 + */ + List getPods(Long id); + + /** + * 获取在线服务的监控信息 + * + * @param id 模型部署信息id + * @return ServingMetricsVO 返回监控信息对象 + */ + ServingMetricsVO getMetricsDetail(Long id); + + /** + * k8s回调在线服务状态 + * + * @param times 回调请求次数 + * @param req 回调请求对象 + * @return boolean 返回是否回调成功 + */ + boolean servingCallback(int times, ServingK8sDeploymentCallbackCreateDTO req); + + /** + * 获取在线服务回滚信息列表 + * + * @param servingId 在线服务id + * @return Map> 返回回滚信息列表 + */ + Map> getRollbackList(Long servingId); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractDeploymentCallback.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractDeploymentCallback.java new file mode 100644 index 0000000..bf0be16 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractDeploymentCallback.java @@ -0,0 +1,85 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.abstracts; + +import org.dubhe.constant.NumberConstant; +import org.dubhe.dto.callback.BaseK8sDeploymentCallbackCreateDTO; +import org.dubhe.enums.LogEnum; +import org.dubhe.service.DeploymentCallbackAsyncService; +import org.dubhe.utils.K8sCallBackTool; +import org.dubhe.utils.LogUtil; + +import javax.annotation.Resource; + +/** + * @description 公共 deployment回调 + * @date 2020-11-27 + */ +public abstract class AbstractDeploymentCallback implements DeploymentCallbackAsyncService { + + @Resource + private K8sCallBackTool k8sCallBackTool; + + /** + * 公共 失败重试策略 + * + * @param k8sDeploymentCallbackCreateDTO + * @param + */ + @Override + public void deploymentCallBack(R k8sDeploymentCallbackCreateDTO) { + int tryTime = 1; + while (!doCallback(tryTime, k8sDeploymentCallbackCreateDTO)) { + if (k8sCallBackTool.continueRetry(++tryTime)) { + // 继续重试 tryTime重试次数+1 + try { + Thread.sleep(tryTime * NumberConstant.NUMBER_1000); + } catch (InterruptedException e) { + LogUtil.error(LogEnum.NOTE_BOOK, "AbstractDeploymentCallback deploymentCallBack InterruptedException : {}", e); + // Restore interrupted state...       + Thread.currentThread().interrupt(); + } + } else { + // 重试超限 tryTime重试次数+1未尝试，因此需要tryTime重试次数-1 + callbackFailed(--tryTime, k8sDeploymentCallbackCreateDTO); + break; + } + } + } + + + /** + * deployment 异步回调具体实现处理类 + * + * @param times 第n次处理 + * @param k8sDeploymentCallbackCreateDTO k8s回调实体类 + * @param BaseK8sDeploymentCallbackReq k8s回调基类 + * @return true：处理成功 false：处理失败 + */ + public abstract boolean doCallback(int times, R k8sDeploymentCallbackCreateDTO); + + + /** + * deployment 异步回调具体实现处理类 + * + * @param retryTimes 总处理次数 + * @param k8sDeploymentCallbackCreateDTO k8s回调实体类 + * @param BaseK8sDeploymentCallbackReq k8s回调基类 + */ + public abstract void callbackFailed(int retryTimes, R k8sDeploymentCallbackCreateDTO); +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java index 00baba8..e14a456 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java index 4c1cef1..092fb92 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java index 219f6a8..77c9d00 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java index 4dba6cd..3627773 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingAsyncServiceImpl.java new file mode 100644 index 0000000..0eafe5d --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingAsyncServiceImpl.java @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; +import org.dubhe.dto.callback.BatchServingK8sPodCallbackCreateDTO; +import org.dubhe.enums.LogEnum; +import org.dubhe.service.BatchServingService; +import org.dubhe.service.PodCallbackAsyncService; +import org.dubhe.service.abstracts.AbstractPodCallback; +import org.dubhe.utils.LogUtil; +import org.springframework.stereotype.Service; +import javax.annotation.Resource; + +/** + * @description 云端Serving批量服务回调 + * @date 2020-11-27 + */ +@Service(value = "batchServingAsyncService") +public class BatchServingAsyncServiceImpl extends AbstractPodCallback implements PodCallbackAsyncService { + + @Resource + private BatchServingService batchServingService; + + @Override + public boolean doCallback(int times, R k8sPodCallbackCreateDTO) { + // 强制转型 + BatchServingK8sPodCallbackCreateDTO req = (BatchServingK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; + LogUtil.info(LogEnum.SERVING, "Thread {} try {} time.Request: {}", Thread.currentThread(), times, req.toString()); + //批量服务回调 + return batchServingService.batchServingCallback(times, req); + } + + @Override + public void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO) { + BatchServingK8sPodCallbackCreateDTO req = (BatchServingK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; + LogUtil.info(LogEnum.SERVING, "Thread {} try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingServiceImpl.java new file mode 100644 index 0000000..5a4b65f --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingServiceImpl.java @@ -0,0 +1,673 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.dubhe.annotation.DataPermissionMethod; +import org.dubhe.constant.NumberConstant; +import org.dubhe.constant.ServingConstant; +import org.dubhe.constant.StringConstant; +import org.dubhe.constant.SymbolConstant; +import org.dubhe.dao.BatchServingMapper; +import org.dubhe.dao.PtModelBranchMapper; +import org.dubhe.dao.PtModelInfoMapper; +import org.dubhe.domain.PtModelBranch; +import org.dubhe.domain.PtModelInfo; +import org.dubhe.domain.dto.BatchServingCreateDTO; +import org.dubhe.domain.dto.BatchServingDeleteDTO; +import org.dubhe.domain.dto.BatchServingDetailDTO; +import org.dubhe.domain.dto.BatchServingQueryDTO; +import org.dubhe.domain.dto.BatchServingStartDTO; +import org.dubhe.domain.dto.BatchServingStopDTO; +import org.dubhe.domain.dto.BatchServingUpdateDTO; +import org.dubhe.domain.dto.RecycleTaskCreateDTO; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.BatchServing; +import org.dubhe.domain.vo.BatchServingCreateVO; +import org.dubhe.domain.vo.BatchServingDeleteVO; +import org.dubhe.domain.vo.BatchServingDetailVO; +import org.dubhe.domain.vo.BatchServingQueryVO; +import org.dubhe.domain.vo.BatchServingStartVO; +import org.dubhe.domain.vo.BatchServingStopVO; +import org.dubhe.domain.vo.BatchServingUpdateVO; +import org.dubhe.domain.vo.ServingInfoQueryVO; +import org.dubhe.dto.callback.BatchServingK8sPodCallbackCreateDTO; +import org.dubhe.enums.BizEnum; +import org.dubhe.enums.DatasetTypeEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.RecycleModuleEnum; +import org.dubhe.enums.RecycleResourceEnum; +import org.dubhe.enums.RecycleTypeEnum; +import org.dubhe.enums.ServingErrorEnum; +import org.dubhe.enums.ServingStatusEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.k8s.domain.dto.PodQueryDTO; +import org.dubhe.k8s.domain.vo.PodVO; +import org.dubhe.k8s.enums.PodPhaseEnum; +import org.dubhe.k8s.service.PodService; +import org.dubhe.service.BatchServingService; +import org.dubhe.service.RecycleTaskService; +import org.dubhe.service.UserService; +import org.dubhe.task.DeployServingAsyncTask; +import org.dubhe.utils.DateUtil; +import org.dubhe.utils.JwtUtils; +import org.dubhe.utils.K8sNameTool; +import org.dubhe.utils.K8sUtil; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.MinioUtil; +import org.dubhe.utils.NfsUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.ReflectionUtils; +import org.dubhe.utils.StringUtils; +import org.dubhe.utils.WrapperHelp; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.text.DecimalFormat; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * @description 批量服务管理 + * @date 2020-08-26 + */ +@Service +public class BatchServingServiceImpl extends ServiceImpl implements BatchServingService { + + @Resource + private BatchServingMapper batchServingMapper; + + @Resource + private PtModelInfoMapper ptModelInfoMapper; + + @Resource + private PtModelBranchMapper ptModelBranchMapper; + + @Resource + private DeployServingAsyncTask deployServingAsyncTask; + + @Resource + private NfsUtil nfsUtil; + + @Resource + private K8sUtil k8sUtil; + + @Resource + private MinioUtil minioUtil; + + @Resource + private K8sNameTool k8sNameTool; + + @Resource + private PodService podService; + + @Resource + private RecycleTaskService recycleTaskService; + + @Resource + private UserService userService; + + @Value("${minio.bucketName}") + private String bucketName; + + + public final static List FILE_NAMES; + + static { + FILE_NAMES = ReflectionUtils.getFieldNames(BatchServingQueryVO.class); + } + + /** + * 批量服务查询 + * + * @param batchServingQueryDTO 批量服务查询参数 + * @return Map 批量服务查询返回分页对象 + */ + @Override + public Map query(BatchServingQueryDTO batchServingQueryDTO) { + String name = batchServingQueryDTO.getName(); + //批量服务名称或id条件非空 + if (StringUtils.isNotBlank(name)) { + //整数匹配 + if (StringConstant.PATTERN_NUM.matcher(name).matches()) { + batchServingQueryDTO.setId(Long.parseLong(name)); + batchServingQueryDTO.setName(null); + Map map = queryBatchServing(batchServingQueryDTO); + if (((List) map.get(StringConstant.RESULT)).size() > NumberConstant.NUMBER_0) { + return map; + } else { + batchServingQueryDTO.setId(null); + batchServingQueryDTO.setName(name); + } + } + } + return queryBatchServing(batchServingQueryDTO); + } + + /** + * 批量服务查询 + * + * @param batchServingQueryDTO 批量服务查询参数 + * @return Map 批量服务查询返回分页对象 + */ + public Map queryBatchServing(BatchServingQueryDTO batchServingQueryDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + LogUtil.info(LogEnum.SERVING, "User {} queried the online service list with the query of{}", userDTO.getUsername(), JSONObject.toJSONString(batchServingQueryDTO)); + + QueryWrapper wrapper = WrapperHelp.getWrapper(batchServingQueryDTO); + wrapper.eq("create_user_id", userDTO.getId()); + if (StringUtils.isNotBlank(batchServingQueryDTO.getName())) { + //按名称搜索时不区分大小写 + wrapper.like("lower(name)", batchServingQueryDTO.getName().toLowerCase()); + } + Page page = new Page(null == batchServingQueryDTO.getCurrent() ? NumberConstant.NUMBER_1 : batchServingQueryDTO.getCurrent(), + null == batchServingQueryDTO.getSize() ? NumberConstant.NUMBER_10 : batchServingQueryDTO.getSize()); + try { + //排序字段，默认按更新时间降序，否则将驼峰转换为下划线 + String column = batchServingQueryDTO.getSort() != null && FILE_NAMES.contains(batchServingQueryDTO.getSort()) ? StringUtils.humpToLine(batchServingQueryDTO.getSort()) : "update_time"; + //排序方式 + boolean isAsc = StringUtils.isBlank(batchServingQueryDTO.getOrder()) || StringUtils.equals(batchServingQueryDTO.getOrder(), StringConstant.SORT_DESC) ? false : true; + wrapper.orderBy(true, isAsc, column); + } catch (Exception e) { + LogUtil.error(LogEnum.SERVING, "Query online service with an exception, query info:{}，exception info:{}", JSONObject.toJSONString(batchServingQueryDTO), e); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + IPage batchServings = batchServingMapper.selectPage(page, wrapper); + List queryVOList = batchServings.getRecords().stream().map(batchServing -> { + BatchServingQueryVO batchServingQueryVO = new BatchServingQueryVO(); + BeanUtils.copyProperties(batchServing, batchServingQueryVO); + return batchServingQueryVO; + }).collect(Collectors.toList()); + LogUtil.info(LogEnum.SERVING, "User {} queried batching service list, the number of batching service is {}", userDTO.getUsername(), queryVOList.size()); + return PageUtil.toPage(page, queryVOList); + } + + /** + * 创建批量服务 + * + * @param batchServingCreateDTO 批量服务创建参数 + * @return BatchServingCreateVO 返回创建后结果 + */ + @Override + @Transactional(rollbackFor = Exception.class) + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + public BatchServingCreateVO create(BatchServingCreateDTO batchServingCreateDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + //参数校验 + assert Objects.nonNull(userDTO); + checkNameExist(batchServingCreateDTO.getName(), userDTO.getId()); + Integer frameType = getFrameType(batchServingCreateDTO.getModelId(), batchServingCreateDTO.getModelAddress()); + checkResourceType(frameType, batchServingCreateDTO.getResourcesPoolType()); + checkInputExist(batchServingCreateDTO.getInputPath()); + BatchServing batchServing = new BatchServing(); + BeanUtils.copyProperties(batchServingCreateDTO, batchServing); + batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); + batchServing.setFrameType(frameType); + String outputPath = ServingConstant.OUTPUT_NFS_PATH + userDTO.getId() + SymbolConstant.SLASH + StringUtils.getTimestamp() + SymbolConstant.SLASH; + batchServing.setOutputPath(outputPath); + saveBatchServing(userDTO, batchServing); + //输入文件定时清理 + createRecycleTask(batchServing.getInputPath(), NumberConstant.NUMBER_1); + deployServingAsyncTask.deployBatchServing(userDTO, batchServing); + return new BatchServingCreateVO(batchServing.getId(), batchServing.getStatus()); + } + + /** + * 创建批量服务并保存数据 + * + * @param userDTO 用户信息 + * @param batchServing 批量服务信息 + */ + @Transactional(rollbackFor = Exception.class) + public void saveBatchServing(UserDTO userDTO, BatchServing batchServing) { + if (!save(batchServing)) { + LogUtil.error(LogEnum.SERVING, "User {} failed to save the batching service info to the database, service name：{}", userDTO.getUsername(), batchServing.getName()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + } + + /** + * 校验资源类型 + * + * @param frameType 模型框架 + * @param resourcePoolType 节点类型 + */ + public void checkResourceType(Integer frameType, Integer resourcePoolType) { + // oneflow 暂不支持cpu + if (NumberConstant.NUMBER_1 == frameType && NumberConstant.NUMBER_0 == resourcePoolType) { + throw new BusinessException(ServingErrorEnum.CPU_NOT_SUPPORTED_BY_ONEFLOW); + } + } + + /** + * 文件定时清理 + * + * @param path 文件路径 + * @param day 保留天数 + */ + public void createRecycleTask(String path, Integer day) { + RecycleTaskCreateDTO recycleTaskCreateDTO = new RecycleTaskCreateDTO() + .setRecycleCustom(RecycleResourceEnum.SERVING_RECYCLE_FILE.getClassName()) + .setRecycleCondition(k8sUtil.getAbsoluteNfsPath(path)) + .setRecycleDelayDate(day) + .setRecycleType(RecycleTypeEnum.FILE.getCode()) + .setRecycleModule(RecycleModuleEnum.BIZ_SERVING.getValue()) + .setRecycleNote(RecycleResourceEnum.SERVING_RECYCLE_FILE.getMessage()); + recycleTaskService.createRecycleTask(recycleTaskCreateDTO); + } + + /** + * 校验名称是否存在 + * + * @param name 服务名称 + * @param userId 用户ID + */ + public void checkNameExist(String name, Long userId) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + wrapper.eq(BatchServing::getName, name); + wrapper.eq(BatchServing::getCreateUserId, userId); + int count = batchServingMapper.selectCount(wrapper); + if (count > NumberConstant.NUMBER_0) { + throw new BusinessException(ServingErrorEnum.SERVING_NAME_EXIST); + } + } + + /** + * 获取模型框架 + * + * @param modelId 模型ID + * @param modelAddress 模型路径 + * @return 模型框架 + */ + public Integer getFrameType(Long modelId, String modelAddress) { + PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(modelId); + String path = k8sUtil.getAbsoluteNfsPath(modelAddress); + if (ptModelInfo == null || nfsUtil.fileOrDirIsEmpty(path)) { + throw new BusinessException(ServingErrorEnum.MODEL_FILE_NOT_EXIST); + } + if (ptModelInfo.getFrameType() > NumberConstant.NUMBER_4) { + throw new BusinessException(ServingErrorEnum.MODEL_FRAME_TYPE_NOT_SUPPORTED); + } + return ptModelInfo.getFrameType(); + } + + /** + * 更新批量服务 + * + * @param batchServingUpdateDTO 批量服务修改参数 + * @return BatchServingUpdateVO 返回更新后结果 + */ + @Override + @Transactional(rollbackFor = Exception.class) + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + public BatchServingUpdateVO update(BatchServingUpdateDTO batchServingUpdateDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + BatchServing batchServing = checkBatchServingExist(batchServingUpdateDTO.getId(), userDTO.getId()); + checkBatchServingStatus(batchServing.getStatus()); + //修改输入路径时，定时删除之前路径下的文件 + if (!batchServing.getInputPath().equals(batchServingUpdateDTO.getInputPath())) { + createRecycleTask(batchServing.getInputPath(), NumberConstant.NUMBER_1); + } + createRecycleTask(batchServing.getOutputPath(), NumberConstant.NUMBER_1); + //修改模型或版本时，重新校验模型 + if (!(batchServing.getModelId().equals(batchServingUpdateDTO.getModelId()) && + batchServing.getModelAddress().equals(batchServingUpdateDTO.getModelAddress()))) { + Integer frameType = getFrameType(batchServingUpdateDTO.getModelId(), batchServingUpdateDTO.getModelAddress()); + batchServing.setFrameType(frameType); + } + checkResourceType(batchServing.getFrameType(), batchServing.getResourcesPoolType()); + BeanUtils.copyProperties(batchServingUpdateDTO, batchServing); + batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); + batchServing.setUpdateTime(DateUtil.getCurrentTimestamp()); + String outputPath = ServingConstant.OUTPUT_NFS_PATH + userDTO.getId() + SymbolConstant.SLASH + StringUtils.getTimestamp() + SymbolConstant.SLASH; + batchServing.setOutputPath(outputPath); + updateBatchServing(userDTO, batchServing); + deployServingAsyncTask.deleteBatchServing(userDTO, batchServing); + deployServingAsyncTask.deployBatchServing(userDTO, batchServing); + createRecycleTask(batchServingUpdateDTO.getInputPath(), NumberConstant.NUMBER_1); + return new BatchServingUpdateVO(batchServing.getId(), batchServing.getStatus()); + } + + /** + * 修改批量服务并保存数据 + * + * @param userDTO 用户信息 + * @param batchServing 批量服务信息 + */ + @Transactional(rollbackFor = Exception.class) + public void updateBatchServing(UserDTO userDTO, BatchServing batchServing) { + int result = batchServingMapper.updateById(batchServing); + if (result < 1) { + LogUtil.error(LogEnum.SERVING, "User {} failed modifying the batching service in the database, service id={}", userDTO.getUsername(), batchServing.getId()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + } + + /** + * 校验批量服务是否存在 + * + * @param id 批量服务id + * @param userId 用户id + * @return BatchServing 批量服务对象 + */ + BatchServing checkBatchServingExist(Long id, Long userId) { + BatchServing batchServing = batchServingMapper.selectById(id); + //校验服务是否存在，或是否为当前用户创建 + if (batchServing == null || (!userId.equals(batchServing.getCreateUserId()))) { + throw new BusinessException(ServingErrorEnum.SERVING_INFO_ABSENT); + } + return batchServing; + } + + /** + * 校验批量服务状态是否为停止状态 + * + * @param status 批量服务状态 + */ + void checkBatchServingStatus(String status) { + if (ServingStatusEnum.WORKING.getStatus().equals(status) || + ServingStatusEnum.IN_DEPLOYMENT.getStatus().equals(status)) { + throw new BusinessException(ServingErrorEnum.OPERATION_NOT_ALLOWED); + } + } + + /** + * 校验输入图片是否为空 + * + * @param inputPath 图片路径 + */ + void checkInputExist(String inputPath) { + if (nfsUtil.fileOrDirIsEmpty(k8sUtil.getAbsoluteNfsPath(inputPath))) { + throw new BusinessException(ServingErrorEnum.INPUT_FILE_NOT_EXIST); + } + } + + /** + * 删除批量服务 + * + * @param batchServingDeleteDTO 批量服务删除参数 + * @return BatchServingDeleteVO 返回删除后结果 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public BatchServingDeleteVO delete(BatchServingDeleteDTO batchServingDeleteDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + BatchServing batchServing = checkBatchServingExist(batchServingDeleteDTO.getId(), userDTO.getId()); + checkBatchServingStatus(batchServing.getStatus()); + deleteBatchServing(batchServingDeleteDTO, userDTO); + createRecycleTask(batchServing.getInputPath(), NumberConstant.NUMBER_1); + createRecycleTask(batchServing.getOutputPath(), NumberConstant.NUMBER_1); + return new BatchServingDeleteVO(batchServing.getId()); + } + + /** + * 删除批量服务并保存数据 + * + * @param batchServingDeleteDTO 批量服务信息 + * @param userDTO 用户信息 + */ + @Transactional(rollbackFor = Exception.class) + public void deleteBatchServing(BatchServingDeleteDTO batchServingDeleteDTO, UserDTO userDTO) { + if (!removeById(batchServingDeleteDTO.getId())) { + LogUtil.error(LogEnum.SERVING, "User {} failed deleting the batching service in the database, service id={}", userDTO.getUsername(), batchServingDeleteDTO.getId()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + } + + /** + * 启动批量服务 + * + * @param batchServingStartDTO 批量服务启动参数 + * @return BatchServingStartVO 返回启动后信息 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public BatchServingStartVO start(BatchServingStartDTO batchServingStartDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + BatchServing batchServing = checkBatchServingExist(batchServingStartDTO.getId(), userDTO.getId()); + if (StringUtils.equalsAny(batchServing.getStatus(), ServingStatusEnum.IN_DEPLOYMENT.getStatus(), ServingStatusEnum.WORKING.getStatus(), ServingStatusEnum.COMPLETED.getStatus())) { + LogUtil.error(LogEnum.SERVING, "User {} failed starting the batching service, service id={}, service name:{}, service status：{}", userDTO.getUsername(), batchServing.getId(), batchServing.getName(), batchServing.getStatus()); + throw new BusinessException(ServingErrorEnum.OPERATION_NOT_ALLOWED); + } + //删除之前的推理结果，重新推理 + createRecycleTask(batchServing.getOutputPath(), NumberConstant.NUMBER_1); + batchServing.setProgress(SymbolConstant.ZERO); + batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); + //生成新的输出路径 + String outputPath = ServingConstant.OUTPUT_NFS_PATH + userDTO.getId() + SymbolConstant.SLASH + StringUtils.getTimestamp() + SymbolConstant.SLASH; + batchServing.setOutputPath(outputPath); + updateBatchServing(userDTO, batchServing); + deployServingAsyncTask.deployBatchServing(userDTO, batchServing); + return new BatchServingStartVO(batchServing.getId(), batchServing.getStatus(), batchServing.getProgress()); + } + + /** + * 停止批量服务 + * + * @param batchServingStopDTO 批量服务停止参数 + * @return BatchServingStopVO 返回停止后信息 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public BatchServingStopVO stop(BatchServingStopDTO batchServingStopDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + BatchServing batchServing = checkBatchServingExist(batchServingStopDTO.getId(), userDTO.getId()); + if (ServingStatusEnum.STOP.getStatus().equals(batchServing.getStatus()) || ServingStatusEnum.EXCEPTION.equals(batchServing.getStatus())) { + LogUtil.error(LogEnum.SERVING, "The service is not running, user {} failed stopping the service. Service id={}, service name:{}, service status:{}", + userDTO.getUsername(), batchServing.getId(), batchServing.getName(), batchServing.getStatus()); + throw new BusinessException(ServingErrorEnum.OPERATION_NOT_ALLOWED); + } + updateBatchServing(userDTO, batchServing); + deployServingAsyncTask.deleteBatchServing(userDTO, batchServing); + return new BatchServingStopVO(batchServing.getId(), batchServing.getStatus()); + } + + /** + * 获取批量服务详情 + * + * @param batchServingDetailDTO 批量服务详情参数 + * @return BatchServingDetailVO 返回批量服务详情 + */ + @Override + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + public BatchServingDetailVO getDetail(BatchServingDetailDTO batchServingDetailDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + BatchServing batchServing = checkBatchServingExist(batchServingDetailDTO.getId(), userDTO.getId()); + BatchServingDetailVO batchServingDetailVO = new BatchServingDetailVO(); + BeanUtils.copyProperties(batchServing, batchServingDetailVO); + PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(batchServingDetailVO.getModelId()); + if (ptModelInfo == null) { + LogUtil.error(LogEnum.SERVING, "User {} failed deploying the batch service. The model is not exist: {}, Service id={}", userDTO.getUsername(), ptModelInfo.getName(), batchServingDetailDTO.getId()); + throw new BusinessException(ServingErrorEnum.MODEL_NOT_EXIST); + } + batchServingDetailVO.setModelName(ptModelInfo.getName()); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper(); + wrapper.eq(PtModelBranch::getParentId, batchServingDetailVO.getModelId()); + wrapper.eq(PtModelBranch::getModelAddress, batchServingDetailVO.getModelAddress()); + PtModelBranch ptModelBranch = ptModelBranchMapper.selectOne(wrapper); + if (ptModelBranch != null) { + batchServingDetailVO.setModelVersion(ptModelBranch.getVersionNum()); + } + //运行中的任务计算进度 + if (batchServing.getStatus().equals(ServingStatusEnum.WORKING.getStatus())) { + String progress = queryProgressByMinIO(batchServing); + batchServingDetailVO.setProgress(progress); + } + return batchServingDetailVO; + } + + /** + * 批量服务回调 + * + * @param times 回调请求次数 + * @param req 回调请求对象 + * @return boolean 返回回调结果 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public boolean batchServingCallback(int times, BatchServingK8sPodCallbackCreateDTO req) { + // 不处理slave + if (req.getPodName().contains(ServingConstant.SLAVE_POD)) { + return true; + } + // 根据namespace和podName找到模型配置 + String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.BATCH_SERVING, req.getResourceName()); + if (StringUtils.isBlank(resourceInfo)) { + LogUtil.warn(LogEnum.SERVING, "Cannot find modelConfig ID! Request: {}", Thread.currentThread(), times, req.toString()); + return false; + } + Long id = Long.parseLong(resourceInfo.substring(NumberConstant.NUMBER_4)); + BatchServing batchServing = batchServingMapper.selectById(id); + if (Objects.isNull(batchServing)) { + LogUtil.warn(LogEnum.SERVING, "Cannot find batchServing! Request: {}", Thread.currentThread(), times, req.toString()); + return false; + } + // 对于当前状态是已完成的不处理 + if (ServingStatusEnum.COMPLETED.getStatus().equals(batchServing.getStatus())) { + return true; + } + if (PodPhaseEnum.PENDING.getPhase().equals(req.getPhase())) { + batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); + } + if (PodPhaseEnum.RUNNING.getPhase().equals(req.getPhase())) { + //从部署中状态转变为运行中时设置开始时间 + if (ServingStatusEnum.IN_DEPLOYMENT.getStatus().equals(batchServing.getStatus())) { + batchServing.setStartTime(DateUtil.getCurrentTimestamp()); + } + batchServing.setStatus(ServingStatusEnum.WORKING.getStatus()); + } + if (PodPhaseEnum.SUCCEEDED.getPhase().equals(req.getPhase())) { + batchServing.setEndTime(DateUtil.getCurrentTimestamp()); + batchServing.setStatus(ServingStatusEnum.COMPLETED.getStatus()); + batchServing.setProgress(String.valueOf(NumberConstant.NUMBER_100)); + // 批量推理成功异步发送短信通知 + deployServingAsyncTask.asyncSendServingMail(userService.findById(batchServing.getCreateUserId()).getEmail(), batchServing.getId()); + } + if (PodPhaseEnum.FAILED.getPhase().equals(req.getPhase())) { + String progress = queryProgressByMinIO(batchServing); + batchServing.setProgress(progress); + batchServing.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); + } + if (PodPhaseEnum.DELETED.getPhase().equals(req.getPhase())) { + String progress = queryProgressByMinIO(batchServing); + batchServing.setProgress(progress); + batchServing.setStatus(ServingStatusEnum.STOP.getStatus()); + } + if (PodPhaseEnum.UNKNOWN.getPhase().equals(req.getPhase())) { + batchServing.setStatus(ServingStatusEnum.UNKNOWN.getStatus()); + } + return updateById(batchServing); + } + + /** + * 获取批量服务所有POD + * + * @param id 服务配置id + * @return List 返回POD列表 + */ + @Override + public List getPods(Long id) { + BatchServing batchServing = batchServingMapper.selectById(id); + if (batchServing == null) { + return Collections.emptyList(); + } + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(user); + String nameSpace = k8sUtil.getNamespace(user); + return podService.getPods(new PodQueryDTO(nameSpace, k8sUtil.getBatchResourceName(batchServing.getResourceInfo()))); + } + + /** + * 查询批量服务状态与进度 + * + * @param id 批量服务id + * @return BatchServingQueryVO 返回查询结果 + */ + @Override + public BatchServingQueryVO queryStatusAndProgress(Long id) { + BatchServing batchServing = batchServingMapper.selectById(id); + if (Objects.isNull(batchServing)) { + throw new BusinessException(ServingErrorEnum.SERVING_INFO_ABSENT); + } + String progress = queryProgressByMinIO(batchServing); + return BatchServingQueryVO.builder() + .id(id) + .name(batchServing.getName()) + .description(batchServing.getDescription()) + .status(batchServing.getStatus()) + .progress(progress) + .startTime(batchServing.getStartTime()) + .endTime(batchServing.getEndTime()) + .outputPath(batchServing.getOutputPath()) + .build(); + } + + /** + * 通过minio查询推理进度 + * + * @param batchServing 批量服务信息 + * @return String 返回进度结果 + */ + private String queryProgressByMinIO(BatchServing batchServing) { + DecimalFormat df = new DecimalFormat(String.valueOf(NumberConstant.NUMBER_0)); + int inputCount = queryCount(batchServing.getInputPath()); + int outputCount = queryCount(batchServing.getOutputPath()); + String progress = String.valueOf(NumberConstant.NUMBER_0); + if (inputCount != NumberConstant.NUMBER_0) { + progress = df.format((float) outputCount / inputCount * NumberConstant.NUMBER_100); + } + return progress; + } + + /** + * 通过minio查询文件数量 + * + * @param path 文件路径 + * @return int 返回文件数量 + */ + private int queryCount(String path) { + try { + if (nfsUtil.fileOrDirIsEmpty(k8sUtil.getAbsoluteNfsPath(path))) { + return 0; + } + return minioUtil.getCount(bucketName, path); + } catch (Exception e) { + LogUtil.error(LogEnum.SERVING, "query count failed by path in minio: {}", path, e); + } + return NumberConstant.NUMBER_0; + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java new file mode 100644 index 0000000..4374c15 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java @@ -0,0 +1,62 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import lombok.extern.slf4j.Slf4j; +import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; +import org.dubhe.dto.callback.ModelOptK8sPodCallbackCreateDTO; +import org.dubhe.enums.LogEnum; +import org.dubhe.service.ModelOptTaskInstanceService; +import org.dubhe.service.PodCallbackAsyncService; +import org.dubhe.service.abstracts.AbstractPodCallback; +import org.dubhe.utils.LogUtil; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +/** + * @description 模型优化-k8s回调处理类 + * @date 2020-06-05 + */ +@Service(value = "modelOptAsyncServiceImpl") +@Slf4j +public class ModelOptAsyncServiceImpl extends AbstractPodCallback implements PodCallbackAsyncService { + + @Resource + private ModelOptTaskInstanceService modelOptTaskInstanceService; + + @Override + public boolean doCallback(int times, R k8sPodCallbackCreateDTO) { + try { + // 强制转型 + ModelOptK8sPodCallbackCreateDTO req = (ModelOptK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; + LogUtil.info(LogEnum.MODEL_OPT, "Thread {} try {} time.Request: {}", Thread.currentThread(), times, req.toString()); + return modelOptTaskInstanceService.modelOptCallBack(req); + } catch (Exception e) { + LogUtil.error(LogEnum.MODEL_OPT, "ModelOpt doCallback error!", e); + return false; + } + } + + @Override + public void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO) { + ModelOptK8sPodCallbackCreateDTO req = (ModelOptK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; + LogUtil.info(LogEnum.SERVING, "Thread {} try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java new file mode 100644 index 0000000..a99cf16 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java @@ -0,0 +1,568 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import cn.hutool.core.collection.CollectionUtil; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.apache.commons.lang3.StringUtils; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.constant.ModelOptConstant; +import org.dubhe.constant.ModelOptErrorEnum; +import org.dubhe.constant.ModelOptInstanceStatusEnum; +import org.dubhe.constant.NumberConstant; +import org.dubhe.constant.SymbolConstant; +import org.dubhe.dao.ModelOptTaskInstanceMapper; +import org.dubhe.domain.dto.DictDTO; +import org.dubhe.domain.dto.DictDetailDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceCancelDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceDeleteDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceDetailDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceQueryDTO; +import org.dubhe.domain.dto.ModelOptTaskInstanceResubmitDTO; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.ModelOptTaskInstance; +import org.dubhe.domain.vo.ModelOptResultQueryVO; +import org.dubhe.domain.vo.ModelOptTaskInstanceQueryVO; +import org.dubhe.dto.callback.ModelOptK8sPodCallbackCreateDTO; +import org.dubhe.enums.BizEnum; +import org.dubhe.enums.BizNfsEnum; +import org.dubhe.enums.DistillCommandEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.OptimizeTypeEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.k8s.api.ModelOptJobApi; +import org.dubhe.k8s.api.NamespaceApi; +import org.dubhe.k8s.domain.bo.PtModelOptimizationJobBO; +import org.dubhe.k8s.domain.dto.PodQueryDTO; +import org.dubhe.k8s.domain.resource.BizJob; +import org.dubhe.k8s.domain.resource.BizNamespace; +import org.dubhe.k8s.domain.vo.PodVO; +import org.dubhe.k8s.enums.K8sResponseEnum; +import org.dubhe.k8s.enums.PodPhaseEnum; +import org.dubhe.k8s.service.PodService; +import org.dubhe.service.DictService; +import org.dubhe.service.ModelOptTaskInstanceService; +import org.dubhe.utils.DateUtil; +import org.dubhe.utils.JwtUtils; +import org.dubhe.utils.K8sNameTool; +import org.dubhe.utils.K8sUtil; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.MinioUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.WrapperHelp; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + +import javax.annotation.Resource; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * @description 模型优化任务实例 + * @date 2020-05-22 + */ +@Service +public class ModelOptTaskInstanceServiceImpl extends ServiceImpl implements ModelOptTaskInstanceService { + + @Resource + private ModelOptTaskInstanceMapper modelOptTaskInstanceMapper; + + @Resource + private ModelOptJobApi modelOptJobApi; + + @Resource + private NamespaceApi namespaceApi; + + @Resource + private DictService dictService; + + @Resource + private K8sNameTool k8sNameTool; + + @Resource + private K8sUtil k8sUtil; + + @Resource + private PodService podService; + + @Resource + private MinioUtil minioUtil; + + @Value("${minio.bucketName}") + private String bucket; + + @Value("${optimize.image}") + private String optimizeImage; + + /** + * 分页查询任务执行记录实例列表 + * + * @param instanceQueryDTO 查询条件 + * @return Map 分页对象 + */ + @Override + public Map queryAll(ModelOptTaskInstanceQueryDTO instanceQueryDTO) { + QueryWrapper wrapper = WrapperHelp.getWrapper(instanceQueryDTO); + wrapper.orderByDesc("id"); + Page page = new Page(null == instanceQueryDTO.getCurrent() ? MagicNumConstant.ONE : instanceQueryDTO.getCurrent() + , null == instanceQueryDTO.getSize() ? MagicNumConstant.TEN : instanceQueryDTO.getSize()); + IPage modelOptTaskInstances = modelOptTaskInstanceMapper.selectPage(page, wrapper); + List list = modelOptTaskInstances.getRecords().stream().map(modelOptTaskInstance -> { + ModelOptTaskInstanceQueryVO modelOptTaskInstanceVO = buildResult(modelOptTaskInstance); + return modelOptTaskInstanceVO; + }).collect(Collectors.toList()); + return PageUtil.toPage(modelOptTaskInstances, list); + } + + /** + * 新增任务实例 + * + * @param modelOptTaskInstance 任务实例对象 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void create(ModelOptTaskInstance modelOptTaskInstance) { + modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.WAITING.getValue()); + if (StringUtils.isBlank(modelOptTaskInstance.getAlgorithmPath()) || StringUtils.isBlank(modelOptTaskInstance.getAlgorithmPath())) { + LogUtil.error(LogEnum.MODEL_OPT, "模型优化实例id={}的算法路径为空，算法：{}", modelOptTaskInstance.getId(), modelOptTaskInstance.getAlgorithmPath()); + throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); + } + modelOptTaskInstanceMapper.insert(modelOptTaskInstance); + if (!runTask(modelOptTaskInstance)) { + throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); + } + } + + /** + * 重新提交任务实例 + * + * @param resubmitDTO 重新提交任务实例参数 + */ + @Override + public void resubmit(ModelOptTaskInstanceResubmitDTO resubmitDTO) { + ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(resubmitDTO.getId()); + checkInstExist(modelOptTaskInstance); + // 校验该任务是否存在进行中和等待中的实例 + if (checkUnfinishedInst(modelOptTaskInstance.getTaskId())) { + LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务实例状态:{}，提交失败，实例id={}", modelOptTaskInstance.getStatus(), resubmitDTO.getId()); + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_EXIST); + } + ModelOptTaskInstance newInstance = rebuild(modelOptTaskInstance); + create(newInstance); + } + + /** + * 取消模型优化任务 + * + * @param cancelDTO 取消模型优化任务参数 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void cancel(ModelOptTaskInstanceCancelDTO cancelDTO) { + ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(cancelDTO.getId()); + checkInstExist(modelOptTaskInstance); + //等待、进行中的实例才能取消 + if (!StringUtils.equalsAny(modelOptTaskInstance.getStatus(), ModelOptInstanceStatusEnum.WAITING.getValue(), ModelOptInstanceStatusEnum.RUNNING.getValue())) { + LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务实例状态:{}，取消失败，实例id={}", modelOptTaskInstance.getStatus(), cancelDTO.getId()); + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_STATUS_ERROR); + } + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + // 调用k8s删除相关资源 + assert userDTO != null; + modelOptJobApi.deleteByResourceName(getNamespace(userDTO), k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, cancelDTO.getId().toString())); + modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.CANCELED.getValue()); + modelOptTaskInstance.setUpdateTime(new Timestamp(System.currentTimeMillis())); + modelOptTaskInstanceMapper.updateById(modelOptTaskInstance); + } + + /** + * 查看单个任务实例详情 + * + * @param detailDTO 查看任务实例详情参数 + * @return ModelOptTaskInstanceVO 任务实例对象 + */ + @Override + public ModelOptTaskInstanceQueryVO getInstDetail(ModelOptTaskInstanceDetailDTO detailDTO) { + ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(detailDTO.getId()); + checkInstExist(modelOptTaskInstance); + return buildResult(modelOptTaskInstance); + } + + /** + * k8s回调模型优化方法 + * + * @param req 模型优化自定义回调参数类 + * @return + */ + @Override + @Transactional(rollbackFor = Exception.class) + public boolean modelOptCallBack(ModelOptK8sPodCallbackCreateDTO req) { + String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.MODEL_OPT, req.getResourceName()); + if (StringUtils.isBlank(resourceInfo)) { + LogUtil.error(LogEnum.MODEL_OPT, "can't resolve resourceName {}", req.getResourceName()); + return false; + } + Long instId = Long.valueOf(resourceInfo); + ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(instId); + checkInstExist(modelOptTaskInstance); + //对当前状态是已完成或已取消的不执行回调操作 + if (ModelOptInstanceStatusEnum.COMPLETED.getValue().equals(modelOptTaskInstance.getStatus()) + || ModelOptInstanceStatusEnum.CANCELED.getValue().equals(modelOptTaskInstance.getStatus())) { + return true; + } + if (PodPhaseEnum.RUNNING.getPhase().equals(req.getPhase())) { + modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.RUNNING.getValue()); + modelOptTaskInstance.setStartTime(DateUtil.getCurrentTimestamp()); + } + if (PodPhaseEnum.FAILED.getPhase().equals(req.getPhase())) { + modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.EXEC_FAILED.getValue()); + } + if (PodPhaseEnum.DELETED.getPhase().equals(req.getPhase())) { + modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.CANCELED.getValue()); + } + if (PodPhaseEnum.SUCCEEDED.getPhase().equals(req.getPhase())) { + modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.COMPLETED.getValue()); + modelOptTaskInstance.setEndTime(DateUtil.getCurrentTimestamp()); + if (StringUtils.isBlank(modelOptTaskInstance.getOptResultBefore()) + || StringUtils.isBlank(modelOptTaskInstance.getOptResultAfter())) { + try { + String optResultBefore = minioUtil.readString(bucket, modelOptTaskInstance.getOptResultJsonPathBefore() + ModelOptConstant.OPTIMIZE_JSON_NAME); + String optResultAfter = minioUtil.readString(bucket, modelOptTaskInstance.getOptResultJsonPathAfter() + ModelOptConstant.OPTIMIZE_JSON_NAME); + modelOptTaskInstance.setOptResultBefore(optResultBefore); + modelOptTaskInstance.setOptResultAfter(optResultAfter); + } catch (Exception e) { + LogUtil.error(LogEnum.MODEL_OPT, "Read json file field by k8s callback，instance id={},Exception={}", instId, e); + } + } + } + return updateById(modelOptTaskInstance); + } + + /** + * 查询K8S同步相关实例状态 + */ + @Override + public void syncInstanceStatus() { + // 获取等待状态超过五分钟的任务实例 + List taskInstanceList = modelOptTaskInstanceMapper.selectWaitingFor5MinutesInstances(); + // 若相应实例已启动失败更新相关实例状态 + for (ModelOptTaskInstance instance : taskInstanceList) { + UserDTO tempUser = new UserDTO(); + tempUser.setId(instance.getCreateUserId()); + tempUser.setUsername("更新优化实例状态定时任务"); + String namespace = getNamespace(tempUser); + String resourceName = k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, instance.getId().toString()); + BizJob bizJob = modelOptJobApi.getWithResourceName(namespace, resourceName); + if (K8sResponseEnum.NOT_FOUND.getCode().equals(bizJob.getCode())) { + // k8s侧无pod存在当做实例已停止 + ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(instance.getId()); + checkInstExist(modelOptTaskInstance); + modelOptTaskInstanceMapper.markInstanceExecFailed(instance.getId()); + } + } + } + + /** + * 校验任务实例是否存在 + * + * @param modelOptTaskInstance 模型优化实例对象 + */ + public void checkInstExist(ModelOptTaskInstance modelOptTaskInstance) { + if (modelOptTaskInstance == null) { + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_ABSENT); + } + } + + /** + * 重新构建任务实例 + * + * @param instance 任务实例参数来源 + * @return 填充参数后的实例对象 + */ + private ModelOptTaskInstance rebuild(ModelOptTaskInstance instance) { + ModelOptTaskInstance newInstance = new ModelOptTaskInstance(); + newInstance.setTaskId(instance.getTaskId()) + .setTaskName(instance.getTaskName()) + .setIsBuiltIn(instance.getIsBuiltIn()) + .setModelId(instance.getModelId()) + .setModelName(instance.getModelName()) + .setModelAddress(instance.getModelAddress()) + .setAlgorithmId(instance.getAlgorithmId()) + .setAlgorithmType(instance.getAlgorithmType()) + .setAlgorithmName(instance.getAlgorithmName()) + .setAlgorithmPath(instance.getAlgorithmPath()) + .setDatasetName(instance.getDatasetName()) + .setDatasetPath(instance.getDatasetPath()) + .setCommand(instance.getCommand()) + .setParams(instance.getParams()); + return newInstance; + } + + /** + * @param instance 运行实例 + * @return boolean 调用k8s是否成功 + */ + private boolean runTask(ModelOptTaskInstance instance) { + PtModelOptimizationJobBO jobBo = new PtModelOptimizationJobBO(); + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + // 检查namespace,不存在就创建 + assert currentUser != null; + String namespace = getNamespace(currentUser); + String command = getCommand(instance); + LogUtil.info(LogEnum.MODEL_OPT, "用户{}执行模型优化输入的命令为{}，实例id={}", currentUser.getUsername(), command, instance.getId()); + String logPath = SymbolConstant.SLASH + BizNfsEnum.MODEL_OPT.getBizNfsPath() + SymbolConstant.SLASH + currentUser.getId() + SymbolConstant.SLASH + instance.getTaskId() + SymbolConstant.SLASH + instance.getId() + ModelOptConstant.OPTIMIZE_LOG; + String outputModelDir = SymbolConstant.SLASH + BizNfsEnum.MODEL_OPT.getBizNfsPath() + SymbolConstant.SLASH + currentUser.getId() + SymbolConstant.SLASH + instance.getTaskId() + SymbolConstant.SLASH + instance.getId() + ModelOptConstant.OPTIMIZE_MODEL; + String optResultJsonPathBefore = SymbolConstant.SLASH + BizNfsEnum.MODEL_OPT.getBizNfsPath() + SymbolConstant.SLASH + currentUser.getId() + SymbolConstant.SLASH + instance.getTaskId() + SymbolConstant.SLASH + instance.getId() + ModelOptConstant.OPTIMIZE_JSON_BEFORE; + String optResultJsonPathAfter = SymbolConstant.SLASH + BizNfsEnum.MODEL_OPT.getBizNfsPath() + SymbolConstant.SLASH + currentUser.getId() + SymbolConstant.SLASH + instance.getTaskId() + SymbolConstant.SLASH + instance.getId() + ModelOptConstant.OPTIMIZE_JSON_AFTER; + jobBo.setNamespace(namespace); + jobBo.setName(k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, instance.getId().toString())); + jobBo.setCpuNum(ModelOptConstant.CPU_NUM); + jobBo.setMemNum(ModelOptConstant.MEMORY_NUM); + jobBo.setGpuNum(ModelOptConstant.GPU_NUM); + jobBo.setCmdLines(Arrays.asList("-c", command)); + jobBo.putNfsMounts(ModelOptConstant.DATASET_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(instance.getDatasetPath())); + jobBo.putNfsMounts(ModelOptConstant.INPUT_MODEL_BEFORE_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(instance.getModelAddress())); + jobBo.putNfsMounts(ModelOptConstant.ALGORITHM_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(instance.getAlgorithmPath())); + jobBo.putNfsMounts(ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(logPath)); + jobBo.putNfsMounts(ModelOptConstant.OUTPUT_MODEL_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(outputModelDir)); + jobBo.putNfsMounts(ModelOptConstant.OUTPUT_RESULT_BEFORE_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(optResultJsonPathBefore)); + jobBo.putNfsMounts(ModelOptConstant.OUTPUT_RESULT_AFTER_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(optResultJsonPathAfter)); + jobBo.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.MODEL_OPT)); + jobBo.setImage(optimizeImage); + + // 调用k8s接口 + BizJob bizJob = modelOptJobApi.create(jobBo); + if (null == bizJob || !bizJob.isSuccess()) { + String message = null == bizJob ? "未知的错误" : bizJob.getMessage(); + LogUtil.error( + LogEnum.MODEL_OPT, + "用户{}创建模型优化任务实例, k8s创建过程中失败, 实例id={}, 传递参数为{}, 错误的信息为{}", + currentUser.getUsername(), instance.getId(), instance, message); + return false; + } + instance.setLogPath(logPath); + instance.setOutputModelDir(outputModelDir); + instance.setOptResultJsonPathBefore(optResultJsonPathBefore); + instance.setOptResultJsonPathAfter(optResultJsonPathAfter); + return updateById(instance); + } + + /** + * 获取模型优化执行命令行 + * + * @param instance 模型优化实例 + * @return 返回执行命令行 + */ + private String getCommand(ModelOptTaskInstance instance) { + String command = SymbolConstant.BLANK; + if (instance.getIsBuiltIn()) { + // 内置为默认命令 + if (OptimizeTypeEnum.DISTILL.getType().equals(instance.getAlgorithmType())) { + command = DistillCommandEnum.getCommandByName(instance.getAlgorithmName()); + assert command != null; + command = String.format(command, ModelOptConstant.DATASET_MOUNT_PATH, + ModelOptConstant.INPUT_MODEL_BEFORE_MOUNT_PATH, + ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, + ModelOptConstant.OUTPUT_RESULT_BEFORE_MOUNT_PATH, + instance.getDatasetName(), + ModelOptConstant.DATASET_MOUNT_PATH, + ModelOptConstant.OUTPUT_MODEL_MOUNT_PATH + ModelOptConstant.OPTIMIZE_JSON_AFTER, + ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, + ModelOptConstant.OUTPUT_RESULT_AFTER_MOUNT_PATH, + instance.getDatasetName()); + } + if (OptimizeTypeEnum.SLIMMING.getType().equals(instance.getAlgorithmType())) { + command = String.format(ModelOptConstant.OPT_START_SLIMMING_COMMAND, + ModelOptConstant.NEURONAL_PRUNING.equals(instance.getAlgorithmName()) ? "bn" : instance.getAlgorithmName(), + instance.getModelName(), + instance.getDatasetName(), + ModelOptConstant.DATASET_MOUNT_PATH, + ModelOptConstant.OUTPUT_MODEL_MOUNT_PATH, + ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, + ModelOptConstant.OUTPUT_RESULT_BEFORE_MOUNT_PATH, + ModelOptConstant.OUTPUT_RESULT_AFTER_MOUNT_PATH); + } + if (OptimizeTypeEnum.QUANTIFY.getType().equals(instance.getAlgorithmType())) { + command = String.format(ModelOptConstant.OPT_START_QUANTIFY_COMMAND, instance.getModelName(), + ModelOptConstant.INPUT_MODEL_BEFORE_MOUNT_PATH, + ModelOptConstant.DATASET_MOUNT_PATH, + ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, + ModelOptConstant.OUTPUT_RESULT_BEFORE_MOUNT_PATH, + instance.getModelName(), + ModelOptConstant.INPUT_MODEL_BEFORE_MOUNT_PATH, + ModelOptConstant.DATASET_MOUNT_PATH, + ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, + ModelOptConstant.OUTPUT_RESULT_AFTER_MOUNT_PATH); + } + } else { + // 非内置为用户输入命令 + command = String.format(ModelOptConstant.MY_OPT_COMMAND, + ModelOptConstant.ALGORITHM_MOUNT_PATH, + instance.getCommand() + getParams(instance.getParams())); + } + return command; + } + + /** + * 解析运行参数 + * + * @return String 运行参数字符串 + */ + public String getParams(JSONObject params) { + StringBuilder paramStr = new StringBuilder(); + for (String key : params.keySet()) { + if (Objects.nonNull(params.get(key))) { + paramStr.append(" --").append(key).append(SymbolConstant.FLAG_EQUAL + SymbolConstant.APOSTROPHE).append(params.get(key)).append(SymbolConstant.APOSTROPHE); + } + } + return paramStr.toString(); + } + + /** + * @param currentUser 当前用户 + * @return namespace k8s的命名空间 + */ + private String getNamespace(UserDTO currentUser) { + String namespaceStr = k8sNameTool.generateNamespace(currentUser.getId()); + BizNamespace bizNamespace = namespaceApi.get(namespaceStr); + if (null == bizNamespace) { + BizNamespace namespace = namespaceApi.create(namespaceStr, null); + if (null == namespace || !namespace.isSuccess()) { + LogUtil.error(LogEnum.MODEL_OPT, "用户{}启动k8s模型优化失败，namespace为空", currentUser.getUsername()); + throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); + } + } + return namespaceStr; + } + + /** + * 校验该任务是否存在进行中和等待中的实例 + * + * @param taskId 任务id + * @return Boolean 是否存在进行中和等待中的实例 + */ + @Override + public Boolean checkUnfinishedInst(Long taskId) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + wrapper.eq(ModelOptTaskInstance::getTaskId, taskId) + .in(ModelOptTaskInstance::getStatus, Arrays.asList(ModelOptInstanceStatusEnum.WAITING.getValue(), ModelOptInstanceStatusEnum.RUNNING.getValue())); + List list = modelOptTaskInstanceMapper.selectList(wrapper); + return CollectionUtil.isNotEmpty(list); + } + + /** + * 根据任务id删除实例 + * + * @param taskId 任务id + * @return Integer 删除实例数量 + */ + @Override + public int deleteByTaskId(Long taskId) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + wrapper.eq(ModelOptTaskInstance::getTaskId, taskId); + return modelOptTaskInstanceMapper.delete(wrapper); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void delete(ModelOptTaskInstanceDeleteDTO modelOptTaskInstanceDeleteDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(modelOptTaskInstanceDeleteDTO.getId()); + checkInstExist(modelOptTaskInstance); + if (StringUtils.equalsAny(modelOptTaskInstance.getStatus(), ModelOptInstanceStatusEnum.RUNNING.getValue(), ModelOptInstanceStatusEnum.WAITING.getValue())) { + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_STATUS_ERROR); + } + modelOptTaskInstanceMapper.deleteById(modelOptTaskInstanceDeleteDTO.getId()); + // 调用k8s删除相关资源 + modelOptJobApi.deleteByResourceName(getNamespace(userDTO), + k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, modelOptTaskInstanceDeleteDTO.getId().toString())); + } + + /** + * 构建优化结果 + * + * @param modelOptTaskInstance 任务实例对象 + * @return ModelOptTaskInstanceQueryVO 模型优化任务实例 + */ + public ModelOptTaskInstanceQueryVO buildResult(ModelOptTaskInstance modelOptTaskInstance) { + //有返回结果，构造返回结果 + ModelOptTaskInstanceQueryVO modelOptTaskInstanceVO = new ModelOptTaskInstanceQueryVO(); + BeanUtils.copyProperties(modelOptTaskInstance, modelOptTaskInstanceVO); + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(user); + String nameSpace = k8sUtil.getNamespace(user); + List pods = podService.getPods(new PodQueryDTO(nameSpace, k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, modelOptTaskInstance.getId().toString()))); + if (!CollectionUtils.isEmpty(pods)) { + modelOptTaskInstanceVO.setPodName(pods.get(NumberConstant.NUMBER_0).getPodName()); + } + if (StringUtils.isEmpty(modelOptTaskInstance.getOptResultBefore()) || + StringUtils.isEmpty(modelOptTaskInstance.getOptResultAfter())) { + return modelOptTaskInstanceVO; + } + //获取模型优化任务结果字典 + DictDTO dictDTO = dictService.findByName(ModelOptConstant.TASK_RESULT_DICT_NAME); + List resultList = new ArrayList<>(); + if (dictDTO == null || CollectionUtils.isEmpty(dictDTO.getDictDetails())) { + LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务结果字典为空，opt_result字典为{}", JSONObject.toJSONString(dictDTO)); + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_RESULT_DICT_EMPTY); + } + //解析优化结果字符串 + JSONObject before = JSONObject.parseObject(modelOptTaskInstance.getOptResultBefore()); + JSONObject after = JSONObject.parseObject(modelOptTaskInstance.getOptResultAfter()); + for (DictDetailDTO dictDetail : dictDTO.getDictDetails()) { + if (StringUtils.isEmpty(dictDetail.getLabel()) || StringUtils.isEmpty(dictDetail.getValue())) { + LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务结果字典详情为空，opt_result字典详情为{}", JSONObject.toJSONString(dictDetail)); + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_RESULT_DICT_EMPTY); + } + //构造模型优化结果 + ModelOptResultQueryVO modelOptResultVO = new ModelOptResultQueryVO(); + modelOptResultVO.setName(dictDetail.getLabel()); + //去掉模型优化结果中的单位 + String beforeStr = before.getOrDefault(dictDetail.getLabel(), SymbolConstant.ZERO).toString().replace(dictDetail.getValue(), SymbolConstant.BLANK); + String afterStr = after.getOrDefault(dictDetail.getLabel(), SymbolConstant.ZERO).toString().replace(dictDetail.getValue(), SymbolConstant.BLANK); + Float beforeValue = Float.parseFloat(beforeStr); + Float afterValue = Float.parseFloat(afterStr); + if (ModelOptConstant.ACCURACY.equals(dictDetail.getLabel())) { + beforeValue = beforeValue * NumberConstant.NUMBER_100; + afterValue = afterValue * NumberConstant.NUMBER_100; + } + modelOptResultVO.setBefore(beforeValue); + modelOptResultVO.setAfter(afterValue); + modelOptResultVO.setUnit(dictDetail.getValue()); + modelOptResultVO.setPositive(Integer.toString(afterValue.compareTo(beforeValue))); + resultList.add(modelOptResultVO); + } + modelOptTaskInstanceVO.setOptResult(resultList); + return modelOptTaskInstanceVO; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java new file mode 100644 index 0000000..1900bd0 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java @@ -0,0 +1,560 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import cn.hutool.core.collection.CollectionUtil; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.base.MagicNumConstant; +import org.dubhe.config.NfsConfig; +import org.dubhe.constant.ModelOptConstant; +import org.dubhe.constant.ModelOptErrorEnum; +import org.dubhe.constant.NumberConstant; +import org.dubhe.constant.SymbolConstant; +import org.dubhe.dao.ModelOptBuiltInMapper; +import org.dubhe.dao.ModelOptDatasetMapper; +import org.dubhe.dao.ModelOptTaskMapper; +import org.dubhe.dao.PtModelInfoMapper; +import org.dubhe.data.constant.Constant; +import org.dubhe.domain.PtModelInfo; +import org.dubhe.domain.dto.ModelOptDatasetCreateDTO; +import org.dubhe.domain.dto.ModelOptTaskCreateDTO; +import org.dubhe.domain.dto.ModelOptTaskDeleteDTO; +import org.dubhe.domain.dto.ModelOptTaskQueryDTO; +import org.dubhe.domain.dto.ModelOptTaskSubmitDTO; +import org.dubhe.domain.dto.ModelOptTaskUpdateDTO; +import org.dubhe.domain.dto.PtModelAlgorithmCreateDTO; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.ModelOptDataset; +import org.dubhe.domain.entity.ModelOptTask; +import org.dubhe.domain.entity.ModelOptTaskInstance; +import org.dubhe.domain.entity.PtTrainAlgorithm; +import org.dubhe.domain.vo.ModelOptAlgorithmQueryVO; +import org.dubhe.domain.vo.ModelOptCreateVO; +import org.dubhe.domain.vo.ModelOptDatasetQueryVO; +import org.dubhe.domain.vo.ModelOptDatasetVO; +import org.dubhe.domain.vo.ModelOptModelQueryVO; +import org.dubhe.domain.vo.ModelOptTaskQueryVO; +import org.dubhe.domain.vo.ModelOptUpdateVO; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.OptimizeTypeEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.service.ModelOptTaskInstanceService; +import org.dubhe.service.ModelOptTaskService; +import org.dubhe.service.PtTrainAlgorithmService; +import org.dubhe.utils.DateUtil; +import org.dubhe.utils.JwtUtils; +import org.dubhe.utils.K8sUtil; +import org.dubhe.utils.LocalFileUtil; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.NfsUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.PtModelUtil; +import org.dubhe.utils.ReflectionUtils; +import org.dubhe.utils.StringUtils; +import org.dubhe.utils.WrapperHelp; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @description 模型优化任务 + * @date 2020-05-22 + */ +@Service +public class ModelOptTaskServiceImpl implements ModelOptTaskService { + + @Resource + private ModelOptTaskMapper modelOptTaskMapper; + + @Resource + private ModelOptTaskInstanceService modelOptTaskInstanceService; + + @Resource + private ModelOptBuiltInMapper modelOptBuiltInMapper; + + @Resource + private PtTrainAlgorithmService ptTrainAlgorithmService; + + @Resource + private NfsUtil nfsUtil; + + @Resource + private K8sUtil k8sUtil; + + @Resource + private LocalFileUtil localFileUtil; + + @Resource + private NfsConfig nfsConfig; + + @Resource + private PtModelInfoMapper ptModelInfoMapper; + + @Resource + private ModelOptDatasetMapper modelOptDatasetMapper; + + public final static List FILED_MANES; + + static { + FILED_MANES = ReflectionUtils.getFieldNames(ModelOptTaskQueryVO.class); + } + + /** + * 查询数据分页 + * + * @param modelOptTaskQueryDTO 模型优化任务查询参数 + * @return Map 模型优化任务分页对象 + */ + @Override + public Map queryAll(ModelOptTaskQueryDTO modelOptTaskQueryDTO) { + String name = modelOptTaskQueryDTO.getName(); + //任务名称或id条件非空 + if (StringUtils.isNotEmpty(name)) { + //整数匹配 + if (Constant.PATTERN_NUM.matcher(name).matches()) { + //查询条件为整数时，先根据id精确查询 + modelOptTaskQueryDTO.setId(Long.parseLong(name)); + modelOptTaskQueryDTO.setName(null); + Map map = queryTask(modelOptTaskQueryDTO); + //根据id查询结果为空时，再根据任务名称模糊查询 + if (((List) map.get(Constant.RESULT)).size() > NumberConstant.NUMBER_0) { + return map; + } else { + modelOptTaskQueryDTO.setName(name); + } + } + } + return queryTask(modelOptTaskQueryDTO); + } + + /** + * 模型优化任务查询 + * + * @param modelOptTaskQueryDTO 模型优化任务查询条件DTO + * @return Map 分页查询结果 + */ + public Map queryTask(ModelOptTaskQueryDTO modelOptTaskQueryDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + LogUtil.info(LogEnum.MODEL_OPT, "用户{}查询模型优化任务列表，查询条件为{}", userDTO.getUsername(), JSONObject.toJSONString(modelOptTaskQueryDTO)); + QueryWrapper wrapper = WrapperHelp.getWrapper(modelOptTaskQueryDTO); + wrapper.eq("create_user_id", userDTO.getId()); + if (StringUtils.isNotEmpty(modelOptTaskQueryDTO.getName())) { + //任务名称模糊搜索时不区分大小写 + wrapper.like("lower(name)", modelOptTaskQueryDTO.getName().toLowerCase()); + } + Page page = new Page(null == modelOptTaskQueryDTO.getCurrent() ? MagicNumConstant.ONE : modelOptTaskQueryDTO.getCurrent(), + null == modelOptTaskQueryDTO.getSize() ? MagicNumConstant.TEN : modelOptTaskQueryDTO.getSize()); + try { + //排序字段，默认按更新时间降序，否则将驼峰转换为下划线 + String column = modelOptTaskQueryDTO.getSort() != null && FILED_MANES.contains(modelOptTaskQueryDTO.getSort()) ? StringUtils.humpToLine(modelOptTaskQueryDTO.getSort()) : "update_time"; + //排序方式 + boolean isAsc = !StringUtils.isEmpty(modelOptTaskQueryDTO.getOrder()) && !StringUtils.equals(modelOptTaskQueryDTO.getOrder(), Constant.SORT_DESC); + wrapper.orderBy(true, isAsc, column); + } catch (Exception e) { + LogUtil.error(LogEnum.MODEL_OPT, "查询模型优化任务列表展示异常,请求信息:{}，异常信息:{}", JSONObject.toJSONString(modelOptTaskQueryDTO), e); + throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); + } + IPage modelOptTasks = modelOptTaskMapper.selectPage(page, wrapper); + List queryVOList = modelOptTasks.getRecords().stream().map(modelOptTask -> { + ModelOptTaskQueryVO queryVO = new ModelOptTaskQueryVO(); + BeanUtils.copyProperties(modelOptTask, queryVO); + return queryVO; + }).collect(Collectors.toList()); + LogUtil.info(LogEnum.MODEL_OPT, "用户{}查询模型优化任务列表，任务数={}", userDTO.getUsername(), queryVOList.size()); + return PageUtil.toPage(modelOptTasks, queryVOList); + } + + /** + * 创建模型优化任务 + * + * @param modelOptTaskCreateDTO 模型优化任务创建对象 + * @return 返回创建成功的对象 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public ModelOptCreateVO create(ModelOptTaskCreateDTO modelOptTaskCreateDTO) { + //参数校验 + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + checkDatasetExist(modelOptTaskCreateDTO.getDatasetPath()); + assert userDTO != null; + checkNameExist(modelOptTaskCreateDTO.getName(), userDTO.getId()); + ModelOptTask modelOptTask = new ModelOptTask(); + BeanUtils.copyProperties(modelOptTaskCreateDTO, modelOptTask); + if (!modelOptTaskCreateDTO.getIsBuiltIn()) { + checkModelExist(modelOptTask, modelOptTaskCreateDTO.getModelId()); + //选择非内置，且算法为内置算法时（算法id为空），需要添加到我的算法中 + if (modelOptTaskCreateDTO.getAlgorithmId() == null) { + PtTrainAlgorithm algorithm = forkAlgorithm(modelOptTaskCreateDTO.getAlgorithmType(), modelOptTaskCreateDTO.getAlgorithmName(), modelOptTaskCreateDTO.getAlgorithmPath(), modelOptTaskCreateDTO.getModelAddress()); + modelOptTask.setAlgorithmType(modelOptTaskCreateDTO.getAlgorithmType()); + modelOptTask.setAlgorithmId(algorithm.getId()); + modelOptTask.setAlgorithmName(algorithm.getAlgorithmName()); + modelOptTask.setAlgorithmPath(algorithm.getCodeDir()); + saveTask(userDTO, modelOptTask); + return new ModelOptCreateVO(algorithm.getId(), algorithm.getCodeDir()); + } + //非内置，算法为我的算法 + localFileUtil.copyFile(nfsConfig.getBucket() + modelOptTaskCreateDTO.getModelAddress(), nfsConfig.getBucket() + modelOptTaskCreateDTO.getAlgorithmPath() + ModelOptConstant.COPY_MODEL_POSTFIX); + saveTask(userDTO, modelOptTask); + return new ModelOptCreateVO(modelOptTaskCreateDTO.getAlgorithmId(), modelOptTaskCreateDTO.getAlgorithmPath()); + } + //内置 + saveTask(userDTO, modelOptTask); + return null; + } + + /** + * 保存模型任务 + * + * @param userDTO 用户信息 + * @param modelOptTask 模型任务信息 + */ + private void saveTask(UserDTO userDTO, ModelOptTask modelOptTask) { + int result = modelOptTaskMapper.insert(modelOptTask); + if (result < MagicNumConstant.ONE) { + LogUtil.error(LogEnum.MODEL_OPT, "用户{}创建模型优化任务:{}, 数据库操作失败", userDTO.getUsername(), modelOptTask.getName()); + throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); + } + } + + /** + * fork内置算法到我的算法 + * + * @param algorithmName 内置算法名称 + * @param algorithmPath 内置算法路径 + * @param modelAddress 模型路径 + * @return PtTrainAlgorithm 我的算法对象 + */ + private PtTrainAlgorithm forkAlgorithm(Integer type, String algorithmName, String algorithmPath, String modelAddress) { + String newAlgorithmName = algorithmName + SymbolConstant.HYPHEN + DateUtil.getTimestampStr(); + String newAlgorithmPath = ModelOptConstant.MY_OPT_ALGORITHM_ROOT_PATH + newAlgorithmName; + String codeDir = SymbolConstant.BLANK; + if (type != null) { + codeDir = OptimizeTypeEnum.getCodeDirByType(type); + } + //copy算法 + localFileUtil.copyFile(nfsConfig.getBucket() + algorithmPath, nfsConfig.getBucket() + newAlgorithmPath); + //剪枝算法 我的模型需要拷贝到算法目录下 + if (OptimizeTypeEnum.SLIMMING.getType().equals(type)) { + localFileUtil.copyDir(nfsConfig.getBucket() + modelAddress, nfsConfig.getBucket() + newAlgorithmPath + codeDir + ModelOptConstant.COPY_MODEL_POSTFIX); + } + PtModelAlgorithmCreateDTO algorithmCreateDTO = new PtModelAlgorithmCreateDTO(); + algorithmCreateDTO.setName(newAlgorithmName); + algorithmCreateDTO.setPath(newAlgorithmPath + codeDir); + return ptTrainAlgorithmService.modelOptimizationUploadAlgorithm(algorithmCreateDTO); + } + + /** + * 提交模型优化任务，创建任务实例 + * + * @param submitDTO 任务提交参数 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void submit(ModelOptTaskSubmitDTO submitDTO) { + ModelOptTask modelOptTask = modelOptTaskMapper.selectById(submitDTO.getId()); + checkTaskExist(modelOptTask); + if (modelOptTaskInstanceService.checkUnfinishedInst(submitDTO.getId())) { + LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务存在等待或进行中的实例，提交失败，任务id={}", submitDTO.getId()); + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_EXIST); + } + ModelOptTaskInstance modelOptTaskInstance = buildInstance(modelOptTask); + modelOptTaskInstanceService.create(modelOptTaskInstance); + } + + /** + * 修改模型优化任务 + * + * @param modelOptTaskUpdateDTO 模型优化任务修改对象 + * @return 返回修改后信息 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public ModelOptUpdateVO update(ModelOptTaskUpdateDTO modelOptTaskUpdateDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + ModelOptTask oldTask = modelOptTaskMapper.selectById(modelOptTaskUpdateDTO.getId()); + checkTaskExist(oldTask); + //用户不能修改其他用户的任务 + assert userDTO != null; + if (!userDTO.getId().equals(oldTask.getCreateUserId())) { + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_ABSENT); + } + //如果修改了任务名，校验新的任务名是否存在 + if (!modelOptTaskUpdateDTO.getName().equals(oldTask.getName())) { + checkNameExist(modelOptTaskUpdateDTO.getName(), userDTO.getId()); + } + checkDatasetExist(modelOptTaskUpdateDTO.getDatasetPath()); + ModelOptTask newTask = new ModelOptTask(); + BeanUtils.copyProperties(modelOptTaskUpdateDTO, newTask); + if (!modelOptTaskUpdateDTO.getIsBuiltIn()) { + checkModelExist(newTask, modelOptTaskUpdateDTO.getModelId()); + //选择非内置，且算法为内置算法时（算法id为空），需要添加到我的算法中 + if (modelOptTaskUpdateDTO.getAlgorithmId() == null) { + PtTrainAlgorithm algorithm = forkAlgorithm(modelOptTaskUpdateDTO.getAlgorithmType(), modelOptTaskUpdateDTO.getAlgorithmName(), modelOptTaskUpdateDTO.getAlgorithmPath(), modelOptTaskUpdateDTO.getModelAddress()); + newTask.setAlgorithmId(algorithm.getId()); + newTask.setAlgorithmName(algorithm.getAlgorithmName()); + newTask.setAlgorithmPath(algorithm.getCodeDir()); + updateTask(userDTO, newTask); + return new ModelOptUpdateVO(algorithm.getId(), algorithm.getCodeDir()); + } + //非内置，算法为我的算法 + localFileUtil.copyFile(nfsConfig.getBucket() + modelOptTaskUpdateDTO.getModelAddress(), nfsConfig.getBucket() + modelOptTaskUpdateDTO.getAlgorithmPath() + "/model"); + updateTask(userDTO, newTask); + return new ModelOptUpdateVO(modelOptTaskUpdateDTO.getAlgorithmId(), modelOptTaskUpdateDTO.getAlgorithmPath()); + } + //内置 + updateTask(userDTO, newTask); + return null; + } + + /** + * 修改任务 + * + * @param userDTO 用户 + * @param newTask 任务 + */ + private void updateTask(UserDTO userDTO, ModelOptTask newTask) { + int result = modelOptTaskMapper.updateById(newTask); + if (result < MagicNumConstant.ONE) { + LogUtil.error(LogEnum.MODEL_OPT, "用户{}修改模型优化任务, 数据库操作失败，任务id={}， 任务名:{}", userDTO.getUsername(), + newTask.getId(), newTask.getName()); + throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); + } + } + + /** + * 删除模型优化任务 + * + * @param modelOptTaskDeleteDTO 模型优化任务删除参数 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void delete(ModelOptTaskDeleteDTO modelOptTaskDeleteDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + ModelOptTask modelOptTask = modelOptTaskMapper.selectById(modelOptTaskDeleteDTO.getId()); + //用户不能删除其他用户的任务 + assert userDTO != null; + if (!userDTO.getId().equals(modelOptTask.getCreateUserId())) { + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_ABSENT); + } + checkTaskExist(modelOptTask); + if (modelOptTaskInstanceService.checkUnfinishedInst(modelOptTaskDeleteDTO.getId())) { + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_DELETE_ERROR); + } + int result = modelOptTaskMapper.deleteById(modelOptTaskDeleteDTO.getId()); + if (result < MagicNumConstant.ONE) { + LogUtil.error(LogEnum.MODEL_OPT, "用户{}删除模型优化任务, 数据库操作失败，任务id={}， 任务名:{}", userDTO.getUsername(), modelOptTaskDeleteDTO.getId(), modelOptTask.getName()); + throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); + } + //删除该任务对应的实例 + modelOptTaskInstanceService.deleteByTaskId(modelOptTaskDeleteDTO.getId()); + } + + /** + * 获取内置模型 + * + * @param type 算法类型 + * @param dataset 数据集 + * @param algorithm 训练算法 + * @return List 内置模型列表 + */ + @Override + public List getBuiltInModel(Integer type, String dataset, String algorithm) { + return modelOptBuiltInMapper.getModel(type, dataset, algorithm); + } + + /** + * 获取优化算法 + * + * @param type 算法类型 + * @param model 模型 + * @param dataset 数据集 + * @return List 获取模型优化算法列表 + */ + @Override + public List getAlgorithm(Integer type, String model, String dataset) { + return modelOptBuiltInMapper.getAlgorithm(type, model, dataset); + } + + /** + * 获取模型优化数据集 + * + * @param type 算法类型 + * @param model 模型 + * @param algorithm 训练算法 + * @return List 模型优化数据集列表 + */ + @Override + public List getDataset(Integer type, String model, String algorithm) { + return modelOptBuiltInMapper.getDataset(type, model, algorithm); + } + + /** + * 获取我的模型优化数据集 + * + * @return List 我的模型优化数据集列表 + */ + @Override + public List getMyDataset() { + List list = modelOptDatasetMapper.getAllDataset(); + if (CollectionUtils.isEmpty(list)) { + return new ArrayList<>(); + } + List voList = new ArrayList<>(); + list.forEach(modelOptDataset -> { + ModelOptDatasetVO vo = new ModelOptDatasetVO(); + BeanUtils.copyProperties(modelOptDataset, vo); + voList.add(vo); + }); + return voList; + } + + /** + * 创建我的模型优化数据集 + * + * @param modelOptDatasetCreateDTO 数据集创建参数 + * @return ModelOptDatasetVO 模型优化模块我的数据集查询 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public ModelOptDatasetVO createMyDataset(ModelOptDatasetCreateDTO modelOptDatasetCreateDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + wrapper.eq(ModelOptDataset::getName, modelOptDatasetCreateDTO.getName()); + assert userDTO != null; + wrapper.eq(ModelOptDataset::getCreateUserId, userDTO.getId()); + List list = modelOptDatasetMapper.selectList(wrapper); + if (!CollectionUtils.isEmpty(list)) { + throw new BusinessException(ModelOptErrorEnum.DATASET_NAME_EXIST); + } + //源文件路径 + String sourcePath = nfsConfig.getBucket() + modelOptDatasetCreateDTO.getPath(); + if (nfsUtil.fileOrDirIsEmpty(sourcePath)) { + LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} upload path or source path {} does not exist", userDTO.getUsername(), sourcePath); + throw new BusinessException("源文件或路径不存在"); + } + ModelOptDataset modelOptDataset = new ModelOptDataset(); + modelOptDataset.setName(modelOptDatasetCreateDTO.getName()); + //校验path是否带有压缩文件，如有，则解压至当前文件夹并删除压缩文件 + String targetPath = ModelOptConstant.MY_OPT_DATASET_ROOT_PATH + StringUtils.getTimestamp() + SymbolConstant.SLASH; + if (modelOptDatasetCreateDTO.getPath().endsWith(PtModelUtil.ZIP)) { + //目标路径 + boolean unzip = localFileUtil.unzipLocalPath(sourcePath, nfsConfig.getBucket() + targetPath); + if (!unzip) { + LogUtil.error(LogEnum.BIZ_MODEL, "用户{}解压模型文件失败", userDTO.getUsername()); + throw new BusinessException("数据集文件解压失败"); + } + modelOptDataset.setPath(targetPath); + } + + int result = modelOptDatasetMapper.insert(modelOptDataset); + if (result < MagicNumConstant.ONE) { + LogUtil.error(LogEnum.MODEL_OPT, "用户{}创建模型优化数据集:{}, 数据库操作失败", userDTO.getUsername(), modelOptDatasetCreateDTO.getName()); + throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); + } + return new ModelOptDatasetVO(modelOptDataset); + } + + /** + * 校验任务是否存在 + * + * @param modelOptTask 模型优化任务对象 + */ + public void checkTaskExist(ModelOptTask modelOptTask) { + if (modelOptTask == null) { + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_ABSENT); + } + } + + /** + * 校验任务名称是否已存在 + * + * @param name 任务名称 + * @param userId 用户id + */ + public void checkNameExist(String name, Long userId) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + wrapper.eq(ModelOptTask::getName, name); + wrapper.eq(ModelOptTask::getCreateUserId, userId); + List list = modelOptTaskMapper.selectList(wrapper); + if (CollectionUtil.isNotEmpty(list)) { + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_NAME_EXIST); + } + } + + /** + * 校验数据集是否存在 + * + * @param datasetPath 数据集路径 + */ + public void checkDatasetExist(String datasetPath) { + String path = k8sUtil.getAbsoluteNfsPath(datasetPath); + if (nfsUtil.fileOrDirIsEmpty(path)) { + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_DATASET_ABSENT); + } + } + + /** + * 校验模型是否存在 + * + * @param modelOptTask 模型优化任务 + * @param modelId 模型id + */ + public void checkModelExist(ModelOptTask modelOptTask, Long modelId) { + PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(modelId); + if (ptModelInfo == null) { + throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_MODEL_NOT_EXIST); + } + modelOptTask.setModelName(ptModelInfo.getName()); + } + + /** + * 根据模型优化任务，生成任务实例 + * + * @param task 模型优化任务对象 + * @return ModelOptTaskInstance 模型优化任务实例 + */ + private ModelOptTaskInstance buildInstance(ModelOptTask task) { + ModelOptTaskInstance instance = new ModelOptTaskInstance(); + instance.setTaskId(task.getId()); + instance.setTaskName(task.getName()); + instance.setIsBuiltIn(task.getIsBuiltIn()); + instance.setModelId(task.getModelId()); + instance.setModelName(task.getModelName()); + instance.setModelAddress(task.getModelAddress()); + instance.setAlgorithmType(task.getAlgorithmType()); + instance.setAlgorithmId(task.getAlgorithmId()); + instance.setAlgorithmName(task.getAlgorithmName()); + instance.setAlgorithmPath(task.getAlgorithmPath()); + instance.setDatasetName(task.getDatasetName()); + instance.setDatasetPath(task.getDatasetPath()); + instance.setCommand(task.getCommand()); + instance.setParams(task.getParams()); + return instance; + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java index 03cf7ad..b80261e 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java index 323168c..0398f52 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java @@ -1,12 +1,12 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. - * + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +19,7 @@ package org.dubhe.service.impl; import cn.hutool.core.collection.CollUtil; import cn.hutool.core.util.RandomUtil; +import cn.hutool.core.util.StrUtil; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @@ -27,47 +28,33 @@ import org.dubhe.base.MagicNumConstant; import org.dubhe.constant.SymbolConstant; import org.dubhe.dao.NoteBookMapper; import org.dubhe.dao.NoteBookModelMapper; -import org.dubhe.domain.dto.NoteBookListQueryDTO; -import org.dubhe.domain.dto.NoteBookQueryDTO; -import org.dubhe.domain.dto.NoteBookStatusDTO; -import org.dubhe.domain.dto.SourceNoteBookDTO; +import org.dubhe.domain.dto.*; import org.dubhe.domain.entity.NoteBook; import org.dubhe.domain.entity.NoteBookModel; import org.dubhe.domain.vo.NoteBookVO; import org.dubhe.enums.*; +import org.dubhe.exception.BusinessException; import org.dubhe.exception.NotebookBizException; -import org.dubhe.harbor.api.HarborApi; -import org.dubhe.k8s.api.PodApi; import org.dubhe.k8s.api.JupyterResourceApi; import org.dubhe.k8s.api.NamespaceApi; +import org.dubhe.k8s.api.PodApi; import org.dubhe.k8s.domain.PtBaseResult; import org.dubhe.k8s.domain.resource.BizNamespace; import org.dubhe.k8s.domain.resource.BizPod; import org.dubhe.k8s.domain.vo.PtJupyterDeployVO; import org.dubhe.k8s.enums.K8sResponseEnum; -import org.dubhe.service.HarborProjectService; import org.dubhe.service.NoteBookService; +import org.dubhe.service.PtImageService; import org.dubhe.service.convert.NoteBookConvert; import org.dubhe.service.convert.PtJupyterResourceConvert; -import org.dubhe.utils.HttpUtils; -import org.dubhe.utils.K8sNameTool; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.NotebookUtil; -import org.dubhe.utils.NumberUtil; -import org.dubhe.utils.PageUtil; -import org.dubhe.utils.WrapperHelp; +import org.dubhe.utils.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; /** @@ -99,10 +86,7 @@ public class NoteBookServiceImpl implements NoteBookService { private K8sNameTool k8sNameTool; @Autowired - private HarborApi harborApi; - - @Autowired - private HarborProjectService harborProjectService; + private PtImageService ptImageService; @Value("${delay.notebook.delete}") private Integer notebookDelayDeleteTime; @@ -121,21 +105,21 @@ public class NoteBookServiceImpl implements NoteBookService { QueryWrapper queryWrapper = WrapperHelp.getWrapper(noteBookListQueryDTO); queryWrapper.ne(true, NoteBook.COLUMN_STATUS, NoteBookStatusEnum.DELETE.getCode()) .ne(true, "deleted", NoteBookStatusEnum.STOP.getCode()); - if (noteBookListQueryDTO.getStatus() != null){ - if (noteBookListQueryDTO.getStatus().equals(NoteBookStatusEnum.RUN.getCode())){ + if (noteBookListQueryDTO.getStatus() != null) { + if (noteBookListQueryDTO.getStatus().equals(NoteBookStatusEnum.RUN.getCode())) { //运行中的notebook必须有url queryWrapper.eq(NoteBook.COLUMN_STATUS, NoteBookStatusEnum.RUN.getCode()) - .ne(NoteBook.COLUMN_URL,SymbolConstant.BLANK); - }else if (noteBookListQueryDTO.getStatus().equals(NoteBookStatusEnum.STARTING.getCode())){ + .ne(NoteBook.COLUMN_URL, SymbolConstant.BLANK); + } else if (noteBookListQueryDTO.getStatus().equals(NoteBookStatusEnum.STARTING.getCode())) { //启动中的notebook还包括运行中但没有url - queryWrapper.and((qw)-> + queryWrapper.and((qw) -> qw.eq(NoteBook.COLUMN_STATUS, NoteBookStatusEnum.RUN.getCode()).eq(NoteBook.COLUMN_URL, SymbolConstant.BLANK) .or() - .eq(NoteBook.COLUMN_STATUS,NoteBookStatusEnum.STARTING.getCode()) + .eq(NoteBook.COLUMN_STATUS, NoteBookStatusEnum.STARTING.getCode()) ); - }else { + } else { // 其他状态照常 - queryWrapper.eq(NoteBook.COLUMN_STATUS, NoteBookStatusEnum.RUN.getCode()); + queryWrapper.eq(NoteBook.COLUMN_STATUS, noteBookListQueryDTO.getStatus()); } } queryWrapper.orderBy(true, false, "id"); @@ -158,22 +142,18 @@ public class NoteBookServiceImpl implements NoteBookService { /** * 获取镜像路径 * - * @param bizEnum - * @return String + * @return String 镜像路径 */ - private String getDefaultImage(BizEnum bizEnum) { - if (bizEnum == null) { - throw new NotebookBizException("业务模块未识别！无法获取默认镜像。"); - } - List projectList = harborProjectService.getHarborProjects(bizEnum.getCreateResource()); - if (CollUtil.isEmpty(projectList)) { - throw new NotebookBizException("此模块" + bizEnum.getBizName() + "未配置Project！无法获取默认镜像。"); - } - List imageList = harborApi.searchImageNames(projectList); - if (CollUtil.isEmpty(imageList)) { - throw new NotebookBizException("此模块" + bizEnum.getBizName() + "未配置镜像！"); + private String getDefaultImage() { + PtImageQueryUrlDTO imageQueryUrlDTO = new PtImageQueryUrlDTO(); + imageQueryUrlDTO.setProjectType(ImageTypeEnum.NOTEBOOK.getType()) + .setImageResource(ImageSourceEnum.PRE.getCode()); + String imageUrl = ptImageService.getImageUrl(imageQueryUrlDTO); + if (StrUtil.isEmpty(imageUrl)) { + throw new BusinessException(ImageTypeEnum.NOTEBOOK.getCode() + "未配置默认镜像！"); } - return imageList.get(MagicNumConstant.ZERO); + + return imageUrl; } /** @@ -181,11 +161,12 @@ public class NoteBookServiceImpl implements NoteBookService { * * @param noteBook * @return NoteBookVO + * */ @Override @Transactional(rollbackFor = Exception.class) public NoteBookVO createNoteBook(NoteBook noteBook) { - if (noteBookMapper.findByNameAndStatus(noteBook.getNoteBookName(),NoteBookStatusEnum.DELETE.getCode()) != null) { + if (noteBookMapper.findByNameAndStatus(noteBook.getNoteBookName(), NoteBookStatusEnum.DELETE.getCode()) != null) { throw new NotebookBizException("Notebook名称已使用过！请重新提交。"); } if (StringUtils.isEmpty(noteBook.getName())) { @@ -575,7 +556,7 @@ public class NoteBookServiceImpl implements NoteBookService { noteBook.setDescription(bizNfsEnum.getBizName()); noteBook.setName(k8sNameTool.getK8sName()); String notebookName = NotebookUtil.generateName(bizNfsEnum, sourceNoteBookDTO.getSourceId()); - if (noteBookMapper.findByNameAndStatus(notebookName,NoteBookStatusEnum.DELETE.getCode()) != null) { + if (noteBookMapper.findByNameAndStatus(notebookName, NoteBookStatusEnum.DELETE.getCode()) != null) { // 重名随机符号拼接 notebookName += RandomUtil.randomString(MagicNumConstant.TWO); } @@ -588,7 +569,7 @@ public class NoteBookServiceImpl implements NoteBookService { noteBook.setAlgorithmId(sourceNoteBookDTO.getSourceId()); noteBook.setK8sPvcPath(k8sPvcPath); - noteBook.setK8sImageName(getDefaultImage(BizEnum.NOTEBOOK)); + noteBook.setK8sImageName(getDefaultImage()); return noteBook; } @@ -737,9 +718,9 @@ public class NoteBookServiceImpl implements NoteBookService { @Override public List getNotebookDetail(Set noteBookIds) { QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.in("id",noteBookIds); - queryWrapper.ne(NoteBook.COLUMN_STATUS,NoteBookStatusEnum.DELETE.getCode()); - List noteBookList = noteBookMapper.selectList(queryWrapper); + queryWrapper.in("id", noteBookIds); + queryWrapper.ne(NoteBook.COLUMN_STATUS, NoteBookStatusEnum.DELETE.getCode()); + List noteBookList = noteBookMapper.selectList(queryWrapper); return noteBookConvert.toDto(noteBookList); } @@ -751,6 +732,6 @@ public class NoteBookServiceImpl implements NoteBookService { */ @Override public List getRunNotUrlList(Page page) { - return noteBookMapper.selectRunNotUrlList(page,NoteBookStatusEnum.RUN.getCode()); + return noteBookMapper.selectRunNotUrlList(page, NoteBookStatusEnum.RUN.getCode()); } } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java index e050baa..10d38b4 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,21 +17,24 @@ package org.dubhe.service.impl; +import cn.hutool.core.collection.CollUtil; import cn.hutool.core.util.StrUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.dubhe.annotation.DataPermissionMethod; import org.dubhe.async.HarborImagePushAsync; +import org.dubhe.base.DataContext; +import org.dubhe.base.MagicNumConstant; import org.dubhe.base.ResponseCode; import org.dubhe.config.NfsConfig; import org.dubhe.config.TrainHarborConfig; -import org.dubhe.dao.HarborProjectMapper; +import org.dubhe.constatnts.UserConstant; import org.dubhe.dao.PtImageMapper; import org.dubhe.data.constant.Constant; import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.HarborProject; import org.dubhe.domain.entity.PtImage; import org.dubhe.domain.vo.PtImageQueryVO; import org.dubhe.enums.*; @@ -58,9 +61,6 @@ public class PtImageServiceImpl implements PtImageService { @Autowired private PtImageMapper ptImageMapper; - @Autowired - private HarborProjectMapper harborProjectMapper; - @Autowired private HarborApi harborApi; @@ -91,22 +91,28 @@ public class PtImageServiceImpl implements PtImageService { //从会话中获取用户信息 UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} query image list display begins, and the received parameters are :{}", user.getId(), ptImageQueryDTO); + LogUtil.info(LogEnum.IMAGE, "The user {} query image list display begins, and the received parameters are :{}", user.getId(), ptImageQueryDTO); Page page = ptImageQueryDTO.toPage(); - //镜像来源判断，默认为0(我的镜像) - if (ptImageQueryDTO.getImageResource() == null) { - ptImageQueryDTO.setImageResource(ImageSourceEnum.MINE.getCode()); - } + QueryWrapper query = new QueryWrapper<>(); + + if (ptImageQueryDTO.getProjectType().equals(BizEnum.NOTEBOOK.getCreateResource())) { + ptImageQueryDTO.setSort("imageResource"); + DataContext.set(CommonPermissionDataDTO.builder().type(true).build()); + } if (ptImageQueryDTO.getImageStatus() != null) { query.eq("image_status", ptImageQueryDTO.getImageStatus()); } - query.eq("image_resource", ptImageQueryDTO.getImageResource()); + if (ptImageQueryDTO.getImageResource() != null) { + query.eq("image_resource", ptImageQueryDTO.getImageResource()); + } + query.eq("project_name", resourcetoName(ptImageQueryDTO.getProjectType())); if (StringUtils.isNotEmpty(ptImageQueryDTO.getImageNameOrId())) { query.and(x -> x.eq("id", ptImageQueryDTO.getImageNameOrId()).or().like("image_name", ptImageQueryDTO.getImageNameOrId())); } + //排序 IPage ptImages; try { @@ -121,7 +127,7 @@ public class PtImageServiceImpl implements PtImageService { } ptImages = ptImageMapper.selectPage(page, query); } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} query mirror list display exception :{}, request information :{}",user.getId() ,e, ptImageQueryDTO); + LogUtil.error(LogEnum.IMAGE, "User {} query mirror list display exception :{}, request information :{}", user.getId(), e, ptImageQueryDTO); throw new BusinessException("查询镜像列表展示异常"); } List ptImageQueryResult = ptImages.getRecords().stream().map(x -> { @@ -129,7 +135,8 @@ public class PtImageServiceImpl implements PtImageService { BeanUtils.copyProperties(x, ptImageQueryVO); return ptImageQueryVO; }).collect(Collectors.toList()); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query mirror list display ends, the result is {}", user.getId(), ptImageQueryResult); + LogUtil.info(LogEnum.IMAGE, "User {} query mirror list display ends, the result is {}", user.getId(), ptImageQueryResult); + DataContext.remove(); return PageUtil.toPage(page, ptImageQueryResult); } @@ -141,40 +148,52 @@ public class PtImageServiceImpl implements PtImageService { @Override @Transactional(rollbackFor = Exception.class) public void uploadImage(PtImageUploadDTO ptImageUploadDTO) { - LogUtil.info(LogEnum.BIZ_TRAIN, "Upload image to harbor to receive parameters :{}", ptImageUploadDTO); + LogUtil.info(LogEnum.IMAGE, "Upload image to harbor to receive parameters :{}", ptImageUploadDTO); UserDTO currentUser = JwtUtils.getCurrentUserDto(); + //notebook镜像只能由管理员上传 + if (ptImageUploadDTO.getProjectType().equals(BizEnum.NOTEBOOK.getBizCode()) && UserConstant.ADMIN_USER_ID != currentUser.getId()) { + throw new BusinessException(ResponseCode.UNAUTHORIZED, "该用户不支持上传noteBook镜像!"); + } + //校验用户自定义镜像不能和预置镜像重名 - List resList = checkUploadImage(ptImageUploadDTO, currentUser, ImageSourceEnum.MINE.getCode()); + List resList = checkUploadImage(ptImageUploadDTO, null, ImageSourceEnum.PRE.getCode()); if (!CollectionUtils.isEmpty(resList)) { - LogUtil.info(LogEnum.BIZ_TRAIN, "Preset image [{}:{}] already exists, no repeat allowed!", ptImageUploadDTO.getImageName(), ptImageUploadDTO.getImageTag()); - throw new BusinessException(ResponseCode.SUCCESS, "不允许和预置镜像信息重复，请重新上传!"); + LogUtil.info(LogEnum.IMAGE, "Preset image [{}:{}] already exists, no repeat allowed!", ptImageUploadDTO.getImageName(), ptImageUploadDTO.getImageTag()); + throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复，请重新上传!"); } //同一用户上传镜像的(userId+imageName+imageTag)存在的情况下是不能重复上传的 - List imageList = checkUploadImage(ptImageUploadDTO, currentUser, ImageSourceEnum.PRE.getCode()); + List imageList = checkUploadImage(ptImageUploadDTO, currentUser, ImageSourceEnum.MINE.getCode()); if (!CollectionUtils.isEmpty(imageList) && ImageStateEnum.SUCCESS.getCode().equals(imageList.get(0).getImageStatus())) { - LogUtil.info(LogEnum.BIZ_TRAIN, "The mirror [id:{}] already exists", imageList.get(0).getId()); - throw new BusinessException(ResponseCode.SUCCESS, "镜像信息已存在，不允许重复上传!"); + LogUtil.info(LogEnum.IMAGE, "The mirror [id:{}] already exists", imageList.get(0).getId()); + throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在，不允许重复上传!"); } - String harborImagePath = trainHarborConfig.getModelName() + StrUtil.SLASH + ptImageUploadDTO.getImageName() + StrUtil.DASHED + currentUser.getId() + + String projectName = ImageTypeEnum.getType(ptImageUploadDTO.getProjectType()); + + String harborImagePath = projectName + StrUtil.SLASH + ptImageUploadDTO.getImageName() + StrUtil.DASHED + currentUser.getId() + StrUtil.COLON + ptImageUploadDTO.getImageTag(); //存储镜像信息 PtImage ptImage = new PtImage(); ptImage.setImageName(ptImageUploadDTO.getImageName()) - .setProjectName(trainHarborConfig.getModelName()) + .setProjectName(projectName) .setImageUrl(harborImagePath) .setImageResource(ImageSourceEnum.MINE.getCode()) .setImageStatus(ImageStateEnum.MAKING.getCode()) .setRemark(ptImageUploadDTO.getRemark()) .setImageTag(ptImageUploadDTO.getImageTag()) .setCreateUserId(currentUser.getId()); - ptImage.setOriginUserId(currentUser.getId()); + if (ptImageUploadDTO.getProjectType().equals(BizEnum.NOTEBOOK.getCreateResource())) { + //notebook镜像所有用户都可以查看和使用 + ptImage.setOriginUserId(MagicNumConstant.ZERO_LONG); + } else { + ptImage.setOriginUserId(currentUser.getId()); + } int count = ptImageMapper.insert(ptImage); if (count < 1) { imagePushAsync.updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} failed to store image information!", currentUser.getUsername()); + LogUtil.info(LogEnum.IMAGE, "User {} failed to store image information!", currentUser.getUsername()); throw new BusinessException("内部错误!"); } //shell脚本上传镜像 @@ -183,51 +202,39 @@ public class PtImageServiceImpl implements PtImageService { String imageNameAndTag = ptImageUploadDTO.getImageName() + StrUtil.DASHED + currentUser.getId() + StrUtil.COLON + ptImageUploadDTO.getImageTag(); imagePushAsync.execShell(imagePath, imageNameAndTag, ptImage); } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "Image upload exception :{}", e); + LogUtil.error(LogEnum.IMAGE, "Image upload exception :{}", e); throw new BusinessException("镜像上传失败!"); } } /** - * 查询所含镜像版本信息 + * 根据镜像获取信息 * * @param imageName 镜像名 * @return List 通过imageName查询所含镜像版本信息 */ @Override @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public List searchImages(String imageName) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("image_name", imageName); - queryWrapper.eq("image_status", ImageStateEnum.SUCCESS.getCode()); + public List searchImages(Integer projectType, String imageName) { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.eq(PtImage::getProjectName, resourcetoName(projectType)) + .eq(PtImage::getImageName, imageName) + .eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()); List ptImages = ptImageMapper.selectList(queryWrapper); if (CollectionUtils.isEmpty(ptImages)) { - LogUtil.info(LogEnum.BIZ_TRAIN, "No mirroring information for imageName is :{}", imageName); + LogUtil.info(LogEnum.IMAGE, "No mirroring information for imageName is :{}", imageName); throw new BusinessException(ResponseCode.SUCCESS, "未查询到镜像信息!"); } - List list = new ArrayList<>(); + List list = new ArrayList<>(); ptImages = ptImages.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(PtImage::getImageTag))), ArrayList::new)); ptImages.stream().forEach(ptImage -> { - list.add(ptImage.getImageTag()); + ptImage.setImageUrl(trainHarborConfig.getAddress() + StrUtil.SLASH + ptImage.getImageUrl()); + list.add(ptImage); }); return list; } - /** - * 查询项目列表 - * - * @return List 获取Harbor - **/ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public List getHarborProjectList() { - LogUtil.info(LogEnum.BIZ_TRAIN, "Query the mirror project list..."); - QueryWrapper queryWrapper = new QueryWrapper<>(); - List list = Arrays.asList(TrainUtil.NUMBER_ONE, TrainUtil.NUMBER_TWO); - queryWrapper.in("create_resource", list); - return harborProjectMapper.selectList(queryWrapper); - } /** * 删除镜像 @@ -236,29 +243,31 @@ public class PtImageServiceImpl implements PtImageService { */ @Override @Transactional(rollbackFor = Exception.class) + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) public void deleteTrainImage(PtImageDeleteDTO imageDeleteDTO) { UserDTO user = JwtUtils.getCurrentUserDto(); List imageList = ptImageMapper.selectList(new LambdaQueryWrapper() - .eq(PtImage::getCreateUserId, user.getId()) .in(PtImage::getId, imageDeleteDTO.getIds())); imageList.forEach(image -> { //禁止删除预置镜像 - if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource())) { + if (!BizEnum.NOTEBOOK.getBizCode().equals(image.getProjectName()) && ImageSourceEnum.PRE.getCode().equals(image.getImageResource())) { throw new BusinessException("禁止删除预置镜像"); } - String imageUrl = trainHarborConfig.getAddress() + StrUtil.SLASH + image.getImageUrl(); - LogUtil.info(LogEnum.BIZ_TRAIN, "delete harbor image url:{}", imageUrl); - //同步删除harbor镜像 - harborApi.deleteImageByTag(imageUrl); + if (ImageStateEnum.SUCCESS.getCode().equals(image.getImageStatus())) { + String imageUrl = trainHarborConfig.getAddress() + StrUtil.SLASH + image.getImageUrl(); + LogUtil.info(LogEnum.IMAGE, "delete harbor image url:{}", imageUrl); + //同步删除harbor镜像 + harborApi.deleteImageByTag(imageUrl); + } }); //删除本地镜像 int deleteSum = ptImageMapper.deleteBatchIds(imageDeleteDTO.getIds()); if (deleteSum < imageDeleteDTO.getIds().size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} failed to delete image, and the pt_image table deletion operation failed according to the id array {}", user.getId(), imageDeleteDTO.getIds()); + LogUtil.error(LogEnum.IMAGE, "The user {} failed to delete image, and the pt_image table deletion operation failed according to the id array {}", user.getId(), imageDeleteDTO.getIds()); throw new BusinessException("内部错误"); } } @@ -270,16 +279,16 @@ public class PtImageServiceImpl implements PtImageService { */ @Override @Transactional(rollbackFor = Exception.class) + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) public void updateTrainImage(PtImageUpdateDTO imageUpdateDTO) { UserDTO user = JwtUtils.getCurrentUserDto(); List imageList = ptImageMapper.selectList(new LambdaQueryWrapper() - .eq(PtImage::getCreateUserId, user.getId()) .in(PtImage::getId, imageUpdateDTO.getIds())); if (CollectionUtils.isEmpty(imageList)) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user{} update image failed,inquire condition ids{} not result", user.getId(), imageUpdateDTO.getIds()); + LogUtil.error(LogEnum.IMAGE, "The user{} update image failed,inquire condition ids{} not result", user.getId(), imageUpdateDTO.getIds()); throw new BusinessException("内部错误"); } for (PtImage image : imageList) { @@ -288,22 +297,22 @@ public class PtImageServiceImpl implements PtImageService { throw new BusinessException("无法修改预置镜像信息"); } image.setRemark(imageUpdateDTO.getRemark()); - LogUtil.info(LogEnum.BIZ_TRAIN, "The user{}update image,update image info:{}", user.getId(), image); + LogUtil.info(LogEnum.IMAGE, "The user{}update image,update image info:{}", user.getId(), image); ptImageMapper.updateById(image); } } /** * 获取镜像名称列表 - * + * @param projectType 镜像项目类型 * @return Set 镜像列表 */ @Override @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public Set getImageNameList() { - - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - List imageList = ptImageMapper.selectList(new LambdaQueryWrapper().eq(PtImage::getCreateUserId, currentUser.getId()).or().eq(PtImage::getImageResource, ImageSourceEnum.PRE.getCode())); + public Set getImageNameList(Integer projectType) { + List imageList = ptImageMapper.selectList(new LambdaQueryWrapper() + .eq(PtImage::getProjectName, ImageTypeEnum.getType(projectType)) + .eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode())); Set imageNames = new HashSet<>(); imageList.forEach(image -> { imageNames.add(image.getImageName()); @@ -311,21 +320,116 @@ public class PtImageServiceImpl implements PtImageService { return imageNames; } + /** + * 修改镜像来源(notebook定制) + * + * @param id 镜像id + */ + @Override + public void updImageResource(Long id) { + UserDTO userDto = JwtUtils.getCurrentUserDto(); + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + //notebook镜像只能由管理员上传 + if (UserConstant.ADMIN_USER_ID != userDto.getId()) { + throw new BusinessException(ResponseCode.UNAUTHORIZED, "该用户无权限修改镜像状态!"); + } + + //校验id是否存在 + PtImage image = ptImageMapper.selectById(id); + if (image == null || !BizEnum.NOTEBOOK.getBizCode().equals(image.getProjectName())) { + throw new BusinessException(ResponseCode.BADREQUEST, "该镜像不存在或镜像类型不支持!"); + } + + //仅支持[制作成功]状态镜像设置为默认镜像 + if (!ImageStateEnum.SUCCESS.getCode().equals(image.getImageStatus())) { + throw new BusinessException(ResponseCode.BADREQUEST, "仅支持[制作成功]状态镜像设置为默认镜像!"); + } + + //修改该用户的notebook镜像为"我的镜像" + updateWrapper.eq("project_name", BizEnum.NOTEBOOK.getBizCode()); + updateWrapper.eq("image_resource", ImageSourceEnum.PRE.getCode()); + updateWrapper.set("image_resource", ImageSourceEnum.MINE.getCode()); + ptImageMapper.update(null, updateWrapper); + + PtImage ptImage = new PtImage(); + ptImage.setId(id); + ptImage.setImageResource(ImageSourceEnum.PRE.getCode()); + ptImageMapper.updateById(ptImage); + } + + /** + * 获取镜像URL + * + * @param imageQueryUrlDTO 查询镜像路径DTO + * @return 镜像完整路径 + */ + @Override + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + public String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO) { + UserDTO userDto = JwtUtils.getCurrentUserDto(); + + if (imageQueryUrlDTO.getProjectType().equals(BizEnum.NOTEBOOK.getCreateResource())) { + DataContext.set(CommonPermissionDataDTO.builder().type(true).build()); + } + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + if (imageQueryUrlDTO.getProjectType() != null && ImageTypeEnum.NOTEBOOK.getType().equals(imageQueryUrlDTO.getProjectType())) { + DataContext.set(CommonPermissionDataDTO.builder().type(true).build()); + } + if (imageQueryUrlDTO.getImageResource() != null) { + queryWrapper.eq(PtImage::getImageResource, imageQueryUrlDTO.getImageResource()); + } + if (StrUtil.isNotEmpty(imageQueryUrlDTO.getImageName())) { + queryWrapper.eq(PtImage::getImageName, imageQueryUrlDTO.getImageName()); + } + if (StrUtil.isNotEmpty(imageQueryUrlDTO.getIamgeTag())) { + queryWrapper.eq(PtImage::getImageTag, imageQueryUrlDTO.getIamgeTag()); + } + queryWrapper.eq(PtImage::getProjectName, resourcetoName(imageQueryUrlDTO.getProjectType())) + .eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()); + List imageList = ptImageMapper.selectList(queryWrapper); + + if (CollUtil.isEmpty(imageList)) { + LogUtil.error(LogEnum.IMAGE, "The user{} update image failed,inquire condition {} not result", userDto.getId(), imageQueryUrlDTO); + throw new BusinessException("未查询到镜像信息"); + } + String imageUrl = trainHarborConfig.getAddress() + StrUtil.SLASH + imageList.get(0).getImageUrl(); + DataContext.remove(); + return imageUrl; + } /** * @param ptImageUploadDTO 镜像上传逻辑校验 * @param user 用户 - * @param source 来源 * @return List 镜像列表 **/ @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - private List checkUploadImage(PtImageUploadDTO ptImageUploadDTO, UserDTO user, int source) { + private List checkUploadImage(PtImageUploadDTO ptImageUploadDTO, UserDTO user, Integer source) { + + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("image_name", ptImageUploadDTO.getImageName()); - queryWrapper.eq("image_tag", ptImageUploadDTO.getImageTag()); + queryWrapper.eq(PtImage::getProjectName, resourcetoName(ptImageUploadDTO.getProjectType())) + .eq(PtImage::getImageName, ptImageUploadDTO.getImageName()) + .eq(PtImage::getImageTag, ptImageUploadDTO.getImageTag()) + .eq(PtImage::getImageResource, source); + + if (user != null) { + queryWrapper.eq(PtImage::getCreateUserId, user.getId()); + } List imageList = ptImageMapper.selectList(queryWrapper); return imageList; } + /** + * 项目类型转换为项目名称 + * + * @param projectType 项目类型 + * @return String 镜像项目名称 + */ + private static String resourcetoName(Integer projectType) { + String projectName = ImageTypeEnum.getType(projectType); + if (StrUtil.isEmpty(projectName)) { + throw new BusinessException("上传镜像项目类型不支持"); + } + return projectName; + } } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtMeasureServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtMeasureServiceImpl.java new file mode 100644 index 0000000..5089df2 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtMeasureServiceImpl.java @@ -0,0 +1,274 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.service.impl; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.dubhe.annotation.DataPermissionMethod; +import org.dubhe.config.NfsConfig; +import org.dubhe.config.RecycleConfig; +import org.dubhe.dao.PtMeasureMapper; +import org.dubhe.data.constant.Constant; +import org.dubhe.domain.dto.*; +import org.dubhe.domain.entity.PtMeasure; +import org.dubhe.domain.vo.PtImageQueryVO; +import org.dubhe.domain.vo.PtMeasureQueryVO; +import org.dubhe.enums.DatasetTypeEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.RecycleModuleEnum; +import org.dubhe.enums.RecycleTypeEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.service.PtMeasureService; +import org.dubhe.service.RecycleTaskService; +import org.dubhe.utils.*; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * @description 度量管理实现类 + * @date 2020-11-16 + */ +@Service +public class PtMeasureServiceImpl implements PtMeasureService { + + + @Autowired + private PtMeasureMapper ptMeasureMapper; + + @Autowired + private NfsUtil nfsUtil; + + @Autowired + private NfsConfig nfsConfig; + + @Autowired + private RecycleTaskService recycleTaskService; + + @Autowired + private RecycleConfig recycleConfig; + + public final static List FIELD_NAMES; + + static { + FIELD_NAMES = ReflectionUtils.getFieldNames(PtImageQueryVO.class); + } + + + /** + * 查询度量信息 + * + * @param ptMeasureQueryDTO 查询条件 + * @return Map 度量列表分页信息 + */ + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + @Override + public Map getMeasure(PtMeasureQueryDTO ptMeasureQueryDTO) { + + //从会话中获取用户信息 + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + Page page = ptMeasureQueryDTO.toPage(); + + QueryWrapper query = new QueryWrapper<>(); + if (StrUtil.isNotEmpty(ptMeasureQueryDTO.getNameOrId())) { + query.and(x -> x.eq("id", ptMeasureQueryDTO.getNameOrId()).or().like("name", ptMeasureQueryDTO.getNameOrId())); + } + + //排序 + IPage ptMeasures; + try { + if (StrUtil.isNotEmpty(ptMeasureQueryDTO.getSort()) && FIELD_NAMES.contains(ptMeasureQueryDTO.getSort())) { + if (Constant.SORT_ASC.equalsIgnoreCase(ptMeasureQueryDTO.getOrder())) { + query.orderByAsc(StringUtils.humpToLine(ptMeasureQueryDTO.getSort())); + } else { + query.orderByDesc(StringUtils.humpToLine(ptMeasureQueryDTO.getSort())); + } + } else { + query.orderByDesc(Constant.ID); + } + ptMeasures = ptMeasureMapper.selectPage(page, query); + } catch (Exception e) { + LogUtil.error(LogEnum.MEASURE, "User {} query measure list display exception :{}, request information :{}", currentUser.getId(), e, ptMeasureQueryDTO); + throw new BusinessException("查询度量列表展示异常"); + } + + List ptMeasureQueryResult = ptMeasures.getRecords().stream().map(x -> { + PtMeasureQueryVO ptMeasureQueryVO = new PtMeasureQueryVO(); + BeanUtils.copyProperties(x, ptMeasureQueryVO); + return ptMeasureQueryVO; + }).collect(Collectors.toList()); + return PageUtil.toPage(page, ptMeasureQueryResult); + } + + /** + * 新建度量 + * + * @param ptMeasureCreateDTO 新建度量入参DTO + */ + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + @Transactional(rollbackFor = Exception.class) + @Override + public void createMeasure(PtMeasureDTO ptMeasureCreateDTO) { + //从会话中获取用户信息 + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + + PtMeasure ptMeasure = new PtMeasure(); + BeanUtils.copyProperties(ptMeasureCreateDTO, ptMeasure); + ptMeasure.setCreateUserId(currentUser.getId()); + try { + ptMeasureMapper.insert(ptMeasure); + } catch (Exception e) { + LogUtil.error(LogEnum.MEASURE, " pt_measure table insert operation failed,Cause of exception is {}", e); + throw new BusinessException("内部错误"); + } + } + + /** + * 修改度量 + * + * @param ptMeasureUpdateDTO 修改度量入参DTO + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void updateMeasure(PtMeasureUpdateDTO ptMeasureUpdateDTO) { + //从会话中获取用户信息 + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + + PtMeasure ptMeasure = new PtMeasure(); + BeanUtils.copyProperties(ptMeasureUpdateDTO, ptMeasure); + ptMeasure.setUpdateUserId(currentUser.getId()); + + try { + PtMeasure measure = ptMeasureMapper.selectById(ptMeasureUpdateDTO.getId()); + if (measure == null) { + LogUtil.error(LogEnum.MEASURE, "The user{} update measure failed,inquire condition id{} no result", currentUser.getId(), ptMeasureUpdateDTO.getId()); + throw new BusinessException("内部错误"); + } + ptMeasureMapper.updateById(ptMeasure); + } catch (BusinessException e) { + LogUtil.error(LogEnum.MEASURE, " pt_measure table update operation failed,Cause of exception is {}", e); + throw new BusinessException("内部错误"); + } + } + + /** + * 根据id删除度量 + * + * @param ptMeasureDeleteDTO 删除度量的条件DTO + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void deleteMeasure(PtMeasureDeleteDTO ptMeasureDeleteDTO) { + //从会话中获取用户信息 + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + try { + + Set idList = ptMeasureDeleteDTO.getIds(); + List measureList = ptMeasureMapper.selectBatchIds(idList); + if (CollUtil.isEmpty(idList)) { + LogUtil.error(LogEnum.MEASURE, "The user:{} delete measure failed,inquire condition id:{} no result", currentUser.getId(), ptMeasureDeleteDTO.getIds()); + throw new BusinessException("您删除的ID不存在或已被删除"); + } + + int count = ptMeasureMapper.deleteBatchIds(idList); + if (count < measureList.size()) { + LogUtil.error(LogEnum.MEASURE, "The user:{} delete measure failed,inquire condition id:{} no result", currentUser.getId(), ptMeasureDeleteDTO.getIds()); + throw new BusinessException("您删除的ID不存在或已被删除"); + } + measureList.forEach(x -> { + delMeasureFile(x.getUrl()); + }); + + } catch (BusinessException e) { + LogUtil.error(LogEnum.MEASURE, " pt_measure table delete operation failed,Cause of exception is {}", e); + throw new BusinessException("内部错误"); + } + } + + /** + * 根据度量名称返回度量文件信息 + * + * @param name 度量名称 + * @return String 度量文件json字符串 + */ + @Override + public String getMeasureByName(String name) { + + List ptMeasureList = ptMeasureMapper.selectList(new LambdaQueryWrapper() + .eq(PtMeasure::getName, name)); + BufferedInputStream bufferedInput = null; + byte[] buffer = new byte[1024]; + StringBuilder sb = new StringBuilder(); + try { + if (CollUtil.isNotEmpty(ptMeasureList)) { + String url = StrUtil.SLASH + nfsConfig.getBucket() + StrUtil.SLASH + ptMeasureList.get(0).getUrl(); + if (!nfsUtil.fileOrDirIsEmpty(url)) { + int bytesRead = 0; + //获取文件输入流 + bufferedInput = nfsUtil.getInputStream(url); + while ((bytesRead = bufferedInput.read(buffer)) != -1) { + //将读取的字节转为字符串对象 + String tmpStr = new String(buffer, 0, bytesRead); + sb.append(tmpStr); + } + } + } + } catch (IOException e) { + LogUtil.error(LogEnum.MEASURE, " getMeasureByName method read jsonFile operation failed,Cause of exception is {}", e); + throw new BusinessException("内部错误"); + } finally { + IOUtil.close(bufferedInput); + } + return JSONUtil.toJsonStr(sb); + } + + /** + * 删除nfs服务器上的度量文件 + * + * @param filePath 度量文件相对路径 + */ + private void delMeasureFile(String filePath) { + String nfsBucket = nfsConfig.getRootDir() + nfsConfig.getBucket() + StrUtil.SLASH; + //度量文件的nfs服务器相对路径 + filePath = filePath.substring(0, filePath.lastIndexOf(StrUtil.SLASH)); + //度量文件的nfs服务器绝对路径 + String recyclePath = nfsUtil.formatPath((nfsBucket + StrUtil.SLASH + filePath)); + + //创建已删除度量的无效文件回收任务 + RecycleTaskCreateDTO recycleTask = new RecycleTaskCreateDTO(); + recycleTask.setRecycleModule(RecycleModuleEnum.BIZ_TRAIN.getValue()) + .setRecycleType(RecycleTypeEnum.FILE.getCode()) + .setRecycleDelayDate(recycleConfig.getTrainValid()) + .setRecycleCondition(recyclePath) + .setRecycleNote("回收已删除度量文件"); + LogUtil.info(LogEnum.MEASURE, "delete measure add recycle task:{}", recycleTask); + recycleTaskService.createRecycleTask(recycleTask); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java index 6dd0216..8aff34a 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,8 +29,8 @@ import org.dubhe.async.TrainAlgorithmUploadAsync; import org.dubhe.base.MagicNumConstant; import org.dubhe.base.ResponseCode; import org.dubhe.config.NfsConfig; -import org.dubhe.constant.AlgorithmSourceEnum; import org.dubhe.config.RecycleConfig; +import org.dubhe.constant.AlgorithmSourceEnum; import org.dubhe.constant.TrainAlgorithmConstant; import org.dubhe.dao.NoteBookMapper; import org.dubhe.dao.PtImageMapper; @@ -52,7 +52,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -176,7 +175,7 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { */ @Override @Transactional(rollbackFor = Exception.class) - public List create(PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO) { + public Long create(PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO) { //从会话中获取用户信息 UserDTO user = JwtUtils.getCurrentUserDto(); LogUtil.info(LogEnum.BIZ_TRAIN, "Save the new algorithm and receive the parameter {}", ptTrainAlgorithmCreateDTO); @@ -228,7 +227,7 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { } LogUtil.info(LogEnum.BIZ_TRAIN, "User {} saves the new algorithm and returns the new algorithm id as {}.", user.getUsername(), ptTrainAlgorithm.getId()); - return Collections.singletonList(ptTrainAlgorithm.getId()); + return ptTrainAlgorithm.getId(); } /** @@ -239,7 +238,7 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { */ @Override @Transactional(rollbackFor = Exception.class) - public List update(PtTrainAlgorithmUpdateDTO ptTrainAlgorithmUpdateDTO) { + public Long update(PtTrainAlgorithmUpdateDTO ptTrainAlgorithmUpdateDTO) { UserDTO currentUser = JwtUtils.getCurrentUserDto(); LogUtil.info(LogEnum.BIZ_TRAIN, "User {} modifies the algorithm and receives {} as the parameter", currentUser.getUsername(), ptTrainAlgorithmUpdateDTO); //权限校验 @@ -287,7 +286,7 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { throw new BusinessException("修改失败"); } LogUtil.info(LogEnum.BIZ_TRAIN, "End of user {} modification algorithm, return modification algorithm ID={}", currentUser.getUsername(), ptTrainAlgorithm.getId()); - return Collections.singletonList(ptTrainAlgorithm.getId()); + return ptTrainAlgorithm.getId(); } /** @@ -438,4 +437,19 @@ public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { } return ptImage.getImageUrl(); } + + /** + * + * @param ptModelAlgorithmCreateDTO 模型优化上传算法入参 + * @return PtTrainAlgorithm 新增算法信息 + */ + @Override + public PtTrainAlgorithm modelOptimizationUploadAlgorithm(PtModelAlgorithmCreateDTO ptModelAlgorithmCreateDTO) { + PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO = new PtTrainAlgorithmCreateDTO(); + ptTrainAlgorithmCreateDTO.setAlgorithmName(ptModelAlgorithmCreateDTO.getName()).setCodeDir(ptModelAlgorithmCreateDTO.getPath()).setAlgorithmUsage("模型优化").setIsTrainOut(false).setIsTrainLog(false).setIsVisualizedLog(false); + Long id = create(ptTrainAlgorithmCreateDTO); + PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(id); + return ptTrainAlgorithm; + } + } diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java index 8ff64df..32a9e75 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java index 1b1a466..f6400ea 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,20 +33,23 @@ import org.dubhe.async.TransactionAsyncManager; import org.dubhe.base.MagicNumConstant; import org.dubhe.base.ResponseCode; import org.dubhe.config.NfsConfig; +import org.dubhe.config.RecycleConfig; import org.dubhe.config.TrainHarborConfig; +import org.dubhe.config.TrainJobConfig; import org.dubhe.constant.AlgorithmSourceEnum; -import org.dubhe.config.RecycleConfig; import org.dubhe.constant.SymbolConstant; -import org.dubhe.config.TrainJobConfig; import org.dubhe.dao.DictDetailMapper; -import org.dubhe.dao.ModelQueryMapper; import org.dubhe.dao.PtJobParamMapper; +import org.dubhe.dao.PtModelBranchMapper; +import org.dubhe.dao.PtModelInfoMapper; import org.dubhe.dao.PtTrainAlgorithmMapper; import org.dubhe.dao.PtTrainJobMapper; import org.dubhe.dao.PtTrainJobSpecsMapper; import org.dubhe.dao.PtTrainMapper; import org.dubhe.dao.PtTrainParamMapper; import org.dubhe.data.constant.Constant; +import org.dubhe.domain.PtModelBranch; +import org.dubhe.domain.PtModelInfo; import org.dubhe.domain.dto.BaseTrainJobDTO; import org.dubhe.domain.dto.PtTrainDataSourceStatusQueryDTO; import org.dubhe.domain.dto.PtTrainJobCreateDTO; @@ -56,30 +59,32 @@ import org.dubhe.domain.dto.PtTrainJobResumeDTO; import org.dubhe.domain.dto.PtTrainJobStopDTO; import org.dubhe.domain.dto.PtTrainJobUpdateDTO; import org.dubhe.domain.dto.PtTrainJobVersionQueryDTO; +import org.dubhe.domain.dto.PtTrainModelDTO; import org.dubhe.domain.dto.PtTrainQueryDTO; import org.dubhe.domain.dto.RecycleTaskCreateDTO; import org.dubhe.domain.dto.UserDTO; import org.dubhe.domain.entity.DictDetail; -import org.dubhe.domain.entity.ModelQuery; -import org.dubhe.domain.entity.ModelQueryBrance; import org.dubhe.domain.entity.PtJobParam; import org.dubhe.domain.entity.PtTrain; import org.dubhe.domain.entity.PtTrainAlgorithm; import org.dubhe.domain.entity.PtTrainJob; import org.dubhe.domain.entity.PtTrainJobSpecs; import org.dubhe.domain.entity.PtTrainParam; +import org.dubhe.domain.vo.ModelVO; import org.dubhe.domain.vo.PtImageAndAlgorithmVO; import org.dubhe.domain.vo.PtJobMetricsGrafanaVO; import org.dubhe.domain.vo.PtTrainDataSourceStatusQueryVO; import org.dubhe.domain.vo.PtTrainJobDeleteVO; import org.dubhe.domain.vo.PtTrainJobDetailQueryVO; import org.dubhe.domain.vo.PtTrainJobDetailVO; +import org.dubhe.domain.vo.PtTrainJobModelVO; import org.dubhe.domain.vo.PtTrainJobStatisticsMineVO; import org.dubhe.domain.vo.PtTrainJobStopVO; import org.dubhe.domain.vo.PtTrainVO; import org.dubhe.enums.AlgorithmStatusEnum; import org.dubhe.enums.DatasetTypeEnum; import org.dubhe.enums.LogEnum; +import org.dubhe.enums.ModelResourceEnum; import org.dubhe.enums.RecycleModuleEnum; import org.dubhe.enums.RecycleTypeEnum; import org.dubhe.enums.TrainJobStatusEnum; @@ -115,9 +120,11 @@ import org.springframework.transaction.annotation.Transactional; import java.io.File; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -181,10 +188,6 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { @Autowired private DictDetailMapper dictDetailMapper; - - @Autowired - private ModelQueryMapper modelQueryMapper; - @Autowired private NfsConfig nfsConfig; @@ -200,6 +203,12 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { @Value("${k8s.pod.metrics.grafanaUrl}") private String k8sPodMetricsGrafanaUrl; + @Autowired + private PtModelInfoMapper ptModelInfoMapper; + + @Autowired + private PtModelBranchMapper ptModelBranchMapper; + public final static List FIELD_NAMES; static { @@ -426,7 +435,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { baseTrainJobDTO.setPtTrainJobSpecs(ptTrainJobSpecs); //结果集处理 - PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, currentUser, images, trainKey, jobName); + PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, currentUser, images, trainKey, baseTrainJobDTO); LogUtil.info(LogEnum.BIZ_TRAIN, "User {} creates training job, returns result {}", currentUser.getUsername(), ptTrainJob.getTrainId()); // 提交job @@ -441,10 +450,11 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { * @param currentUser 用户 * @param imageName 镜像名称 * @param trainKey 训练key - * @param jobName 训练名称 + * @param baseTrainJobDTO 基础训练参数 * @return PtTrain 训练 */ - private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserDTO currentUser, String imageName, String trainKey, String jobName) { + private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserDTO currentUser, + String imageName, String trainKey, BaseTrainJobDTO baseTrainJobDTO) { // 添加train表 PtTrain ptTrain = new PtTrain(); ptTrain.setTrainName(ptTrainJobCreateDTO.getTrainName()) @@ -455,25 +465,16 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { LogUtil.error(LogEnum.BIZ_TRAIN, "User {} creates training job, pt Train table insert data failed", currentUser.getUsername()); throw new BusinessException("内部错误"); } + + //检查模型是否合法,合法则保存其路径地址 + checkModelAndSavePath(currentUser, baseTrainJobDTO); + // 添加train_job表 PtTrainJob ptTrainJob = new PtTrainJob(); - //查询modelName值 - ModelQuery modelNameById = modelQueryMapper.findModelNameById(ptTrainJobCreateDTO.getModelId()); - ModelQueryBrance modelVersionByUrl = modelQueryMapper.findModelVersionByUrl(ptTrainJobCreateDTO.getModelLoadPathDir()); - if (modelNameById != null) { - String name = modelNameById.getName(); - if (modelVersionByUrl != null) { - ptTrainJobCreateDTO.setModelName(name + SymbolConstant.COLON + modelVersionByUrl.getVersion()); - } else { - //设置预置模型的url路径 - ptTrainJobCreateDTO.setModelLoadPathDir(modelNameById.getUrl()); - ptTrainJobCreateDTO.setModelName(name); - } - } BeanUtil.copyProperties(ptTrainJobCreateDTO, ptTrainJob); ptTrainJob.setTrainId(ptTrain.getId()) - .setTrainVersion(trainJobConfig.getVersionLabel().toUpperCase() + String.format(TrainUtil.FOUR_DECIMAL, 1)) - .setJobName(jobName) + .setTrainVersion(trainJobConfig.getVersionLabel().toUpperCase() + String.format(TrainUtil.FOUR_DECIMAL, TrainUtil.NUMBER_ONE)) + .setJobName(baseTrainJobDTO.getJobName()) .setCreateUserId(currentUser.getId()); int jobResult = ptTrainJobMapper.insert(ptTrainJob); if (jobResult < 1) { @@ -509,6 +510,123 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { return ptTrainJob; } + /** + * 检查模型是否合法,合法则保存其路径地址 + * + * @param currentUser 用户 + * @param baseTrainJobDTO 基础训练参数 + */ + private void checkModelAndSavePath(UserDTO currentUser, BaseTrainJobDTO baseTrainJobDTO) { + + Integer modelResource = baseTrainJobDTO.getModelResource(); + if(null == modelResource ) { + if(null == baseTrainJobDTO.getModelId() && + StringUtils.isBlank(baseTrainJobDTO.getStudentModelIds()) && + StringUtils.isBlank(baseTrainJobDTO.getStudentModelIds())) { + return; + } else { + logErrorInfoOnModel(currentUser.getUsername()); + } + } + switch (ModelResourceEnum.getType(modelResource)) { + case MINE: + if(null == baseTrainJobDTO.getModelBranchId() || null == baseTrainJobDTO.getModelId() || + StringUtils.isNotBlank(baseTrainJobDTO.getTeacherModelIds()) || + StringUtils.isNotBlank(baseTrainJobDTO.getStudentModelIds())) { + logErrorInfoOnModel(currentUser.getUsername()); + } + PtModelBranch ptModelBranch = ptModelBranchMapper.selectById(baseTrainJobDTO.getModelBranchId()); + if(null == ptModelBranch || ptModelBranch.getParentId().compareTo(baseTrainJobDTO.getModelId()) != 0 || + StringUtils.isBlank(ptModelBranch.getModelAddress())) { + logErrorInfoOnModel(currentUser.getUsername()); + } + PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(ptModelBranch.getParentId()); + if(null == ptModelInfo || ptModelInfo.getModelResource().compareTo(baseTrainJobDTO.getModelResource()) != 0){ + logErrorInfoOnModel(currentUser.getUsername()); + } + baseTrainJobDTO.setModelPath(adjustmentUrl(ptModelBranch.getModelAddress())); + break; + case PRESET: + if(null == baseTrainJobDTO.getModelId() || StringUtils.isNotBlank(baseTrainJobDTO.getTeacherModelIds()) || + StringUtils.isNotBlank(baseTrainJobDTO.getStudentModelIds())) { + logErrorInfoOnModel(currentUser.getUsername()); + } + PtModelInfo ptModelInfoPreset = ptModelInfoMapper.selectById(baseTrainJobDTO.getModelId()); + if(null == ptModelInfoPreset || StringUtils.isBlank(ptModelInfoPreset.getUrl()) || + ptModelInfoPreset.getModelResource().compareTo(baseTrainJobDTO.getModelResource()) != 0) { + logErrorInfoOnModel(currentUser.getUsername()); + } + baseTrainJobDTO.setModelPath(adjustmentUrl(ptModelInfoPreset.getUrl())); + break; + case ATLAS: + if(StringUtils.isBlank(baseTrainJobDTO.getTeacherModelIds()) || null != baseTrainJobDTO.getModelId() ){ + logErrorInfoOnModel(currentUser.getUsername()); + } + Set ids = new HashSet<>(); + Set teacherModelList = new HashSet<>(); + Arrays.stream(baseTrainJobDTO.getTeacherModelIds().trim().split(SymbolConstant.COMMA)) + .forEach(id -> teacherModelList.add(Long.parseLong(id))); + ids.addAll(teacherModelList); + + Set studentModelList = new HashSet<>(); + if(StringUtils.isNotBlank(baseTrainJobDTO.getStudentModelIds())) { + Arrays.stream(baseTrainJobDTO.getStudentModelIds().trim().split(SymbolConstant.COMMA)) + .forEach(id -> studentModelList.add(Long.parseLong(id))); + ids.addAll(studentModelList); + } + if(ids.isEmpty()) { + logErrorInfoOnModel(currentUser.getUsername()); + } + LambdaQueryWrapper query = new LambdaQueryWrapper<>(); + query.eq(PtModelInfo::getModelResource, baseTrainJobDTO.getModelResource()) + .in(PtModelInfo::getId, ids).isNotNull(PtModelInfo::getUrl).ne(PtModelInfo::getUrl, SymbolConstant.BLANK); + List modelInfoList = ptModelInfoMapper.selectList(query); + if(null == modelInfoList || modelInfoList.size() < ids.size()) { + logErrorInfoOnModel(currentUser.getUsername()); + } + + //保存炼知教师模型路径地址 + baseTrainJobDTO.setTeacherModelPathList(new ArrayList<>()); + modelInfoList.stream() + .filter(modelInfo -> teacherModelList.contains(modelInfo.getId())) + .forEach(modelInfo -> baseTrainJobDTO.getTeacherModelPathList().add(adjustmentUrl(modelInfo.getUrl()))); + + //保存炼知学生模型路径地址 + if(!studentModelList.isEmpty()) { + baseTrainJobDTO.setStudentModelPathList(new ArrayList<>()); + modelInfoList.stream() + .filter(modelInfo -> studentModelList.contains(modelInfo.getId())) + .forEach(modelInfo -> baseTrainJobDTO.getStudentModelPathList().add(adjustmentUrl(modelInfo.getUrl()))); + } + break; + } + } + + + + /** + * 调整模型地址 + * + * @param modelUrl 模型地址 + * @return + */ + private String adjustmentUrl(String modelUrl) { + if(modelUrl.endsWith(SymbolConstant.SLASH)) { + modelUrl = modelUrl.substring(TrainUtil.NUMBER_ZERO, modelUrl.length() - TrainUtil.NUMBER_ONE); + } + return modelUrl; + } + + /** + * 打印训练任务中模型相关的错误日志 + * + * @param username + */ + private void logErrorInfoOnModel(String username){ + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} operating training job, error on model......", username); + throw new BusinessException("模型参数参数不合法"); + } + /** * 保存任务参数到数据库 * @@ -666,7 +784,7 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { baseTrainJobDTO.setJobName(jobName); baseTrainJobDTO.setPtTrainJobSpecs(ptTrainJobSpecs); //结果集处理 - PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, currentUser, existPtTrainJob, images, ptTrain, jobName); + PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, currentUser, existPtTrainJob, images, ptTrain, baseTrainJobDTO); //提交job asyncManager.execute(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); @@ -681,30 +799,20 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { * @param existPtTrainJob 存在的训练任务 * @param imageName 镜像名称 * @param ptTrain 训练 - * @param jobName 任务名称 + * @param baseTrainJobDTO 基本训练信息 * @return PtTrainJob 训练任务 */ private PtTrainJob updateTrainJobTableData(PtTrainJobUpdateDTO ptTrainJobUpdateDTO, UserDTO - currentUser, PtTrainJob existPtTrainJob, String imageName, PtTrain ptTrain, String jobName) { + currentUser, PtTrainJob existPtTrainJob, String imageName, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) { + + //检查模型是否合法,合法则保存其路径地址 + checkModelAndSavePath(currentUser, baseTrainJobDTO); + //添加train_job表 PtTrainJob ptTrainJob = new PtTrainJob(); - //根据id查询model的name值 - ModelQuery modelName = modelQueryMapper.findModelNameById(ptTrainJobUpdateDTO.getModelId()); - //根据Url查询版本的路径值 - ModelQueryBrance modelVersion = modelQueryMapper.findModelVersionByUrl(ptTrainJobUpdateDTO.getModelLoadPathDir()); - if (modelName != null) { - String name = modelName.getName(); - if (modelVersion != null) { - ptTrainJobUpdateDTO.setModelName(name + SymbolConstant.COLON + modelVersion.getVersion()); - } else { - //设置预置模型的url - ptTrainJobUpdateDTO.setModelLoadPathDir(modelName.getUrl()); - ptTrainJobUpdateDTO.setModelName(name); - } - } BeanUtil.copyProperties(ptTrainJobUpdateDTO, ptTrainJob); ptTrainJob.setTrainId(ptTrain.getId()).setTrainVersion(trainJobConfig.getVersionLabel().toUpperCase() + String.format(TrainUtil.FOUR_DECIMAL, ptTrain.getTotalNum() + 1)) - .setJobName(jobName).setParentTrainVersion(existPtTrainJob.getTrainVersion()) + .setJobName(baseTrainJobDTO.getJobName()).setParentTrainVersion(existPtTrainJob.getTrainVersion()) .setCreateUserId(currentUser.getId()); int jobResult = ptTrainJobMapper.insert(ptTrainJob); if (jobResult < 1) { @@ -1272,6 +1380,106 @@ public class PtTrainJobServiceImpl implements PtTrainJobService { return list; } + /** + * 获取训练使用的模型信息 + * + * @param ptTrainModelDTO + * @return PtTrainJobModelVO + */ + @Override + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + public PtTrainJobModelVO getTrainJobModel(PtTrainModelDTO ptTrainModelDTO) { + + PtTrainJobModelVO ptTrainJobModelVO = new PtTrainJobModelVO(); + UserDTO currentUser = JwtUtils.getCurrentUserDto(); + Integer modelResource = ptTrainModelDTO.getModelResource(); + switch (ModelResourceEnum.getType(modelResource)) { + case MINE: + if(null == ptTrainModelDTO.getModelBranchId() || null == ptTrainModelDTO.getModelId()) { + logErrorInfoOnModel(currentUser.getUsername()); + } + PtModelBranch ptModelBranch = ptModelBranchMapper.selectById(ptTrainModelDTO.getModelBranchId()); + if(null == ptModelBranch) { + break; + } + if(ptModelBranch.getParentId().compareTo(ptTrainModelDTO.getModelId()) != 0 || + StringUtils.isBlank(ptModelBranch.getModelAddress())) { + logErrorInfoOnModel(currentUser.getUsername()); + } + PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(ptModelBranch.getParentId()); + if(null == ptModelInfo || ptModelInfo.getModelResource().compareTo(ptTrainModelDTO.getModelResource()) != 0){ + logErrorInfoOnModel(currentUser.getUsername()); + } + ptTrainJobModelVO.setModelList(new ArrayList<>()); + ptTrainJobModelVO.getModelList() + .add(new ModelVO(ptModelInfo.getName(), ptModelBranch.getVersionNum(), adjustmentUrl(ptModelBranch.getModelAddress()))); + break; + case PRESET: + if(null == ptTrainModelDTO.getModelId()) { + logErrorInfoOnModel(currentUser.getUsername()); + } + PtModelInfo ptModelInfoPreset = ptModelInfoMapper.selectById(ptTrainModelDTO.getModelId()); + if(null == ptModelInfoPreset) { + break; + } + if(StringUtils.isBlank(ptModelInfoPreset.getUrl()) || + ptModelInfoPreset.getModelResource().compareTo(ptTrainModelDTO.getModelResource()) != 0) { + logErrorInfoOnModel(currentUser.getUsername()); + } + ptTrainJobModelVO.setModelList(new ArrayList<>()); + ptTrainJobModelVO.getModelList() + .add(new ModelVO(ptModelInfoPreset.getName(), ptModelInfoPreset.getVersionNum(), adjustmentUrl(ptModelInfoPreset.getUrl()))); + break; + case ATLAS: + if(StringUtils.isBlank(ptTrainModelDTO.getTeacherModelIds())){ + logErrorInfoOnModel(currentUser.getUsername()); + } + Set ids = new HashSet<>(); + Set teacherModelList = new HashSet<>(); + Arrays.stream(ptTrainModelDTO.getTeacherModelIds().trim().split(SymbolConstant.COMMA)) + .forEach(id -> teacherModelList.add(Long.parseLong(id))); + ids.addAll(teacherModelList); + + Set studentModelList = new HashSet<>(); + if(StringUtils.isNotBlank(ptTrainModelDTO.getStudentModelIds())) { + Arrays.stream(ptTrainModelDTO.getStudentModelIds().trim().split(SymbolConstant.COMMA)) + .forEach(id -> studentModelList.add(Long.parseLong(id))); + ids.addAll(studentModelList); + } + if(ids.isEmpty()) { + logErrorInfoOnModel(currentUser.getUsername()); + } + LambdaQueryWrapper query = new LambdaQueryWrapper<>(); + query.eq(PtModelInfo::getModelResource, ptTrainModelDTO.getModelResource()) + .in(PtModelInfo::getId, ids).isNotNull(PtModelInfo::getUrl).ne(PtModelInfo::getUrl, SymbolConstant.BLANK); + List modelInfoList = ptModelInfoMapper.selectList(query); + if(null == modelInfoList || modelInfoList.isEmpty()) { + break; + } + + //保存炼知教师模型信息 + ptTrainJobModelVO.setTeacherModelList(new ArrayList<>()); + List teacherModelVOS = ptTrainJobModelVO.getTeacherModelList(); + modelInfoList.stream() + .filter(modelInfo -> teacherModelList.contains(modelInfo.getId())) + .forEach(modelInfo -> teacherModelVOS + .add(new ModelVO(modelInfo.getName(), modelInfo.getVersionNum(), adjustmentUrl(modelInfo.getUrl())))); + + //保存炼知学生模型信息 + if(!studentModelList.isEmpty()) { + ptTrainJobModelVO.setStudentModelList(new ArrayList<>()); + List studentModelVOS = ptTrainJobModelVO.getStudentModelList(); + modelInfoList.stream() + .filter(modelInfo -> studentModelList.contains(modelInfo.getId())) + .forEach(modelInfo -> studentModelVOS + .add(new ModelVO(modelInfo.getName(), modelInfo.getVersionNum(), adjustmentUrl(modelInfo.getUrl())))); + } + break; + } + + return ptTrainJobModelVO; + } + public void recycleTaskWithTrain(String recyclePath) { //创建已删除训练任务的无效文件回收任务 RecycleTaskCreateDTO recycleTask = new RecycleTaskCreateDTO(); diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java index 198449f..32b4623 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java index 4ccea5e..f5c5f41 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java index d74f118..36b6b65 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,36 +17,55 @@ package org.dubhe.service.impl; +import cn.hutool.core.bean.BeanUtil; import cn.hutool.core.util.StrUtil; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.dubhe.annotation.DataPermissionMethod; import org.dubhe.base.MagicNumConstant; import org.dubhe.base.ResponseCode; -import org.dubhe.constant.SymbolConstant; import org.dubhe.config.TrainJobConfig; -import org.dubhe.dao.ModelQueryMapper; +import org.dubhe.constant.SymbolConstant; +import org.dubhe.dao.PtModelBranchMapper; +import org.dubhe.dao.PtModelInfoMapper; import org.dubhe.dao.PtTrainAlgorithmMapper; import org.dubhe.dao.PtTrainParamMapper; import org.dubhe.data.constant.Constant; -import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.ModelQuery; -import org.dubhe.domain.entity.ModelQueryBrance; +import org.dubhe.domain.PtModelBranch; +import org.dubhe.domain.PtModelInfo; +import org.dubhe.domain.dto.BaseTrainParamDTO; +import org.dubhe.domain.dto.PtTrainParamCreateDTO; +import org.dubhe.domain.dto.PtTrainParamDeleteDTO; +import org.dubhe.domain.dto.PtTrainParamQueryDTO; +import org.dubhe.domain.dto.PtTrainParamUpdateDTO; +import org.dubhe.domain.dto.UserDTO; import org.dubhe.domain.entity.PtTrainAlgorithm; import org.dubhe.domain.entity.PtTrainParam; import org.dubhe.domain.vo.PtTrainParamQueryVO; import org.dubhe.enums.DatasetTypeEnum; import org.dubhe.enums.LogEnum; +import org.dubhe.enums.ModelResourceEnum; import org.dubhe.exception.BusinessException; import org.dubhe.service.PtTrainParamService; -import org.dubhe.utils.*; +import org.dubhe.utils.ImageUtil; +import org.dubhe.utils.JwtUtils; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -66,7 +85,11 @@ public class PtTrainParamServiceImpl implements PtTrainParamService { private ImageUtil imageUtil; @Autowired - private ModelQueryMapper modelQueryMapper; + private PtModelInfoMapper ptModelInfoMapper; + + @Autowired + private PtModelBranchMapper ptModelBranchMapper; + /** * 参数列表展示 * @@ -148,20 +171,11 @@ public class PtTrainParamServiceImpl implements PtTrainParamService { Integer algorithmSource = ptTrainAlgorithm.getAlgorithmSource(); //保存任务参数 PtTrainParam ptTrainParam = new PtTrainParam(); - //模型名称 - ModelQuery modelName = modelQueryMapper.findModelNameById(ptTrainParamCreateDTO.getModelId()); - //模型版本 - ModelQueryBrance modelVersion = modelQueryMapper.findModelVersionByUrl(ptTrainParamCreateDTO.getModelLoadPathDir()); - if(modelName!=null){ - String name = modelName.getName(); - if(modelVersion!=null){ - ptTrainParamCreateDTO.setModelName(name+ SymbolConstant.COLON +modelVersion.getVersion()); - }else { - //设置预置模型的url路径 - ptTrainParamCreateDTO.setModelLoadPathDir(modelName.getUrl()); - ptTrainParamCreateDTO.setModelName(name); - } - } + //模型检测 + BaseTrainParamDTO baseTrainParamDTO = new BaseTrainParamDTO(); + BeanUtil.copyProperties(ptTrainParamCreateDTO, baseTrainParamDTO); + checkModel(user, baseTrainParamDTO); + BeanUtils.copyProperties(ptTrainParamCreateDTO, ptTrainParam); //获取镜像 String images = imageUtil.getImageUrl(ptTrainParamCreateDTO, user); @@ -177,6 +191,92 @@ public class PtTrainParamServiceImpl implements PtTrainParamService { return Collections.singletonList(ptTrainParam.getId()); } + /** + * 检查模型是否合法 + * + * @param currentUser 用户 + * @param baseTrainParamDTO 基础训练模板参数 + */ + private void checkModel(UserDTO currentUser, BaseTrainParamDTO baseTrainParamDTO) { + + Integer modelResource = baseTrainParamDTO.getModelResource(); + if(null == modelResource ) { + if(null == baseTrainParamDTO.getModelId() && + StringUtils.isBlank(baseTrainParamDTO.getStudentModelIds()) && + StringUtils.isBlank(baseTrainParamDTO.getStudentModelIds())) { + return; + } else { + logErrorInfoOnModel(currentUser.getUsername()); + } + } + switch (ModelResourceEnum.getType(modelResource)) { + case MINE: + if(null == baseTrainParamDTO.getModelBranchId() || null == baseTrainParamDTO.getModelId() || + StringUtils.isNotBlank(baseTrainParamDTO.getTeacherModelIds()) || + StringUtils.isNotBlank(baseTrainParamDTO.getStudentModelIds())) { + logErrorInfoOnModel(currentUser.getUsername()); + } + PtModelBranch ptModelBranch = ptModelBranchMapper.selectById(baseTrainParamDTO.getModelBranchId()); + if(null == ptModelBranch || ptModelBranch.getParentId().compareTo(baseTrainParamDTO.getModelId()) != 0 || + StringUtils.isBlank(ptModelBranch.getModelAddress())) { + logErrorInfoOnModel(currentUser.getUsername()); + } + PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(ptModelBranch.getParentId()); + if(null == ptModelInfo || ptModelInfo.getModelResource().compareTo(baseTrainParamDTO.getModelResource()) != 0){ + logErrorInfoOnModel(currentUser.getUsername()); + } + break; + case PRESET: + if(null == baseTrainParamDTO.getModelId() || StringUtils.isNotBlank(baseTrainParamDTO.getTeacherModelIds()) || + StringUtils.isNotBlank(baseTrainParamDTO.getStudentModelIds())) { + logErrorInfoOnModel(currentUser.getUsername()); + } + PtModelInfo ptModelInfoPreset = ptModelInfoMapper.selectById(baseTrainParamDTO.getModelId()); + if(null == ptModelInfoPreset || StringUtils.isBlank(ptModelInfoPreset.getUrl()) || + ptModelInfoPreset.getModelResource().compareTo(baseTrainParamDTO.getModelResource()) != 0) { + logErrorInfoOnModel(currentUser.getUsername()); + } + break; + case ATLAS: + if(StringUtils.isBlank(baseTrainParamDTO.getTeacherModelIds()) || null != baseTrainParamDTO.getModelId()){ + logErrorInfoOnModel(currentUser.getUsername()); + } + Set ids = new HashSet<>(); + Set teacherModelList = new HashSet<>(); + Arrays.stream(baseTrainParamDTO.getTeacherModelIds().trim().split(SymbolConstant.COMMA)) + .forEach(id -> teacherModelList.add(Long.parseLong(id))); + ids.addAll(teacherModelList); + + Set studentModelList = new HashSet<>(); + if(StringUtils.isNotBlank(baseTrainParamDTO.getStudentModelIds())) { + Arrays.stream(baseTrainParamDTO.getStudentModelIds().trim().split(SymbolConstant.COMMA)) + .forEach(id -> studentModelList.add(Long.parseLong(id))); + ids.addAll(studentModelList); + } + if(ids.isEmpty()) { + logErrorInfoOnModel(currentUser.getUsername()); + } + LambdaQueryWrapper query = new LambdaQueryWrapper<>(); + query.eq(PtModelInfo::getModelResource, baseTrainParamDTO.getModelResource()) + .in(PtModelInfo::getId, ids).isNotNull(PtModelInfo::getUrl).ne(PtModelInfo::getUrl, SymbolConstant.BLANK); + List modelInfoList = ptModelInfoMapper.selectList(query); + if(null == modelInfoList || modelInfoList.size() < ids.size()) { + logErrorInfoOnModel(currentUser.getUsername()); + } + break; + } + } + + /** + * 打印训练模板中模型相关的错误日志 + * + * @param username + */ + private void logErrorInfoOnModel(String username){ + LogUtil.error(LogEnum.BIZ_TRAIN, "User {} operating training param, error on model......", username); + throw new BusinessException("模型参数不合法"); + } + /** * 修改任务参数 * @@ -194,21 +294,10 @@ public class PtTrainParamServiceImpl implements PtTrainParamService { checkUpdateTrainParam(ptTrainParamUpdateDTO, user); //修改任务参数 PtTrainParam ptTrainParam = new PtTrainParam(); - //模型名称 - ModelQuery modelName = modelQueryMapper.findModelNameById(ptTrainParamUpdateDTO.getModelId()); - //模型版本 - ModelQueryBrance modelVersion = modelQueryMapper.findModelVersionByUrl(ptTrainParamUpdateDTO.getModelLoadPathDir()); - //设置版本 - if(modelName!=null){ - String name=modelName.getName(); - if(modelVersion!=null){ - ptTrainParamUpdateDTO.setModelName(name+SymbolConstant.COLON+modelVersion.getVersion()); - }else { - //设置预置模型的url值 - ptTrainParamUpdateDTO.setModelLoadPathDir(modelName.getUrl()); - ptTrainParamUpdateDTO.setModelName(name); - } - } + //模型检测 + BaseTrainParamDTO baseTrainParamDTO = new BaseTrainParamDTO(); + BeanUtil.copyProperties(ptTrainParamUpdateDTO, baseTrainParamDTO); + checkModel(user, baseTrainParamDTO); BeanUtils.copyProperties(ptTrainParamUpdateDTO, ptTrainParam); ptTrainParam.setUpdateUserId(user.getId()); diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingAsyncServiceImpl.java new file mode 100644 index 0000000..2060c48 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingAsyncServiceImpl.java @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import org.dubhe.dto.callback.BaseK8sDeploymentCallbackCreateDTO; +import org.dubhe.dto.callback.ServingK8sDeploymentCallbackCreateDTO; +import org.dubhe.enums.LogEnum; +import org.dubhe.service.DeploymentCallbackAsyncService; +import org.dubhe.service.ServingService; +import org.dubhe.service.abstracts.AbstractDeploymentCallback; +import org.dubhe.utils.LogUtil; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +/** + * @description 云端Serving回调 + * @date 2020-09-16 + */ +@Service(value = "servingAsyncService") +public class ServingAsyncServiceImpl extends AbstractDeploymentCallback implements DeploymentCallbackAsyncService { + + @Resource + private ServingService servingService; + + @Override + public boolean doCallback(int times, R k8sDeploymentCallbackCreateDTO) { + // 强制转型 + ServingK8sDeploymentCallbackCreateDTO req = (ServingK8sDeploymentCallbackCreateDTO) k8sDeploymentCallbackCreateDTO; + LogUtil.info(LogEnum.SERVING, "Thread {} try {} time.Request: {}", Thread.currentThread(), times, req.toString()); + //在线服务回调 + return servingService.servingCallback(times, req); + } + + @Override + public void callbackFailed(int retryTimes, R k8sDeploymentCallbackCreateDTO) { + ServingK8sDeploymentCallbackCreateDTO req = (ServingK8sDeploymentCallbackCreateDTO) k8sDeploymentCallbackCreateDTO; + LogUtil.info(LogEnum.SERVING, "Thread {} try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingServiceImpl.java new file mode 100644 index 0000000..7078a2a --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingServiceImpl.java @@ -0,0 +1,951 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.service.impl; + +import cn.hutool.core.bean.BeanUtil; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import io.grpc.ManagedChannel; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections4.CollectionUtils; +import org.dubhe.annotation.DataPermissionMethod; +import org.dubhe.constant.NumberConstant; +import org.dubhe.constant.ServingConstant; +import org.dubhe.constant.StringConstant; +import org.dubhe.constant.SymbolConstant; +import org.dubhe.dao.PtModelBranchMapper; +import org.dubhe.dao.PtModelInfoMapper; +import org.dubhe.dao.ServingInfoMapper; +import org.dubhe.dao.ServingModelConfigMapper; +import org.dubhe.domain.PtModelBranch; +import org.dubhe.domain.PtModelInfo; +import org.dubhe.domain.dto.PredictParamDTO; +import org.dubhe.domain.dto.ServingInfoCreateDTO; +import org.dubhe.domain.dto.ServingInfoDeleteDTO; +import org.dubhe.domain.dto.ServingInfoDetailDTO; +import org.dubhe.domain.dto.ServingInfoQueryDTO; +import org.dubhe.domain.dto.ServingInfoUpdateDTO; +import org.dubhe.domain.dto.ServingModelConfigDTO; +import org.dubhe.domain.dto.ServingStartDTO; +import org.dubhe.domain.dto.ServingStopDTO; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.ImageInfo; +import org.dubhe.domain.entity.ServingInfo; +import org.dubhe.domain.entity.ServingModelConfig; +import org.dubhe.domain.vo.PredictParamVO; +import org.dubhe.domain.vo.ServingConfigMetricsVO; +import org.dubhe.domain.vo.ServingInfoCreateVO; +import org.dubhe.domain.vo.ServingInfoDeleteVO; +import org.dubhe.domain.vo.ServingInfoDetailVO; +import org.dubhe.domain.vo.ServingInfoQueryVO; +import org.dubhe.domain.vo.ServingInfoUpdateVO; +import org.dubhe.domain.vo.ServingMetricsVO; +import org.dubhe.domain.vo.ServingModelConfigVO; +import org.dubhe.domain.vo.ServingPodMetricsVO; +import org.dubhe.domain.vo.ServingStartVO; +import org.dubhe.domain.vo.ServingStopVO; +import org.dubhe.dto.callback.ServingK8sDeploymentCallbackCreateDTO; +import org.dubhe.enums.BizEnum; +import org.dubhe.enums.DatasetTypeEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.ServingErrorEnum; +import org.dubhe.enums.ServingRouteEventEnum; +import org.dubhe.enums.ServingStatusEnum; +import org.dubhe.enums.ServingTypeEnum; +import org.dubhe.exception.BusinessException; +import org.dubhe.k8s.api.MetricsApi; +import org.dubhe.k8s.domain.dto.PodQueryDTO; +import org.dubhe.k8s.domain.vo.PodVO; +import org.dubhe.k8s.domain.vo.PtPodsVO; +import org.dubhe.k8s.service.PodService; +import org.dubhe.service.ServingLuaScriptService; +import org.dubhe.service.ServingModelConfigService; +import org.dubhe.service.ServingService; +import org.dubhe.task.DeployServingAsyncTask; +import org.dubhe.utils.DateUtil; +import org.dubhe.utils.GrpcClient; +import org.dubhe.utils.JwtUtils; +import org.dubhe.utils.K8sNameTool; +import org.dubhe.utils.K8sUtil; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.NfsUtil; +import org.dubhe.utils.PageUtil; +import org.dubhe.utils.ReflectionUtils; +import org.dubhe.utils.StringUtils; +import org.dubhe.utils.WrapperHelp; +import org.springframework.amqp.rabbit.connection.CorrelationData; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.connection.stream.StreamRecords; +import org.springframework.data.redis.connection.stream.StringRecord; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + + +/** + * @description 在线服务管理 + * @date 2020-08-25 + */ +@Service +public class ServingServiceImpl implements ServingService { + + @Value("${serving.gateway-uri-postfix}") + private String GATEWAY_URI_POSTFIX; + @Value("${k8s.pod.metrics.grafanaUrl}") + private String k8sPodMetricsGrafanaUrl; + @Value("${serving.group}") + private String servingGroup; + @Resource + private ServingInfoMapper servingInfoMapper; + @Resource + private ServingModelConfigService servingModelConfigService; + @Resource + private ServingModelConfigMapper servingModelConfigMapper; + @Resource + private PtModelInfoMapper ptModelInfoMapper; + @Resource + private PtModelBranchMapper ptModelBranchMapper; + @Resource + private DeployServingAsyncTask deployServingAsyncTask; + @Resource + private K8sUtil k8sUtil; + @Resource + private ServingLuaScriptService servingLuaScriptService; + @Resource + private PodService podService; + @Resource + private MetricsApi metricsApi; + @Resource + private NfsUtil nfsUtil; + @Resource + private K8sNameTool k8sNameTool; + @Resource + private StringRedisTemplate stringRedisTemplate; + + public final static List FILE_NAMES; + + static { + FILE_NAMES = ReflectionUtils.getFieldNames(ServingInfoQueryVO.class); + } + + /** + * 查询分页数据 + * + * @param servingInfoQueryDTO 服务查询参数 + * @return Map云端服务分页对象 + */ + @Override + public Map query(ServingInfoQueryDTO servingInfoQueryDTO) { + String name = servingInfoQueryDTO.getName(); + //服务名称或id条件非空 + if (StringUtils.isNotBlank(name)) { + //整数匹配 + if (StringConstant.PATTERN_NUM.matcher(name).matches()) { + servingInfoQueryDTO.setId(Long.parseLong(name)); + servingInfoQueryDTO.setName(null); + Map map = queryServing(servingInfoQueryDTO); + if (((List) map.get(StringConstant.RESULT)).size() > NumberConstant.NUMBER_0) { + return map; + } else { + servingInfoQueryDTO.setId(null); + servingInfoQueryDTO.setName(name); + } + } + } + return queryServing(servingInfoQueryDTO); + } + + /** + * 分页查询在线服务 + * + * @param servingInfoQueryDTO 查询条件 + * @return Map 分页结果 + */ + public Map queryServing(ServingInfoQueryDTO servingInfoQueryDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + LogUtil.info(LogEnum.SERVING, "User {} queried online service list, with the query {}", userDTO.getUsername(), JSONObject.toJSONString(servingInfoQueryDTO)); + QueryWrapper wrapper = WrapperHelp.getWrapper(servingInfoQueryDTO); + wrapper.eq("create_user_id", userDTO.getId()); + if (StringUtils.isNotBlank(servingInfoQueryDTO.getName())) { + //按名称搜索时不区分大小写 + wrapper.like("lower(name)", servingInfoQueryDTO.getName().toLowerCase()); + } + Page page = new Page(null == servingInfoQueryDTO.getCurrent() ? NumberConstant.NUMBER_1 : servingInfoQueryDTO.getCurrent(), + null == servingInfoQueryDTO.getSize() ? NumberConstant.NUMBER_10 : servingInfoQueryDTO.getSize()); + try { + //排序字段，默认按更新时间降序，否则将驼峰转换为下划线 + String column = servingInfoQueryDTO.getSort() != null && FILE_NAMES.contains(servingInfoQueryDTO.getSort()) + ? StringUtils.humpToLine(servingInfoQueryDTO.getSort()) : "update_time"; + //排序方式 + boolean isAsc = !StringUtils.isBlank(servingInfoQueryDTO.getOrder()) && !StringUtils.equals(servingInfoQueryDTO.getOrder(), StringConstant.SORT_DESC); + wrapper.orderBy(true, isAsc, column); + } catch (Exception e) { + LogUtil.error(LogEnum.SERVING, "User queried online service with an exception, request info: {}，exception info: {}", JSONObject.toJSONString(servingInfoQueryDTO), e); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + IPage servingInfos = servingInfoMapper.selectPage(page, wrapper); + List queryVOList = servingInfos.getRecords().stream().map(servingInfo -> { + ServingInfoQueryVO servingInfoQueryVO = new ServingInfoQueryVO(); + BeanUtils.copyProperties(servingInfo, servingInfoQueryVO); + servingInfoQueryVO.setUrl(servingInfo.getUuid() + GATEWAY_URI_POSTFIX); + Map statistics = servingLuaScriptService.countCallsByServingInfoId(servingInfo.getId()); + servingInfoQueryVO.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO)); + servingInfoQueryVO.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO)); + return servingInfoQueryVO; + }).collect(Collectors.toList()); + LogUtil.info(LogEnum.SERVING, "User {} queried online service list, online service count = {}", userDTO.getUsername(), queryVOList.size()); + return PageUtil.toPage(page, queryVOList); + } + + /** + * 创建服务 + * + * @param servingInfoCreateDTO 服务创建参数 + * @return ServingInfoCreateVO 服务创建返回对象 + */ + @Override + @Transactional(rollbackFor = Exception.class) + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + public ServingInfoCreateVO create(ServingInfoCreateDTO servingInfoCreateDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + //参数校验 + assert Objects.nonNull(userDTO); + checkNameExist(servingInfoCreateDTO.getName(), userDTO.getId()); + //gRPC不支持灰度发布 + if (NumberConstant.NUMBER_1 == servingInfoCreateDTO.getType() && servingInfoCreateDTO.getModelConfigList().size() > NumberConstant.NUMBER_1) { + throw new BusinessException(ServingErrorEnum.GRPC_PROTOCOL_NOT_SUPPORTED); + } + ServingInfo servingInfo = buildServingInfo(servingInfoCreateDTO); + List modelConfigList = insertServing(servingInfoCreateDTO, userDTO, servingInfo); + //异步部署容器 + deployServingAsyncTask.deployServing(userDTO, servingInfo, modelConfigList); + return new ServingInfoCreateVO(servingInfo.getId(), servingInfo.getStatus()); + } + + /** + * 创建服务并保存数据 + * + * @param servingInfoCreateDTO 在线服务信息创建对象 + * @param userDTO 用户信息 + * @param servingInfo 在线服务信息对象 + * @return List 在线服务模型配置信息列表 + */ + @Transactional(rollbackFor = Exception.class) + public List insertServing(ServingInfoCreateDTO servingInfoCreateDTO, UserDTO userDTO, ServingInfo servingInfo) { + int result = servingInfoMapper.insert(servingInfo); + if (result < NumberConstant.NUMBER_1) { + LogUtil.error(LogEnum.SERVING, "User {} failed saving service into into the database. Service name: {}", userDTO.getUsername(), servingInfoCreateDTO.getName()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + return saveModelConfig(servingInfoCreateDTO.getModelConfigList(), servingInfo, userDTO.getUsername()); + } + + + /** + * 校验名称是否存在 + * + * @param name 服务名称 + * @param userId 用户ID + */ + public void checkNameExist(String name, Long userId) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + wrapper.eq(ServingInfo::getName, name); + wrapper.eq(ServingInfo::getCreateUserId, userId); + int count = servingInfoMapper.selectCount(wrapper); + if (count > NumberConstant.NUMBER_0) { + throw new BusinessException(ServingErrorEnum.SERVING_NAME_EXIST); + } + } + + /** + * 构建服务信息 + * + * @param servingInfoCreateDTO 服务创建参数 + * @return ServingInfo 服务信息 + */ + public ServingInfo buildServingInfo(ServingInfoCreateDTO servingInfoCreateDTO) { + ServingInfo servingInfo = new ServingInfo(); + BeanUtils.copyProperties(servingInfoCreateDTO, servingInfo); + servingInfo.setUuid(StringUtils.getUUID()); + servingInfo.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); + int totalNode = servingInfoCreateDTO.getModelConfigList().stream().mapToInt(ServingModelConfigDTO::getResourcesPoolNode).sum(); + servingInfo.setTotalNode(totalNode); + return servingInfo; + } + + /** + * 校验资源类型 + * + * @param servingModelConfig 模型配置 + */ + public void checkResourceType(ServingModelConfig servingModelConfig) { + // oneflow 暂不支持cpu + if (servingModelConfig.getFrameType() == NumberConstant.NUMBER_1 && servingModelConfig.getResourcesPoolType() == NumberConstant.NUMBER_0) { + throw new BusinessException(ServingErrorEnum.CPU_NOT_SUPPORTED_BY_ONEFLOW); + } + } + + /** + * 保存模型配置对象列表 + * + * @param modelConfigDTOList 模型配置列表 + * @param servingInfo 服务信息 + * @return List 模型配置对象列表 + */ + List saveModelConfig(List modelConfigDTOList, ServingInfo servingInfo, String userName) { + List list = new ArrayList<>(); + //生成4位随机字符串，作为部署id + String deployId = StringUtils.getTimestamp(); + for (ServingModelConfigDTO servingModelConfigDTO : modelConfigDTOList) { + ServingModelConfig servingModelConfig = new ServingModelConfig(); + BeanUtils.copyProperties(servingModelConfigDTO, servingModelConfig); + servingModelConfig.setServingId(servingInfo.getId()); + servingModelConfig.setDeployId(deployId); + PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(servingModelConfig.getModelId()); + //校验框架 + Integer frameType = getFrameType(ptModelInfo, servingModelConfigDTO.getModelAddress()); + servingModelConfig.setFrameType(frameType); + checkResourceType(servingModelConfig); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper(); + wrapper.eq(PtModelBranch::getParentId, servingModelConfigDTO.getModelId()); + wrapper.eq(PtModelBranch::getModelAddress, servingModelConfigDTO.getModelAddress()); + PtModelBranch ptModelBranch = ptModelBranchMapper.selectOne(wrapper); + if (ptModelBranch != null) { + servingModelConfig.setModelVersion(ptModelBranch.getVersionNum()); + } + servingModelConfig.setModelName(ptModelInfo.getName()); + String path = k8sUtil.getAbsoluteNfsPath(servingModelConfig.getModelAddress()); + if (nfsUtil.fileOrDirIsEmpty(path)) { + throw new BusinessException(ServingErrorEnum.MODEL_FILE_NOT_EXIST); + } + if (servingModelConfigMapper.insert(servingModelConfig) < NumberConstant.NUMBER_1) { + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + list.add(servingModelConfig); + } + return list; + } + + /** + * 获取模型框架 + * + * @param ptModelInfo 模型信息 + * @param modelAddress 模型路径 + * @return 模型框架 + */ + public Integer getFrameType(PtModelInfo ptModelInfo, String modelAddress) { + String path = k8sUtil.getAbsoluteNfsPath(modelAddress); + if (ptModelInfo == null || nfsUtil.fileOrDirIsEmpty(path)) { + throw new BusinessException(ServingErrorEnum.MODEL_FILE_NOT_EXIST); + } + if (ptModelInfo.getFrameType() > NumberConstant.NUMBER_4) { + throw new BusinessException(ServingErrorEnum.MODEL_FRAME_TYPE_NOT_SUPPORTED); + } + return ptModelInfo.getFrameType(); + } + + /** + * 在线服务修改 + * + * @param servingInfoUpdateDTO 服务对象修改 + * @return ServingInfoUpdateVO 返回修改后的状态 + */ + @Override + @Transactional(rollbackFor = Exception.class) + @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) + public ServingInfoUpdateVO update(ServingInfoUpdateDTO servingInfoUpdateDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + ServingInfo servingInfo = checkServingInfoExist(servingInfoUpdateDTO.getId(), userDTO.getId()); + checkRunningNode(servingInfo.getRunningNode()); + //如果修改了任务名，校验新的任务名是否存在 + if (!servingInfo.getName().equals(servingInfoUpdateDTO.getName())) { + checkNameExist(servingInfoUpdateDTO.getName(), userDTO.getId()); + } + servingInfo.setName(servingInfoUpdateDTO.getName()) + .setDescription(servingInfoUpdateDTO.getDescription()) + .setType(servingInfoUpdateDTO.getType()) + .setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()) + .setUpdateTime(DateUtil.getCurrentTimestamp()); + servingInfo.setUuid(StringUtils.getUUID()); + int totalNode = servingInfoUpdateDTO.getModelConfigList().stream().mapToInt(ServingModelConfigDTO::getResourcesPoolNode).sum(); + servingInfo.setTotalNode(totalNode); + Set oldIds = servingModelConfigService.getIdsByServingId(servingInfoUpdateDTO.getId()); + //删除去掉的模型配置信息 + if (CollectionUtils.isNotEmpty(oldIds)) { + if (!servingModelConfigService.removeByIds(oldIds)) { + LogUtil.error(LogEnum.SERVING, "User {} modified online service model config but failed deleting online service model config. Model config ids={}", userDTO.getUsername(), oldIds); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + List oldModelConfigList = servingModelConfigService.listByIds(oldIds); + deployServingAsyncTask.deleteServing(userDTO, servingInfo, oldModelConfigList); + //删除路由信息 + if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { + this.notifyUpdateServingRoute(Collections.emptyList(), oldModelConfigList.stream().map(ServingModelConfig::getId).collect(Collectors.toList())); + } + } + List modelConfigList = updateServing(servingInfoUpdateDTO, userDTO, servingInfo); + //异步部署容器 + deployServingAsyncTask.deployServing(userDTO, servingInfo, modelConfigList); + return new ServingInfoUpdateVO(servingInfo.getId(), servingInfo.getStatus()); + } + + /** + * 修改服务并保存数据 + * + * @param servingInfoUpdateDTO 服务对象修改 + * @param userDTO 用户信息 + * @param servingInfo 在线服务信息 + * @return List 模型配置对象列表 + */ + @Transactional(rollbackFor = Exception.class) + public List updateServing(ServingInfoUpdateDTO servingInfoUpdateDTO, UserDTO userDTO, ServingInfo servingInfo) { + int result = servingInfoMapper.updateById(servingInfo); + if (result < NumberConstant.NUMBER_1) { + LogUtil.error(LogEnum.SERVING, "User {} failed deleting online service from the database, service id={}", userDTO.getUsername(), servingInfo.getId()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + return saveModelConfig(servingInfoUpdateDTO.getModelConfigList(), servingInfo, userDTO.getUsername()); + } + + /** + * 校验是否有节点在运行 + * + * @param runningNode 运行节点数 + */ + void checkRunningNode(Integer runningNode) { + if (runningNode != null && runningNode > NumberConstant.NUMBER_0) { + throw new BusinessException(ServingErrorEnum.OPERATION_NOT_ALLOWED); + } + } + + /** + * 校验服务信息是否存在 + * + * @param id 服务id + * @param userId 用户id + * @return ServingInfo 服务信息对象 + */ + ServingInfo checkServingInfoExist(Long id, Long userId) { + ServingInfo servingInfo = servingInfoMapper.selectById(id); + //校验服务是否存在，或是否为当前用户创建 + if (servingInfo == null || (!userId.equals(servingInfo.getCreateUserId()))) { + throw new BusinessException(ServingErrorEnum.SERVING_INFO_ABSENT); + } + return servingInfo; + } + + /** + * 在线服务删除 + * + * @param servingInfoDeleteDTO 服务对象删除 + * @return ServingInfoDeleteVO 返回删除对象的id + */ + @Override + @Transactional(rollbackFor = Exception.class) + public ServingInfoDeleteVO delete(ServingInfoDeleteDTO servingInfoDeleteDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + ServingInfo servingInfo = checkServingInfoExist(servingInfoDeleteDTO.getId(), userDTO.getId()); + List modelConfigList = getModelConfigByServingId(servingInfo.getId()); + deleteServing(servingInfoDeleteDTO, userDTO, servingInfo); + Map map = new HashMap<>(NumberConstant.NUMBER_2); + map.put("serving_id", servingInfo.getId()); + if (!servingModelConfigService.removeByMap(map)) { + LogUtil.error(LogEnum.SERVING, "User {} failed update online service in the database, service id={}, service name:{}", + userDTO.getUsername(), servingInfoDeleteDTO.getId(), servingInfo.getName()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + deployServingAsyncTask.deleteServing(userDTO, servingInfo, modelConfigList); + //删除路由信息 + if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { + this.notifyUpdateServingRoute(Collections.emptyList(), modelConfigList.stream().map(ServingModelConfig::getId).collect(Collectors.toList())); + } + return new ServingInfoDeleteVO(servingInfo.getId()); + } + + /** + * 删除服务并保存数据 + * + * @param servingInfoDeleteDTO 服务信息删除对象 + * @param userDTO 用户信息 + * @param servingInfo 在线服务信息 + */ + @Transactional(rollbackFor = Exception.class) + public void deleteServing(ServingInfoDeleteDTO servingInfoDeleteDTO, UserDTO userDTO, ServingInfo servingInfo) { + int result = servingInfoMapper.deleteById(servingInfo.getId()); + if (result < NumberConstant.NUMBER_1) { + LogUtil.error(LogEnum.SERVING, "User {} failed deleting online service from the database, service id={}, service name:{}", + userDTO.getUsername(), servingInfoDeleteDTO.getId(), servingInfo.getName()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + } + + /** + * 在线服务获取详情 + * + * @param servingInfoDetailDTO 获取服务详情参数 + * @return ServingInfoDetailVO 返回详情对象 + */ + @Override + public ServingInfoDetailVO getDetail(ServingInfoDetailDTO servingInfoDetailDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + ServingInfo servingInfo = checkServingInfoExist(servingInfoDetailDTO.getId(), userDTO.getId()); + Map map = new HashMap<>(NumberConstant.NUMBER_2); + map.put("serving_id", servingInfo.getId()); + List servingModelConfigList = servingModelConfigService.listByMap(map); + ServingInfoDetailVO servingInfoDetailVO = new ServingInfoDetailVO(); + BeanUtils.copyProperties(servingInfo, servingInfoDetailVO); + List servingModelConfigVOList = new ArrayList<>(); + List configIdList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(servingModelConfigList)) { + for (ServingModelConfig servingModelConfig : servingModelConfigList) { + ServingModelConfigVO vo = new ServingModelConfigVO(); + BeanUtils.copyProperties(servingModelConfig, vo); + servingModelConfigVOList.add(vo); + configIdList.add(servingModelConfig.getId()); + } + } + servingInfoDetailVO.setModelConfigList(servingModelConfigVOList); + // 获取调用次数 + Map countCalls = servingLuaScriptService.countCalls(configIdList); + servingInfoDetailVO.setTotalNum(countCalls.getOrDefault("callCount", SymbolConstant.ZERO)); + servingInfoDetailVO.setFailNum(countCalls.getOrDefault("failedCount", SymbolConstant.ZERO)); + return servingInfoDetailVO; + } + + /** + * 启动在线服务 + * + * @param servingStartDTO 启动服务参数 + * @return ServingStartVO 返回启动后对象 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public ServingStartVO start(ServingStartDTO servingStartDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + ServingInfo servingInfo = checkServingInfoExist(servingStartDTO.getId(), userDTO.getId()); + checkRunningNode(servingInfo.getRunningNode()); + List modelConfigList = getModelConfigByServingId(servingInfo.getId()); + servingInfo.setUuid(StringUtils.getUUID()); + servingInfo.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); + updateServingStart(userDTO, servingInfo); + //异步部署容器 + deployServingAsyncTask.deployServing(userDTO, servingInfo, modelConfigList); + return new ServingStartVO(servingInfo.getId(), servingInfo.getStatus()); + } + + /** + * 启动服务并保存数据 + * + * @param userDTO 用户信息 + * @param servingInfo 在线服务信息 + */ + @Transactional(rollbackFor = Exception.class) + public void updateServingStart(UserDTO userDTO, ServingInfo servingInfo) { + int result = servingInfoMapper.updateById(servingInfo); + if (result < NumberConstant.NUMBER_1) { + LogUtil.error(LogEnum.SERVING, "User {} failed update online service in the database, service id={}", userDTO.getUsername(), servingInfo.getId()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + } + + /** + * 在线服务推理 + * + * @param id 预测服务ID + * @param url 预测地址 + * @param files 需要预测的图片文件 + * @return String 返回推理结果 + */ + @Override + public String predict(Long id, String url, MultipartFile[] files) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + ServingInfo servingInfo = checkServingInfoExist(id, userDTO.getId()); + //校验服务状态是否为运行中 + if (servingInfo.getRunningNode() == NumberConstant.NUMBER_0) { + throw new BusinessException(ServingErrorEnum.SERVING_NOT_WORKING); + } + if (StringUtils.isBlank(url)) { + throw new BusinessException(ServingErrorEnum.SERVING_NOT_WORKING); + } + //校验推理图片 + if (files == null || files.length == NumberConstant.NUMBER_0) { + throw new BusinessException(ServingErrorEnum.PREDICT_IMAGE_EMPTY); + } + List imageInfoList = new ArrayList<>(); + for (MultipartFile file : files) { + String fileName = file.getOriginalFilename(); + //获取文件后缀名 + assert Objects.nonNull(fileName); + int begin = fileName.lastIndexOf(SymbolConstant.DOT); + String suffix = fileName.substring(begin).toLowerCase(); + if (!ServingConstant.IMAGE_FORMAT.contains(suffix)) { + throw new BusinessException(ServingErrorEnum.IMAGE_FORMAT_ERROR); + } + String base64File; + try { + base64File = Base64.encodeBase64String(file.getBytes()); + } catch (Exception e) { + throw new BusinessException(ServingErrorEnum.IMAGE_CONVERT_BASE64_FAIL); + } + if (StringUtils.isNotBlank(base64File)) { + ImageInfo imageInfo = new ImageInfo(); + imageInfo.setImageName(fileName); + imageInfo.setImageFile(base64File); + imageInfoList.add(imageInfo); + } + } + if (imageInfoList.isEmpty()) { + throw new BusinessException(ServingErrorEnum.IMAGE_FORMAT_ERROR); + } + ManagedChannel channel = GrpcClient.getChannel(id, url); + return GrpcClient.getResult(channel, imageInfoList).getJsonResult(); + } + + /** + * 在线服务停止 + * + * @param servingStopDTO 停止服务参数 + * @return ServingStopVO 返回停止后对象 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public ServingStopVO stop(ServingStopDTO servingStopDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + ServingInfo servingInfo = checkServingInfoExist(servingStopDTO.getId(), userDTO.getId()); + //修改状态及可用节点数 + servingInfo.setStatus(ServingStatusEnum.STOP.getStatus()); + servingInfo.setRunningNode(NumberConstant.NUMBER_0); + List modelConfigList = getModelConfigByServingId(servingInfo.getId()); + updateServingStop(userDTO, servingInfo, modelConfigList); + deployServingAsyncTask.deleteServing(userDTO, servingInfo, modelConfigList); + //删除路由信息 + if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { + this.notifyUpdateServingRoute(Collections.emptyList(), modelConfigList.stream().map(ServingModelConfig::getId).collect(Collectors.toList())); + } + return new ServingStopVO(servingInfo.getId(), servingInfo.getStatus()); + } + + /** + * 停止服务并保存数据 + * + * @param userDTO 用户信息 + * @param servingInfo 在线服务信息 + * @param modelConfigList 模型配置列表 + */ + @Transactional(rollbackFor = Exception.class) + public void updateServingStop(UserDTO userDTO, ServingInfo servingInfo, List modelConfigList) { + int result = servingInfoMapper.updateById(servingInfo); + if (result < NumberConstant.NUMBER_1) { + LogUtil.error(LogEnum.SERVING, "User {} failed stopping the online service and failed to update the service in the database. Service id={}, service name:{}, running node number:{}", + userDTO.getUsername(), servingInfo.getId(), servingInfo.getName(), servingInfo.getRunningNode()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + modelConfigList.forEach(servingModelConfig -> { + servingModelConfig.setReadyReplicas(NumberConstant.NUMBER_0); + servingModelConfigMapper.updateById(servingModelConfig); + }); + if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { + GrpcClient.shutdownChannel(servingInfo.getId(), userDTO); + } + } + + /** + * 获取推理参数 + * + * @param predictParamDTO 获取预测参数服务 + * @return PredictParamVO 返回推理参数对象 + */ + @Override + public PredictParamVO getPredictParam(PredictParamDTO predictParamDTO) { + UserDTO userDTO = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(userDTO); + ServingInfo servingInfo = checkServingInfoExist(predictParamDTO.getId(), userDTO.getId()); + PredictParamVO predictParamVO = new PredictParamVO(); + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + wrapper.eq(ServingModelConfig::getServingId, predictParamDTO.getId()); + List servingModelConfigList = servingModelConfigService.list(wrapper); + if (CollectionUtils.isEmpty(servingModelConfigList)) { + return predictParamVO; + } + //grpc协议 + if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { + predictParamVO.setRequestMethod("gRPC"); + String url = servingModelConfigList.get(0).getUrl(); + predictParamVO.setUrl(url); + Map inputs = new HashMap<>(); + inputs.put("DataRequest", "List"); + predictParamVO.setInputs(inputs); + Map> other = new HashMap<>(); + Map image = new HashMap<>(); + image.put("image_file", "String"); + image.put("image_name", "String"); + other.put("Image", image); + Map outputs = new HashMap<>(); + outputs.put("DataResponse", "String"); + predictParamVO.setOutputs(outputs); + predictParamVO.setOther(other); + } else if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { + String url = "http://" + servingInfo.getUuid() + GATEWAY_URI_POSTFIX + ServingConstant.INFERENCE_INTERFACE_NAME; + predictParamVO.setUrl(url); + Map inputs = new HashMap<>(); + inputs.put("image_files", "File"); + predictParamVO.setInputs(inputs); + + Map outputs = new HashMap<>(); + outputs.put("label", "String"); + outputs.put("probability", "Float"); + predictParamVO.setOutputs(outputs); + predictParamVO.setRequestMethod(SymbolConstant.POST); + } else { + throw new BusinessException(ServingErrorEnum.PROTOCOL_NOT_SUPPORTED); + } + return predictParamVO; + } + + /** + * 根据servingId获取模型配置列表 + * + * @param servingId 在线服务id + * @return List 模型配置列表 + */ + public List getModelConfigByServingId(Long servingId) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); + wrapper.eq(ServingModelConfig::getServingId, servingId); + return servingModelConfigService.list(wrapper); + } + + /** + * 获取容器监控信息 + * + * @param id 模型部署信息id + * @return ServingMetricsVO 返回监控信息对象 + */ + @Override + public ServingMetricsVO getMetricsDetail(Long id) { + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + ServingInfoDetailDTO queryDTO = new ServingInfoDetailDTO(); + queryDTO.setId(id); + ServingInfoDetailVO detail = this.getDetail(queryDTO); + List configMetricsVOS = new ArrayList<>(); + for (ServingModelConfigVO servingModelConfigVO : detail.getModelConfigList()) { + // 获取id、模型等信息 + ServingConfigMetricsVO metricsTemp = new ServingConfigMetricsVO(); + BeanUtil.copyProperties(servingModelConfigVO, metricsTemp); + // 获取实时监控数据 + assert Objects.nonNull(user); + List podsVOS = metricsApi.getPodMetricsRealTime(k8sUtil.getNamespace(user), k8sUtil.getResourceName(servingModelConfigVO.getResourceInfo())); + if (CollectionUtils.isNotEmpty(podsVOS)) { + List podMetricsVOS = podsVOS.stream().map(podsVO -> { + ServingPodMetricsVO servingPodMetricsVO = new ServingPodMetricsVO(); + BeanUtil.copyProperties(podsVO, servingPodMetricsVO); + servingPodMetricsVO.setGrafanaUrl(k8sPodMetricsGrafanaUrl.concat(podsVO.getPodName())); + return servingPodMetricsVO; + }).collect(Collectors.toList()); + metricsTemp.setPodList(podMetricsVOS); + } else { + metricsTemp.setPodList(Collections.emptyList()); + } + // 接口调用信息 + Map statistics = servingLuaScriptService.countCallsByServingConfigId(servingModelConfigVO.getId()); + metricsTemp.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO)); + metricsTemp.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO)); + configMetricsVOS.add(metricsTemp); + } + return new ServingMetricsVO().setId(id).setServingConfigList(configMetricsVOS); + } + + /** + * 新增和删除在线服务路由通知 + * + * @param saveIdList 新增的路由ID列表 + * @param deleteIdList 删除的路由ID列表 + */ + @Override + public void notifyUpdateServingRoute(List saveIdList, List deleteIdList) { + String message = StringUtils.EMPTY; + // 处理新增路由消息 + if (CollectionUtils.isNotEmpty(saveIdList)) { + String idString = StringUtils.join(saveIdList.toArray(), SymbolConstant.COMMA); + message = ServingRouteEventEnum.SAVE.getCode() + SymbolConstant.COLON + idString + SymbolConstant.EVENT_SEPARATOR; + LogUtil.info(LogEnum.SERVING, "Serving start success and seed notify {}", message); + } + // 处理删除路由消息 + if (CollectionUtils.isNotEmpty(deleteIdList)) { + String idString = StringUtils.join(deleteIdList.toArray(), SymbolConstant.COMMA); + message = message + ServingRouteEventEnum.DELETE.getCode() + SymbolConstant.COLON + idString; + LogUtil.info(LogEnum.SERVING, "Serving stop success and seed notify {}", message); + } + if (StringUtils.isNotBlank(message)) { + LogUtil.info(LogEnum.SERVING, "Start send message to stream with notify {}", message); + StringRecord stringRecord = StreamRecords.string(Collections.singletonMap(servingGroup, message)).withStreamKey(ServingConstant.SERVING_STREAM); + stringRedisTemplate.opsForStream().add(stringRecord); + } + } + + /** + * 获取在线服务所有POD + * + * @param id 服务配置id + * @return List 返回POD列表 + */ + @Override + public List getPods(Long id) { + ServingModelConfig modelConfig = servingModelConfigService.getById(id); + if (modelConfig == null) { + return Collections.emptyList(); + } + //从会话中获取用户信息 + UserDTO user = JwtUtils.getCurrentUserDto(); + assert Objects.nonNull(user); + String nameSpace = k8sUtil.getNamespace(user); + List list = podService.getPods(new PodQueryDTO(nameSpace, k8sUtil.getResourceName(modelConfig.getResourceInfo()))); + list.sort((pod1, pod2) -> { + Integer pod1Index = Integer.parseInt(pod1.getDisplayName().substring(NumberConstant.NUMBER_3)); + Integer pod2Index = Integer.parseInt(pod2.getDisplayName().substring(NumberConstant.NUMBER_3)); + return pod1Index - pod2Index; + }); + return list; + } + + /** + * 在线服务回调 + * + * @param times 回调请求次数 + * @param req 回调请求对象 + * @return boolean 返回是否回调成功 + */ + @Override + public boolean servingCallback(int times, ServingK8sDeploymentCallbackCreateDTO req) { + // 根据namespace和podName找到模型配置 + String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.SERVING, req.getResourceName()); + if (StringUtils.isBlank(resourceInfo)) { + LogUtil.warn(LogEnum.SERVING, "Cannot find modelConfig ID! Request: {}", Thread.currentThread(), times, req.toString()); + return false; + } + String idStr = resourceInfo.substring(NumberConstant.NUMBER_4); + ServingModelConfig servingModelConfig = servingModelConfigService.getById(Long.parseLong(idStr)); + if (Objects.isNull(servingModelConfig)) { + LogUtil.warn(LogEnum.SERVING, "Cannot find modelConfig! Request: {}", Thread.currentThread(), times, req.toString()); + return false; + } + ServingInfo servingInfo = servingInfoMapper.selectById(servingModelConfig.getServingId()); + if (Objects.isNull(servingInfo)) { + LogUtil.warn(LogEnum.SERVING, "Cannot find servingInfo! Request: {}", Thread.currentThread(), times, req.toString()); + return false; + } + //不处理已停止服务的回调 + if (ServingStatusEnum.STOP.getStatus().equals(servingInfo.getStatus())) { + return false; + } + //更新信息 + if (updateByCallback(req, servingModelConfig, servingInfo)) { + return false; + } + //增加发送路由信息 + if (req.getReadyReplicas() > NumberConstant.NUMBER_0 && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { + this.notifyUpdateServingRoute(Collections.singletonList(servingModelConfig.getId()), Collections.emptyList()); + } + //增加删除路由信息 + if (req.getReadyReplicas() == NumberConstant.NUMBER_0 && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { + this.notifyUpdateServingRoute(Collections.emptyList(), Collections.singletonList(servingModelConfig.getId())); + } + return true; + } + + @Transactional(rollbackFor = Exception.class) + public boolean updateByCallback(ServingK8sDeploymentCallbackCreateDTO req, ServingModelConfig servingModelConfig, ServingInfo servingInfo) { + //更新当前模型配置有效节点数 + servingModelConfig.setReadyReplicas(req.getReadyReplicas()); + int result = servingModelConfigMapper.updateById(servingModelConfig); + if (result < NumberConstant.NUMBER_1) { + return true; + } + //更新当前服务有效总节点数 + ServingModelConfig another = servingModelConfigMapper.selectAnother(servingInfo.getId(), servingModelConfig.getId()); + if (Objects.nonNull(another)) { + servingInfo.setRunningNode(req.getReadyReplicas() + another.getReadyReplicas()); + } else { + servingInfo.setRunningNode(req.getReadyReplicas()); + } + //运行中的服务回调运行节点数为0则说明服务运行失败 + if (ServingStatusEnum.WORKING.getStatus().equals(servingInfo.getStatus()) && servingInfo.getRunningNode() == NumberConstant.NUMBER_0) { + servingInfo.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); + } + if (servingInfo.getRunningNode() > NumberConstant.NUMBER_0) { + servingInfo.setStatus(ServingStatusEnum.WORKING.getStatus()); + } + servingInfo.setUpdateTime(DateUtil.getCurrentTimestamp()); + return servingInfoMapper.updateById(servingInfo) < NumberConstant.NUMBER_1; + } + + /** + * 获取在线服务回滚信息列表 + * + * @param servingId 在线服务id + * @return Map> 返回回滚信息列表 + */ + @Override + public Map> getRollbackList(Long servingId) { + List servingModelConfigList = servingModelConfigMapper.getRollbackList(servingId); + Map> map = new HashMap<>(); + if (CollectionUtils.isEmpty(servingModelConfigList)) { + return map; + } + servingModelConfigList.forEach(servingModelConfig -> { + if (!map.containsKey(servingModelConfig.getDeployId())) { + ServingModelConfigVO servingModelConfigVO = new ServingModelConfigVO(); + BeanUtils.copyProperties(servingModelConfig, servingModelConfigVO); + List list = new ArrayList<>(); + list.add(servingModelConfigVO); + map.put(servingModelConfig.getDeployId(), list); + } else { + ServingModelConfigVO servingModelConfigVO = new ServingModelConfigVO(); + BeanUtils.copyProperties(servingModelConfig, servingModelConfigVO); + List list = map.get(servingModelConfig.getDeployId()); + list.add(servingModelConfigVO); + } + }); + return map; + } + +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/DeployServingAsyncTask.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/DeployServingAsyncTask.java new file mode 100644 index 0000000..78ffbc4 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/DeployServingAsyncTask.java @@ -0,0 +1,391 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.task; + +import org.dubhe.constant.NumberConstant; +import org.dubhe.constant.ServingConstant; +import org.dubhe.dao.BatchServingMapper; +import org.dubhe.dao.ServingInfoMapper; +import org.dubhe.domain.dto.UserDTO; +import org.dubhe.domain.entity.BatchServing; +import org.dubhe.domain.entity.ServingInfo; +import org.dubhe.domain.entity.ServingModelConfig; +import org.dubhe.enums.BizEnum; +import org.dubhe.enums.LogEnum; +import org.dubhe.enums.ResourcesPoolTypeEnum; +import org.dubhe.enums.ServingErrorEnum; +import org.dubhe.enums.ServingFrameTypeEnum; +import org.dubhe.enums.ServingStatusEnum; +import org.dubhe.enums.ServingTypeEnum; +import org.dubhe.exception.BaseErrorCode; +import org.dubhe.exception.BusinessException; +import org.dubhe.k8s.api.DistributeTrainApi; +import org.dubhe.k8s.api.ModelServingApi; +import org.dubhe.k8s.api.TrainJobApi; +import org.dubhe.k8s.domain.PtBaseResult; +import org.dubhe.k8s.domain.bo.DistributeTrainBO; +import org.dubhe.k8s.domain.bo.ModelServingBO; +import org.dubhe.k8s.domain.bo.PtJupyterJobBO; +import org.dubhe.k8s.domain.bo.PtMountDirBO; +import org.dubhe.k8s.domain.resource.BizDistributeTrain; +import org.dubhe.k8s.domain.resource.BizIngressRule; +import org.dubhe.k8s.domain.vo.ModelServingVO; +import org.dubhe.k8s.domain.vo.PtJupyterJobVO; +import org.dubhe.service.MailService; +import org.dubhe.service.ServingModelConfigService; +import org.dubhe.utils.GrpcClient; +import org.dubhe.utils.K8sNameTool; +import org.dubhe.utils.K8sUtil; +import org.dubhe.utils.LogUtil; +import org.dubhe.utils.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +/** + * @description 部署服务异步处理 + * @date 2020-09-02 + */ +@Component +public class DeployServingAsyncTask { + + @Resource + private ServingInfoMapper servingInfoMapper; + + @Resource + private BatchServingMapper batchServingMapper; + + @Resource + private ServingModelConfigService servingModelConfigService; + + @Resource + private ModelServingApi modelServingApi; + + @Resource + private K8sUtil k8sUtil; + + @Resource + private K8sNameTool k8sNameTool; + + @Resource + private DistributeTrainApi distributeTrainApi; + + @Resource + private TrainJobApi trainJobApi; + + @Resource + private MailService mailService; + + /** + * Service源码路径 + */ + @Value("${serving.sourcePath}") + private String sourcePath; + + @Value("${serving.gpu-image}") + private String gpuImage; + + @Value("${serving.cpu-image}") + private String cpuImage; + + private final static String TRUE = "True"; + + private final static String FALSE = "False"; + + /** + * 创建pod，部署模型 + * + * @param userDTO 用户信息 + * @param servingInfo 在线服务信息 + * @param modelConfigList 在线服务模型部署信息集合 + */ + @Async("servingExecutor") + @Transactional(rollbackFor = Exception.class) + public void deployServing(UserDTO userDTO, ServingInfo servingInfo, List modelConfigList) { + boolean flag = false; + for (ServingModelConfig servingModelConfig : modelConfigList) { + try { + ModelServingBO bo = buildModelServingBO(userDTO, servingInfo, servingModelConfig); + //创建pod + ModelServingVO modelServingVO = modelServingApi.create(bo); + if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { + // 获取pod对应的url，并修改模型部署状态 + List rules = modelServingVO.getBizIngress().getRules(); + //取第一个url + String url = rules.get(NumberConstant.NUMBER_0).getHost(); + if (StringUtils.isNotBlank(url)) { + servingModelConfig.setUrl(url); + if (servingModelConfigService.updateById(servingModelConfig)) { + flag = true; + } else { + LogUtil.error(LogEnum.SERVING, "User {} failed saving online service model config. Database update FAILED, service id={}", userDTO.getUsername(), servingModelConfig.getId()); + } + } + } + } catch (Exception e) { + LogUtil.error(LogEnum.SERVING, "User {} create serving failed.The name of serving is {}", userDTO.getUsername(), servingInfo.getName(), e); + } + } + + //修改服务状态 + if (!flag) { + servingInfo.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); + } + //grpc协议，创建对应的通道 + if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { + String url = modelConfigList.get(0).getUrl(); + if (StringUtils.isNotBlank(url)) { + GrpcClient.createChannel(servingInfo.getId(), url, userDTO); + } + } + int result = servingInfoMapper.updateById(servingInfo); + if (result < NumberConstant.NUMBER_1) { + LogUtil.error(LogEnum.SERVING, "User {} failed to update the servingInfo table.The name of serving is {}", userDTO.getUsername(), servingInfo.getName()); + } + } + + /** + * 构建模型部署参数 + * + * @param userDTO 用户信息 + * @param servingInfo 在线服务信息 + * @param servingModelConfig 在线服务模型部署信息 + * @return ModelServingBO 返回构建后对象 + */ + private ModelServingBO buildModelServingBO(UserDTO userDTO, ServingInfo servingInfo, ServingModelConfig servingModelConfig) { + ModelServingBO bo = new ModelServingBO(); + //容器端口 + if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { + bo.setGrpcPort(ServingConstant.POD_GRPC_PORT); + } else { + bo.setHttpPort(ServingConstant.POD_HTTP_PORT); + } + //推理脚本名称，不同协议采用不同的推理脚本 + String scriptName = ServingConstant.HTTP_SCRIPT; + if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { + scriptName = ServingConstant.GRPC_SCRIPT; + } + + // 构建参数 + String command = String.format(ServingConstant.SERVING_COMMAND, ServingConstant.TS_SERVING_PATH, scriptName, ServingFrameTypeEnum.getFrameName(servingModelConfig.getFrameType()), + ServingConstant.MODEL_PATH, ResourcesPoolTypeEnum.isGpuCode(servingModelConfig.getResourcesPoolType()) ? TRUE : FALSE) + servingModelConfig.getDeployParam(); + String resourceInfo = StringUtils.getRandomString() + servingModelConfig.getId(); + servingModelConfig.setResourceInfo(resourceInfo); + bo.setNamespace(k8sUtil.getNamespace(userDTO)) + .setResourceName(k8sUtil.getResourceName(resourceInfo)) + .setReplicas(servingModelConfig.getResourcesPoolNode()) + .setGpuNum(servingModelConfig.getGpuNum()) + .setCpuNum(servingModelConfig.getCpuNum()) + .setMemNum(servingModelConfig.getMemNum()) + .setImage(ResourcesPoolTypeEnum.isGpuCode(servingModelConfig.getResourcesPoolType()) ? gpuImage : cpuImage) + .setCmdLines(Arrays.asList("-c", command)); + bo.setNfsMounts(new HashMap(NumberConstant.NUMBER_4) {{ + put(ServingConstant.MODEL_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(servingModelConfig.getModelAddress()))); + put(ServingConstant.TS_SERVING_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(sourcePath))); + }}); + bo.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.SERVING)); + return bo; + } + + /** + * 删除pod + * + * @param userDTO 用户信息 + * @param servingInfo 在线服务信息 + * @param modelConfigList 在线服务模型部署信息集合 + */ + @Async("servingExecutor") + @Transactional(rollbackFor = Exception.class) + public void deleteServing(UserDTO userDTO, ServingInfo servingInfo, List modelConfigList) { + boolean flag = true; + for (ServingModelConfig servingModelConfig : modelConfigList) { + String namespace = k8sUtil.getNamespace(userDTO); + String resourceName = k8sUtil.getResourceName(servingModelConfig.getResourceInfo()); + PtBaseResult ptBaseResult = modelServingApi.delete(namespace, resourceName); + if (!ServingConstant.SUCCESS_CODE.equals(ptBaseResult.getCode())) { + flag = false; + } + } + if (!flag) { + servingInfo.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); + LogUtil.error(LogEnum.SERVING, "An Exception occurred when user {} stopping the service, service name:{}", userDTO.getUsername(), servingInfo.getName()); + } + LogUtil.info(LogEnum.SERVING, "User {} stopped the service with SUCCESS, service name:{}", userDTO.getUsername(), servingInfo.getName()); + //grpc协议关闭对应通道 + if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { + GrpcClient.shutdownChannel(servingInfo.getId(), userDTO); + } + int result = servingInfoMapper.updateById(servingInfo); + if (result < NumberConstant.NUMBER_1) { + LogUtil.error(LogEnum.SERVING, "User {} FAILED stopping the online service. Database update FAILED. Service id={}, service name:{}，service status:{}", userDTO.getUsername(), servingInfo.getId(), servingInfo.getName(), servingInfo.getStatus()); + throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); + } + } + + /** + * 异步部署批量服务 + * + * @param userDTO 用户信息 + * @param batchServing 批量服务信息 + */ + @Async("servingExecutor") + @Transactional(rollbackFor = Exception.class) + public void deployBatchServing(UserDTO userDTO, BatchServing batchServing) { + if (batchServing.getResourcesPoolNode() == NumberConstant.NUMBER_1) { + //单节点 + PtJupyterJobBO ptJupyterJobBO = buildJobBo(userDTO, batchServing); + if (ptJupyterJobBO != null) { + PtJupyterJobVO vo = trainJobApi.create(ptJupyterJobBO); + if (vo.isSuccess()) { + LogUtil.info(LogEnum.SERVING, "User {} deployed batching service with SUCCESS. Service name：{}", userDTO.getUsername(), batchServing.getName()); + batchServingMapper.updateById(batchServing); + return; + } + } + + } else { + //多节点分布式 + DistributeTrainBO distributeTrainBO = buildDistributeTrainBO(userDTO, batchServing); + if (distributeTrainBO != null) { + BizDistributeTrain distribute = distributeTrainApi.create(distributeTrainBO); + if (distribute.isSuccess()) { + LogUtil.info(LogEnum.SERVING, "User {} deployed batching service with SUCCESS. Service name：{}", userDTO.getUsername(), batchServing.getName()); + batchServingMapper.updateById(batchServing); + return; + } + } + } + batchServing.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); + batchServingMapper.updateById(batchServing); + LogUtil.error(LogEnum.SERVING, "User {} FAILED deployed batching service. Service name：{}", userDTO.getUsername(), batchServing.getName()); + } + + /** + * 异步删除批量服务 + * + * @param userDTO 用户信息 + * @param batchServing 批量服务信息 + */ + @Async("servingExecutor") + @Transactional(rollbackFor = Exception.class) + public void deleteBatchServing(UserDTO userDTO, BatchServing batchServing) { + String namespace = k8sUtil.getNamespace(userDTO); + String resourceName = k8sUtil.getBatchResourceName(batchServing.getResourceInfo()); + if (batchServing.getResourcesPoolNode() == NumberConstant.NUMBER_1) { + if (trainJobApi.delete(namespace, resourceName)) { + return; + } + } else { + PtBaseResult result = distributeTrainApi.deleteByResourceName(namespace, resourceName); + if (ServingConstant.SUCCESS_CODE.equals(result.getCode())) { + return; + } + } + batchServing.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); + batchServingMapper.updateById(batchServing); + LogUtil.error(LogEnum.SERVING, "User {} FAILED deployed batching service. Service name：{}", userDTO.getUsername(), batchServing.getName()); + } + + /** + * 构建单节点推理参数 + * + * @param userDTO 用户信息 + * @param batchServing 批量服务信息 + * @return PtJupyterJobBO 返回构建后对象 + */ + public PtJupyterJobBO buildJobBo(UserDTO userDTO, BatchServing batchServing) { + String command = String.format(ServingConstant.BATCH_COMMAND, ServingConstant.TS_SERVING_PATH, ServingFrameTypeEnum.getFrameName(batchServing.getFrameType()), + ServingConstant.MODEL_PATH, ServingConstant.INPUT_PATH, ServingConstant.OUTPUT_PATH, FALSE, ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType()) ? TRUE : FALSE) + batchServing.getDeployParam(); + String resourceInfo = StringUtils.getRandomString() + batchServing.getId(); + batchServing.setResourceInfo(resourceInfo); + PtJupyterJobBO bo = new PtJupyterJobBO() + .setNamespace(k8sUtil.getNamespace(userDTO)) + .setName(k8sUtil.getBatchResourceName(resourceInfo)) + .setCpuNum(batchServing.getCpuNum()) + .setGpuNum(batchServing.getGpuNum()) + .setUseGpu(ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType())) + .setMemNum(batchServing.getMemNum()) + .setCmdLines(Arrays.asList("-c", command)) + .setNfsMounts(new HashMap(NumberConstant.NUMBER_6) {{ + put(ServingConstant.MODEL_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getModelAddress()))); + put(ServingConstant.INPUT_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getInputPath()))); + put(ServingConstant.OUTPUT_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getOutputPath()))); + put(ServingConstant.TS_SERVING_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(sourcePath))); + }}) + .setImage(ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType()) ? gpuImage : cpuImage) + .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.BATCH_SERVING)); + return bo; + } + + /** + * 构建分布式推理参数 + * + * @param userDTO 用户信息 + * @param batchServing 批量服务信息 + * @return DistributeTrainBO 返回构建后对象 + */ + public DistributeTrainBO buildDistributeTrainBO(UserDTO userDTO, BatchServing batchServing) { + String command = String.format(ServingConstant.BATCH_COMMAND, ServingConstant.TS_SERVING_PATH, ServingFrameTypeEnum.getFrameName(batchServing.getFrameType()), + ServingConstant.MODEL_PATH, ServingConstant.INPUT_PATH, ServingConstant.OUTPUT_PATH, TRUE, ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType()) ? TRUE : FALSE) + batchServing.getDeployParam(); + String resourceInfo = StringUtils.getRandomString() + batchServing.getId(); + batchServing.setResourceInfo(resourceInfo); + DistributeTrainBO bo = new DistributeTrainBO() + .setName(k8sUtil.getBatchResourceName(resourceInfo)) + .setNamespace(k8sUtil.getNamespace(userDTO)) + .setSize(batchServing.getResourcesPoolNode()) + .setImage(ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType()) ? gpuImage : cpuImage) + .setMemNum(batchServing.getMemNum()) + .setCpuNum(batchServing.getCpuNum()) + .setGpuNum(batchServing.getGpuNum()) + .setMasterCmd(command) + .setSlaveCmd(command) + .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.BATCH_SERVING)); + bo.setNfsMounts(new HashMap(NumberConstant.NUMBER_6) {{ + put(ServingConstant.MODEL_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getModelAddress()))); + put(ServingConstant.INPUT_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getInputPath()))); + put(ServingConstant.OUTPUT_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getOutputPath()))); + put(ServingConstant.TS_SERVING_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(sourcePath))); + }}); + return bo; + } + + /** + * 批量服务推理成功异步发送邮件通知 + * + * @param receiverMailAddress 邮件地址 + * @param id 批量服务ID + */ + @Async("servingExecutor") + public void asyncSendServingMail(String receiverMailAddress, Long id) { + try { + final StringBuffer sb = new StringBuffer(); + sb.append("

" + "亲爱的").append(receiverMailAddress).append("您好！

"); + sb.append("

ID为：" + id + "的批量服务推理成功，请及时查看！

"); + mailService.sendHtmlMail(receiverMailAddress, "推理成功通知", sb.toString()); + } catch (Exception e) { + LogUtil.error(LogEnum.SERVING, "UserServiceImpl sendMail error , param:{} error:{}", receiverMailAddress, e); + throw new BusinessException(BaseErrorCode.ERROR_SYSTEM.getCode(), BaseErrorCode.ERROR_SYSTEM.getMsg()); + } + } +} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java index 8f1c483..4d90455 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java index 38913ba..6b14dd4 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java @@ -1,12 +1,12 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. - * + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -224,7 +224,7 @@ public class NotebookUtil { /** * 获取超时时间秒 * - * @return Long + * @return Long */ public static Long getTimeoutSecondLong() { return K8sCallBackTool.getTimeoutSecondLong(NOTEBOOK_TIMEOUT_SECOND); diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java index 8182789..20dc4c7 100644 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java +++ b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/main/resources/config/application.yml b/dubhe-server/dubhe-admin/src/main/resources/config/application.yml index 94630c0..b69f95a 100644 --- a/dubhe-server/dubhe-admin/src/main/resources/config/application.yml +++ b/dubhe-server/dubhe-admin/src/main/resources/config/application.yml @@ -79,6 +79,12 @@ train-job: docker-log-path: "train_log=${train-job.docker-train-path}/${train-job.log-path}" docker-visualized-log-path: "train_visualized_log=${train-job.docker-train-path}/${train-job.visualized-log-path}" docker-dataset: "data_url=${train-job.docker-dataset-path}" + docker-teacher-model-key: "teacher_path_list" + docker-teacher-model-path: "/model/teacher" + docker-student-model-key: "student_path_list" + docker-student-model-path: "/model/student" + atlas-anaconda: "cd /root/anaconda3/bin && source activate pytorch_1_3" + atlas-pythonioencoding: "PYTHONIOENCODING=utf-8 " eight: "8" plus-eight: "+8" node-ips: "node_ips" @@ -111,7 +117,7 @@ train-algorithm: fork: false -# 配置slq打印日志 +# 配置sql打印日志 logging: level: com: @@ -138,6 +144,15 @@ delay: #模型开发延时关闭时间 delete: 240 +# 模型优化容器镜像配置 +optimize: + image: harbor.dubhe.ai/optimize/oneflow-gpu:base + +# # serving模块容器镜像配置 +serving: + gpu-image: harbor.dubhe.ai/serving/serving-gpu:base + cpu-image: harbor.dubhe.ai/serving/serving-cpu:base + #垃圾回收默认配置 recycle: #过期时间设置 diff --git a/dubhe-server/dubhe-admin/src/main/resources/mapper/ModelOptBuiltInMapper.xml b/dubhe-server/dubhe-admin/src/main/resources/mapper/ModelOptBuiltInMapper.xml new file mode 100644 index 0000000..8569f4d --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/resources/mapper/ModelOptBuiltInMapper.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + diff --git a/dubhe-server/dubhe-admin/src/main/resources/server.crt b/dubhe-server/dubhe-admin/src/main/resources/server.crt new file mode 100644 index 0000000..d36060c --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/resources/server.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkTCCAnmgAwIBAgIJALnQkyWsbGjVMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV +BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg +Q29tcGFueSBMdGQxGzAZBgNVBAMMEiouc2VydmluZy5kdWJoZS5haTAeFw0yMDEw +MTYwMzAwMzVaFw0zMDEwMTQwMzAwMzVaMF8xCzAJBgNVBAYTAlhYMRUwEwYDVQQH +DAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxGzAZ +BgNVBAMMEiouc2VydmluZy5kdWJoZS5haTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAOjZBySFMRPMjAgdDffVUrU0ZeZNlpZo97l4Malv5iyjbx5ULNm7 +WZT8PXbBupUw32Y0JA8Sea276Zzs6eluMmx98v7R7b5h0zHIO3j/ObRiH6wOwUsN +Xb5B6vTjcrX02rXC+jBO4wh2URJRafAtnPK7f0QsS0a+n6g3LpG1bT1MUIEr74iB +AuwrmVw2BHGy3qudUC9X4FXJoRcPhEFIoAM2A6Fj20R8euYIXZYEgG73TzxxhEPy +TY5kRqVIHhLSd16RUkrdVuY7TQR2ZjT4ZS1ZQkNgYekSF8VgB2AzeymdyPks3OBR +xfpyMcvsX+tVCYnbJ47rDWkk12mrIYFI0TsCAwEAAaNQME4wHQYDVR0OBBYEFDMT +UzFb/OMzwkAwfGVieAo6hapXMB8GA1UdIwQYMBaAFDMTUzFb/OMzwkAwfGVieAo6 +hapXMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAm8L2WkH3m5PkG2 +nrPQj1UPWfsGbpuUVnRO4qYr+IlUXK683a0+vl++iLkqlIL2MgkbDhFBwAUFVI3P +Bw7jtq9/kKWYb6ItQXWC7LmLYk0onEQACbQ41E6DbzJD0225L4BQFDiF+8OZUB3v +NtCrbE1u+5D2+X4ig0SW/FZhoPW87g28Mw+4Feyx29EOwzUaGz9ZObd+Vnw8ff5B +rUJFBSSXV0A7Iyd/QZt/GAQrcCwi8VDIbFXHm/ekyZcHaISwMQadNMMTKDCH+BNH +Wcr5py6d/NPYZJa1GqI1EUIMqpea8oxl9CahwbpIgO76AVXEditIGqRct/dIeq8M +Hv3qcSs= +-----END CERTIFICATE----- diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java index 150d571..6f756eb 100755 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java index 2cfb705..db56668 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java index d88d4c7..e95e64c 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java @@ -1,4 +1,4 @@ -/** Copyright 2020 Zhejiang Lab. All Rights Reserved. +/** Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +16,6 @@ package org.dubhe; -import com.alibaba.fastjson.JSON; -import org.dubhe.domain.dto.PtTrainAlgorithmQueryDTO; import org.junit.Test; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.result.MockMvcResultMatchers; @@ -30,8 +28,7 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers; public class ForwardTest extends BaseTest { @Test public void ForwardControllerTest() throws Exception { - PtTrainAlgorithmQueryDTO ptTrainAlgorithmQueryDTO = new PtTrainAlgorithmQueryDTO(); - ptTrainAlgorithmQueryDTO.setAlgorithmSource(1).setCurrent(1).setSize(10).setSort("id").setOrder("asc"); - mockMvcTest(MockMvcRequestBuilders.get("/forward/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmQueryDTO), MockMvcResultMatchers.status().isOk(), 200); + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithm").param("algorithmSource", "2")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); } } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java index eef1b5c..b4051af 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ import org.springframework.test.context.junit4.SpringRunner; * @description K8sCallBackToolTest测试类 * @date 2020-05-28 */ -@ActiveProfiles(value = "notebook") +@ActiveProfiles(value = "dev") @RunWith(SpringRunner.class) @SpringBootTest(classes= AppRun.class,webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) public class K8sCallBackToolTest{ @@ -51,7 +51,7 @@ public class K8sCallBackToolTest{ @Test public void getPodCallbackUrl(){ - Assert.assertEquals("http://xxx.xxx.xxx.xxx:xxxx/api/k8s/callback/pod/notebook",k8sCallBackTool.getPodCallbackUrl(k8sNameTool.getPodLabel(BizEnum.NOTEBOOK))); + Assert.assertEquals("localhost:8000/api/k8s/callback/pod/notebook",k8sCallBackTool.getPodCallbackUrl(k8sNameTool.getPodLabel(BizEnum.NOTEBOOK))); } } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java index f38d6a1..5c181f7 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ import org.springframework.test.context.junit4.SpringRunner; * @description K8sNameToolTest测试类 * @date 2020-05-13 */ -@ActiveProfiles(value = "notebook") +@ActiveProfiles(value = "dev") @RunWith(SpringRunner.class) @SpringBootTest public class K8sNameToolTest { diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java index 09773a7..98ae925 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtMeasureTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtMeasureTest.java new file mode 100644 index 0000000..001fcb8 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtMeasureTest.java @@ -0,0 +1,107 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe; + +import com.alibaba.fastjson.JSON; +import org.dubhe.domain.dto.PtMeasureDTO; +import org.dubhe.domain.dto.PtMeasureDeleteDTO; +import org.dubhe.domain.dto.PtMeasureUpdateDTO; +import org.junit.Test; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * @description 度量管理接口单元测试 + * @date 2020-11-16 + */ +public class PtMeasureTest extends BaseTest { + + /** + * 查询度量 + */ + @Test + public void getMeasureTest() throws Exception { + MultiValueMap params = new LinkedMultiValueMap<>(); + params.add("current", "1"); + params.add("size", "10"); + params.add("sort", "id"); + params.add("order", "desc"); + params.add("name", "COCO1"); + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/ptMeasure/info").params(params)) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } + + + /** + * 新建度量 + */ + @Test + public void addMeasureTest() throws Exception { + PtMeasureDTO ptMeasureCreateDTO = PtMeasureDTO.builder() + .name("测试111701") + .description("测试新建度量") + .url("/nfs/dubhe-dev/upload-temp/1/5b07d66ad27a11eab7cffa3a5ae24f00.json") + .build(); + + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/ptMeasure"), JSON.toJSONString(ptMeasureCreateDTO), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + } + + /** + * 修改度量信息 + */ + @Test + public void updateMeasureTest() throws Exception { + PtMeasureUpdateDTO ptMeasureUpdateDTO = PtMeasureUpdateDTO.builder() + .id(1L) + .name("COCO1") + .description("测试修改度量") + .url("E:/chromeDownload/5b07d66ad27a11eab7cffa3a5ae24f00.json") + .build(); + + mockMvcTest(MockMvcRequestBuilders.put("/api/v1/ptMeasure"), JSON.toJSONString(ptMeasureUpdateDTO), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + } + + /** + * 删除度量 + */ + @Test + public void deleteMeasureTest() throws Exception { + Set ids = Stream.of(1L, 2L).collect(Collectors.toSet()); + PtMeasureDeleteDTO ptMeasureDeleteDTO = PtMeasureDeleteDTO.builder().ids(ids).build(); + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/ptMeasure"), JSON.toJSONString(ptMeasureDeleteDTO), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + } + + /** + * 根据度量名称返回度量文件信息 + */ + @Test + public void getMeasureByNameTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/ptMeasure/byName").param("name", "COCO")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); + } +} + diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java index 5c6c230..bb550a4 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ package org.dubhe; import com.alibaba.fastjson.JSON; import org.dubhe.domain.dto.PtTrainAlgorithmUsageCreateDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmUsageDeleteDTO; import org.dubhe.domain.dto.PtTrainAlgorithmUsageUpdateDTO; import org.junit.Test; import org.junit.runner.RunWith; @@ -57,7 +58,7 @@ public class PtTrainAlgorithmUsageTest extends BaseTest { public void createTest() throws Exception { PtTrainAlgorithmUsageCreateDTO ptTrainAlgorithmUsageCreateDTO = new PtTrainAlgorithmUsageCreateDTO(); - ptTrainAlgorithmUsageCreateDTO.setAuxInfo("测试"); + ptTrainAlgorithmUsageCreateDTO.setAuxInfo("untilTesting"); mockMvcTest(MockMvcRequestBuilders.post("/api/v1/algorithmUsage"), JSON.toJSONString(ptTrainAlgorithmUsageCreateDTO), MockMvcResultMatchers.status().is2xxSuccessful(), @@ -69,8 +70,10 @@ public class PtTrainAlgorithmUsageTest extends BaseTest { @Test public void deleteTest() throws Exception { Long[] longs = new Long[1]; - longs[0] = 13L; - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/algorithmUsage"), JSON.toJSONString(longs), + longs[0] = 38L; + PtTrainAlgorithmUsageDeleteDTO ptTrainAlgorithmUsageDeleteDTO = new PtTrainAlgorithmUsageDeleteDTO(); + ptTrainAlgorithmUsageDeleteDTO.setIds(longs); + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/algorithmUsage"), JSON.toJSONString(ptTrainAlgorithmUsageDeleteDTO), MockMvcResultMatchers.status().is2xxSuccessful(), 200); } @@ -79,7 +82,7 @@ public class PtTrainAlgorithmUsageTest extends BaseTest { public void updateTest() throws Exception { PtTrainAlgorithmUsageUpdateDTO ptTrainAlgorithmUsageUpdateDTO = new PtTrainAlgorithmUsageUpdateDTO(); - ptTrainAlgorithmUsageUpdateDTO.setId(12L); + ptTrainAlgorithmUsageUpdateDTO.setId(38L); ptTrainAlgorithmUsageUpdateDTO.setAuxInfo("更新测试"); mockMvcTest(MockMvcRequestBuilders.put("/api/v1/algorithmUsage"), diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java index 74634d3..568ddde 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java index c55b0a2..0d42599 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,51 +40,37 @@ import org.springframework.transaction.annotation.Transactional; @SpringBootTest public class PtTrainModelOptJobApiTest extends BaseTest { - /** - * 修改任务参数 算法id=2在算法表中runcommand为空 + * 作业列表展示 * + * @throws Exception */ - @Test - @Transactional(rollbackFor = Exception.class) - public void createTrainJobTest() throws Exception { - PtTrainJobCreateDTO ptTrainJobCreateDTO = new PtTrainJobCreateDTO(); - ptTrainJobCreateDTO.setAlgorithmId(18L); - ptTrainJobCreateDTO.setDataSourceName("dataset/68"); - ptTrainJobCreateDTO.setDataSourcePath("dataset/68"); - ptTrainJobCreateDTO.setDescription("job描述"); - ptTrainJobCreateDTO.setTrainJobSpecsName("11111111111111").setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); - JSONObject runParams = new JSONObject(); - runParams.put("key1", 33); - runParams.put("key2", 33); - runParams.put("key3", 33); - runParams.put("key4", 33); - ptTrainJobCreateDTO.setRunParams(runParams); - ptTrainJobCreateDTO.setSaveParams(true); - ptTrainJobCreateDTO.setTrainName("trainjobtest"); - ptTrainJobCreateDTO.setTrainParamDesc("job描述"); - ptTrainJobCreateDTO.setTrainParamName("paramname5"); - - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobCreateDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } - @Test public void getTrainJobTest() throws Exception { mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob")) .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); } + /** + * 规格展示 + * + * @throws Exception + */ @Test public void getTrainJobSpecsTest() throws Exception { mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/trainJobSpecs").param("resourcesPoolType", "0")) .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); } + /** + * 根据jobId查询训练任务详情 + * + * @throws Exception + */ @Test public void getTrainJobDetailTest() throws Exception { mockMvcWithNoRequestBody( - mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/jobDetail").param("id", "20")) + mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/jobDetail").param("id", "538")) .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); } @@ -101,6 +87,8 @@ public class PtTrainModelOptJobApiTest extends BaseTest { } /** + * 作业不同版本任务列表展示 + * * @param @throws Exception 入参 * @return void 返回类型 * @throws @date 2020年6月16日 上午10:19:12 @@ -109,12 +97,14 @@ public class PtTrainModelOptJobApiTest extends BaseTest { */ @Test public void getTrainJobVersionTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/10")) + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/trainJobVersionDetail").param("trainId", String.valueOf(371L))) .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); } /** + * 数据集状态展示 + * * @param @throws Exception 入参 * @return void 返回类型 * @throws @date 2020年6月16日 上午10:31:25 @@ -123,13 +113,38 @@ public class PtTrainModelOptJobApiTest extends BaseTest { */ @Test public void getTrainDataSourceStatusTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc - .perform(MockMvcRequestBuilders.get("/api/v1/trainJob/dataSourceStatus").param("dataSourcePath", - "dataset/68,dataset/20741/versionFile/V0003")) + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/dataSourceStatus").param("dataSourcePath", + "dataset/68,dataset/20741/versionFile/V0003")) .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); } + /** + * 创建训练任务 + * + */ + @Test + @Transactional(rollbackFor = Exception.class) + public void createTrainJobTest() throws Exception { + PtTrainJobCreateDTO ptTrainJobCreateDTO = new PtTrainJobCreateDTO(); + ptTrainJobCreateDTO.setTrainName("创建训练任务单元测试"); + ptTrainJobCreateDTO.setAlgorithmId(134L); + ptTrainJobCreateDTO.setDataSourceName("CUB-200-2011:V0002"); + ptTrainJobCreateDTO.setDataSourcePath("dataset/242/versionFile/V0002/ofrecord/train"); + ptTrainJobCreateDTO.setDescription("创建训练任务单元测试"); + ptTrainJobCreateDTO.setResourcesPoolType(1).setResourcesPoolNode(1).setTrainType(0); + ptTrainJobCreateDTO.setTrainJobSpecsName("1Core4GB 1TITAN V").setTrainJobSpecsInfo("1Core4GB 1TITAN V").setRunCommand("python atlas_knowledge_distillation.py").setImageName("atlas").setImageTag("2.1"); + JSONObject runParams = new JSONObject(); + runParams.put("epochs", 100); + runParams.put("batch_size", 16); + runParams.put("weight_decay", 1e-4); + runParams.put("learning_rate", 1e-4); + ptTrainJobCreateDTO.setRunParams(runParams); + + mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobCreateDTO), + MockMvcResultMatchers.status().is2xxSuccessful(), 200); + } + /** * 更新训练任务测试 * @@ -143,11 +158,19 @@ public class PtTrainModelOptJobApiTest extends BaseTest { public void updateTrainJobTest() throws Exception { // TODO Auto-generated method stub PtTrainJobUpdateDTO ptTrainJobUpdateDTO = new PtTrainJobUpdateDTO(); - ptTrainJobUpdateDTO.setId(39L); - ptTrainJobUpdateDTO.setAlgorithmId(91L); - ptTrainJobUpdateDTO.setDataSourceName("dataset/68"); - ptTrainJobUpdateDTO.setDataSourcePath("dataset/68"); - ptTrainJobUpdateDTO.setTrainJobSpecsName("").setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + ptTrainJobUpdateDTO.setId(537L); + ptTrainJobUpdateDTO.setAlgorithmId(134L); + ptTrainJobUpdateDTO.setDataSourceName("CUB-200-2011:V0002"); + ptTrainJobUpdateDTO.setDataSourcePath("dataset/242/versionFile/V0002/ofrecord/train"); + ptTrainJobUpdateDTO.setDescription("修改训练任务单元测试"); + ptTrainJobUpdateDTO.setResourcesPoolType(1).setResourcesPoolNode(1).setTrainType(0); + ptTrainJobUpdateDTO.setTrainJobSpecsName("1Core4GB 1TITAN V").setTrainJobSpecsInfo("1Core4GB 1TITAN V").setRunCommand("python atlas_knowledge_distillation.py").setImageName("atlas").setImageTag("2.1"); + JSONObject runParams = new JSONObject(); + runParams.put("epochs", 100); + runParams.put("batch_size", 16); + runParams.put("weight_decay", 1e-4); + runParams.put("learning_rate", 1e-4); + ptTrainJobUpdateDTO.setRunParams(runParams); mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobUpdateDTO), MockMvcResultMatchers.status().is2xxSuccessful(), 200); @@ -167,13 +190,18 @@ public class PtTrainModelOptJobApiTest extends BaseTest { public void deleteTrainJobTest() throws Exception { // TODO Auto-generated method stub PtTrainJobDeleteDTO ptTrainJobDeleteDTO = new PtTrainJobDeleteDTO(); - ptTrainJobDeleteDTO.setId(38L); - ptTrainJobDeleteDTO.setTrainId(36L); + ptTrainJobDeleteDTO.setId(118L); + ptTrainJobDeleteDTO.setTrainId(87L); mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobDeleteDTO), MockMvcResultMatchers.status().is2xxSuccessful(), 200); } + /** + * + * + * @throws Exception + */ @Test public void deleteTrainJobWithNoIdTest() throws Exception { // TODO Auto-generated method stub diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java index 353bae5..560ef35 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package org.dubhe; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.dubhe.domain.dto.PtTrainAlgorithmCreateDTO; +import org.dubhe.domain.dto.PtTrainAlgorithmDeleteDTO; import org.dubhe.domain.dto.PtTrainAlgorithmUpdateDTO; import org.junit.Assert; import org.junit.Test; @@ -27,6 +28,9 @@ import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import java.util.HashSet; +import java.util.Set; + /** * @description 算法管理模块算法管理单元测试 * @date 2020-06-18 @@ -63,14 +67,6 @@ public class TrainAlgorithmTest extends BaseTest { } - /** - * 模型管理查询对应路径的算法 - */ - @Test - public void getTrainAlgorithmFromPathTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithm/fromPath").param("codeDir", "/algorithm-manage/common/resnet50v1.5/")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } /** * 新增算法 @@ -79,13 +75,9 @@ public class TrainAlgorithmTest extends BaseTest { public void ptTrainAlgorithmCreateTest() throws Exception { PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO = new PtTrainAlgorithmCreateDTO(); JSONObject jsonObject = new JSONObject(); - ptTrainAlgorithmCreateDTO.setAlgorithmName("test") - .setDescription("test") - .setImageName("tensorflow/tensorflow:latest") - .setCodeDir("algorithm-manage/18/20200617073954700eqhl/") - .setRunParams(jsonObject) - .setAlgorithmUsage("test") - .setRunCommand("test"); + ptTrainAlgorithmCreateDTO.setAlgorithmName("untilTesting") + .setDescription("untilTesting") + .setCodeDir("upload-temp/1/20201202135732212Bp8F/OneFlow算法.zip"); mockMvcTest(MockMvcRequestBuilders.post("/api/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmCreateDTO), MockMvcResultMatchers.status().isOk(), 200); } @@ -95,9 +87,9 @@ public class TrainAlgorithmTest extends BaseTest { @Test public void ptTrainAlgorithmUpdateTest() throws Exception { PtTrainAlgorithmUpdateDTO ptTrainAlgorithmUpdateDTO = new PtTrainAlgorithmUpdateDTO(); - ptTrainAlgorithmUpdateDTO.setId(1L) - .setAlgorithmName("test") - .setDescription("test"); + ptTrainAlgorithmUpdateDTO.setId(138L) + .setAlgorithmName("untilTesting"+ System.currentTimeMillis()) + .setDescription("untilTesting"); mockMvcTest(MockMvcRequestBuilders.put("/api/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmUpdateDTO), MockMvcResultMatchers.status().isOk(), 200); } @@ -107,8 +99,11 @@ public class TrainAlgorithmTest extends BaseTest { */ @Test public void ptTrainAlgorithmDeleteTest() throws Exception { - Long[] ids = {(long) 1, (long) 2}; - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/algorithm"), JSON.toJSONString(ids), MockMvcResultMatchers.status().isOk(), 200); + Set ids = new HashSet<>(); + ids.add(138L); + PtTrainAlgorithmDeleteDTO ptTrainAlgorithmDeleteDTO = new PtTrainAlgorithmDeleteDTO(); + ptTrainAlgorithmDeleteDTO.setIds(ids); + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmDeleteDTO), MockMvcResultMatchers.status().isOk(), 200); } } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainModelOptJobApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainModelOptJobApiTest.java deleted file mode 100644 index 0cd2c20..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainModelOptJobApiTest.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import org.dubhe.domain.dto.PtTrainJobCreateDTO; -import org.dubhe.domain.dto.PtTrainJobStopDTO; -import org.dubhe.domain.dto.PtTrainQueryDTO; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.http.MediaType; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; - -/** - * @description - * @date 2020-04-17 - */ -public class TrainModelOptJobApiTest extends BaseTest { - - /** - * 停止任务测试 - * - * @throws Exception - */ - @Test - public void testStopTrainJob() throws Exception { - - PtTrainJobStopDTO dto = new PtTrainJobStopDTO(); - dto.setId(1L); - dto.setTrainId(10L); - MockHttpServletResponse response = this.mockMvc.perform( - MockMvcRequestBuilders. - post("/api/train_job/1") - .contentType(MediaType.APPLICATION_JSON).content(JSON.toJSONString(dto) - )) - .andExpect(MockMvcResultMatchers.status().isOk()) - .andReturn().getResponse(); - JSONObject jsonObject = JSON.parseObject(response.getContentAsString()); - Assert.assertNotNull(jsonObject.get("code")); - } - - - /** - * 我的训练任务统计 - * - * @throws Exception - */ - @Test - public void statisticsMine() throws Exception { - - MockHttpServletResponse response = this.mockMvc.perform( - MockMvcRequestBuilders. - get("/api/train_job/mine") - ) - .andExpect(MockMvcResultMatchers.status().isOk()) - .andReturn().getResponse(); - JSONObject jsonObject = JSON.parseObject(response.getContentAsString()); - Assert.assertNotNull(jsonObject.get("code")); - } - - /** - * 分页查询训练任务 - */ - @Test - public void testGetTrainJob() throws Exception { - PtTrainQueryDTO dto = new PtTrainQueryDTO(); - MockHttpServletResponse response = this.mockMvc.perform( - MockMvcRequestBuilders. - get("/api/v1/trainJob") - .contentType(MediaType.APPLICATION_JSON) - .content(JSON.toJSONString(dto)) - ) - .andExpect(MockMvcResultMatchers.status().isOk()) - .andReturn().getResponse(); - JSONObject jsonObject = JSON.parseObject(response.getContentAsString()); - Assert.assertNotNull(jsonObject.get("code")); - } - - /** - * 分页查询训练任务 - */ - @Test - public void testCreateTrainJob() throws Exception { - PtTrainJobCreateDTO dto = new PtTrainJobCreateDTO(); - dto.setAlgorithmId(3L); - dto.setRunParams(new JSONObject()); - dto.setDataSourceName("dataset/68"); - dto.setDataSourcePath("dataset/68"); - dto.setTrainName("test-train"+System.currentTimeMillis()); - dto.setTrainParamDesc("test-train"); - dto.setTrainJobSpecsName("11111").setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); - - MockHttpServletResponse response = this.mockMvc.perform( - MockMvcRequestBuilders. - post("/api/v1/trainJob") - .contentType(MediaType.APPLICATION_JSON) - .content(JSON.toJSONString(dto)) - ) - .andExpect(MockMvcResultMatchers.status().isOk()) - .andReturn().getResponse(); - JSONObject jsonObject = JSON.parseObject(response.getContentAsString()); - Assert.assertNotNull(jsonObject.get("code")); - } -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java index 9f69df5..3c368dd 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package org.dubhe; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import org.dubhe.domain.dto.PtTrainParamCreateDTO; +import org.dubhe.domain.dto.PtTrainParamDeleteDTO; import org.dubhe.domain.dto.PtTrainParamUpdateDTO; import org.junit.Test; import org.junit.runner.RunWith; @@ -30,6 +31,9 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.result.MockMvcResultMatchers; import org.springframework.transaction.annotation.Transactional; +import java.util.HashSet; +import java.util.Set; + /** * @description 训练任务管理模块任务参数管理单元测试 * @date 2020-5-11 @@ -47,14 +51,6 @@ public class TrainParamApiTest extends BaseTest { .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); } - /** - * 查询任务参数列表 - */ - @Test - public void ptTrainParamQueryTest1() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainParams").param("paramName", "lpf").param("resourcesPoolType", "0")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } /** * 新增任务参数 @@ -66,56 +62,13 @@ public class TrainParamApiTest extends BaseTest { PtTrainParamCreateDTO ptTrainParamCreateDTO = new PtTrainParamCreateDTO(); JSONObject runParams = new JSONObject(); runParams.put("data_url", "/nfs/testuser1/mnist/MNIST_data"); - ptTrainParamCreateDTO.setParamName("新增任务参数名称").setAlgorithmId((long) 1).setRunParams(runParams) - .setDescription("描述信息").setDataSourcePath("1") - .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); + ptTrainParamCreateDTO.setParamName("新增任务参数名称单元测试").setAlgorithmId(133L).setRunParams(runParams) + .setDescription("新增任务参数名称").setDataSourceName("T1:V0004").setDataSourcePath("dataset/159/versionFile/V0004/ofrecord/train") + .setResourcesPoolType(1).setTrainType(0).setResourcesPoolNode(1).setRunCommand("python p.py").setTrainJobSpecsName("1Core4GB 1TITAN V").setImageName("oneflow").setImageTag("cu102-py37-dist"); mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamCreateDTO), MockMvcResultMatchers.status().isOk(), 200); } - /** - * 新增任务参数 - */ - @Test - @Transactional(rollbackFor = Exception.class) - public void ptTrainParamCreateTest1() throws Exception { - PtTrainParamCreateDTO ptTrainParamCreateDTO = new PtTrainParamCreateDTO(); - JSONObject runParams = new JSONObject(); - runParams.put("key1", 2); - runParams.put("key2", 2); - runParams.put("key3", 2); - runParams.put("key4", 2); - ptTrainParamCreateDTO.setParamName("新增任务参数名称") - .setAlgorithmId((long) 1) - .setRunParams(runParams) - .setDescription("描述信息") - .setDataSourcePath("/usr/local/data/out.json") - .setDataSourceName("out.json") - .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamCreateDTO), MockMvcResultMatchers.status().is4xxClientError(), 400); - - } - - /** - * 新增任务参数 - */ - @Test - @Transactional(rollbackFor = Exception.class) - public void ptTrainParamCreateTest2() throws Exception { - PtTrainParamCreateDTO ptTrainParamCreateDTO = new PtTrainParamCreateDTO(); - JSONObject runParams = new JSONObject(); - runParams.put("key1", 3); - runParams.put("key2", 3); - runParams.put("key3", 3); - runParams.put("key4", 3); - ptTrainParamCreateDTO.setParamName("新增任务参数名称" + System.currentTimeMillis()).setAlgorithmId(Long.MAX_VALUE).setRunParams(runParams) - .setDescription("描述信息") - .setDataSourcePath("/usr/local/data/out.json") - .setDataSourceName("out.json") - .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamCreateDTO), MockMvcResultMatchers.status().is4xxClientError(), 400); - - } /** * 修改任务参数 @@ -130,52 +83,12 @@ public class TrainParamApiTest extends BaseTest { runParams.put("key2", 11); runParams.put("key3", 11); runParams.put("key4", 11); - ptTrainParamUpdateDTO.setId((long) 1).setParamName("修改任务参数名称" + System.currentTimeMillis()).setAlgorithmId((long) 1) - .setRunParams(runParams) - .setDescription("描述信息") - .setDataSourcePath("/usr/local/data/out.json") - .setDataSourceName("out.json") - .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("harbor.dubhe.ai/tensorflow/tensorflow:latest"); + ptTrainParamUpdateDTO.setId(125L).setParamName("修改任务参数名称单元测试" + System.currentTimeMillis()).setAlgorithmId(133L).setRunParams(runParams) + .setDescription("修改任务参数名称单元测试").setDataSourceName("T1:V0004").setDataSourcePath("dataset/159/versionFile/V0004/ofrecord/train") + .setResourcesPoolType(1).setTrainType(0).setResourcesPoolNode(1).setRunCommand("python p.py").setTrainJobSpecsName("1Core4GB 1TITAN V").setImageName("oneflow").setImageTag("cu102-py37-dist"); mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamUpdateDTO), MockMvcResultMatchers.status().isOk(), 200); } - /** - * 修改任务参数 - */ - @Test - @Transactional(rollbackFor = Exception.class) - public void ptTrainParamUpdateTest2() throws Exception { - PtTrainParamUpdateDTO ptTrainParamUpdateDTO = new PtTrainParamUpdateDTO(); - JSONObject runParams = new JSONObject(); - runParams.put("key1", 22); - runParams.put("key2", 22); - runParams.put("key3", 22); - runParams.put("key4", 22); - ptTrainParamUpdateDTO.setId((long) 2).setParamName("修改测试").setAlgorithmId((long) 1).setRunParams(runParams) - .setDescription("描述信息").setDataSourcePath("/usr/local/data/out.json") - .setDataSourceName("out.json") - .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); - mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamUpdateDTO), MockMvcResultMatchers.status().is4xxClientError(), 400); - } - - /** - * 修改任务参数 - */ - @Test - @Transactional(rollbackFor = Exception.class) - public void ptTrainParamUpdateTest1() throws Exception { - PtTrainParamUpdateDTO ptTrainParamUpdateDTO = new PtTrainParamUpdateDTO(); - JSONObject runParams = new JSONObject(); - runParams.put("key1", 33); - runParams.put("key2", 33); - runParams.put("key3", 33); - runParams.put("key4", 33); - ptTrainParamUpdateDTO.setId((long) 1).setParamName("修改任务参数名称").setAlgorithmId((long) 10).setRunParams(runParams) - .setDescription("描述信息").setDataSourcePath("/usr/local/data/out.json") - .setDataSourceName("out.json") - .setResourcesPoolType(1).setRunCommand("python p.py").setImageName("tensorflow").setImageTag("latest"); - mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamUpdateDTO), MockMvcResultMatchers.status().is4xxClientError(), 400); - } /** * 删除任务参数 @@ -184,30 +97,10 @@ public class TrainParamApiTest extends BaseTest { @Transactional(rollbackFor = Exception.class) @Rollback(false) public void ptTrainParamDeleteTest() throws Exception { - Long[] ids = {1L}; - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainParams"), JSON.toJSONString(ids), MockMvcResultMatchers.status().isOk(), 200); + Set ids = new HashSet<>(); + ids.add(125L); + PtTrainParamDeleteDTO ptTrainParamDeleteDTO=new PtTrainParamDeleteDTO(); + ptTrainParamDeleteDTO.setIds(ids); + mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamDeleteDTO), MockMvcResultMatchers.status().isOk(), 200); } - - /** - * 删除任务参数 - */ - @Test - @Transactional(rollbackFor = Exception.class) - public void ptTrainParamDeleteTest0() throws Exception { - Long[] ids = {}; - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainParams"), JSON.toJSONString(ids), MockMvcResultMatchers.status().is4xxClientError(), 400); - } - - /** - * 删除任务参数 - * 传入id数组在映射表中有id不存在测试 - * id=10在表中不存在 - */ - @Test - @Transactional(rollbackFor = Exception.class) - public void ptTrainParamDeleteTest1() throws Exception { - Long[] ids = {1L, 10L}; - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainParams"), JSON.toJSONString(ids), MockMvcResultMatchers.status().is4xxClientError(), 400); - } - } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java index 00a7cb8..bf196ce 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java index 3f638f7..cd12405 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/TaskServiceServiceImplTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/TaskServiceServiceImplTest.java index ee74465..c1a2ea9 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/TaskServiceServiceImplTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/TaskServiceServiceImplTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,24 +17,19 @@ package org.dubhe.data.service; - import cn.hutool.core.lang.UUID; import org.dubhe.BaseTest; import org.dubhe.data.domain.dto.AutoAnnotationCreateDTO; import org.dubhe.data.domain.dto.DatasetCreateDTO; import org.dubhe.data.domain.entity.File; import org.dubhe.data.domain.vo.ProgressVO; -import org.dubhe.data.service.http.AnnotationHttpService; import org.dubhe.data.service.impl.DatasetServiceImpl; import org.dubhe.data.service.impl.FileServiceImpl; import org.dubhe.data.service.impl.TaskServiceImpl; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.MockBean; - import java.util.*; /** @@ -49,8 +44,6 @@ public class TaskServiceServiceImplTest extends BaseTest { private DatasetServiceImpl datasetService; @Autowired private FileServiceImpl fileService; - @MockBean - private AnnotationHttpService annotationHttpService; public static final int FILE_SIZE = 100; @@ -64,9 +57,6 @@ public class TaskServiceServiceImplTest extends BaseTest { TASK_IDS.add(id); return id; }; - - Mockito.when(annotationHttpService.annotate(Mockito.any())).then(answer); - dsId = initDataset(); addFile(dsId); } @@ -107,9 +97,5 @@ public class TaskServiceServiceImplTest extends BaseTest { } } - @Test - public void fail() { - taskService.fail(); - } } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/DataFactory.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/DataFactory.java index 94bd3b1..45f1a08 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/DataFactory.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/DataFactory.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/DatasetApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/DatasetApiTest.java index 9c6d38b..c53b1f6 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/DatasetApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/DatasetApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/MockUtil.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/MockUtil.java index 72a9473..dc22ac9 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/MockUtil.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/dataset/MockUtil.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/store/MinioStoreServiceImplTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/store/MinioStoreServiceImplTest.java index 8343d41..e6f2662 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/store/MinioStoreServiceImplTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/store/MinioStoreServiceImplTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/PodCallbackTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/PodCallbackTest.java new file mode 100644 index 0000000..da10709 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/PodCallbackTest.java @@ -0,0 +1,93 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.k8s; + +import com.alibaba.fastjson.JSON; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.*; +import org.dubhe.AppRun; +import org.dubhe.k8s.enums.WatcherActionEnum; +import org.dubhe.k8s.event.callback.PodCallback; +import org.dubhe.k8s.utils.BizConvertUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * @description WatcherCallbackTest测试类 + * @date 2020-6-3 + */ +@RunWith(SpringRunner.class) +@SpringBootTest(classes= AppRun.class,webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) +public class PodCallbackTest { + @Autowired + private PodCallback podCallback; + + /** + * 连接k8s master服务器 + * + * @return + */ + public static KubernetesClient connectK8s() { + String namespace = "local"; + String master = "http://xxx.xxx.xxx.xxx:8080"; + KubernetesClient client = null; + Config config = new ConfigBuilder().withMasterUrl(master) + .withTrustCerts(true) + .withNamespace(namespace).build(); + try { + client = new DefaultKubernetesClient(config); + + } catch (Exception e) { + } + return client; + } + + @Test + public void watch(){ + KubernetesClient client = connectK8s(); + try { + client.pods().inNamespace("watch").watch(new Watcher() { + @Override + public void eventReceived(Action action, Pod pod) { + System.out.println("action = "+action.name()); + System.out.println("pod = "+ JSON.toJSONString(pod)); + podCallback.podCallback(WatcherActionEnum.get(action.name()), BizConvertUtils.toBizPod(pod)); + } + @Override + public void onClose(KubernetesClientException e) { + } + }); + + Thread.sleep(Integer.MAX_VALUE); + } catch (Exception interruptedException) { + Thread.currentThread().interrupt(); + } + } + + @Test + public void callback(){ + String podStr = "{\"additionalProperties\":{},\"apiVersion\":\"v1\",\"kind\":\"Pod\",\"metadata\":{\"additionalProperties\":{},\"annotations\":{\"cni.projectcalico.org/podIP\":\"172.16.29.81/32\"},\"creationTimestamp\":\"2020-09-28T07:20:31Z\",\"finalizers\":[],\"generateName\":\"serving-rn-173-wan40byz-64d88c69b4-\",\"labels\":{\"platform/business\":\"serving\",\"platform/create-by\":\"platform\",\"platform/p-kind\":\"Deployment\",\"platform/p-name\":\"serving-rn-173-wan40byz\",\"platform/resource-name\":\"serving-rn-173\",\"platform/runtime-env\":\"serving\",\"pod-template-hash\":\"64d88c69b4\"},\"managedFields\":[],\"name\":\"serving-rn-173-wan40byz-64d88c69b4-8zq4t\",\"namespace\":\"namespace-1\",\"ownerReferences\":[{\"additionalProperties\":{},\"apiVersion\":\"apps/v1\",\"blockOwnerDeletion\":true,\"controller\":true,\"kind\":\"ReplicaSet\",\"name\":\"serving-rn-173-wan40byz-64d88c69b4\",\"uid\":\"c8e7e4c2-c19f-4ee1-9dd7-7e32e258086c\"}],\"resourceVersion\":\"20702829\",\"selfLink\":\"/api/v1/namespaces/namespace-1/pods/serving-rn-173-wan40byz-64d88c69b4-8zq4t\",\"uid\":\"15a1f63c-5bd4-4c6f-90bd-5c984a3b3086\"},\"spec\":{\"additionalProperties\":{},\"containers\":[{\"additionalProperties\":{},\"args\":[\"-c\",\"python /usr/local/TS_Serving/serving/server.py --platform='tensorflow' --model_name='resnet50' --model_path='/usr/local/TS_Serving/models/resnet50' --log_dir='/usr/local/TS_Serving/logs/'\"],\"command\":[\"/bin/bash\"],\"env\":[],\"envFrom\":[],\"image\":\"harbor.dubhe.ai/serving/serving:v1.0\",\"imagePullPolicy\":\"IfNotPresent\",\"name\":\"serving-rn-173-wan40byz\",\"ports\":[{\"additionalProperties\":{},\"containerPort\":5000,\"name\":\"http\",\"protocol\":\"TCP\"}],\"resources\":{\"additionalProperties\":{},\"limits\":{\"cpu\":{\"additionalProperties\":{},\"amount\":\"1\",\"format\":\"\"},\"nvidia.com/gpu\":{\"additionalProperties\":{},\"amount\":\"1\",\"format\":\"\"}},\"requests\":{\"cpu\":{\"additionalProperties\":{},\"amount\":\"1\",\"format\":\"\"},\"nvidia.com/gpu\":{\"additionalProperties\":{},\"amount\":\"1\",\"format\":\"\"}}},\"terminationMessagePath\":\"/dev/termination-log\",\"terminationMessagePolicy\":\"File\",\"volumeDevices\":[],\"volumeMounts\":[{\"additionalProperties\":{},\"mountPath\":\"/usr/local/TS_Serving/models/resnet50\",\"name\":\"volume-0\"},{\"additionalProperties\":{},\"mountPath\":\"/usr/local/TS_Serving/serving\",\"name\":\"volume-1\"},{\"additionalProperties\":{},\"mountPath\":\"/var/run/secrets/kubernetes.io/serviceaccount\",\"name\":\"default-token-l7kbz\",\"readOnly\":true}]}],\"dnsPolicy\":\"ClusterFirst\",\"enableServiceLinks\":true,\"ephemeralContainers\":[],\"hostAliases\":[],\"imagePullSecrets\":[],\"initContainers\":[],\"nodeName\":\"qjy-ai05\",\"nodeSelector\":{\"gpu\":\"gpu\"},\"priority\":0,\"readinessGates\":[],\"restartPolicy\":\"Always\",\"schedulerName\":\"default-scheduler\",\"securityContext\":{\"additionalProperties\":{},\"supplementalGroups\":[],\"sysctls\":[]},\"serviceAccount\":\"default\",\"serviceAccountName\":\"default\",\"terminationGracePeriodSeconds\":30,\"tolerations\":[{\"additionalProperties\":{},\"effect\":\"NoExecute\",\"key\":\"node.kubernetes.io/not-ready\",\"operator\":\"Exists\",\"tolerationSeconds\":300},{\"additionalProperties\":{},\"effect\":\"NoExecute\",\"key\":\"node.kubernetes.io/unreachable\",\"operator\":\"Exists\",\"tolerationSeconds\":300}],\"topologySpreadConstraints\":[],\"volumes\":[{\"additionalProperties\":{},\"name\":\"volume-0\",\"nfs\":{\"additionalProperties\":{},\"path\":\"/nfs/dubhe-dev/serving/models/tensorflow_models/resnet50/\",\"server\":\"10.5.26.234\"}},{\"additionalProperties\":{},\"name\":\"volume-1\",\"nfs\":{\"additionalProperties\":{},\"path\":\"/nfs/dubhe-dev/serving/TS_Serving\",\"server\":\"10.5.26.234\"}},{\"additionalProperties\":{},\"name\":\"default-token-l7kbz\",\"secret\":{\"additionalProperties\":{},\"defaultMode\":420,\"items\":[],\"secretName\":\"default-token-l7kbz\"}}]},\"status\":{\"additionalProperties\":{},\"conditions\":[{\"additionalProperties\":{},\"lastTransitionTime\":\"2020-09-28T07:20:02Z\",\"status\":\"True\",\"type\":\"Initialized\"},{\"additionalProperties\":{},\"lastTransitionTime\":\"2020-09-28T07:20:05Z\",\"status\":\"True\",\"type\":\"Ready\"},{\"additionalProperties\":{},\"lastTransitionTime\":\"2020-09-28T07:20:05Z\",\"status\":\"True\",\"type\":\"ContainersReady\"},{\"additionalProperties\":{},\"lastTransitionTime\":\"2020-09-28T07:20:31Z\",\"status\":\"True\",\"type\":\"PodScheduled\"}],\"containerStatuses\":[{\"additionalProperties\":{},\"containerID\":\"docker://777ee925f078ed62d77902117de2043ffd2323894c00d6ea4d9d5d0bee1fad50\",\"image\":\"harbor.dubhe.ai/serving/serving:v1.0\",\"imageID\":\"docker-pullable://harbor.dubhe.ai/serving/serving@sha256:f09d3b1a7e854582c33235d25569ef7ea6eff12446ce6bd8662147d68c6a88ee\",\"lastState\":{\"additionalProperties\":{}},\"name\":\"serving-rn-173-wan40byz\",\"ready\":true,\"restartCount\":0,\"started\":true,\"state\":{\"additionalProperties\":{},\"running\":{\"additionalProperties\":{},\"startedAt\":\"2020-09-28T07:20:04Z\"}}}],\"ephemeralContainerStatuses\":[],\"hostIP\":\"10.5.24.134\",\"initContainerStatuses\":[],\"phase\":\"Running\",\"podIP\":\"172.16.29.81\",\"podIPs\":[{\"additionalProperties\":{},\"ip\":\"172.16.29.81\"}],\"qosClass\":\"Burstable\",\"startTime\":\"2020-09-28T07:20:02Z\"}}"; + Pod pod = JSON.parseObject(podStr,Pod.class); + WatcherActionEnum watcherActionEnum = WatcherActionEnum.get("ADDED"); + podCallback.podCallback(watcherActionEnum, BizConvertUtils.toBizPod(pod)); + } + +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/ResourceCacheTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/ResourceCacheTest.java index 4fc7030..a54aa63 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/ResourceCacheTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/ResourceCacheTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/WatcherCallbackTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/WatcherCallbackTest.java deleted file mode 100644 index f5bff05..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/WatcherCallbackTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.k8s; - -import com.alibaba.fastjson.JSON; -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.client.*; -import org.dubhe.AppRun; -import org.dubhe.k8s.enums.WatcherActionEnum; -import org.dubhe.k8s.event.callback.WatcherCallback; -import org.dubhe.k8s.utils.BizConvertUtils; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -/** - * @description WatcherCallbackTest测试类 - * @date 2020-6-3 - */ -@RunWith(SpringRunner.class) -@SpringBootTest(classes= AppRun.class,webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) -public class WatcherCallbackTest { - @Autowired - private WatcherCallback watcherCallback; - - /** - * 连接k8s master服务器 - * - * @return - */ - public static KubernetesClient connectK8s() { - String namespace = "local"; - String master = "http://xxx.xxx.xxx.xxx:8080"; - KubernetesClient client = null; - Config config = new ConfigBuilder().withMasterUrl(master) - .withTrustCerts(true) - .withNamespace(namespace).build(); - try { - client = new DefaultKubernetesClient(config); - - } catch (Exception e) { - } - return client; - } - - @Test - public void watch(){ - KubernetesClient client = connectK8s(); - try { - client.pods().inNamespace("watch").watch(new Watcher() { - @Override - public void eventReceived(Action action, Pod pod) { - System.out.println("action = "+action.name()); - System.out.println("pod = "+ JSON.toJSONString(pod)); - watcherCallback.podCallback(WatcherActionEnum.get(action.name()), BizConvertUtils.toBizPod(pod)); - } - @Override - public void onClose(KubernetesClientException e) { - } - }); - - Thread.sleep(Integer.MAX_VALUE); - } catch (Exception interruptedException) { - Thread.currentThread().interrupt(); - } - } - -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/DeploymentApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/DeploymentApiTest.java index 59eee53..24e8994 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/DeploymentApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/DeploymentApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/DistributeTrainApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/DistributeTrainApiTest.java index 597b76f..5657570 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/DistributeTrainApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/DistributeTrainApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import cn.hutool.core.io.FileUtil; import com.alibaba.fastjson.JSON; import org.dubhe.k8s.domain.PtBaseResult; import org.dubhe.k8s.domain.bo.DistributeTrainBO; +import org.dubhe.k8s.domain.bo.PtMountDirBO; import org.dubhe.k8s.domain.resource.BizDistributeTrain; import org.junit.Test; import org.junit.runner.RunWith; @@ -28,6 +29,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; import javax.annotation.Resource; +import java.util.HashMap; /** * @description NodeApiTest测试类 @@ -42,27 +44,30 @@ public class DistributeTrainApiTest { @Test public void testCreate(){ DistributeTrainBO bo = new DistributeTrainBO(); - bo.setName("yepeng11"); - bo.setNamespace("yep"); - bo.setSize(3); + bo.setName("batchserving-rn-wdma9"); + bo.setNamespace("sunjd"); + bo.setSize(30); bo.setImage("harbor.dubhe.ai/oneflow/oneflow-cuda:py36-v3"); bo.setMasterCmd("export NODE_IPS=`cat /home/hostfile.json |jq -r '.[]|.ip'|paste -d \",\" -s` && cd /workspace/Classification/cnns && rm -rf core.* && rm -rf ./output/snapshots/* && python3 of_cnn_train_val.py --train_data_dir=$DATA_ROOT/train --train_data_part_num=$TRAIN_DATA_PART_NUM --val_data_dir=$DATA_ROOT/validation --val_data_part_num=$VAL_DATA_PART_NUM --num_nodes=$NODE_NUM --node_ips=\"$NODE_IPS\" --gpu_num_per_node=$GPU_NUM_PER_NODE --model_update=\"momentum\" --learning_rate=0.256 --loss_print_every_n_iter=1 --batch_size_per_device=1 --val_batch_size_per_device=1 --num_epoch=1 --model=\"resnet50\" --model_save_dir=/model"); bo.setMemNum(8192); bo.setCpuNum(4000); bo.setGpuNum(2); bo.setSlaveCmd("export NODE_IPS=`cat /home/hostfile.json |jq -r '.[]|.ip'|paste -d \",\" -s` && cd /workspace/Classification/cnns && rm -rf core.* && rm -rf ./output/snapshots/* && python3 of_cnn_train_val.py --train_data_dir=$DATA_ROOT/train --train_data_part_num=$TRAIN_DATA_PART_NUM --val_data_dir=$DATA_ROOT/validation --val_data_part_num=$VAL_DATA_PART_NUM --num_nodes=$NODE_NUM --node_ips=\"$NODE_IPS\" --gpu_num_per_node=$GPU_NUM_PER_NODE --model_update=\"momentum\" --learning_rate=0.256 --loss_print_every_n_iter=1 --batch_size_per_device=1 --val_batch_size_per_device=1 --num_epoch=1 --model=\"resnet50\" --model_save_dir=/model"); - bo.setDatasetStoragePath("/nfs/sunjd/dataset/of_dataset"); - bo.setWorkspaceStoragePath("/nfs/sunjd/workspace"); - bo.setModelStoragePath("/nfs/sunjd/model"); + bo.setNfsMounts(new HashMap(){{ + put("/dataset",new PtMountDirBO("/nfs/sunjd/dataset/of_dataset")); + put("/workspace",new PtMountDirBO("/nfs/sunjd/workspace")); + put("/model",new PtMountDirBO("/nfs/sunjd/model")); + }}); bo.setBusinessLabel("train"); bo.setDelayCreateTime(10); bo.setDelayDeleteTime(10); - distributeTrainApi.create(bo); + System.out.println(JSON.toJSONString(distributeTrainApi.create(bo))); + System.out.println("123"); } @Test public void deleteByResourceName() { - PtBaseResult result = distributeTrainApi.deleteByResourceName("tianlong", "tianlong-dt"); + PtBaseResult result = distributeTrainApi.deleteByResourceName("namespace-1", "yepeng11"); System.out.println(JSON.toJSONString(result)); } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/HarborApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/HarborApiTest.java index 5ee3dd8..9cf9a03 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/HarborApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/HarborApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/JupyterResourceApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/JupyterResourceApiTest.java index 039eefa..ba66ac3 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/JupyterResourceApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/JupyterResourceApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -142,7 +142,7 @@ public class JupyterResourceApiTest { @Test public void delete() { - PtBaseResult ptBaseResult = jupyterResourceApi.delete("namespace", "notebook1"); + PtBaseResult ptBaseResult = jupyterResourceApi.delete("namespace-75", "notebook-rn-20200819142637099n6t8"); System.out.println(ptBaseResult); } } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/LimitRangeApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/LimitRangeApiTest.java index f3a9af8..cae9736 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/LimitRangeApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/LimitRangeApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/LogMonitoringApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/LogMonitoringApiTest.java index 9100ecf..5a7038b 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/LogMonitoringApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/LogMonitoringApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/MetricsApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/MetricsApiTest.java index fe4b1f9..a75a131 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/MetricsApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/MetricsApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,4 +43,10 @@ public class MetricsApiTest { public void getContainerMetrics(){ System.out.println(JSON.toJSONString(metricsApi.getContainerMetrics("namespace-1"))); } + + @Test + public void getPodMetricsRealTime(){ + System.out.println(JSON.toJSONString(metricsApi.getPodMetricsRealTime("namespace-1","serving-rn-212"))); + } + } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ModelOptJobApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ModelOptJobApiTest.java index 9608f76..c8d8441 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ModelOptJobApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ModelOptJobApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ModelServingApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ModelServingApiTest.java new file mode 100644 index 0000000..8bb2508 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ModelServingApiTest.java @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.k8s.api; + +import com.alibaba.fastjson.JSON; +import org.dubhe.k8s.domain.bo.ModelServingBO; +import org.dubhe.k8s.domain.bo.PtMountDirBO; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.Arrays; +import java.util.HashMap; + +/** + * @description + * @date 2020-09-11 + */ +@SpringBootTest +@RunWith(SpringRunner.class) +public class ModelServingApiTest { + @Autowired + private ModelServingApi modelServingApi; + + @Test + public void create(){ + ModelServingBO bo = new ModelServingBO(); + bo.setNamespace("sunjd"); + bo.setResourceName("model-serving"); + bo.setReplicas(10); + bo.setMemNum(4000); + bo.setCpuNum(1); + bo.setGpuNum(3); + bo.setImage("quay.io/kubernetes-ingress-controller/grpc-fortune-teller:0.1"); + bo.setGrpcPort(50051); + bo.setBusinessLabel("serving"); + bo.setCmdLines(Arrays.asList("-c","while true; do echo hello; sleep 10;done")); + bo.setNfsMounts(new HashMap(){{ + put("/usr/local/TS_Serving/serving",new PtMountDirBO("/nfs/dubhe-dev/serving/TS_Serving")); + put("/usr/local/TS_Serving/models/resnet50",new PtMountDirBO("/nfs/dubhe-dev/serving/models/oneflow_models/resnet50/")); + }}); + System.out.println(JSON.toJSONString(modelServingApi.create(bo))); + } + + @Test + public void delete(){ + modelServingApi.delete("namespace-1","serving-rn-nogn35"); + modelServingApi.delete("namespace-1","serving-rn-nojb33"); + modelServingApi.delete("namespace-1","serving-rn-nxvr86"); + modelServingApi.delete("namespace-1","serving-rn-nyfe87"); + + modelServingApi.delete("namespace-1","serving-rn-rije34"); + modelServingApi.delete("namespace-1","serving-rn-ubvo102"); + + System.out.println(JSON.toJSONString(modelServingApi.delete("namespace-1","serving-rn-nyfe87"))); + } +} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NamespaceApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NamespaceApiTest.java index 56f3aa5..189f914 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NamespaceApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NamespaceApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NativeResourceApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NativeResourceApiTest.java index 694b599..8e524e7 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NativeResourceApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NativeResourceApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NodeApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NodeApiTest.java index 3a7d50a..071ecbb 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NodeApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/NodeApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ package org.dubhe.k8s.api; import com.alibaba.fastjson.JSON; +import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; +import org.dubhe.k8s.constant.K8sParamConstants; import org.dubhe.k8s.domain.resource.BizNode; import org.dubhe.k8s.enums.LackOfResourcesEnum; import org.junit.Test; @@ -31,6 +33,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * @description NodeApiTest测试类 @@ -94,8 +97,12 @@ public class NodeApiTest { @Test public void isAllocatable(){ LackOfResourcesEnum flag; - flag = nodeApi.isAllocatable(10000,300000 ,30 ); + flag = nodeApi.isAllocatable(10000,300000 ,30); System.out.println(flag.getMessage()); } + @Test + public void isOutOfTotalAllocatableGpu(){ + System.out.println(nodeApi.isOutOfTotalAllocatableGpu(3)); + } } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/PersistentVolumeClaimApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/PersistentVolumeClaimApiTest.java index 09f77dc..d0084b6 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/PersistentVolumeClaimApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/PersistentVolumeClaimApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/PodApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/PodApiTest.java index a268825..0ffbad2 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/PodApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/PodApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ResourceQuotaApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ResourceQuotaApiTest.java index fb821ff..cd8138d 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ResourceQuotaApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/ResourceQuotaApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/TrainJobApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/TrainJobApiTest.java index ef48b64..4cc0166 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/TrainJobApiTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/k8s/api/TrainJobApiTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/model/ModelBranchTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/model/ModelBranchTest.java index abad582..b5e19d1 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/model/ModelBranchTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/model/ModelBranchTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,19 +18,18 @@ package org.dubhe.model; +import com.alibaba.fastjson.JSON; +import org.dubhe.BaseTest; import org.dubhe.domain.dto.PtModelBranchCreateDTO; import org.dubhe.domain.dto.PtModelBranchDeleteDTO; -import org.dubhe.domain.dto.PtModelBranchQueryDTO; -import org.dubhe.domain.dto.PtModelBranchUpdateDTO; -import org.dubhe.domain.vo.PtModelBranchCreateVO; -import org.dubhe.service.impl.PtModelBranchServiceImpl; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; - -import java.util.Map; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.transaction.annotation.Transactional; /** * @description 分支管理 @@ -38,21 +37,16 @@ import java.util.Map; */ @RunWith(SpringRunner.class) @SpringBootTest -public class ModelBranchTest { - @Autowired - private PtModelBranchServiceImpl ptModelBranchServiceImpl; +public class ModelBranchTest extends BaseTest { /** * 查询ModelBranch */ @Test - public void ptModelBranchQueryTest() { - PtModelBranchQueryDTO ptModelBranchQueryCriteria = new PtModelBranchQueryDTO(); - ptModelBranchQueryCriteria.setParentId(1); - + public void ptModelBranchQueryTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/ptModelBranch").param("parentId", String.valueOf(111))) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - Map results = ptModelBranchServiceImpl.queryAll(ptModelBranchQueryCriteria); - System.out.println(results.toString()); } @@ -60,34 +54,27 @@ public class ModelBranchTest { * 新增ModelBranch */ @Test - public void ptModelBranchCreateTest() { + @Transactional(rollbackFor = Exception.class) + @Rollback(false) + public void ptModelBranchCreateTest() throws Exception { PtModelBranchCreateDTO ptModelBranchCreateDTO = new PtModelBranchCreateDTO(); - ptModelBranchCreateDTO.setParentId((long) 1) - .setModelAddress("http://10.0.0.1"); - PtModelBranchCreateVO ptModelBranchCreateVO = ptModelBranchServiceImpl.create(ptModelBranchCreateDTO); - System.out.println(ptModelBranchCreateVO.toString()); + ptModelBranchCreateDTO.setParentId(114L) + .setModelAddress("/model/1/20201130170019114pij1/").setModelSource(1); + mockMvcTest(MockMvcRequestBuilders.post("/api/ptModelBranch"), JSON.toJSONString(ptModelBranchCreateDTO), MockMvcResultMatchers.status().isOk(), 200); } - /** - * 修改ModelBranch - */ - @Test - public void ptModelBranchUpdateTest() { - PtModelBranchUpdateDTO ptModelBranchUpdateDTO = new PtModelBranchUpdateDTO(); - ptModelBranchUpdateDTO.setParentId((long) 1) - .setModelAddress("http://10.0.0.1"); - ptModelBranchServiceImpl.update(ptModelBranchUpdateDTO); - } /** * 删除ModelBranch */ @Test - public void ptModelBranchDeleteTest() { - Long[] ids = {(long) 1, (long) 2}; - PtModelBranchDeleteDTO ptModelBranchDeleteDTO =new PtModelBranchDeleteDTO(); + @Transactional(rollbackFor = Exception.class) + @Rollback(false) + public void ptModelBranchDeleteTest() throws Exception { + Long[] ids = {94L}; + PtModelBranchDeleteDTO ptModelBranchDeleteDTO = new PtModelBranchDeleteDTO(); ptModelBranchDeleteDTO.setIds(ids); + mockMvcTest(MockMvcRequestBuilders.delete("/api/ptModelBranch"), JSON.toJSONString(ptModelBranchDeleteDTO), MockMvcResultMatchers.status().isOk(), 200); - ptModelBranchServiceImpl.deleteAll(ptModelBranchDeleteDTO); } } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/model/ModelInfoTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/model/ModelInfoTest.java index 703c117..55d2f9c 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/model/ModelInfoTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/model/ModelInfoTest.java @@ -1,12 +1,12 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. - * + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,19 +17,19 @@ package org.dubhe.model; +import com.alibaba.fastjson.JSON; +import org.dubhe.BaseTest; import org.dubhe.domain.dto.PtModelInfoCreateDTO; import org.dubhe.domain.dto.PtModelInfoDeleteDTO; -import org.dubhe.domain.dto.PtModelInfoQueryDTO; import org.dubhe.domain.dto.PtModelInfoUpdateDTO; -import org.dubhe.domain.vo.PtModelInfoCreateVO; -import org.dubhe.service.impl.PtModelInfoServiceImpl; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; - -import java.util.Map; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.result.MockMvcResultMatchers; +import org.springframework.transaction.annotation.Transactional; /** * @description 模型管理模块单元测试 @@ -37,61 +37,63 @@ import java.util.Map; */ @RunWith(SpringRunner.class) @SpringBootTest -public class ModelInfoTest { - @Autowired - private PtModelInfoServiceImpl ptModelInfoServiceImpl; +public class ModelInfoTest extends BaseTest { /** * 查询任务参数列表 * 无条件分页查询 */ @Test - public void ptModelInfoQueryTest() { - PtModelInfoQueryDTO ptModelInfoQueryCriteria = new PtModelInfoQueryDTO(); + public void ptModelInfoQueryTest() throws Exception { + mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/ptModelInfo")) + .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - Map results = ptModelInfoServiceImpl.queryAll(ptModelInfoQueryCriteria); - System.out.println(results.toString()); } /** * 新增ModelInfo */ @Test - public void ptModelInfoCreateTest() { + @Transactional(rollbackFor = Exception.class) + @Rollback(false) + public void ptModelInfoCreateTest() throws Exception { PtModelInfoCreateDTO ptModelInfoCreateDTO = new PtModelInfoCreateDTO(); - ptModelInfoCreateDTO.setName("test") + ptModelInfoCreateDTO.setName("untilTestingCreateTest") .setFrameType(1) .setModelType(1) .setModelClassName("测试") - .setModelDescription("test"); - PtModelInfoCreateVO ptModelInfoCreateVO = ptModelInfoServiceImpl.create(ptModelInfoCreateDTO); - System.out.println(ptModelInfoCreateVO.toString()); + .setModelDescription("untilTestingCreateTest"); + mockMvcTest(MockMvcRequestBuilders.post("/api/ptModelInfo"), JSON.toJSONString(ptModelInfoCreateDTO), MockMvcResultMatchers.status().isOk(), 200); } /** * 更新ModelInfo */ @Test - public void ptModelInfoUpdateTest() { + @Transactional(rollbackFor = Exception.class) + @Rollback(false) + public void ptModelInfoUpdateTest() throws Exception { PtModelInfoUpdateDTO ptModelInfoUpdateDTO = new PtModelInfoUpdateDTO(); - ptModelInfoUpdateDTO.setId((long) 1) - .setName("test") + ptModelInfoUpdateDTO.setId(117L) + .setName("untilTestingUpdateTest" + System.currentTimeMillis()) .setFrameType(1) .setModelType(1) .setModelClassName("测试") - .setModelDescription("test"); - ptModelInfoServiceImpl.update(ptModelInfoUpdateDTO); + .setModelDescription("untilTestingUpdateTest"); + mockMvcTest(MockMvcRequestBuilders.put("/api/ptModelInfo"), JSON.toJSONString(ptModelInfoUpdateDTO), MockMvcResultMatchers.status().isOk(), 200); } /** * 删除任务参数 */ @Test - public void ptTrainAlgorithmDeleteTest() { - Long[] ids = {(long) 1, (long) 2}; + @Transactional(rollbackFor = Exception.class) + @Rollback(false) + public void ptTrainAlgorithmDeleteTest() throws Exception { + Long[] ids = {117L}; PtModelInfoDeleteDTO ptModelInfoDeleteDTO = new PtModelInfoDeleteDTO(); ptModelInfoDeleteDTO.setIds(ids); - ptModelInfoServiceImpl.deleteAll(ptModelInfoDeleteDTO); + mockMvcTest(MockMvcRequestBuilders.delete("/api/ptModelInfo"), JSON.toJSONString(ptModelInfoDeleteDTO), MockMvcResultMatchers.status().isOk(), 200); } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/spring/SpringContextHolderTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/spring/SpringContextHolderTest.java index f4f29d0..bfa9e52 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/spring/SpringContextHolderTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/spring/SpringContextHolderTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,13 +18,10 @@ package org.dubhe.spring; -import org.dubhe.enums.BizNfsEnum; -import org.dubhe.utils.K8sNameTool; import org.dubhe.utils.SpringContextHolder; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; @@ -33,14 +30,14 @@ import org.springframework.test.context.junit4.SpringRunner; /** * @date 2020.05.26 */ -@ActiveProfiles(value = "notebook") +@ActiveProfiles(value = "dev") @RunWith(SpringRunner.class) @SpringBootTest public class SpringContextHolderTest { @Test public void getActiveProfile(){ - Assert.assertEquals( "notebook", SpringContextHolder.getActiveProfile()); + Assert.assertEquals( "dev", SpringContextHolder.getActiveProfile()); } diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/utils/LogUtilTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/utils/LogUtilTest.java index 395c703..2cd882c 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/utils/LogUtilTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/utils/LogUtilTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/utils/NotebookUtilTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/utils/NotebookUtilTest.java index 46b9214..d561539 100644 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/utils/NotebookUtilTest.java +++ b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/utils/NotebookUtilTest.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.setup.MockMvcBuilders; @@ -42,6 +43,7 @@ import java.util.Date; * * @date 2020.04.27 */ +@ActiveProfiles(value = "dev") @RunWith(SpringRunner.class) @SpringBootTest @SpringBootConfiguration diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/AnnotateTypeEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/AnnotateTypeEnum.java index 4d035e0..fb54d23 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/AnnotateTypeEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/AnnotateTypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,13 @@ public enum AnnotateTypeEnum { /** * 目标跟踪 */ - OBJECT_TRACK(5, "目标跟踪"); + OBJECT_TRACK(5, "目标跟踪"), + + /** + * 文本分类 + */ + TEXT_CLASSIFICATION(6, "文本分类"); + AnnotateTypeEnum(Integer value, String msg) { this.value = value; diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/Constant.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/Constant.java index f8fb37c..0c76ef9 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/Constant.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/Constant.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -93,7 +93,7 @@ public class Constant { public static final String DATA_TYPE_RULE = "数据类型参数不对,请使用: 0-图片 1-视频"; - public static final String ANNOTATE_TYPE_RULE = "数据标注类型参数不对,请使用: 2-分类 1-目标检测 5-目标跟踪"; + public static final String ANNOTATE_TYPE_RULE = "数据标注类型参数不对,请使用: 2-分类 1-目标检测 5-目标跟踪 6 文本分类"; /** @@ -145,7 +145,7 @@ public class Constant { /** * 数据集名称路径 */ - public static final String DATASET_PATH_NAME = File.separator + "dataset" + File.separator; + public static final String DATASET_PATH_NAME = "dataset" + File.separator; /** * 版本文件路径 @@ -229,6 +229,11 @@ public class Constant { */ public static final Long IMAGENET_ID = 2L; + /** + * 文本数据集预置标签组ID + */ + public static final Long TXT_PREPARE_ID = 748L; + /** * 大数据默认删除数量 @@ -243,4 +248,19 @@ public class Constant { public final static String DATASET_LABEL_PUB_KEY = "dateset:label:pub"; + + /** + * 数据集预置标签组类型 + */ + public static final Integer COMMON_LABEL_GROUP_TYPE = 1; + + + + /** + * 抽象名称前缀 + */ + public static final String ABSTRACT_NAME_PREFIX = "abstract_"; + + + } diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ConversionStatusEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ConversionStatusEnum.java index 87a46a8..a390c43 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ConversionStatusEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ConversionStatusEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,7 +42,12 @@ public enum ConversionStatusEnum { /** * 转换状态：无法转换 */ - UNABLE_CONVERSION(3, "无法转换"); + UNABLE_CONVERSION(3, "无法转换"), + + /** + * 转换状态：发布中 + */ + PUBLISHING(4,"发布中"); ConversionStatusEnum(Integer value, String msg) { this.value = value; diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DataStatusEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DataStatusEnum.java index 5e30978..2cdab8e 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DataStatusEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DataStatusEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DataTaskTypeEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DataTaskTypeEnum.java index ed13875..2cad2e5 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DataTaskTypeEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DataTaskTypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,7 +49,19 @@ public enum DataTaskTypeEnum { /** * 视频采样 */ - VIDEO_SAMPLE(5, "视频采样") + VIDEO_SAMPLE(5, "视频采样"), + /** + * 医学标注 + */ + MEDICINE_ANNOTATION(6,"医学标注"), + /** + * 文本分类 + */ + TEXT_CLASSIFICATION(7, "文本分类"), + /** + * 重新自动标注 + */ + AGAIN_ANNOTATION(8, "重新自动标注") ; DataTaskTypeEnum(Integer value, String msg) { diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetDecompressStateEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetDecompressStateEnum.java index 32d85ca..769136d 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetDecompressStateEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetDecompressStateEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetLabelEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetLabelEnum.java index 5722282..37d1a51 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetLabelEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetLabelEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,7 +43,11 @@ public enum DatasetLabelEnum { /** * MS COCO */ - MS_COCO(3, "MS COCO"); + MS_COCO(3, "MS COCO"), + /** + * 文本 + */ + TXT(4, "文本"); DatasetLabelEnum(Integer type, String name) { this.type = type; diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetStatusEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetStatusEnum.java index 1697854..12fc998 100755 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetStatusEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatasetStatusEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatatypeEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatatypeEnum.java index c87667e..7d03362 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatatypeEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/DatatypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,11 @@ public enum DatatypeEnum { /** * 视频 */ - VIDEO(1, "视频"); + VIDEO(1, "视频"), + /** + * 文本 + */ + TEXT(2, "文本"); DatatypeEnum(Integer value, String msg) { this.value = value; @@ -58,4 +62,23 @@ public enum DatatypeEnum { return false; } + /** + * 获取数据类型枚举 + * + * @param value 获取数据类型枚举值 + * @return 数据类型枚举 + */ + public static DatatypeEnum getEnumValue(Integer value) { + switch (value) { + case 0: + return IMAGE; + case 1: + return VIDEO; + case 2: + return TEXT; + default: + return IMAGE; + } + } + } diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/EnhanceTypeEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/EnhanceTypeEnum.java index 2d830a9..7ee9527 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/EnhanceTypeEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/EnhanceTypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ErrorEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ErrorEnum.java index 9b633d4..28bc34a 100755 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ErrorEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/ErrorEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -80,10 +80,12 @@ public enum ErrorEnum implements ErrorCode { */ LABEL_ERROR(1600, "标签名不能为空或非系统自动标注支持的标签"), LABEL_NAME_EXIST(1601, "本数据集已有同名标签"), - LABEL_NAME_DUPLICATION(1602,"标签名重复，请检查"), + LABEL_NAME_DUPLICATION(1602,"标签名称或标签ID重复，请检查"), LABEL_NOT_EXISTS(1603,"标签不存在"), LABEL_NAME_COLOR_NOT_NULL(1604,"JSON文件中标签名称和颜色不能为空"), LABEL_PUBLIC_EORROR(1605,"不允许操作公共标签"), + LABEL_GROUP_ID_IS_NULL(1606,"标签组ID为null"), + LABEL_FORMAT_IS_ERROR(1607,"标签格式不正确"), /** * 数据集操作错误 @@ -102,6 +104,9 @@ public enum ErrorEnum implements ErrorCode { DATASET_ENHANCEMENT(1711, "该数据集正在增强中,请稍等"), DATASET_TRACK_TYPE_ERROR(1712, "数据集类型只能是目标跟踪才能进行跟踪!"), DATASET_DELETE_ERROR(1713, "数据集数据大数据删除异常!"), + DATASET_PUBLISH_ERROR(1714, "数据集必须是发布后才能操作!"), + DATASET_SWITCH_VERSION_ERROR(1715, "数据集必须版本回退后才能操作!"), + DATASET_VERSION_FILE_IS_ERROR(1716, "数据集版本文件不存在!"), /** * 数据集版本校验 @@ -127,7 +132,17 @@ public enum ErrorEnum implements ErrorCode { LABELGROUP_LABEL_NAME_ERROR(1911,"请输入正确预置标签组标签"), LABELGROUP_LABEL_GROUP_EDIT_ERROR(1912,"标签组下标签不许修改"), LABELGROUP_LABEL_GROUP_QUOTE_DEL_ERROR(1913,"标签组已被数据集引用,无法删除!"), - ; + LABEL_NAME_REPEAT(1914,"标签名称已存在!"), + LABEL_PREPARE_IS_TXT(1915,"请选择文本预制标签进行自动标注!"), + + /** + * 医学数据集错误 + */ + MEDICINE_AUTO_DATASET_ERROR(2001,"当前医学数据集不可自动标注"), + MEDICINE_MEDICAL_ALREADY_EXISTS(2002,"当前类型的医学数据集已存在"), + DATAMEDICINE_ABSENT(2003, "医学数据集不存在"), + DATAMEDICINE_AUTOMATIC(2004,"数据集正在自动标注中,请稍等!"), + MEDICINE_NAME_ERROR(2005,"当前名称已存在") ; ErrorEnum(int code, String msg) { diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/FileTypeEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/FileTypeEnum.java index 25fb80d..4578389 100755 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/FileTypeEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/FileTypeEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/LabelGroupTypeEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/LabelGroupTypeEnum.java new file mode 100644 index 0000000..2740d0f --- /dev/null +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/LabelGroupTypeEnum.java @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.data.constant; + +import lombok.Getter; + +/** + * @description 标签组数据类型 + * @date 2020-12-11 + */ +@Getter +public enum LabelGroupTypeEnum { + + /** + * 视觉 + */ + VISUAL(0, "视觉"), + /** + * 文本 + */ + TXT(1, "文本"); + + LabelGroupTypeEnum(Integer value, String msg) { + this.value = value; + this.msg = msg; + } + + private Integer value; + private String msg; + + + /** + * 标签组类型转换 + * + * @param datatypeEnum 数据类型 + * @return 标签组类型 + */ + public static LabelGroupTypeEnum convertGroup(DatatypeEnum datatypeEnum){ + LabelGroupTypeEnum labelGroupTypeEnum; + switch (datatypeEnum){ + case TEXT: + labelGroupTypeEnum = LabelGroupTypeEnum.TXT; + break; + default: + labelGroupTypeEnum = LabelGroupTypeEnum.VISUAL; + break; + } + return labelGroupTypeEnum; + } + +} diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/TaskSplitStatusEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/TaskSplitStatusEnum.java index eca7b03..93692a2 100755 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/TaskSplitStatusEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/TaskSplitStatusEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/TaskStatusEnum.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/TaskStatusEnum.java index e897c94..930ba73 100755 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/TaskStatusEnum.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/constant/TaskStatusEnum.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DataFileAnnotationMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DataFileAnnotationMapper.java new file mode 100644 index 0000000..8dc529d --- /dev/null +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DataFileAnnotationMapper.java @@ -0,0 +1,72 @@ +/** + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.data.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import io.lettuce.core.dynamic.annotation.Param; +import org.dubhe.data.domain.entity.DataFileAnnotation; + +import java.util.List; + +/** + * @description 数据文件标注服务Mapper + * @date 2021-01-06 + */ +public interface DataFileAnnotationMapper extends BaseMapper { + + /** + * 批量删除数据标注信息 + * + * @param ids 数据集文件标注ID列表 + */ + void deleteBatch(List ids); + + + /** + * 根据版本ID查询标签列表 + * + * @param versionFileId 版本ID + * @return 标签ID列表 + */ + List findInfoByVersionId(@Param("versionFileId") Long versionFileId); + + /** + * 批量保存数据文件标注信息 + * + * @param dataFileAnnotations 数据文件标注实体 + */ + void insertBatch(List dataFileAnnotations); + + /** + * 批量修改标注文件数据 + * + * @param versionFileId 版本文件ID + * @param fileLabelIds 标注标签Ids + * @param fileLabelIds 标注标签Ids + * @param prediction 预测值 + */ + void updateAnnotationFileByVersionIdAndLabelIds(@Param("versionFileId") Long versionFileId, + @Param("fileLabelIds") List fileLabelIds,@Param("prediction") Double prediction); + + /** + * 批量删除标注文件数据 + * + * @param versionFileId 版本文件ID + * @param fileLabelIds 标注标签Ids + */ + void deleteAnnotationFileByVersionIdAndLabelIds(@Param("versionFileId")Long versionFileId, @Param("fileLabelIds") List fileLabelIds); +} diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetGroupLabelMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetGroupLabelMapper.java index a5fee80..1022056 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetGroupLabelMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetGroupLabelMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,8 @@ package org.dubhe.data.dao; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.dubhe.annotation.DataPermission; +import io.lettuce.core.dynamic.annotation.Param; +import org.apache.ibatis.annotations.Delete; import org.dubhe.data.domain.entity.DatasetGroupLabel; /** @@ -26,4 +27,14 @@ import org.dubhe.data.domain.entity.DatasetGroupLabel; * @date 2020-09-22 */ public interface DatasetGroupLabelMapper extends BaseMapper { + + + /** + * 根据标签组ID删除标签和标签组的关联关系 + * + * @param groupId 标签组ID + */ + @Delete("delete from data_group_label where label_group_id = #{groupId} ") + void deleteByGroupId(@Param("groupId") Long groupId); + } diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetLabelMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetLabelMapper.java index 7db507a..55d26df 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetLabelMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetLabelMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetMapper.java index 99c8eeb..30e099b 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,10 +20,10 @@ package org.dubhe.data.dao; import com.baomidou.mybatisplus.core.conditions.Wrapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.ibatis.annotations.*; -import org.apache.ibatis.mapping.FetchType; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; import org.dubhe.annotation.DataPermission; -import org.dubhe.constant.PermissionConstant; import org.dubhe.data.domain.entity.Dataset; /** @@ -49,7 +49,7 @@ public interface DatasetMapper extends BaseMapper { * @param id 数据集ID * @param versionName 数据集版本名称 */ - @Update("update data_dataset set current_version_name = #{versionName} where id = #{id}") + @Update("update data_dataset set current_version_name = #{versionName} where id = #{id}") void updateVersionName(@Param("id") Long id, @Param("versionName") String versionName); /** @@ -61,21 +61,12 @@ public interface DatasetMapper extends BaseMapper { @Update("update data_dataset set status = #{status} where id = #{datasetId}") void updateStatus(@Param("datasetId") Long datasetId, @Param("status") Integer status); - /** - * 更新数据集解压状态 - * @param datasetId 数据集ID - * @param sourceState 压缩开始状态 - * @param targetState 压缩结束状态 - * @return int 被修改行数 - */ - @Update("update data_dataset set decompress_state = #{targetState} where id = #{datasetId} and decompress_state = #{sourceState}") - int updateDecompressState(@Param("datasetId") Long datasetId, @Param("sourceState") Integer sourceState, @Param("targetState") Integer targetState); /** * 获取指定类型数据集的数量 * * @param type 数据集类型 - * @return 公共数据集的数量 + * @return int 公共数据集的数量 */ @Select("SELECT count(1) FROM data_dataset where type = #{type}") int selectCountByPublic(@Param("type") Integer type); @@ -85,7 +76,7 @@ public interface DatasetMapper extends BaseMapper { * 根据标签组ID查询关联的数据集数量 * * @param labelGroupId 标签组ID - * @return 数量 + * @return int 数量 */ @Select("SELECT count(1) FROM data_dataset where label_group_id = #{labelGroupId}") int getCountByLabelGroupId(@Param("labelGroupId")Long labelGroupId); diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionFileMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionFileMapper.java index 68e7ba8..acd6867 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionFileMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionFileMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,12 @@ package org.dubhe.data.dao; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.ibatis.annotations.*; +import org.dubhe.data.domain.dto.DatasetVersionFileDTO; import org.dubhe.data.domain.entity.Dataset; import org.dubhe.data.domain.entity.DatasetVersionFile; import org.dubhe.data.domain.entity.File; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -40,9 +42,8 @@ public interface DatasetVersionFileMapper extends BaseMapper * @param versionName 数据集版本名称 * @return List 数据集ID和版本名称获取正常状态下对应文件列表 */ - @Select("select * from data_dataset_version_file where dataset_id = #{datasetId} " + - "and version_name = #{versionName} and status = 2 ") - List findByDatasetIdAndVersionName(Long datasetId, String versionName); + List findByDatasetIdAndVersionName(@Param("datasetId") Long datasetId, + @Param("versionName") String versionName); /** * 新增关系版本变更 @@ -64,13 +65,7 @@ public interface DatasetVersionFileMapper extends BaseMapper void deleteShip(@Param("datasetId") Long datasetId, @Param("versionName") String versionName, @Param("fileIds") List fileIds); - /** - * 删除数据集下所有版本文件关系 - * - * @param datasetId 数据集ID - */ - @Delete("delete from data_dataset_version_file where dataset_id = #{datasetId}") - void datasetDelete(@Param("datasetId") Long datasetId); + /** * 按数据集和版本查找文件状态列表 @@ -136,7 +131,7 @@ public interface DatasetVersionFileMapper extends BaseMapper /** * 更新数据集版本文件状态 * - * @param datasetVersionFile datasetVersionFile + * @param datasetVersionFile 数据集版本文件实体 * @param status 数据集文件状态 */ @Update("update data_dataset_version_file set annotation_status = #{status} where dataset_id = #{datasetVersionFile.datasetId} " + @@ -147,7 +142,7 @@ public interface DatasetVersionFileMapper extends BaseMapper * 获取数据集的版本文件数据 * * @param datasetIds 数据集IDS - * @return 数据集的版本文件数据 + * @return List 数据集的版本文件数据 */ List listDatasetVersionFileByDatasetIds(List datasetIds); @@ -156,23 +151,11 @@ public interface DatasetVersionFileMapper extends BaseMapper * * @param datasetId 数据集ID * @param versionName 数据集版本名称 - * @return 数据集文件状态统计数据 + * @return Map 数据集文件状态统计数据 */ @MapKey("status") Map getDatasetVersionFileCount(@Param("datasetId") Long datasetId, @Param("versionName") String versionName); - /** - * 分页查询数据集文件中间表 - * - * @param datasetId 数据集ID - * @param versionName 版本名称 - * @param status 文件状态 - * @param offset 偏移量 - * @param limit 页容量 - * @return 数据集版本文件列表 - */ - List getListByDatasetIdAndAnnotationStatus(@Param("datasetId") Long datasetId, @Param("versionName") String versionName, @Param("status") Set status, @Param("offset") Long offset, @Param("limit") Integer limit); - /** * 根据数据集id,版本查询状态为删除的数据版本文件中间表 * @@ -187,8 +170,89 @@ public interface DatasetVersionFileMapper extends BaseMapper * * @param datasetId 数据集ID * @param limitNumber 删除数量 - * @return 成功删除条数 + * @return int 成功删除条数 */ @Delete("delete from data_dataset_version_file where dataset_id = #{datasetId} limit #{limitNumber} ") int deleteBydatasetId(@Param("datasetId") Long datasetId, @Param("limitNumber") int limitNumber); -} + + + + + /** + * 通过数据集ID和状态获取ID + * + * @param datasetId 数据集ID + * @param versionName 版本名称 + * @param status 文件状态 + * @param orderByName 排序字段 + * @param offset 偏移量 + * @param limit 页容量 + * @param order 排序方式 + * @return List 数据集版本文件ID列表 + */ + LinkedList getIdByDatasetIdAndAnnotationStatus(@Param("datasetId") Long datasetId, + @Param("versionName") String versionName, + @Param("status") Set status, + @Param("orderByName")String orderByName, + @Param("offset") Long offset, + @Param("limit") Integer limit, + @Param("order") String order, + @Param("labelId")Long labelId); + + /** + * 通过数据集ID和注释状态获取列表 + * + * @param datasetId 数据集ID + * @param orderByName 排序字段 + * @param versionFileIdList 版本文件ID列表 + * @return List 版本文件列表 + */ + LinkedList getListByDatasetIdAndAnnotationStatus(@Param("datasetId") Long datasetId, + @Param("orderByName")String orderByName, + @Param("versionFileIdList")List versionFileIdList, + @Param("order") String order); + + + + /** + * 根据当前数据集和当前版本号修改数据集是否改变 + * + * @param id 数据集id + * @param versionName 版本号 + */ + @Update("update data_dataset_version_file set changed = 1 where dataset_id = #{datasetId} and version_name = #{currentVersionName}") + void updateChanged(@Param("datasetId") Long id, + @Param("currentVersionName") String versionName); + + /** + * 根据文件id 和 当前版本修改数据 + * + * @param prediction 预测值 + * @param fileId 文件ID + * @param currentVersionName 当前版本 + * @param datasetId 数据集ID + */ + void updateByFileIdAndCurrentVersionName(@Param("prediction")int prediction, @Param("fileId")Long fileId, + @Param("currentVersionName")String currentVersionName, @Param("datasetId")Long datasetId); + + /** + * 根据数据集ID/版本名称/文件ID 查询数据集版本文件数据 + * + * @param datasetId 数据集ID + * @param versionName 版本名称 + * @param fileIds 文件IDS + * @return + */ + List selectByDatasetIdAndVersionNameAndFileIds( + @Param("datasetId")Long datasetId, @Param("versionName")String versionName, @Param("fileIds")List fileIds); + + /** + * 修改标注状态接口 + * + * @param annotationStatus 标注状态 + * @param datasetId 数据集id + * @param id Id + */ + @Update("update data_dataset_version_file set annotation_status = #{annotationStatus} where dataset_id = #{datasetId} and id = #{id}") + void updateAnnotationStatusById(@Param("annotationStatus")Integer annotationStatus, @Param("datasetId") Long datasetId, @Param("id")Long id); +} \ No newline at end of file diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionMapper.java index 9da18c3..31665cc 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/DatasetVersionMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,9 +18,10 @@ package org.dubhe.data.dao; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.apache.ibatis.annotations.*; +import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; import org.dubhe.annotation.DataPermission; -import org.dubhe.constant.PermissionConstant; import org.dubhe.data.domain.entity.DatasetVersion; import java.util.List; diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/FileMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/FileMapper.java index 8c077f5..fc2e905 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/FileMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/FileMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; import org.apache.ibatis.annotations.Update; import org.dubhe.annotation.DataPermission; -import org.dubhe.constant.PermissionConstant; import org.dubhe.data.domain.dto.FileCreateDTO; import org.dubhe.data.domain.entity.File; @@ -34,29 +33,9 @@ import java.util.List; * @description 文件信息 Mapper 接口 * @date 2020-04-10 */ -@DataPermission(ignoresMethod = {"insert","getOneById","selectFile"}) +@DataPermission(ignoresMethod = {"insert", "getOneById", "selectFile"}) public interface FileMapper extends BaseMapper { - /** - * 根据offset, limit查询 - * - * @param offset 偏移量 - * @param limit 页容量 - * @param queryWrapper 查询条件 - * @return List File列表 - */ - @Select("select * from data_file ${ew.customSqlSegment} limit #{offset}, #{limit}") - List selectListByLimit(@Param("offset") long offset, @Param("limit") int limit, @Param("ew") Wrapper queryWrapper); - - /** - * 将文件状态改为采样中 - * - * @param id 文件ID - * @param status 文件状态 - * @return updateSampleStatus 执行次数 - */ - @Update("update data_file set status=1 where id = #{id} and status = #{status}") - int updateSampleStatus(@Param("id") Long id, @Param("status") int status); /** * 根据文件ID获取文件 @@ -65,7 +44,7 @@ public interface FileMapper extends BaseMapper { * @return File 文件对象 */ @Select("select * from data_file where id = #{fileId} and dataset_id = #{datasetId}") - File getOneById(@Param("fileId") Long fileId,@Param("datasetId") long datasetId); + File getOneById(@Param("fileId") Long fileId, @Param("datasetId") long datasetId); /** * 批量保存 @@ -94,7 +73,7 @@ public interface FileMapper extends BaseMapper { * @return File 文件详情 */ @Select("select * from data_file where id = #{fileId} and dataset_id=#{datasetId} and deleted=0") - File selectFile(@Param("fileId") Long fileId,@Param("datasetId") Long datasetId); + File selectFile(@Param("fileId") Long fileId, @Param("datasetId") Long datasetId); /** * 分页获取数据集文件 @@ -108,13 +87,13 @@ public interface FileMapper extends BaseMapper { @Select("") - List selectListOne(@Param("datasetId") Long datasetId,@Param("currentVersionName") String currentVersionName,@Param("offset") int offset,@Param("batchSize") int batchSize); + List selectListOne(@Param("datasetId") Long datasetId, @Param("currentVersionName") String currentVersionName, @Param("offset") int offset, @Param("batchSize") int batchSize); /** * 更新文件状态 @@ -131,8 +110,32 @@ public interface FileMapper extends BaseMapper { * * @param datasetId 数据集ID * @param limitNumber 删除数量 - * @return 成功删除条数 + * @return int 成功删除条数 */ @Delete("delete from data_file where dataset_id = #{datasetId} limit #{limitNumber} ") int deleteBydatasetId(@Param("datasetId") Long datasetId, @Param("limitNumber") int limitNumber); + + /** + * 根据版本和数据集ID获取文件url + * + * @Param datasetId 数据集ID + * @Param versionName 版本名 + * @return List url列表 + */ + @Select("select df.url from data_file df left join data_dataset_version_file ddvf on df.id = ddvf.file_id " + + "where ddvf.dataset_id = #{datasetId} and df.dataset_id = #{datasetId} " + + "and ddvf.version_name = #{versionName}") + List selectUrls(@Param("datasetId") Long datasetId, @Param("versionName") String versionName); + + /** + * 根据version.changed获取文件name列表 + * + * @Param datasetId 数据集ID + * @Param changed 版本文件是否改动 + * @return List 名称列表 + */ + @Select("select df.name from data_file df left join data_dataset_version_file ddvf on df.id = ddvf.file_id " + + "where ddvf.dataset_id = #{datasetId} and df.dataset_id = #{datasetId} " + + "and ddvf.changed = #{changed} and ddvf.version_name = #{versionName}") + List selectNames(@Param("datasetId") Long datasetId, @Param("changed") Integer changed,@Param("versionName")String versionName); } diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/LabelGroupMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/LabelGroupMapper.java index e96fe86..664baf1 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/LabelGroupMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/LabelGroupMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ package org.dubhe.data.dao; import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; import org.dubhe.annotation.DataPermission; import org.dubhe.data.domain.entity.LabelGroup; @@ -25,6 +27,18 @@ import org.dubhe.data.domain.entity.LabelGroup; * @description 标签组管理 Mapper 接口 * @date 2020-09-22 */ -@DataPermission(ignoresMethod = {"insert"}) +@DataPermission(ignoresMethod = {"insert","getLabelGroupByDataId","selectById"}) public interface LabelGroupMapper extends BaseMapper { + + + /** + * 根据数据集ID查询标签组信息 + * + * @param datasetId 数据集id + * @return LabelGroup 标签组实体 + */ + @Select("select dlg.id , dlg.label_group_type from data_label_group dlg left join data_dataset dd on dd.label_group_id = dlg.id\n" + + "where dd.id = #{datasetId}") + LabelGroup getLabelGroupByDataId(@Param("datasetId") Long datasetId); + } diff --git a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/LabelMapper.java b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/LabelMapper.java index 9dc490a..465554e 100644 --- a/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/LabelMapper.java +++ b/dubhe-server/dubhe-data/src/main/java/org/dubhe/data/dao/LabelMapper.java @@ -1,5 +1,5 @@ /** - * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * Copyright 2020 Tianshu AI Platform. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; import org.dubhe.data.domain.dto.LabelDTO; import org.dubhe.data.domain.entity.Label; -import org.springframework.security.core.parameters.P; import java.util.List; @@ -38,7 +37,7 @@ public interface LabelMapper extends BaseMapper