From 13065d7141efcc415271b5c0049104ce6635ad00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=8B=E6=B1=9F=E5=AE=9E=E9=AA=8C=E5=AE=A4?= Date: Tue, 27 Oct 2020 14:21:01 +0800 Subject: [PATCH] update --- .../src/main/resources/application-dev.yml | 32 ++++ .../src/main/resources/application-test.yml | 32 ++++ .../src/main/resources/application.yml | 70 ++++++++ .../src/main/resources/application.yml | 15 ++ .../main/resources/config/application-dev.yml | 168 ++++++++++++++++++ .../src/main/resources/config/application.yml | 152 ++++++++++++++++ .../main/resources/config/application-dev.yml | 168 ++++++++++++++++++ .../src/main/resources/config/application.yml | 132 ++++++++++++++ 8 files changed, 769 insertions(+) create mode 100644 dataset-util/src/main/resources/application-dev.yml create mode 100644 dataset-util/src/main/resources/application-test.yml create mode 100644 dataset-util/src/main/resources/application.yml create mode 100644 distribute-train-operator/src/main/resources/application.yml create mode 100644 dubhe-server/dubhe-admin/src/main/resources/config/application-dev.yml create mode 100644 dubhe-server/dubhe-admin/src/main/resources/config/application.yml create mode 100644 dubhe-server/dubhe-task/src/main/resources/config/application-dev.yml create mode 100644 dubhe-server/dubhe-task/src/main/resources/config/application.yml diff --git a/dataset-util/src/main/resources/application-dev.yml b/dataset-util/src/main/resources/application-dev.yml new file mode 100644 index 0000000..423bf89 --- /dev/null +++ b/dataset-util/src/main/resources/application-dev.yml @@ -0,0 +1,32 @@ +#应用名称 +spring: + datasource: + #数据源URL + url: jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true + #数据源用户名 + username: * + #数据源密码 + password: * + + #配置Sharding-JDBC数据源 + shardingsphere: + datasource: + master: + #数据源URL + url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true + #数据源用户名 + username: * + #数据源密码 + password: * + +minio: + #minio地址 + endpoint: 127.0.0.1 + #minio端口号 + port: 9000 + #minio accessKey + accessKey: abcd + #minio secretKey + secretKey: abcd + #minio bucketName + bucketName: dubhe-dev diff --git a/dataset-util/src/main/resources/application-test.yml b/dataset-util/src/main/resources/application-test.yml new file mode 100644 index 0000000..a03e5a1 --- /dev/null +++ b/dataset-util/src/main/resources/application-test.yml @@ -0,0 +1,32 @@ +#应用名称 +spring: + datasource: + #数据源URL + url: jdbc:mysql://127.0.0.1:3306/dubhe-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true + #数据源用户名 + username: * + #数据源密码 + password: * + + #配置Sharding-JDBC数据源 + shardingsphere: + datasource: + master: + #数据源URL + url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true + #数据源用户名 + username: * + #数据源密码 + password: * + +minio: + #minio地址 + endpoint: 127.0.0.1 + #minio端口号 + port: 9000 + #minio accessKey + accessKey: abcd + #minio secretKey + secretKey: abcd + #minio bucketName + bucketName: dubhe-test diff --git a/dataset-util/src/main/resources/application.yml b/dataset-util/src/main/resources/application.yml new file mode 100644 index 0000000..03000e0 --- /dev/null +++ b/dataset-util/src/main/resources/application.yml @@ -0,0 +1,70 @@ +spring: + profiles: + active: dev + + application: + name: dataset-util + datasource: + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: com.mysql.jdbc.Driver + + # 初始化配置 + initial-size: 3 + # 最小连接数 + min-idle: 3 + # 最大连接数 + max-active: 15 + # 获取连接超时时间 + max-wait: 5000 + # 连接有效性检测时间 + time-between-eviction-runs-millis: 90000 + # 最大空闲时间 + min-evictable-idle-time-millis: 1800000 + test-while-idle: true + test-on-borrow: false + test-on-return: false + + validation-query: select 1 + # 配置允许后面的Bean覆盖前面名称重复的Bean + main: + allow-bean-definition-overriding: true + # 配置Sharding-JDBC数据源名称，可配置多个 + shardingsphere: + datasource: + name: master + master: + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy + test-while-idle: true + validation-query: select 1 + + #指定data_dataset表 主键id 生成策略 + sharding: + tables: + data_file: + actual-data-nodes: master.data_file_$->{1..100000} + table-strategy: + standard: + sharding-column: dataset_id + precise-algorithm-class-name: org.dubhe.datasetutil.common.util.MyPreciseShardingAlgorithm + + data_dataset_version_file: + actual-data-nodes: master.data_dataset_version_file_$->{1..100000} + table-strategy: + standard: + sharding-column: dataset_id + precise-algorithm-class-name: org.dubhe.datasetutil.common.util.MyPreciseShardingAlgorithm + props: + sql: + show: true + +minio: + secure: false + +#logback +logging.config: + classpath:logback-spring-dev.xml + +mybatis-plus: + global-config: + banner: false \ No newline at end of file diff --git a/distribute-train-operator/src/main/resources/application.yml b/distribute-train-operator/src/main/resources/application.yml new file mode 100644 index 0000000..6b9c4c5 --- /dev/null +++ b/distribute-train-operator/src/main/resources/application.yml @@ -0,0 +1,15 @@ +server: + port: 8090 + +spring: + redis: + #数据库索引 + database: 0 + host: + port: 6379 + password: + #连接超时时间 + timeout: 5000 + +k8s: + kubeconfig: kubeconfig \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/resources/config/application-dev.yml b/dubhe-server/dubhe-admin/src/main/resources/config/application-dev.yml new file mode 100644 index 0000000..c6d98c7 --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/resources/config/application-dev.yml @@ -0,0 +1,168 @@ +server: + port: 8000 + # rest API 版本号 + rest-version: v1 + +#配置数据源 +spring: + redis: + #数据库索引 + database: 0 + host: + port: 6379 + password: + #连接超时时间 + timeout: 5000 + datasource: + druid: + db-type: com.alibaba.druid.pool.DruidDataSource + driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy + url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true + username: + password: + + # 初始化配置 + initial-size: 3 + # 最小连接数 + min-idle: 3 + # 最大连接数 + max-active: 15 + # 获取连接超时时间 + max-wait: 5000 + # 连接有效性检测时间 + time-between-eviction-runs-millis: 90000 + # 最大空闲时间 + min-evictable-idle-time-millis: 1800000 + test-while-idle: true + test-on-borrow: false + test-on-return: false + + validation-query: select 1 + # 配置监控统计拦截的filters + filters: stat + stat-view-servlet: + url-pattern: /druid/* + reset-enable: false + + web-stat-filter: + url-pattern: /* + exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" + # 配置允许后面的Bean覆盖前面名称重复的Bean + main: + allow-bean-definition-overriding: true + + # 配置Sharding-JDBC数据源名称，可配置多个 + shardingsphere: + datasource: + name: master + master: + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy + url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true + username: + password: + + #指定data_dataset表 主键id 生成策略 + sharding: + tables: + data_file: + actual-data-nodes: master.data_file_$->{1..100000} + table-strategy: + standard: + sharding-column: dataset_id + precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm + + data_dataset_version_file: + actual-data-nodes: master.data_dataset_version_file_$->{1..100000} + table-strategy: + standard: + sharding-column: dataset_id + precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm + props: + sql: + show: true + +k8s: + # k8s集群配置文件 + kubeconfig: kubeconfig + # nfs服务暴露的IP地址 如需测试需修改为合适的地址 + nfs: + #nfs服务端 共享目录 + nfs-root-path: /nfs/ + nfs-root-windows-path: "Z:" + # 命名空间关键字 + namespace: namespace + # k8s ingress域名 如需测试需修改为合适的域名 + host: + # k8s ingress-controller 对外port + port: + # elasticsearch暴露的服务地址 + elasticsearch: + hostlist: ${eshostlist::} + # 日志采集配置信息 + log: + type: _doc + # 过滤源字段 + source_field: log,@timestamp,kubernetes.pod_name + # 异步回调 + callback: + # boot 单机部署即回调本机实例 + url: localhost:${server.port} + token: + # 秘钥 + secret-key: 1qaz2wsx + # 过期时间（秒） + expire-seconds: 300 + # 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址 + pod: + metrics: + grafanaUrl: http://127.0.0.1:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod= + nfs-storage-class-name: zjlab-nfs-storage +#配置harbor +harbor: + address: + username: + password: + model-name: train +# data模块配置 +data: + annotation: + # 自动标注服务url,最后一位需加'/' + endpoint: http://127.0.0.1/annotation-dev/ + task: + # 自动标注任务分割的文件split size + splitSize: 16 + # 单位ms + retryInterval: 5000 + # 任务不更新置为失败的时间单位秒 + failTime: 43200 + track: + # 自动追踪服务url + endpoint: http://127.0.0.1:7091/ + server: + # 文件存储服务器用户名 + userName: root + enhance: + # 数据增强 + endpoint: http://127.0.0.1:8070/ + imageNet: + # imageNet + endpoint: http://127.0.0.1:8092/ + ofrecord: + # 二进制转换 + endpoint: http://127.0.0.1:8093/ + #数据集训练配置 + ptversion: http://127.0.0.1:8000/ + +# minio配置 +minio: + url: http://127.0.0.1:9000/ + accessKey: + secretKey: + bucketName: dubhe-dev + presignedUrlExpiryTime: 300 + annotation: /annotation/ + +#logback +logging.config: + classpath:logback-spring-dev.xml \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/resources/config/application.yml b/dubhe-server/dubhe-admin/src/main/resources/config/application.yml new file mode 100644 index 0000000..23f74de --- /dev/null +++ b/dubhe-server/dubhe-admin/src/main/resources/config/application.yml @@ -0,0 +1,152 @@ +spring: + profiles: + active: dev + jackson: + time-zone: GMT+8 + date-format: yyyy-MM-dd HH:mm:ss + data: + redis: + repositories: + enabled: false + + # 邮箱配置 + mail: + host: + # 邮件的发送者 163邮箱(开发测试时使用 发送延时 20秒，每天发送量限制 50) + username: + # SMTP授权密码 + password: + protocol: smtp + properties.mail.smtp.auth: true + properties.mail.smtp.port: 465 #465或者994 + properties.mail.smtp.starttls.enable: true + properties.mail.smtp.starttls.required: true + properties.mail.smtp.ssl.enable: true + default-encoding: utf-8 + +#登录图形验证码有效时间/分钟 +loginCode: + expiration: 2 + codeKey: "validate_code" + width: 108 + height: 28 + length: 4 + +#密码加密传输，前端公钥加密，后端私钥解密 +rsa: + private_key: + public_key: + +#jwt +jwt: + header: Authorization + # 令牌前缀 + token-start-with: Bearer + # 必须使用最少88位的Base64对该令牌进行编码 + base64-secret: + # 令牌过期时间 此处单位/毫秒 ，默认24小时，可在此网站生成 https://www.convertworld.com/zh-hans/time/milliseconds.html + token-validity-in-seconds: 86400000 + # 在线用户key + online-key: online-token + # 验证码 + code-key: code-key + +#是否开启 swagger-ui +swagger: + enabled: true + +# 后台添加用户的初始密码 +initial_password: "" + +train-job: + namespace: "namespace-" + separator: "-" + version-label: "v" + pod-name: "platform/resource-name" + python-format: " --" + manage: "train-manage" + out-path: "out" + log-path: "log" + load-path: "load" + load-key: "model_load_dir" + load-val-dataset-key: "val_data_url" + visualized-log-path: "visualizedlog" + docker-dataset-path: "/dataset" + docker-train-path: "/workspace" + docker-model-path: "/modeldir" + docker-val-dataset-path: "/valdataset" + docker-out-path: "train_out=${train-job.docker-train-path}/${train-job.out-path}" + docker-log-path: "train_log=${train-job.docker-train-path}/${train-job.log-path}" + docker-visualized-log-path: "train_visualized_log=${train-job.docker-train-path}/${train-job.visualized-log-path}" + docker-dataset: "data_url=${train-job.docker-dataset-path}" + eight: "8" + plus-eight: "+8" + node-ips: "node_ips" + node-num: "num_nodes" + gpu-num-per-node: "gpu_num_per_node" + + + +minioweb: + GetToken: + url: minio/webrpc + param: + id: 2 + jsonrpc: 2.0 + method: Web.Login + zip: + url: minio/zip?token= + + +train-algorithm: + #是否输出训练结果 + is-train-out: true + #是否输出训练日志 + is-train-log: true + #是否输出可视化日志 + is-visualized-log: false + #设置默认算法来源(1为我的算法，2为预置算法) + algorithm-source: 1 + #设置fork默认值 + fork: false + + +# 配置slq打印日志 +logging: + level: + com: + lawt: + repository: + mapper: debug + +# 配置转发 +dubhe-proxy: + visual: + keyword: visual + server: + port: + refine: + keyword: refine + server: localhost + port: + +# 延时全局配置 +delay: + notebook: + #模型开发延时关闭时间 + delete: 240 + +#垃圾回收默认配置 +recycle: + #过期时间设置 + timeout: + #用户上传文件至临时路径下后文件最大有效时长，以小时为单位 + file-valid: 24 + #用户删除数据后，默认其文件最大有效时长，以天为单位 + date: 7 + #用户删除某一算法后，其算法文件最大有效时长，以天为单位 + algorithm-valid: 3 + #用户删除某一模型后，其模型文件最大有效时长，以天为单位 + model-valid: 3 + #用户删除训练任务后，其训练管理文件最大有效时长，以天为单位 + train-valid: 3 diff --git a/dubhe-server/dubhe-task/src/main/resources/config/application-dev.yml b/dubhe-server/dubhe-task/src/main/resources/config/application-dev.yml new file mode 100644 index 0000000..6f688ed --- /dev/null +++ b/dubhe-server/dubhe-task/src/main/resources/config/application-dev.yml @@ -0,0 +1,168 @@ +server: + port: 9527 + # rest API 版本号 + rest-version: v1 + +#配置数据源 +spring: + redis: + #数据库索引 + database: 0 + host: + port: 6379 + password: + #连接超时时间 + timeout: 5000 + datasource: + druid: + db-type: com.alibaba.druid.pool.DruidDataSource + driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy + url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true + username: + password: + + # 初始化配置 + initial-size: 3 + # 最小连接数 + min-idle: 3 + # 最大连接数 + max-active: 15 + # 获取连接超时时间 + max-wait: 5000 + # 连接有效性检测时间 + time-between-eviction-runs-millis: 90000 + # 最大空闲时间 + min-evictable-idle-time-millis: 1800000 + test-while-idle: true + test-on-borrow: false + test-on-return: false + + validation-query: select 1 + # 配置监控统计拦截的filters + filters: stat + stat-view-servlet: + url-pattern: /druid/* + reset-enable: false + + web-stat-filter: + url-pattern: /* + exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" + # 配置允许后面的Bean覆盖前面名称重复的Bean + main: + allow-bean-definition-overriding: true + + # 配置Sharding-JDBC数据源名称，可配置多个 + shardingsphere: + datasource: + name: master + master: + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy + url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true + username: + password: + + #指定data_dataset表 主键id 生成策略 + sharding: + tables: + data_file: + actual-data-nodes: master.data_file_$->{1..100000} + table-strategy: + standard: + sharding-column: dataset_id + precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm + + data_dataset_version_file: + actual-data-nodes: master.data_dataset_version_file_$->{1..100000} + table-strategy: + standard: + sharding-column: dataset_id + precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm + props: + sql: + show: true + +k8s: + # k8s集群配置文件 + kubeconfig: kubeconfig + # nfs服务暴露的IP地址 如需测试需修改为合适的地址 + nfs: + #nfs服务端 共享目录 + nfs-root-path: /nfs/ + nfs-root-windows-path: "Z:" + # 命名空间关键字 + namespace: namespace + # k8s ingress域名 如需测试需修改为合适的域名 + host: + # k8s ingress-controller 对外port + port: 32493 + # elasticsearch暴露的服务地址 + elasticsearch: + hostlist: ${eshostlist::32321} + # 日志采集配置信息 + log: + type: _doc + # 过滤源字段 + source_field: log,@timestamp,kubernetes.pod_name + # 异步回调 + callback: + # boot 单机部署即回调本机实例 + url: localhost:${server.port} + token: + # 秘钥 + secret-key: 1qaz2wsx + # 过期时间（秒） + expire-seconds: 300 + # 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址 + pod: + metrics: + grafanaUrl: http://127.0.0.1:30006/d/Sx0_4-WGk/jian-kong-xin-xi?orgId=1&refresh=5s&kiosk&var-pod= + nfs-storage-class-name: zjlab-nfs-storage +#配置harbor +harbor: + address: + username: + password: + model-name: train +# data模块配置 +data: + annotation: + # 自动标注服务url,最后一位需加'/' + endpoint: http://127.0.0.1/annotation-dev/ + task: + # 自动标注任务分割的文件split size + splitSize: 16 + # 单位ms + retryInterval: 5000 + # 任务不更新置为失败的时间单位秒 + failTime: 43200 + track: + # 自动追踪服务url + endpoint: http://127.0.0.1:7091/ + server: + # 文件存储服务器用户名 + userName: root + enhance: + # 数据增强 + endpoint: http://127.0.0.1:8070/ + imageNet: + # imageNet + endpoint: http://127.0.0.1:8092/ + ofrecord: + # 二进制转换 + endpoint: http://127.0.0.1:8093/ + #数据集训练配置 + ptversion: http://127.0.0.1:8000/ + +# minio配置 +minio: + url: http://127.0.0.1:9000/ + accessKey: + secretKey: + bucketName: dubhe-dev + presignedUrlExpiryTime: 300 + annotation: /annotation/ + +#logback +logging.config: + classpath:logback-spring-dev.xml \ No newline at end of file diff --git a/dubhe-server/dubhe-task/src/main/resources/config/application.yml b/dubhe-server/dubhe-task/src/main/resources/config/application.yml new file mode 100644 index 0000000..26b5dd3 --- /dev/null +++ b/dubhe-server/dubhe-task/src/main/resources/config/application.yml @@ -0,0 +1,132 @@ +spring: + profiles: + active: dev + jackson: + time-zone: GMT+8 + date-format: yyyy-MM-dd HH:mm:ss + data: + redis: + repositories: + enabled: false + # 邮箱配置 + mail: + host: + # 邮件的发送者 163邮箱(开发测试时使用 发送延时 20秒，每天发送量限制 50) + username: + # SMTP授权密码 + password: + protocol: smtp + properties.mail.smtp.auth: true + properties.mail.smtp.port: 465 #465或者994 + properties.mail.smtp.starttls.enable: true + properties.mail.smtp.starttls.required: true + properties.mail.smtp.ssl.enable: true + default-encoding: utf-8 + +#登录图形验证码有效时间/分钟 +loginCode: + expiration: 2 + codeKey: "validate_code" + width: 108 + height: 28 + length: 4 + +#密码加密传输，前端公钥加密，后端私钥解密 +rsa: + private_key: + public_key: + +#jwt +jwt: + header: Authorization + # 令牌前缀 + token-start-with: Bearer + # 必须使用最少88位的Base64对该令牌进行编码 + base64-secret: + # 令牌过期时间 此处单位/毫秒 ，默认24小时，可在此网站生成 https://www.convertworld.com/zh-hans/time/milliseconds.html + token-validity-in-seconds: 86400000 + # 在线用户key + online-key: online-token + # 验证码 + code-key: code-key + +#是否开启 swagger-ui +swagger: + enabled: true + +# 后台添加用户的初始密码 +initial_password: "" + +train-job: + namespace: "namespace-" + separator: "-" + version-label: "v" + pod-name: "platform/resource-name" + python-format: " --" + manage: "train-manage" + out-path: "out" + log-path: "log" + load-path: "load" + load-key: "model_load_dir" + visualized-log-path: "visualizedlog" + docker-dataset-path: "/dataset" + docker-train-path: "/train" + docker-out-path: "train_out=${train-job.docker-train-path}/${train-job.out-path}" + docker-log-path: "train_log=${train-job.docker-train-path}/${train-job.log-path}" + docker-visualized-log-path: "train_visualized_log=${train-job.docker-train-path}/${train-job.visualized-log-path}" + docker-dataset: "data_url=${train-job.docker-dataset-path}" + eight: "8" + plus-eight: "+8" + +minioweb: + GetToken: + url: minio/webrpc + param: + id: 2 + jsonrpc: 2.0 + method: Web.Login + zip: + url: minio/zip?token= + +train-algorithm: + #是否输出训练结果 + is-train-out: true + #是否输出训练日志 + is-train-log: true + #是否输出可视化日志 + is-visualized-log: false + #设置默认算法来源(1为我的算法，2为预置算法) + algorithm-source: 1 + #设置fork默认值 + fork: false + #上传算法文件路径名 + upload-algorithm-path: "upload-temp/algorithm-manage" + +docker: + host: + port: + +# 配置slq打印日志 +logging: + level: + com: + lawt: + repository: + mapper: debug + +# 配置转发 +dubhe-proxy: + visual: + keyword: visual + server: 10.5.18.239 + port: 9898 + refine: + keyword: refine + server: localhost + port: 9797 + +# 延时全局配置 +delay: + notebook: + #模型开发延时关闭时间 + delete: 240 \ No newline at end of file