update

dataset-util/src/main/resources/application-dev.yml

@@ -0,0 +1,32 @@
+#应用名称
+spring:
+  datasource:
+    #数据源URL
+    url: jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
+    #数据源用户名
+    username: *
+    #数据源密码
+    password: *
+
+  #配置Sharding-JDBC数据源
+  shardingsphere:
+    datasource:
+      master:
+        #数据源URL
+        url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
+        #数据源用户名
+        username: *
+        #数据源密码
+        password: *
+
+minio:
+  #minio地址
+  endpoint: 127.0.0.1
+  #minio端口号
+  port: 9000
+  #minio accessKey
+  accessKey: abcd
+  #minio secretKey
+  secretKey: abcd
+  #minio bucketName
+  bucketName: dubhe-dev

dataset-util/src/main/resources/application-test.yml

@@ -0,0 +1,32 @@
+#应用名称
+spring:
+  datasource:
+    #数据源URL
+    url: jdbc:mysql://127.0.0.1:3306/dubhe-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
+    #数据源用户名
+    username: *
+    #数据源密码
+    password: *
+
+  #配置Sharding-JDBC数据源
+  shardingsphere:
+    datasource:
+      master:
+        #数据源URL
+        url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-test?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
+        #数据源用户名
+        username: *
+        #数据源密码
+        password: *
+
+minio:
+  #minio地址
+  endpoint: 127.0.0.1
+  #minio端口号
+  port: 9000
+  #minio accessKey
+  accessKey: abcd
+  #minio secretKey
+  secretKey: abcd
+  #minio bucketName
+  bucketName: dubhe-test

dataset-util/src/main/resources/application.yml

@@ -0,0 +1,70 @@
+spring:
+  profiles:
+    active: dev
+
+  application:
+    name: dataset-util
+    datasource:
+      type: com.alibaba.druid.pool.DruidDataSource
+      driver-class-name: com.mysql.jdbc.Driver
+
+      # 初始化配置
+      initial-size: 3
+      # 最小连接数
+      min-idle: 3
+      # 最大连接数
+      max-active: 15
+      # 获取连接超时时间
+      max-wait: 5000
+      # 连接有效性检测时间
+      time-between-eviction-runs-millis: 90000
+      # 最大空闲时间
+      min-evictable-idle-time-millis: 1800000
+      test-while-idle: true
+      test-on-borrow: false
+      test-on-return: false
+
+      validation-query: select 1
+  # 配置允许后面的Bean覆盖前面名称重复的Bean
+  main:
+    allow-bean-definition-overriding: true
+  # 配置Sharding-JDBC数据源名称，可配置多个
+  shardingsphere:
+    datasource:
+      name: master
+      master:
+        type: com.alibaba.druid.pool.DruidDataSource
+        driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
+        test-while-idle: true
+        validation-query: select 1
+
+    #指定data_dataset表 主键id 生成策略
+    sharding:
+      tables:
+        data_file:
+          actual-data-nodes: master.data_file_$->{1..100000}
+          table-strategy:
+            standard:
+              sharding-column: dataset_id
+              precise-algorithm-class-name: org.dubhe.datasetutil.common.util.MyPreciseShardingAlgorithm
+
+        data_dataset_version_file:
+          actual-data-nodes: master.data_dataset_version_file_$->{1..100000}
+          table-strategy:
+            standard:
+              sharding-column: dataset_id
+              precise-algorithm-class-name: org.dubhe.datasetutil.common.util.MyPreciseShardingAlgorithm
+      props:
+        sql:
+          show: true
+
+minio:
+  secure: false
+
+#logback
+logging.config:
+  classpath:logback-spring-dev.xml
+
+mybatis-plus:
+  global-config:
+    banner: false

distribute-train-operator/src/main/resources/application.yml

@@ -0,0 +1,15 @@
+server:
+  port: 8090
+
+spring:
+  redis:
+    #数据库索引
+    database: 0
+    host:
+    port: 6379
+    password:
+    #连接超时时间
+    timeout: 5000
+
+k8s:
+  kubeconfig: kubeconfig

dubhe-server/dubhe-admin/src/main/resources/config/application-dev.yml

@@ -0,0 +1,168 @@
+server:
+  port: 8000
+  # rest API 版本号
+  rest-version: v1
+
+#配置数据源
+spring:
+  redis:
+    #数据库索引
+    database: 0
+    host:
+    port: 6379
+    password:
+    #连接超时时间
+    timeout: 5000
+  datasource:
+    druid:
+      db-type: com.alibaba.druid.pool.DruidDataSource
+      driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
+      url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
+      username:
+      password:
+
+      # 初始化配置
+      initial-size: 3
+      # 最小连接数
+      min-idle: 3
+      # 最大连接数
+      max-active: 15
+      # 获取连接超时时间
+      max-wait: 5000
+      # 连接有效性检测时间
+      time-between-eviction-runs-millis: 90000
+      # 最大空闲时间
+      min-evictable-idle-time-millis: 1800000
+      test-while-idle: true
+      test-on-borrow: false
+      test-on-return: false
+
+      validation-query: select 1
+      # 配置监控统计拦截的filters
+      filters: stat
+      stat-view-servlet:
+        url-pattern: /druid/*
+        reset-enable: false
+
+      web-stat-filter:
+        url-pattern: /*
+        exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*"
+  # 配置允许后面的Bean覆盖前面名称重复的Bean
+  main:
+    allow-bean-definition-overriding: true
+
+  # 配置Sharding-JDBC数据源名称，可配置多个
+  shardingsphere:
+    datasource:
+      name: master
+      master:
+        type: com.alibaba.druid.pool.DruidDataSource
+        driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
+        url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
+        username:
+        password:
+
+    #指定data_dataset表 主键id 生成策略
+    sharding:
+      tables:
+        data_file:
+          actual-data-nodes: master.data_file_$->{1..100000}
+          table-strategy:
+            standard:
+              sharding-column: dataset_id
+              precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm
+
+        data_dataset_version_file:
+          actual-data-nodes: master.data_dataset_version_file_$->{1..100000}
+          table-strategy:
+            standard:
+              sharding-column: dataset_id
+              precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm
+      props:
+        sql:
+          show: true
+
+k8s:
+  # k8s集群配置文件
+  kubeconfig: kubeconfig
+  # nfs服务暴露的IP地址 如需测试需修改为合适的地址
+  nfs:
+  #nfs服务端 共享目录
+  nfs-root-path: /nfs/
+  nfs-root-windows-path: "Z:"
+  # 命名空间关键字
+  namespace: namespace
+  # k8s ingress域名 如需测试需修改为合适的域名
+  host:
+  # k8s ingress-controller 对外port
+  port:
+  # elasticsearch暴露的服务地址
+  elasticsearch:
+    hostlist: ${eshostlist::}
+    # 日志采集配置信息
+    log:
+      type: _doc
+      # 过滤源字段
+      source_field: log,@timestamp,kubernetes.pod_name
+  # 异步回调
+  callback:
+    # boot 单机部署即回调本机实例
+    url: localhost:${server.port}
+    token:
+      # 秘钥
+      secret-key: 1qaz2wsx
+      # 过期时间（秒）
+      expire-seconds: 300
+  # 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址
+  pod:
+    metrics:
+      grafanaUrl: http://127.0.0.1:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod=
+  nfs-storage-class-name: zjlab-nfs-storage
+#配置harbor
+harbor:
+  address:
+  username:
+  password:
+  model-name: train
+# data模块配置
+data:
+  annotation:
+    # 自动标注服务url,最后一位需加'/'
+    endpoint: http://127.0.0.1/annotation-dev/
+    task:
+      # 自动标注任务分割的文件split size
+      splitSize: 16
+      # 单位ms
+      retryInterval: 5000
+      # 任务不更新置为失败的时间单位秒
+      failTime: 43200
+  track:
+    # 自动追踪服务url
+    endpoint: http://127.0.0.1:7091/
+  server:
+    # 文件存储服务器用户名
+    userName: root
+  enhance:
+    # 数据增强
+    endpoint: http://127.0.0.1:8070/
+  imageNet:
+    # imageNet
+    endpoint: http://127.0.0.1:8092/
+  ofrecord:
+    # 二进制转换
+    endpoint: http://127.0.0.1:8093/
+  #数据集训练配置
+  ptversion: http://127.0.0.1:8000/
+
+# minio配置
+minio:
+  url: http://127.0.0.1:9000/
+  accessKey:
+  secretKey:
+  bucketName: dubhe-dev
+  presignedUrlExpiryTime: 300
+  annotation: /annotation/
+
+#logback
+logging.config:
+  classpath:logback-spring-dev.xml

dubhe-server/dubhe-admin/src/main/resources/config/application.yml

@@ -0,0 +1,152 @@
+spring:
+  profiles:
+    active: dev
+  jackson:
+    time-zone: GMT+8
+    date-format: yyyy-MM-dd HH:mm:ss
+  data:
+    redis:
+      repositories:
+        enabled: false
+
+  # 邮箱配置
+  mail:
+    host:
+    # 邮件的发送者 163邮箱(开发测试时使用 发送延时 20秒，每天发送量限制 50)
+    username:
+    # SMTP授权密码
+    password:
+    protocol: smtp
+    properties.mail.smtp.auth: true
+    properties.mail.smtp.port: 465 #465或者994
+    properties.mail.smtp.starttls.enable: true
+    properties.mail.smtp.starttls.required: true
+    properties.mail.smtp.ssl.enable: true
+    default-encoding: utf-8
+
+#登录图形验证码有效时间/分钟
+loginCode:
+  expiration: 2
+  codeKey: "validate_code"
+  width: 108
+  height: 28
+  length: 4
+
+#密码加密传输，前端公钥加密，后端私钥解密
+rsa:
+  private_key:
+  public_key:
+
+#jwt
+jwt:
+  header: Authorization
+  # 令牌前缀
+  token-start-with: Bearer
+  # 必须使用最少88位的Base64对该令牌进行编码
+  base64-secret:
+  # 令牌过期时间 此处单位/毫秒 ，默认24小时，可在此网站生成 https://www.convertworld.com/zh-hans/time/milliseconds.html
+  token-validity-in-seconds: 86400000
+  # 在线用户key
+  online-key: online-token
+  # 验证码
+  code-key: code-key
+
+#是否开启 swagger-ui
+swagger:
+  enabled: true
+
+# 后台添加用户的初始密码
+initial_password: ""
+
+train-job:
+  namespace: "namespace-"
+  separator: "-"
+  version-label: "v"
+  pod-name: "platform/resource-name"
+  python-format: " --"
+  manage: "train-manage"
+  out-path: "out"
+  log-path: "log"
+  load-path: "load"
+  load-key: "model_load_dir"
+  load-val-dataset-key: "val_data_url"
+  visualized-log-path: "visualizedlog"
+  docker-dataset-path: "/dataset"
+  docker-train-path: "/workspace"
+  docker-model-path: "/modeldir"
+  docker-val-dataset-path: "/valdataset"
+  docker-out-path: "train_out=${train-job.docker-train-path}/${train-job.out-path}"
+  docker-log-path: "train_log=${train-job.docker-train-path}/${train-job.log-path}"
+  docker-visualized-log-path: "train_visualized_log=${train-job.docker-train-path}/${train-job.visualized-log-path}"
+  docker-dataset: "data_url=${train-job.docker-dataset-path}"
+  eight: "8"
+  plus-eight: "+8"
+  node-ips: "node_ips"
+  node-num: "num_nodes"
+  gpu-num-per-node: "gpu_num_per_node"
+
+
+
+minioweb:
+  GetToken:
+    url: minio/webrpc
+    param:
+      id: 2
+      jsonrpc: 2.0
+      method: Web.Login
+  zip:
+    url: minio/zip?token=
+
+
+train-algorithm:
+  #是否输出训练结果
+  is-train-out: true
+  #是否输出训练日志
+  is-train-log: true
+  #是否输出可视化日志
+  is-visualized-log: false
+  #设置默认算法来源(1为我的算法，2为预置算法)
+  algorithm-source: 1
+  #设置fork默认值
+  fork: false
+
+
+# 配置slq打印日志
+logging:
+  level:
+    com:
+      lawt:
+        repository:
+          mapper: debug
+
+# 配置转发
+dubhe-proxy:
+  visual:
+    keyword: visual
+    server:
+    port:
+  refine:
+    keyword: refine
+    server: localhost
+    port:
+
+# 延时全局配置
+delay:
+  notebook:
+    #模型开发延时关闭时间
+    delete: 240
+
+#垃圾回收默认配置
+recycle:
+  #过期时间设置
+  timeout:
+    #用户上传文件至临时路径下后文件最大有效时长，以小时为单位
+    file-valid: 24
+    #用户删除数据后，默认其文件最大有效时长，以天为单位
+    date: 7
+    #用户删除某一算法后，其算法文件最大有效时长，以天为单位
+    algorithm-valid: 3
+    #用户删除某一模型后，其模型文件最大有效时长，以天为单位
+    model-valid: 3
+    #用户删除训练任务后，其训练管理文件最大有效时长，以天为单位
+    train-valid: 3

dubhe-server/dubhe-task/src/main/resources/config/application-dev.yml

@@ -0,0 +1,168 @@
+server:
+  port: 9527
+  # rest API 版本号
+  rest-version: v1
+
+#配置数据源
+spring:
+  redis:
+    #数据库索引
+    database: 0
+    host:
+    port: 6379
+    password:
+    #连接超时时间
+    timeout: 5000
+  datasource:
+    druid:
+      db-type: com.alibaba.druid.pool.DruidDataSource
+      driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
+      url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true
+      username:
+      password:
+
+      # 初始化配置
+      initial-size: 3
+      # 最小连接数
+      min-idle: 3
+      # 最大连接数
+      max-active: 15
+      # 获取连接超时时间
+      max-wait: 5000
+      # 连接有效性检测时间
+      time-between-eviction-runs-millis: 90000
+      # 最大空闲时间
+      min-evictable-idle-time-millis: 1800000
+      test-while-idle: true
+      test-on-borrow: false
+      test-on-return: false
+
+      validation-query: select 1
+      # 配置监控统计拦截的filters
+      filters: stat
+      stat-view-servlet:
+        url-pattern: /druid/*
+        reset-enable: false
+
+      web-stat-filter:
+        url-pattern: /*
+        exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*"
+  # 配置允许后面的Bean覆盖前面名称重复的Bean
+  main:
+    allow-bean-definition-overriding: true
+
+  # 配置Sharding-JDBC数据源名称，可配置多个
+  shardingsphere:
+    datasource:
+      name: master
+      master:
+        type: com.alibaba.druid.pool.DruidDataSource
+        driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy
+        url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true
+        username:
+        password:
+
+    #指定data_dataset表 主键id 生成策略
+    sharding:
+      tables:
+        data_file:
+          actual-data-nodes: master.data_file_$->{1..100000}
+          table-strategy:
+            standard:
+              sharding-column: dataset_id
+              precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm
+
+        data_dataset_version_file:
+          actual-data-nodes: master.data_dataset_version_file_$->{1..100000}
+          table-strategy:
+            standard:
+              sharding-column: dataset_id
+              precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm
+      props:
+        sql:
+          show: true
+
+k8s:
+  # k8s集群配置文件
+  kubeconfig: kubeconfig
+  # nfs服务暴露的IP地址 如需测试需修改为合适的地址
+  nfs:
+  #nfs服务端 共享目录
+  nfs-root-path: /nfs/
+  nfs-root-windows-path: "Z:"
+  # 命名空间关键字
+  namespace: namespace
+  # k8s ingress域名 如需测试需修改为合适的域名
+  host:
+  # k8s ingress-controller 对外port
+  port: 32493
+  # elasticsearch暴露的服务地址
+  elasticsearch:
+    hostlist: ${eshostlist::32321}
+    # 日志采集配置信息
+    log:
+      type: _doc
+      # 过滤源字段
+      source_field: log,@timestamp,kubernetes.pod_name
+  # 异步回调
+  callback:
+    # boot 单机部署即回调本机实例
+    url: localhost:${server.port}
+    token:
+      # 秘钥
+      secret-key: 1qaz2wsx
+      # 过期时间（秒）
+      expire-seconds: 300
+  # 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址
+  pod:
+    metrics:
+      grafanaUrl: http://127.0.0.1:30006/d/Sx0_4-WGk/jian-kong-xin-xi?orgId=1&refresh=5s&kiosk&var-pod=
+  nfs-storage-class-name: zjlab-nfs-storage
+#配置harbor
+harbor:
+  address:
+  username:
+  password:
+  model-name: train
+# data模块配置
+data:
+  annotation:
+    # 自动标注服务url,最后一位需加'/'
+    endpoint: http://127.0.0.1/annotation-dev/
+    task:
+      # 自动标注任务分割的文件split size
+      splitSize: 16
+      # 单位ms
+      retryInterval: 5000
+      # 任务不更新置为失败的时间单位秒
+      failTime: 43200
+  track:
+    # 自动追踪服务url
+    endpoint: http://127.0.0.1:7091/
+  server:
+    # 文件存储服务器用户名
+    userName: root
+  enhance:
+    # 数据增强
+    endpoint: http://127.0.0.1:8070/
+  imageNet:
+    # imageNet
+    endpoint: http://127.0.0.1:8092/
+  ofrecord:
+    # 二进制转换
+    endpoint: http://127.0.0.1:8093/
+  #数据集训练配置
+  ptversion: http://127.0.0.1:8000/
+
+# minio配置
+minio:
+  url: http://127.0.0.1:9000/
+  accessKey:
+  secretKey:
+  bucketName: dubhe-dev
+  presignedUrlExpiryTime: 300
+  annotation: /annotation/
+
+#logback
+logging.config:
+  classpath:logback-spring-dev.xml

dubhe-server/dubhe-task/src/main/resources/config/application.yml

@@ -0,0 +1,132 @@
+spring:
+  profiles:
+    active: dev
+  jackson:
+    time-zone: GMT+8
+    date-format: yyyy-MM-dd HH:mm:ss
+  data:
+    redis:
+      repositories:
+        enabled: false
+  # 邮箱配置
+  mail:
+    host:
+    # 邮件的发送者 163邮箱(开发测试时使用 发送延时 20秒，每天发送量限制 50)
+    username:
+    # SMTP授权密码
+    password:
+    protocol: smtp
+    properties.mail.smtp.auth: true
+    properties.mail.smtp.port: 465 #465或者994
+    properties.mail.smtp.starttls.enable: true
+    properties.mail.smtp.starttls.required: true
+    properties.mail.smtp.ssl.enable: true
+    default-encoding: utf-8
+
+#登录图形验证码有效时间/分钟
+loginCode:
+  expiration: 2
+  codeKey: "validate_code"
+  width: 108
+  height: 28
+  length: 4
+
+#密码加密传输，前端公钥加密，后端私钥解密
+rsa:
+  private_key:
+  public_key:
+
+#jwt
+jwt:
+  header: Authorization
+  # 令牌前缀
+  token-start-with: Bearer
+  # 必须使用最少88位的Base64对该令牌进行编码
+  base64-secret:
+  # 令牌过期时间 此处单位/毫秒 ，默认24小时，可在此网站生成 https://www.convertworld.com/zh-hans/time/milliseconds.html
+  token-validity-in-seconds: 86400000
+  # 在线用户key
+  online-key: online-token
+  # 验证码
+  code-key: code-key
+
+#是否开启 swagger-ui
+swagger:
+  enabled: true
+
+# 后台添加用户的初始密码
+initial_password: ""
+
+train-job:
+  namespace: "namespace-"
+  separator: "-"
+  version-label: "v"
+  pod-name: "platform/resource-name"
+  python-format: " --"
+  manage: "train-manage"
+  out-path: "out"
+  log-path: "log"
+  load-path: "load"
+  load-key: "model_load_dir"
+  visualized-log-path: "visualizedlog"
+  docker-dataset-path: "/dataset"
+  docker-train-path: "/train"
+  docker-out-path: "train_out=${train-job.docker-train-path}/${train-job.out-path}"
+  docker-log-path: "train_log=${train-job.docker-train-path}/${train-job.log-path}"
+  docker-visualized-log-path: "train_visualized_log=${train-job.docker-train-path}/${train-job.visualized-log-path}"
+  docker-dataset: "data_url=${train-job.docker-dataset-path}"
+  eight: "8"
+  plus-eight: "+8"
+
+minioweb:
+  GetToken:
+    url: minio/webrpc
+    param:
+      id: 2
+      jsonrpc: 2.0
+      method: Web.Login
+  zip:
+    url: minio/zip?token=
+
+train-algorithm:
+  #是否输出训练结果
+  is-train-out: true
+  #是否输出训练日志
+  is-train-log: true
+  #是否输出可视化日志
+  is-visualized-log: false
+  #设置默认算法来源(1为我的算法，2为预置算法)
+  algorithm-source: 1
+  #设置fork默认值
+  fork: false
+  #上传算法文件路径名
+  upload-algorithm-path: "upload-temp/algorithm-manage"
+
+docker:
+  host:
+  port:
+
+# 配置slq打印日志
+logging:
+  level:
+    com:
+      lawt:
+        repository:
+          mapper: debug
+
+# 配置转发
+dubhe-proxy:
+  visual:
+    keyword: visual
+    server: 10.5.18.239
+    port: 9898
+  refine:
+    keyword: refine
+    server: localhost
+    port: 9797
+
+# 延时全局配置
+delay:
+  notebook:
+    #模型开发延时关闭时间
+    delete: 240