diff --git a/dataset-util/.gitignore b/dataset-util/.gitignore
new file mode 100644
index 0000000..df214c3
--- /dev/null
+++ b/dataset-util/.gitignore
@@ -0,0 +1,6 @@
+# Created by .ignore support plugin (hsz.mobi)
+.idea
+*.iml
+logs
+HELP.md
+target/
diff --git a/dataset-util/README.md b/dataset-util/README.md
new file mode 100644
index 0000000..124d4d9
--- /dev/null
+++ b/dataset-util/README.md
@@ -0,0 +1,94 @@
+# 之江天枢-数据集导入脚本
+
+**之江天枢一站式人工智能开源平台**(简称:**之江天枢**),为了实现其他平台已标注完成的数据集在「一站式开发平台」上进行开发,我们增加了数据集导入的功能,实现对数据集的全流程功能操作。
+
+## 源码部署
+
+安装如下软件环境。
+- OpenJDK:1.8+
+
+## 下载脚本
+
+- 数据集模板:http://tianshu.org.cn/static/upload/file/dubhe-dataset-template.zip
+- 上传数据集脚本:http://tianshu.org.cn/static/upload/file/upload_dataset.zip
+
+
+## 脚本使用说明:
+
+- 登录天枢系统深度学习平台,在数据管理菜单下的数据集管理中创建数据集。获取数据集ID
+- 需要自行准备图片文件、标注文件、标签文件
+
+## 运行脚本:
+
+1.解压下载的zip文件,需要自行配置数据源、MinIO相关配置
+
+2.运行脚本Windows 运行 run.bat; macOS/Linux 系统运行 run.sh
+
+注:可自行配置'application-{env}.xml'文件,执行命令后面添加 'run.bat {env}'即可执行对应的application-{env}.xml;然后按提示进行操作
+
+3.输入数据集ID
+
+4.输入待导入数据集绝对路径
+
+
+## 目录结构:
+
+```
+标签文件: label_{name}.json ({name} 代表标签组名,可自行定义; 只读标签组文件夹下的第一个标签组文件,标签文件仅支持:.json 支持大小写;文件内容为JSON字符串)
+图片文件目录: origin (图片文件需要有后缀名,支持四种格式:.jpg,.png,.bmp,.jpeg 支持大小写)
+标注文件目录: annotation (标注文件需要有后缀名,仅支持格式:.json 支持大小写; 文件内容为JSON字符串)
+```
+## 文件格式
+
+- 标签文件内容样例:
+```
+ name: 名称
+ color: 颜色(16进制编码)
+```
+详细示例:
+```
+[{
+ "name": "行人",
+ "color": "#ffbb96"
+},
+{
+ "name": "自行车",
+ "color": "#fcffe6"
+},
+{
+ "name": "汽车",
+ "color": "#f4ffb8"
+}]
+```
+
+- 标注文件内容样例:
+```
+ name: 名称
+ bbox: 标注位置
+ score:分数
+```
+详细示例:
+```
+[{
+ "name": "行人",
+ "bbox": [321.6755762696266, 171.32076993584633, 185.67924201488495, 145.02639323472977],
+ "score": 0.6922634840011597
+},
+{
+ "name": "自行车",
+ "bbox": [40.88740050792694, 22.707078605890274, 451.21362805366516, 326.0102793574333],
+ "score": 0.6069411635398865
+}]
+```
+
+## 了解更多
+
+http://docs.dubhe.ai/docs/module/dataset/import-dataset
+
+
+
+
+
+
+
+
diff --git a/dataset-util/pom.xml b/dataset-util/pom.xml
new file mode 100644
index 0000000..171c082
--- /dev/null
+++ b/dataset-util/pom.xml
@@ -0,0 +1,142 @@
+
+
+ 4.0.0
+ org.dubhe
+ dataset-util
+ 0.0.1-SNAPSHOT
+ dataset-util
+ 数据处理模块工具
+
+
+ 1.8
+ UTF-8
+ UTF-8
+ 2.3.0.RELEASE
+
+
+
+
+ org.springframework.boot
+ spring-boot-starter
+
+
+ com.alibaba
+ druid
+ 1.1.6
+
+
+ mysql
+ mysql-connector-java
+ 8.0.17
+
+
+ org.projectlombok
+ lombok
+ true
+
+
+ com.baomidou
+ mybatis-plus-boot-starter
+ 3.2.0
+
+
+ com.alibaba
+ fastjson
+ 1.2.47
+
+
+ io.minio
+ minio
+ 7.0.2
+
+
+ com.xiaoleilu
+ hutool-all
+ 3.0.1
+
+
+
+
+ org.apache.shardingsphere
+ sharding-jdbc-spring-boot-starter
+ 4.0.0-RC1
+
+
+
+ org.apache.commons
+ commons-lang3
+
+
+
+ commons-io
+ commons-io
+ 1.3.2
+
+
+
+ org.aspectj
+ aspectjweaver
+ 1.8.9
+
+
+
+
+ org.bgee.log4jdbc-log4j2
+ log4jdbc-log4j2-jdbc4.1
+ 1.16
+
+
+
+
+ cn.hutool
+ hutool-all
+ 5.0.6
+
+
+
+
+
+
+
+ org.springframework.boot
+ spring-boot-dependencies
+ ${spring-boot.version}
+ pom
+ import
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.8.1
+
+
+ 1.8
+ UTF-8
+
+
+
+ org.springframework.boot
+ spring-boot-maven-plugin
+ 2.3.0.RELEASE
+
+ org.dubhe.datasetutil.DatasetUtilApplication
+
+
+
+ repackage
+
+ repackage
+
+
+
+
+
+
+
+
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/DatasetUtilApplication.java b/dataset-util/src/main/java/org/dubhe/datasetutil/DatasetUtilApplication.java
new file mode 100644
index 0000000..958b170
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/DatasetUtilApplication.java
@@ -0,0 +1,119 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil;
+
+import lombok.extern.slf4j.Slf4j;
+import org.dubhe.datasetutil.common.util.SpringContextHolder;
+import org.dubhe.datasetutil.handle.DatasetImageUploadHandle;
+import org.dubhe.datasetutil.handle.DatasetImportHandle;
+import org.dubhe.datasetutil.common.util.PrintUtils;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ApplicationContext;
+
+import java.util.Scanner;
+
+/**
+ * @description 文档导入工程启动类
+ * @date 2020-09-17
+ */
+@Slf4j
+@SpringBootApplication
+@MapperScan("org.dubhe.datasetutil.dao")
+public class DatasetUtilApplication {
+
+ /**
+ * 主函数
+ *
+ * @param args 入参
+ */
+ public static void main(String[] args) {
+ ApplicationContext applicationContext = SpringApplication.run(DatasetUtilApplication.class, args);
+ SpringContextHolder springContextHolder = new SpringContextHolder();
+ springContextHolder.setApplicationContext(applicationContext);
+ execute(applicationContext);
+ }
+
+ /**
+ * 执行脚本
+ *
+ * @param applicationContext 请求上下文
+ */
+ public static void execute(ApplicationContext applicationContext) {
+ while (true) {
+ Scanner scanner = new Scanner(System.in);
+ log.warn("###################请输入需要执行的任务#############");
+ log.warn("# 输入1.执行上传图片 ");
+ log.warn("# 输入2.执行导入数据集 ");
+ log.warn("# 输入命令 :exit 退出 ");
+ log.warn("################################################");
+ String a = scanner.nextLine();
+ switch (a) {
+ case "1":
+ uploadDatasetImage(scanner, applicationContext);
+ break;
+ case "2":
+ importDataset(scanner, applicationContext);
+ break;
+ case "exit":
+ default:
+ System.exit(0);
+ break;
+ }
+ }
+ }
+
+ /**
+ * 导入图片
+ *
+ * @param scanner 输入控制台
+ * @param applicationContext 请求上下文
+ */
+ public static void uploadDatasetImage(Scanner scanner, ApplicationContext applicationContext) {
+ log.warn("# 请输入数据集ID #");
+ String datasetIdStr = scanner.nextLine();
+ Long datasetId = Long.parseLong(datasetIdStr);
+ log.warn("# 请输入要上传的图片地址 #");
+ String filePath = scanner.nextLine();
+ DatasetImageUploadHandle datasetImageUploadHandle = (DatasetImageUploadHandle) applicationContext.getBean("datasetImageUploadHandle");
+ try {
+ datasetImageUploadHandle.execute(filePath, datasetId);
+ } catch (Exception e) {
+ log.error("", e);
+ log.error("# 数据集上传失败,请重新尝试.....");
+ }
+ }
+
+ /**
+ * 导入数据集
+ *
+ * @param scanner 输入控制台
+ * @param applicationContext 请求上下文
+ */
+ public static void importDataset(Scanner scanner, ApplicationContext applicationContext) {
+ DatasetImportHandle datasetImportHandle = (DatasetImportHandle) applicationContext.getBean("datasetImportHandle");
+ try{
+ datasetImportHandle.importDataset(scanner);
+ } catch (Exception e) {
+ log.error("");
+ PrintUtils.printLine(" Error:" + e.getMessage(), PrintUtils.RED);
+ log.error("");
+ }
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/aspect/LogAspect.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/aspect/LogAspect.java
new file mode 100644
index 0000000..60a3acc
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/aspect/LogAspect.java
@@ -0,0 +1,76 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.dubhe.datasetutil.common.aspect;
+
+import lombok.extern.slf4j.Slf4j;
+import org.aspectj.lang.JoinPoint;
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Pointcut;
+import org.dubhe.datasetutil.common.enums.LogEnum;
+import org.dubhe.datasetutil.common.util.LogUtil;
+import org.slf4j.MDC;
+import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
+
+import java.util.UUID;
+
+/**
+ * @description 日志切面
+ * @date 2020-04-10
+ */
+@Component
+@Aspect
+@Slf4j
+public class LogAspect {
+
+ public static final String TRACE_ID = "traceId";
+
+ @Pointcut("execution(* org.dubhe..service..*.*(..))) ")
+ public void serviceAspect() {
+ }
+
+ @Pointcut(" serviceAspect() ")
+ public void aroundAspect() {
+ }
+
+ @Around("aroundAspect()")
+ public Object around(JoinPoint joinPoint) throws Throwable {
+ if (StringUtils.isEmpty(MDC.get(TRACE_ID))) {
+ MDC.put(TRACE_ID, UUID.randomUUID().toString());
+ }
+ return combineLogInfo(joinPoint);
+ }
+
+ /**
+ * 根据连接点返回结果
+ *
+ * @param joinPoint 连接点
+ * @return Object 返回结果
+ */
+ private Object combineLogInfo(JoinPoint joinPoint) throws Throwable {
+ Object[] param = joinPoint.getArgs();
+ LogUtil.info(LogEnum.REST_REQ, "uri:{},input:{},==>begin", joinPoint.getSignature(), param);
+ long start = System.currentTimeMillis();
+ Object result = ((ProceedingJoinPoint) joinPoint).proceed();
+ long end = System.currentTimeMillis();
+ LogUtil.info(LogEnum.REST_REQ, "uri:{},output:{},proc_time:{},<==end", joinPoint.getSignature().toString(),
+ result, end - start);
+ return result;
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/BaseEntity.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/BaseEntity.java
new file mode 100644
index 0000000..c09fefe
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/BaseEntity.java
@@ -0,0 +1,66 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.base;
+
+import com.baomidou.mybatisplus.annotation.FieldFill;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableLogic;
+import lombok.Data;
+import java.io.Serializable;
+import java.sql.Timestamp;
+
+/**
+ * @description Entity基础类
+ * @date 2020-03-15
+ */
+@Data
+public class BaseEntity implements Serializable {
+
+ private static final long serialVersionUID = 4936056317364745513L;
+
+ /**
+ * 删除标识
+ **/
+ @TableField(value = "deleted",fill = FieldFill.INSERT)
+ @TableLogic
+ private Boolean deleted = false;
+
+ /**
+ * 创建人id
+ **/
+ @TableField(value = "create_user_id",fill = FieldFill.INSERT)
+ private Long createUserId;
+
+ /**
+ * 修改人id
+ **/
+ @TableField(value = "update_user_id",fill = FieldFill.INSERT_UPDATE)
+ private Long updateUserId;
+
+ /**
+ * 创建时间
+ **/
+ @TableField(value = "create_time",fill = FieldFill.INSERT)
+ private Timestamp createTime;
+
+ /**
+ * 修改时间
+ **/
+ @TableField(value = "update_time",fill = FieldFill.INSERT_UPDATE)
+ private Timestamp updateTime;
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/DataResponseBody.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/DataResponseBody.java
new file mode 100644
index 0000000..83ae2aa
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/DataResponseBody.java
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.base;
+
+
+import lombok.Data;
+import org.slf4j.MDC;
+
+import java.io.Serializable;
+
+/**
+ * @description 统一的公共响应体
+ * @date 2020-03-16
+ */
+@Data
+public class DataResponseBody implements Serializable {
+
+ /**
+ * 返回状态码
+ */
+ private Integer code;
+ /**
+ * 返回信息
+ */
+ private String msg;
+ /**
+ * 泛型数据
+ */
+ private T data;
+ /**
+ * 链路追踪ID
+ */
+ private String traceId;
+
+ public DataResponseBody() {
+ this(ResponseCode.SUCCESS, null);
+ }
+
+ public DataResponseBody(T data) {
+ this(ResponseCode.SUCCESS, null, data);
+ }
+
+ public DataResponseBody(Integer code, String msg) {
+ this(code, msg, null);
+ }
+
+ public DataResponseBody(Integer code, String msg, T data) {
+ this.code = code;
+ this.msg = msg;
+ this.data = data;
+ this.traceId = MDC.get("traceId");
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/MagicNumConstant.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/MagicNumConstant.java
new file mode 100644
index 0000000..01472cb
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/MagicNumConstant.java
@@ -0,0 +1,103 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.base;
+
+/**
+ * @description 魔数数值类
+ * @date 2020-09-17
+ */
+public final class MagicNumConstant {
+
+ public static final int NEGATIVE_ONE = -1;
+ public static final int ZERO = 0;
+ public static final int ONE = 1;
+ public static final int TWO = 2;
+ public static final int THREE = 3;
+ public static final int FOUR = 4;
+ public static final int FIVE = 5;
+ public static final int SIX = 6;
+ public static final int SEVEN = 7;
+ public static final int EIGHT = 8;
+ public static final int NINE = 9;
+ public static final int TEN = 10;
+
+ public static final int ELEVEN = 11;
+ public static final int SIXTEEN = 16;
+ public static final int TWENTY = 20;
+ public static final int FIFTY = 50;
+ public static final int SIXTY = 60;
+ public static final int SIXTY_TWO = 62;
+ public static final int SIXTY_FOUR = 64;
+ public static final int INTEGER_TWO_HUNDRED_AND_FIFTY_FIVE = 255;
+ public static final int ONE_HUNDRED = 100;
+ public static final int ONE_HUNDRED_TWENTY_EIGHT = 128;
+ public static final int TWO_HUNDRED = 200;
+ public static final int FIVE_HUNDRED = 500;
+ public static final int FIVE_HUNDRED_AND_SIXTEEN = 516;
+ public static final int ONE_THOUSAND = 1000;
+ public static final int BINARY_TEN_EXP = 1024;
+ public static final int ONE_THOUSAND_ONE_HUNDRED = 1100;
+ public static final int ONE_THOUSAND_ONE_HUNDRED_ONE = 1101;
+ public static final int ONE_THOUSAND_TWO_HUNDRED = 1200;
+ public static final int ONE_THOUSAND_TWO_HUNDRED_ONE = 1201;
+ public static final int ONE_THOUSAND_TWENTY_FOUR = 1024;
+ public static final int ONE_THOUSAND_THREE_HUNDRED = 1300;
+ public static final int ONE_THOUSAND_THREE_HUNDRED_ONE = 1301;
+ public static final int ONE_THOUSAND_THREE_HUNDRED_NINE = 1309;
+ public static final int ONE_THOUSAND_FIVE_HUNDRED = 1500;
+ public static final int TWO_THOUSAND = 2000;
+ public static final int TWO_THOUSAND_TWENTY_EIGHT = 2048;
+ public static final int THREE_THOUSAND = 3000;
+ public static final int FOUR_THOUSAND = 4000;
+ public static final int FIVE_THOUSAND = 5000;
+ public static final int NINE_THOUSAND = 9000;
+ public static final int NINE_THOUSAND_NINE_HUNDRED_NINTY_NINE = 9999;
+ public static final int TEN_THOUSAND = 10000;
+ public static final int FIFTEEN_THOUSAND = 15000;
+ public static final int HUNDRED_THOUSAND = 100000;
+ public static final int MILLION = 1000000;
+ public static final int ONE_MINUTE = 60000;
+
+ public static final long NEGATIVE_ONE__LONG = -1L;
+ public static final long ZERO_LONG = 0L;
+ public static final long ONE_LONG = 1L;
+ public static final long TWO_LONG = 2L;
+ public static final long THREE_LONG = 3L;
+ public static final long FOUR_LONG = 4L;
+ public static final long FIVE_LONG = 5L;
+ public static final long SIX_LONG = 6L;
+ public static final long SEVEN_LONG = 7L;
+ public static final long EIGHT_LONG = 8L;
+ public static final long NINE_LONG = 9L;
+ public static final long TEN_LONG = 10L;
+
+ public static final long TWELVE_LONG = 12L;
+ public static final long SIXTY_LONG = 60L;
+ public static final long FIFTY_LONG = 50L;
+ public static final long THOUSAND_LONG = 1000L;
+ public static final long TEN_THOUSAND_LONG = 10000L;
+ public static final long ONE_ZERO_ONE_ZERO_ONE_ZERO_LONG = 101010L;
+ public static final long NINE_ZERO_NINE_ZERO_NINE_ZERO_LONG = 909090L;
+ public static final long ONE_YEAR_BEFORE_LONG = 1552579200000L;
+
+ public static final int SIXITY_0XFF = 0xFF;
+
+
+ private MagicNumConstant() {
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/ResponseCode.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/ResponseCode.java
new file mode 100644
index 0000000..1985743
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/ResponseCode.java
@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.base;
+
+/**
+ * @description 返回状态码
+ * @date 2020-02-23
+ */
+public class ResponseCode {
+
+ public static Integer SUCCESS = 200;
+ public static Integer UNAUTHORIZED = 401;
+ public static Integer ERROR = 10000;
+ public static Integer ENTITY_NOT_EXIST = 10001;
+ public static Integer BADREQUEST = 10002;
+ public static Integer SERVICE_ERROR = 10003;
+ public static Integer DOCKER_ERROR = 10004;
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MinioConfig.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MinioConfig.java
new file mode 100644
index 0000000..ac938ad
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MinioConfig.java
@@ -0,0 +1,58 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.config;
+
+import io.minio.MinioClient;
+import io.minio.errors.InvalidEndpointException;
+import io.minio.errors.InvalidPortException;
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+/**
+ * @description MinIO 配置类
+ * @date 2020-09-17
+ */
+@Data
+@Component
+@ConfigurationProperties(prefix = "minio")
+public class MinioConfig {
+
+ private String endpoint;
+
+ private int port;
+
+ private String accessKey;
+
+ private String secretKey;
+
+ private Boolean secure;
+
+ private String bucketName;
+
+ /**
+ * 获取Minio客户端信息
+ *
+ * @return Minio客户端信息
+ */
+ @Bean
+ public MinioClient getMinioClient() throws InvalidEndpointException, InvalidPortException {
+ return new MinioClient(endpoint, port, accessKey, secretKey,secure);
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MybatisPlusConfig.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MybatisPlusConfig.java
new file mode 100644
index 0000000..69857be
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MybatisPlusConfig.java
@@ -0,0 +1,37 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.config;
+
+import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @description 配置分页插件
+ * @date 2020-09-17
+ */
+@Configuration
+public class MybatisPlusConfig {
+
+ /**
+ * 分页插件
+ */
+ @Bean
+ public PaginationInterceptor paginationInterceptor() {
+ return new PaginationInterceptor();
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/BusinessConstant.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/BusinessConstant.java
new file mode 100644
index 0000000..9c4d1d9
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/BusinessConstant.java
@@ -0,0 +1,86 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.constant;
+
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+
+/**
+ * @description 常量
+ * @date 2020-10-19
+ */
+public class BusinessConstant {
+
+ private BusinessConstant(){}
+
+ /**
+ * 分表业务编码 - 文件表
+ */
+ public static final String DATA_FILE = "DATA_FILE";
+
+ /**
+ * 分表业务编码 - 文件版本关系表
+ */
+ public static final String DATA_VERSION_FILE = "DATA_VERSION_FILE";
+ /**
+ * 图片文件路径
+ */
+ public static final String IMAGE_ORIGIN = "origin";
+ /**
+ * 标注文件路径
+ */
+ public static final String ANNOTATION = "annotation";
+ /**
+ * 标签文件路径
+ */
+ public static final String LABEL = "label";
+ /**
+ * 分隔符
+ */
+ public static final String FILE_SEPARATOR = "/";
+ /**
+ * 后缀.
+ */
+ public static final String SPOT = ".";
+
+ /**
+ * JSON后缀名
+ */
+ public static final String SUFFIX_JSON = ".JSON";
+ /**
+ * minio根目录
+ */
+ public static final String MINIO_ROOT_PATH = "dataset";
+ /**
+ * 下划线
+ */
+ public static final String UNDERLINE = "_";
+
+ /**
+ * 分段ID范围区间 50表示 50间隔ID存一张表
+ */
+ public static final long INTERVAL_NUMBER = MagicNumConstant.FIFTY_LONG;
+ /**
+ * 分批长度
+ */
+ public static final int SUB_LENGTH = MagicNumConstant.FIVE_THOUSAND;
+
+ /**
+ * 字母Y
+ */
+ public static final String Y = "Y";
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/DataStateCodeConstant.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/DataStateCodeConstant.java
new file mode 100644
index 0000000..b040eab
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/DataStateCodeConstant.java
@@ -0,0 +1,84 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.constant;
+
+/**
+ * @description 数据集状态码
+ * @date 2020-09-03
+ */
+public class DataStateCodeConstant {
+
+ private DataStateCodeConstant() {
+ }
+
+ /**
+ * 未标注
+ */
+ public static final Integer NOT_ANNOTATION_STATE = 101;
+ /**
+ * 手动标注中
+ */
+ public static final Integer MANUAL_ANNOTATION_STATE = 102;
+ /**
+ * 自动标注中
+ */
+ public static final Integer AUTOMATIC_LABELING_STATE = 103;
+ /**
+ * 自动标注完成
+ */
+ public static final Integer AUTO_TAG_COMPLETE_STATE = 104;
+ /**
+ * 标注完成
+ */
+ public static final Integer ANNOTATION_COMPLETE_STATE = 105;
+
+ /**
+ * 目标跟踪中
+ */
+ public static final Integer TARGET_FOLLOW_STATE = 201;
+ /**
+ * 目标跟踪完成
+ */
+ public static final Integer TARGET_COMPLETE_STATE = 202;
+ /**
+ * 目标跟踪失败
+ */
+ public static final Integer TARGET_FAILURE_STATE = 203;
+
+ /**
+ * 未采样
+ */
+ public static final Integer NOT_SAMPLED_STATE = 301;
+ /**
+ * 采样中
+ */
+ public static final Integer SAMPLING_STATE = 302;
+ /**
+ * 采样失败
+ */
+ public static final Integer SAMPLED_FAILURE_STATE = 303;
+
+ /**
+ * 增强中
+ */
+ public static final Integer STRENGTHENING_STATE = 401;
+ /**
+ * 导入中
+ */
+ public static final Integer IN_THE_IMPORT_STATE = 402;
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/FileStateCodeConstant.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/FileStateCodeConstant.java
new file mode 100644
index 0000000..1662e85
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/FileStateCodeConstant.java
@@ -0,0 +1,54 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.constant;
+
+/**
+ * @description 数据集状态码
+ * @date 2020-09-03
+ */
+public class FileStateCodeConstant {
+
+ private FileStateCodeConstant(){
+
+ }
+
+ /**
+ * 未标注
+ */
+ public static final Integer NOT_ANNOTATION_FILE_STATE = 101;
+ /**
+ * 手动标注中
+ */
+ public static final Integer MANUAL_ANNOTATION_FILE_STATE = 102;
+ /**
+ * 自动标注完成
+ */
+ public static final Integer AUTO_TAG_COMPLETE_FILE_STATE = 103;
+ /**
+ * 标注完成
+ */
+ public static final Integer ANNOTATION_COMPLETE_FILE_STATE = 104;
+ /**
+ * 标注未识别
+ */
+ public static final Integer ANNOTATION_NOT_DISTINGUISH_FILE_STATE = 105;
+ /**
+ * 目标跟踪完成
+ */
+ public static final Integer TARGET_COMPLETE_FILE_STATE = 201;
+
+}
\ No newline at end of file
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/enums/LogEnum.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/enums/LogEnum.java
new file mode 100644
index 0000000..857d6eb
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/enums/LogEnum.java
@@ -0,0 +1,52 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.enums;
+
+import lombok.Getter;
+
+/**
+ * @description 日志类型枚举类
+ * @date 2020-06-23
+ */
+@Getter
+public enum LogEnum {
+
+ //系统报错日志
+ SYS_ERR,
+ //用户请求日志
+ REST_REQ,
+ //全局请求日志
+ GLOBAL_REQ,
+ //数据集模块
+ BIZ_DATASET,
+ //分表模块
+ DATA_SEQUENCE;
+
+ /**
+ * 判断日志类型不能为空
+ *
+ * @param logType 日志类型
+ * @return boolean 返回类型
+ */
+ public static boolean isLogType(LogEnum logType) {
+ if (logType != null) {
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/BusinessException.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/BusinessException.java
new file mode 100644
index 0000000..766ab02
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/BusinessException.java
@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.exception;
+
+import lombok.Getter;
+import org.dubhe.datasetutil.common.base.DataResponseBody;
+import org.dubhe.datasetutil.common.base.ResponseCode;
+
+/**
+ * @description 业务异常
+ * @date 2020-03-13
+ */
+@Getter
+public class BusinessException extends RuntimeException {
+
+ private DataResponseBody responseBody;
+
+ public BusinessException(String msg) {
+ super(msg);
+ this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
+ }
+
+ public BusinessException(String msg, Throwable cause) {
+ super(msg,cause);
+ this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
+ }
+
+ public BusinessException(Throwable cause) {
+ super(cause);
+ this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST);
+ }
+
+ public BusinessException(Integer code, String msg, String info, Throwable cause) {
+ super(msg,cause);
+ if (info == null) {
+ this.responseBody = new DataResponseBody(code, msg);
+ } else {
+ this.responseBody = new DataResponseBody(code, msg + ":" + info);
+ }
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/DataSequenceException.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/DataSequenceException.java
new file mode 100644
index 0000000..6ddf93c
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/DataSequenceException.java
@@ -0,0 +1,31 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.exception;
+
+import lombok.Getter;
+
+/**
+ * @description 获取序列异常
+ * @date 2020-09-23
+ */
+@Getter
+public class DataSequenceException extends BusinessException {
+
+ public DataSequenceException(String msg) {
+ super(msg);
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ErrorCode.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ErrorCode.java
new file mode 100644
index 0000000..28c7f60
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ErrorCode.java
@@ -0,0 +1,39 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.exception;
+
+
+/**
+ * @description 异常code
+ * @date 2020-03-26
+ */
+public interface ErrorCode {
+
+ /**
+ * 错误码
+ * @return code
+ */
+ Integer getCode();
+
+ /**
+ * error info
+ * @return
+ */
+ String getMsg();
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ImportDatasetException.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ImportDatasetException.java
new file mode 100644
index 0000000..b8f884d
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ImportDatasetException.java
@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.exception;
+
+import lombok.Getter;
+
+/**
+ * @description 数据集导入异常处理
+ * @date 2020-10-14
+ */
+@Getter
+public class ImportDatasetException extends BusinessException {
+
+ /**
+ * 数据集导入异常处理
+ *
+ * @param msg 信息
+ */
+ public ImportDatasetException(String msg) {
+ super(msg);
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/BaseLogFilter.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/BaseLogFilter.java
new file mode 100644
index 0000000..b315a2f
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/BaseLogFilter.java
@@ -0,0 +1,82 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.filter;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.filter.AbstractMatcherFilter;
+import ch.qos.logback.core.spi.FilterReply;
+import org.slf4j.Marker;
+
+/**
+ * @description 自定义日志过滤器
+ * @date 2020-07-21
+ */
+public class BaseLogFilter extends AbstractMatcherFilter {
+
+ Level level;
+
+ /**
+ * 重写decide方法
+ *
+ * @param iLoggingEvent 待决定的事件
+ * @return FilterReply 过滤器
+ */
+ @Override
+ public FilterReply decide(ILoggingEvent iLoggingEvent) {
+ if (!isStarted()) {
+ return FilterReply.NEUTRAL;
+ }
+ final String msg = iLoggingEvent.getMessage();
+ //自定义级别
+ if (checkLevel(iLoggingEvent) && msg != null && msg.startsWith("{") && msg.endsWith("}")) {
+ final Marker marker = iLoggingEvent.getMarker();
+ if (marker != null && this.getName() != null && this.getName().contains(marker.getName())) {
+ return onMatch;
+ }
+ }
+
+ return onMismatch;
+ }
+
+ /**
+ * 检查等级
+ *
+ * @param iLoggingEvent 待决定的事件
+ * @return boolean 检查结果
+ */
+ protected boolean checkLevel(ILoggingEvent iLoggingEvent) {
+ return this.level != null
+ && iLoggingEvent.getLevel() != null
+ && iLoggingEvent.getLevel().toInt() == this.level.toInt();
+ }
+
+ public void setLevel(Level level) {
+ this.level = level;
+ }
+
+ /**
+ * 启动
+ */
+ @Override
+ public void start() {
+ if (this.level != null) {
+ super.start();
+ }
+ }
+}
\ No newline at end of file
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/ConsoleLogFilter.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/ConsoleLogFilter.java
new file mode 100644
index 0000000..70d52f6
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/ConsoleLogFilter.java
@@ -0,0 +1,59 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.filter;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.spi.FilterReply;
+import org.dubhe.datasetutil.common.util.LogUtil;
+import org.slf4j.MarkerFactory;
+
+/**
+ * @description 自定义日志过滤器
+ * @date 2020-07-21
+ */
+public class ConsoleLogFilter extends BaseLogFilter {
+
+ /**
+ * 重写decide方法
+ *
+ * @param iLoggingEvent 待决定的事件
+ * @return FilterReply 过滤器
+ */
+ @Override
+ public FilterReply decide(ILoggingEvent iLoggingEvent) {
+ if (!isStarted()) {
+ return FilterReply.NEUTRAL;
+ }
+ return checkLevel(iLoggingEvent) ? onMatch : onMismatch;
+ }
+
+ /**
+ * 检查等级
+ *
+ * @param iLoggingEvent 待决定的事件
+ * @return boolean 检查结果
+ */
+ @Override
+ protected boolean checkLevel(ILoggingEvent iLoggingEvent) {
+ return this.level != null
+ && iLoggingEvent.getLevel() != null
+ && iLoggingEvent.getLevel().toInt() >= this.level.toInt()
+ && !MarkerFactory.getMarker(LogUtil.SCHEDULE_LEVEL).equals(iLoggingEvent.getMarker())
+ && !"log4jdbc.log4j2".equals(iLoggingEvent.getLoggerName());
+ }
+}
\ No newline at end of file
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/GlobalRequestLogFilter.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/GlobalRequestLogFilter.java
new file mode 100644
index 0000000..50b4ded
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/GlobalRequestLogFilter.java
@@ -0,0 +1,40 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.filter;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+
+/**
+ * @description 全局请求 日志过滤器
+ * @date 2020-08-13
+ */
+public class GlobalRequestLogFilter extends BaseLogFilter {
+
+
+ /**
+ * 日志等级
+ *
+ * @param iLoggingEvent 日志等级
+ * @return boolean 执行结果
+ */
+ @Override
+ public boolean checkLevel(ILoggingEvent iLoggingEvent) {
+ return this.level != null && true;
+ }
+
+}
\ No newline at end of file
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/DateUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/DateUtil.java
new file mode 100644
index 0000000..be7d8f9
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/DateUtil.java
@@ -0,0 +1,45 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+/**
+ * @description 时间 配置类
+ * @date 2020-09-17
+ */
+public class DateUtil {
+
+ private DateUtil(){
+ }
+
+ /**
+ * 时间格式
+ */
+ private static final String DATA_FORMAT_STR = "yyyy年MM月dd日 HH时mm分ss秒";
+
+ /**
+ * 获取开始时间
+ *
+ * @return String 返回控制台开始时间
+ */
+ public static String getNowStr() {
+ return new SimpleDateFormat(DATA_FORMAT_STR).format(new Date());
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/GeneratorKeyUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/GeneratorKeyUtil.java
new file mode 100644
index 0000000..5aadb7f
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/GeneratorKeyUtil.java
@@ -0,0 +1,118 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import cn.hutool.core.util.ObjectUtil;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+import org.dubhe.datasetutil.common.enums.LogEnum;
+import org.dubhe.datasetutil.common.exception.DataSequenceException;
+import org.dubhe.datasetutil.domain.dto.IdAlloc;
+import org.dubhe.datasetutil.domain.entity.DataSequence;
+import org.dubhe.datasetutil.service.DataSequenceService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.StringUtils;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @description 生成ID工具类
+ * @date 2020-09-23
+ */
+@Component
+public class GeneratorKeyUtil {
+
+ @Autowired
+ private DataSequenceService dataSequenceService;
+
+ private ConcurrentHashMap idAllocConcurrentHashMap = new ConcurrentHashMap<>();
+
+ /**
+ * 根据业务编码,数量获取序列号
+ *
+ * @param businessCode 业务编码
+ * @param number 数量
+ * @return Long 起始位置
+ */
+ @Transactional(rollbackFor = Exception.class)
+ public synchronized Long getSequenceByBusinessCode(String businessCode, int number) {
+ if (StringUtils.isEmpty(businessCode)) {
+ throw new DataSequenceException("业务编码不可为空");
+ }
+ if (number == MagicNumConstant.ZERO) {
+ throw new DataSequenceException("需要获取的序列号长度不可为0或者空");
+ }
+ IdAlloc idAlloc = idAllocConcurrentHashMap.get(businessCode);
+ if (ObjectUtil.isNull(idAlloc)) {
+ idAlloc = new IdAlloc();
+ idAllocConcurrentHashMap.put(businessCode, idAlloc);
+ }
+
+ if (idAlloc.getUsedNumber() == MagicNumConstant.ZERO) {
+ DataSequence dataSequence = getDataSequence(businessCode);
+ updateDataSequence(businessCode);
+ idAlloc.setStartNumber(dataSequence.getStart());
+ idAlloc.setEndNumber(dataSequence.getStart() + dataSequence.getStep() - MagicNumConstant.ONE);
+ idAlloc.setUsedNumber(idAlloc.getEndNumber() - idAlloc.getStartNumber() + MagicNumConstant.ONE);
+ }
+ if (idAlloc.getUsedNumber() <= number) {
+ expansionUsedNumber(businessCode, number);
+ }
+ long returnStartNumber = idAlloc.getStartNumber();
+ idAlloc.setStartNumber(idAlloc.getStartNumber() + number);
+ idAlloc.setUsedNumber(idAlloc.getUsedNumber() - number);
+ return returnStartNumber;
+ }
+
+ /**
+ * 根据业务编码获取配置信息
+ * @param businessCode 业务编码
+ * @return DataSequence 数据索引
+ */
+ private DataSequence getDataSequence(String businessCode) {
+ DataSequence dataSequence = dataSequenceService.getSequence(businessCode);
+ if (dataSequence == null || dataSequence.getStart() == null || dataSequence.getStep() == null) {
+ throw new DataSequenceException("配置出错,请检查data_sequence表配置");
+ }
+ return dataSequence;
+ }
+
+ /**
+ * 根据业务编码更新起始值
+ * @param businessCode 业务编码
+ */
+ private void updateDataSequence(String businessCode) {
+ dataSequenceService.updateSequenceStart(businessCode);
+ }
+
+ /**
+ * 多次扩容
+ * @param businessCode 业务编码
+ * @param number 数量
+ */
+ private void expansionUsedNumber(String businessCode, int number) {
+ IdAlloc idAlloc = idAllocConcurrentHashMap.get(businessCode);
+ updateDataSequence(businessCode);
+ DataSequence dataSequenceNew = getDataSequence(businessCode);
+ idAlloc.setEndNumber(idAlloc.getEndNumber() + dataSequenceNew.getStep());
+ idAlloc.setUsedNumber(idAlloc.getEndNumber() - idAlloc.getStartNumber() + MagicNumConstant.ONE);
+ if (idAlloc.getUsedNumber() <= number) {
+ expansionUsedNumber(businessCode, number);
+ }
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/HandleFileUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/HandleFileUtil.java
new file mode 100644
index 0000000..baafdb9
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/HandleFileUtil.java
@@ -0,0 +1,91 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * @description 文件工具类
+ * @date 2020-10-15
+ */
+public class HandleFileUtil {
+
+ /**
+ * 读取文件内容
+ *
+ * @param file 文件对象
+ * @return String 文件内容
+ */
+ public static String readFile(File file) throws IOException{
+ StringBuilder stringBuffer = new StringBuilder();
+ LineIterator fileContext = FileUtils.lineIterator(file,"UTF-8");
+ while (fileContext.hasNext()) {
+ stringBuffer.append(fileContext.nextLine());
+ }
+ return stringBuffer.toString();
+ }
+
+
+ /**
+ * 获取文件名后缀名
+ *
+ * @param fileName 文件名
+ * @return String 文件后缀名
+ */
+ public static String readFileSuffixName(String fileName){
+ return fileName.substring(fileName.lastIndexOf("."));
+ }
+
+
+ /**
+ * 获取文件名(踢除后缀名)
+ *
+ * @param fileName 文件名
+ * @return String 文件名(踢除后缀名)
+ */
+ public static String readFileName(String fileName){
+ return fileName.substring(0,fileName.lastIndexOf("."));
+ }
+
+
+ /**
+ * 生成文件路径
+ *
+ * @param businessCode 业务类型
+ * @return String 文件路径
+ */
+ public static String generateFilePath(String businessCode){
+ return BusinessConstant.FILE_SEPARATOR + businessCode;
+ }
+
+
+ /**
+ * 获取标签组名称
+ *
+ * @param fileName 文件名称
+ * @return String 标签组名
+ */
+ public static String getLabelGroupName(String fileName){
+ return fileName.substring(fileName.indexOf(BusinessConstant.UNDERLINE) + 1,fileName.lastIndexOf(BusinessConstant.SPOT));
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/LogUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/LogUtil.java
new file mode 100644
index 0000000..1b429f9
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/LogUtil.java
@@ -0,0 +1,300 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.util;
+
+import ch.qos.logback.classic.Level;
+import com.alibaba.fastjson.JSON;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.dubhe.datasetutil.common.aspect.LogAspect;
+import org.dubhe.datasetutil.common.enums.LogEnum;
+import org.dubhe.datasetutil.domain.entity.LogInfo;
+import org.slf4j.MDC;
+import org.slf4j.MarkerFactory;
+import org.slf4j.helpers.MessageFormatter;
+
+import java.util.Arrays;
+import java.util.UUID;
+
+/**
+ * @description 日志工具类
+ * @date 2020-06-29
+ */
+@Slf4j
+public class LogUtil {
+
+ private static final String TRACE_TYPE = "TRACE_TYPE";
+
+ public static final String SCHEDULE_LEVEL = "SCHEDULE";
+
+ private static final String GLOBAL_REQUEST_LEVEL = "GLOBAL_REQUEST";
+
+ private static final String TRACE_LEVEL = "TRACE";
+
+ private static final String DEBUG_LEVEL = "DEBUG";
+
+ private static final String INFO_LEVEL = "INFO";
+
+ private static final String WARN_LEVEL = "WARN";
+
+ private static final String ERROR_LEVEL = "ERROR";
+
+
+ public static void startScheduleTrace() {
+ MDC.put(TRACE_TYPE, SCHEDULE_LEVEL);
+ }
+
+ public static void cleanTrace() {
+ MDC.clear();
+ }
+
+ /**
+ * info级别的日志
+ *
+ * @param logType 日志类型
+ * @param object 打印的日志参数
+ */
+
+ public static void info(LogEnum logType, Object... object) {
+
+ logHandle(logType, Level.INFO, object);
+ }
+
+ /**
+ * debug级别的日志
+ *
+ * @param logType 日志类型
+ * @param object 打印的日志参数
+ */
+ public static void debug(LogEnum logType, Object... object) {
+ logHandle(logType, Level.DEBUG, object);
+ }
+
+ /**
+ * error级别的日志
+ *
+ * @param logType 日志类型
+ * @param object 打印的日志参数
+ */
+ public static void error(LogEnum logType, Object... object) {
+ errorObjectHandle(object);
+ logHandle(logType, Level.ERROR, object);
+ }
+
+ /**
+ * warn级别的日志
+ *
+ * @param logType 日志类型
+ * @param object 打印的日志参数
+ */
+ public static void warn(LogEnum logType, Object... object) {
+ logHandle(logType, Level.WARN, object);
+ }
+
+ /**
+ * trace级别的日志
+ *
+ * @param logType 日志类型
+ * @param object 打印的日志参数
+ */
+ public static void trace(LogEnum logType, Object... object) {
+ logHandle(logType, Level.TRACE, object);
+ }
+
+ /**
+ * 日志处理
+ *
+ * @param logType 日志类型
+ * @param level 日志级别
+ * @param object 打印的日志参数
+ */
+ private static void logHandle(LogEnum logType, Level level, Object[] object) {
+
+ LogInfo logInfo = generateLogInfo(logType, level, object);
+
+ switch (logInfo.getLevel()) {
+ case TRACE_LEVEL:
+ log.trace(MarkerFactory.getMarker(TRACE_LEVEL), logJsonStringLengthLimit(logInfo));
+ break;
+ case DEBUG_LEVEL:
+ log.debug(MarkerFactory.getMarker(DEBUG_LEVEL), logJsonStringLengthLimit(logInfo));
+ break;
+ case GLOBAL_REQUEST_LEVEL:
+ logInfo.setLevel(null);
+ logInfo.setType(null);
+ logInfo.setLocation(null);
+ log.info(MarkerFactory.getMarker(GLOBAL_REQUEST_LEVEL), logJsonStringLengthLimit(logInfo));
+ break;
+ case SCHEDULE_LEVEL:
+ log.info(MarkerFactory.getMarker(SCHEDULE_LEVEL), logJsonStringLengthLimit(logInfo));
+ break;
+ case INFO_LEVEL:
+ log.info(MarkerFactory.getMarker(INFO_LEVEL), logJsonStringLengthLimit(logInfo));
+ break;
+ case WARN_LEVEL:
+ log.warn(MarkerFactory.getMarker(WARN_LEVEL), logJsonStringLengthLimit(logInfo));
+ break;
+ case ERROR_LEVEL:
+ log.error(MarkerFactory.getMarker(ERROR_LEVEL), logJsonStringLengthLimit(logInfo));
+ break;
+ default:
+ }
+
+ }
+
+
+ /**
+ * 日志信息组装的内部方法
+ *
+ * @param logType 日志类型
+ * @param level 日志级别
+ * @param object 打印的日志参数
+ * @return LogInfo 日志对象
+ */
+ private static LogInfo generateLogInfo(LogEnum logType, Level level, Object[] object) {
+
+
+ LogInfo logInfo = new LogInfo();
+ // 日志类型检测
+ if (!LogEnum.isLogType(logType)) {
+ level = Level.ERROR;
+ object = new Object[1];
+ object[0] = "日志类型【".concat(String.valueOf(logType)).concat("】不正确!");
+ logType = LogEnum.SYS_ERR;
+ }
+
+ // 获取trace_id
+ if (StringUtils.isEmpty(MDC.get(LogAspect.TRACE_ID))) {
+ MDC.put(LogAspect.TRACE_ID, UUID.randomUUID().toString());
+ }
+ // 设置logInfo的level,type,traceId属性
+ logInfo.setLevel(level.levelStr)
+ .setType(logType.toString())
+ .setTraceId(MDC.get(LogAspect.TRACE_ID));
+
+
+ //自定义日志级别
+ //LogEnum、 MDC中的 TRACE_TYPE 做日志分流标识
+ if (Level.INFO.toInt() == level.toInt()) {
+ if (LogEnum.GLOBAL_REQ.equals(logType)) {
+ //info全局请求
+ logInfo.setLevel(GLOBAL_REQUEST_LEVEL);
+ } else {
+ //schedule定时等 链路记录
+ String traceType = MDC.get(TRACE_TYPE);
+ if (StringUtils.isNotBlank(traceType)) {
+ logInfo.setLevel(traceType);
+ }
+ }
+ }
+
+ // 设置logInfo的堆栈信息
+ setLogStackInfo(logInfo);
+ // 设置logInfo的info信息
+ setLogInfo(logInfo, object);
+ // 截取logInfo的长度并转换成json字符串
+ return logInfo;
+ }
+
+ /**
+ * 设置loginfo的堆栈信息
+ *
+ * @param logInfo 日志对象
+ */
+ private static void setLogStackInfo(LogInfo logInfo) {
+ StackTraceElement[] elements = Thread.currentThread().getStackTrace();
+ if (elements.length >= 6) {
+ StackTraceElement element = elements[5];
+ logInfo.setLocation(String.format("%s#%s:%s", element.getClassName(), element.getMethodName(), element.getLineNumber()));
+ }
+ }
+
+ /**
+ * 限制log日志的长度并转换成json
+ *
+ * @param logInfo 日志对象
+ * @return String 转换的json
+ */
+ private static String logJsonStringLengthLimit(LogInfo logInfo) {
+ try {
+
+ String jsonString = JSON.toJSONString(logInfo);
+ if (StringUtils.isBlank(jsonString)) {
+ return "";
+ }
+ if (jsonString.length() > 10000) {
+ String trunk = logInfo.getInfo().toString().substring(0, 9000);
+ logInfo.setInfo(trunk);
+ jsonString = JSON.toJSONString(logInfo);
+ }
+ return jsonString;
+
+ } catch (Exception e) {
+ logInfo.setLevel(Level.ERROR.levelStr).setType(LogEnum.SYS_ERR.toString())
+ .setInfo("cannot serialize exception: " + ExceptionUtils.getStackTrace(e));
+ return JSON.toJSONString(logInfo);
+ }
+ }
+
+ /**
+ * 设置日志对象的info信息
+ *
+ * @param logInfo 日志对象
+ * @param object 打印的日志参数
+ */
+ private static void setLogInfo(LogInfo logInfo, Object[] object) {
+
+ if (object.length > 1) {
+ logInfo.setInfo(MessageFormatter.arrayFormat(object[0].toString(),
+ Arrays.copyOfRange(object, 1, object.length)).getMessage());
+
+ } else if (object.length == 1 && object[0] instanceof Exception) {
+ logInfo.setInfo((ExceptionUtils.getStackTrace((Exception) object[0])));
+ log.error((ExceptionUtils.getStackTrace((Exception) object[0])));
+ } else if (object.length == 1) {
+ logInfo.setInfo(object[0] == null ? "" : object[0]);
+ } else {
+ logInfo.setInfo("");
+ }
+
+ }
+
+ /**
+ * 处理Exception的情况
+ *
+ * @param object 打印的日志参数
+ */
+ private static void errorObjectHandle(Object[] object) {
+
+ if (object.length == 2 && object[1] instanceof Exception) {
+ log.error(String.valueOf(object[0]), (Exception) object[1]);
+ object[1] = ExceptionUtils.getStackTrace((Exception) object[1]);
+
+ } else if (object.length >= 3) {
+ log.error(String.valueOf(object[0]),
+ Arrays.copyOfRange(object, 1, object.length));
+ for (int i = 0; i < object.length; i++) {
+ if (object[i] instanceof Exception) {
+ object[i] = ExceptionUtils.getStackTrace((Exception) object[i]);
+ }
+
+ }
+ }
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MinioUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MinioUtil.java
new file mode 100644
index 0000000..f5bb17e
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MinioUtil.java
@@ -0,0 +1,64 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import io.minio.MinioClient;
+import io.minio.PutObjectOptions;
+import org.dubhe.datasetutil.common.config.MinioConfig;
+import org.dubhe.datasetutil.common.enums.LogEnum;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.io.InputStream;
+
+/**
+ * @description Minio工具类
+ * @date 2020-09-17
+ */
+@Component
+public class MinioUtil {
+
+ @Autowired
+ private MinioClient minioClient;
+
+ @Autowired
+ private MinioConfig minioConfig;
+
+ /**
+ * 上传文件
+ *
+ * @param objectName 对象名称
+ * @param inputStream 文件流
+ * @throws Exception 上传异常
+ */
+ public void upLoadFile(String objectName, InputStream inputStream) throws Exception {
+ LogUtil.info(LogEnum.BIZ_DATASET,"文件上传名称为: 【" + objectName + "】");
+ PutObjectOptions options = new PutObjectOptions(inputStream.available(), -1);
+ minioClient.putObject(minioConfig.getBucketName(), objectName, inputStream, options);
+ }
+
+ /**
+ * 获取文件URL
+ *
+ * @param objectName 对象名称
+ * @return String 文件路径
+ */
+ public String getUrl(String objectName) {
+ return minioConfig.getBucketName() + "/" + objectName;
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MyPreciseShardingAlgorithm.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MyPreciseShardingAlgorithm.java
new file mode 100644
index 0000000..fb0c9e4
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MyPreciseShardingAlgorithm.java
@@ -0,0 +1,72 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm;
+import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.service.DataSequenceService;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Collection;
+
+/**
+ * @description 数据分片
+ * @date 2020-09-21
+ */
+public class MyPreciseShardingAlgorithm implements PreciseShardingAlgorithm{
+
+ @Autowired
+ private DataSequenceService dataSequenceService;
+
+ /**
+ * 数据表分片
+ *
+ * @param collection 集合
+ * @param preciseShardingValue 分片值
+ * @return 字符串
+ */
+ @Override
+ public String doSharding(Collection collection, PreciseShardingValue preciseShardingValue) {
+ long startIndex = MagicNumConstant.ONE;
+ long endIndex = MagicNumConstant.FIFTY;
+ dataSequenceService = SpringContextHolder.getBean(DataSequenceService.class);
+ String tableName = preciseShardingValue.getLogicTableName()+ BusinessConstant.UNDERLINE + preciseSharding(preciseShardingValue.getValue(),startIndex ,endIndex);
+ if(!dataSequenceService.checkTableExist(tableName)){
+ dataSequenceService.createTable(tableName);
+ }
+ return tableName;
+ }
+
+ /**
+ * 分片实现
+ *
+ * @param indexId 起始位置
+ * @param startIndex 起始值
+ * @param endIndex 结束值
+ * @return long 返回截止值
+ */
+ public long preciseSharding(long indexId,long startIndex , long endIndex){
+ if(indexId > endIndex){
+ startIndex = startIndex + BusinessConstant.INTERVAL_NUMBER;
+ endIndex = endIndex + BusinessConstant.INTERVAL_NUMBER;
+ return preciseSharding(indexId,startIndex,endIndex);
+ }
+ return endIndex / BusinessConstant.INTERVAL_NUMBER;
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/PrintUtils.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/PrintUtils.java
new file mode 100644
index 0000000..938cce5
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/PrintUtils.java
@@ -0,0 +1,156 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+/**
+ * @description 控制台字体颜色
+ * @date 2020-10-22
+ */
+public class PrintUtils {
+ /**
+ * 白色
+ */
+ public static final int WHITE = 30;
+
+ /**
+ * 白色背景
+ */
+ public static final int WHITE_BACKGROUND = 40;
+
+ /**
+ * 红色
+ */
+ public static final int RED = 31;
+
+ /**
+ * 红色背景
+ */
+ public static final int RED_BACKGROUND = 41;
+
+ /**
+ * 绿色
+ */
+ public static final int GREEN = 32;
+
+ /**
+ * 绿色背景
+ */
+ public static final int GREEN_BACKGROUND = 42;
+
+ /**
+ * 黄色
+ */
+ public static final int YELLOW = 33;
+
+ /**
+ * 黄色背景
+ */
+ public static final int YELLOW_BACKGROUND = 43;
+
+ /**
+ * 蓝色
+ */
+ public static final int BLUE = 34;
+
+ /**
+ * 蓝色背景
+ */
+ public static final int BLUE_BACKGROUND = 44;
+
+ /**
+ * 品红(洋红)
+ */
+ public static final int MAGENTA = 35;
+
+ /**
+ * 品红背景
+ */
+ public static final int MAGENTA_BACKGROUND = 45;
+
+ /**
+ * 蓝绿
+ */
+ public static final int CYAN = 36;
+
+ /**
+ * 蓝绿背景
+ */
+ public static final int CYAN_BACKGROUND = 46;
+
+ /**
+ * 黑色
+ */
+ public static final int BLACK = 37;
+
+ /**
+ * 黑色背景
+ */
+ public static final int BLACK_BACKGROUND = 47;
+
+ /**
+ * 粗体
+ */
+ public static final int BOLD = 1;
+
+ /**
+ * 斜体
+ */
+ public static final int ITATIC = 3;
+
+ /**
+ * 下划线
+ */
+ public static final int UNDERLINE = 4;
+
+ /**
+ * 反转
+ */
+ public static final int REVERSE = 7;
+
+ /**
+ * 格式化
+ *
+ * @param txt 文本
+ * @param codes 信息
+ * @return String 格式化后的内容
+ */
+ private static String FMT(String txt, int... codes) {
+ StringBuffer sb = new StringBuffer();
+ for (int code : codes) {
+ sb.append(code + ";");
+ }
+ String _code = sb.toString();
+ if (_code.endsWith(";")) {
+ _code = _code.substring(0, _code.length() - 1);
+ }
+ return (char) 27 + "[" + _code + "m" + txt + (char) 27 + "[0m";
+ }
+
+ /**
+ * 打印并换行
+ */
+ public static void printLine(String txt, int... codes) {
+ System.out.println(FMT(txt, codes));
+ }
+
+ /**
+ * 默认打印红色文字
+ */
+ public static void PN(String txt) {
+ System.out.println(FMT(txt, new int[]{RED}));
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/SpringContextHolder.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/SpringContextHolder.java
new file mode 100644
index 0000000..7f3e52b
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/SpringContextHolder.java
@@ -0,0 +1,95 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.util;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+
+/**
+ * @description 上下文工具类
+ * @date 2020-03-25
+ */
+@Slf4j
+public class SpringContextHolder implements ApplicationContextAware, DisposableBean {
+
+ private static ApplicationContext applicationContext = null;
+
+ /**
+ * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型.
+ *
+ * @param name bean名称
+ * @return T 根据Bean名称转型的对象
+ */
+ @SuppressWarnings("unchecked")
+ public static T getBean(String name) {
+ assertContextInjected();
+ return (T) applicationContext.getBean(name);
+ }
+
+ /**
+ * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型.
+ *
+ * @param requiredType 要求类型
+ * @return T 根据Bean名称转型的对象
+ */
+ public static T getBean(Class requiredType) {
+ assertContextInjected();
+ return applicationContext.getBean(requiredType);
+ }
+
+ /**
+ * 检查ApplicationContext不为空.
+ */
+ private static void assertContextInjected() {
+ if (applicationContext == null) {
+ throw new IllegalStateException("applicaitonContext属性未注入, 请在applicationContext" +
+ ".xml中定义SpringContextHolder或在SpringBoot启动类中注册SpringContextHolder.");
+ }
+ }
+
+ /**
+ * 清除SpringContextHolder中的ApplicationContext为Null.
+ */
+ private static void clearHolder() {
+ log.debug("清除SpringContextHolder中的ApplicationContext:"
+ + applicationContext);
+ applicationContext = null;
+ }
+
+ @Override
+ public void destroy() { SpringContextHolder.clearHolder();
+ }
+
+ @Override
+ public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+ if (SpringContextHolder.applicationContext != null) {
+ log.warn("SpringContextHolder中的ApplicationContext被覆盖, 原有ApplicationContext为:" + SpringContextHolder.applicationContext);
+ }
+ SpringContextHolder.applicationContext = applicationContext;
+ }
+
+ /**
+ * 获取当前环境
+ */
+ public static String getActiveProfile(){
+ return applicationContext.getEnvironment().getActiveProfiles()[0];
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/ThreadUtils.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/ThreadUtils.java
new file mode 100644
index 0000000..8e58cfc
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/ThreadUtils.java
@@ -0,0 +1,72 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.List;
+import java.util.concurrent.*;
+
+/**
+ * @description 线程工具
+ * @date 2020-10-19
+ */
+@Slf4j
+public class ThreadUtils {
+
+ private ThreadUtils(){}
+
+ /**
+ * 根据需要处理的数量创建线程数
+ *
+ * @param listSize 集合数量
+ * @return int 数量
+ */
+ public static int createThread(int listSize) {
+ return listSize / getNeedThreadNumber() == 0 ? 1 : listSize / getNeedThreadNumber();
+ }
+
+
+ /**
+ * 获取需要创建的线程数
+ *
+ * @return int 数量
+ */
+ public static int getNeedThreadNumber() {
+ final int numOfCores = Runtime.getRuntime().availableProcessors();
+ final double blockingCoefficient = 0.8;
+ return (int) (numOfCores / (1 - blockingCoefficient));
+ }
+
+ /**
+ * 按要求分多线程执行
+ *
+ * @param partitions 分线程集合
+ * @throws Exception 线程执行异常
+ */
+ public static void runMultiThread(List> partitions) throws Exception {
+ final ExecutorService executorService = Executors.newFixedThreadPool(ThreadUtils.getNeedThreadNumber());
+ final List> valueOfStocks = executorService.invokeAll(partitions);
+ Integer endCount = 0;
+ for (final Future value : valueOfStocks) {
+ endCount += value.get();
+ }
+ log.warn("#-------------处理结束,成功处理文件 【" + endCount + "】个-------------#");
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataFileMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataFileMapper.java
new file mode 100644
index 0000000..0229e77
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataFileMapper.java
@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.entity.DataFile;
+import java.util.List;
+
+/**
+ * @description 数据集文件 Mapper接口
+ * @date 2020-09-17
+ */
+public interface DataFileMapper extends BaseMapper {
+
+ /**
+ * 插入文件数据
+ *
+ * @param listDataFile 文件数据集合
+ */
+ void saveBatchDataFile(@Param("listDataFile") List listDataFile);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataGroupLabelMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataGroupLabelMapper.java
new file mode 100644
index 0000000..2b92942
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataGroupLabelMapper.java
@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.entity.DataGroupLabel;
+
+import java.util.List;
+
+/**
+ * @description 标签组与标签的关系MAPPER
+ * @date 2020-10-21
+ */
+public interface DataGroupLabelMapper extends BaseMapper {
+ /**
+ * 插入标签组与标签的关数据
+ *
+ * @param listDataGroupLabel 标签组与标签数据集合
+ */
+ void saveDataGroupLabel(@Param("list") List listDataGroupLabel);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelGroupMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelGroupMapper.java
new file mode 100644
index 0000000..f1ee840
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelGroupMapper.java
@@ -0,0 +1,38 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+import org.dubhe.datasetutil.domain.entity.DataLabelGroup;
+
+/**
+ * @description 数据集标签组
+ * @date 2020-10-14
+ */
+public interface DataLabelGroupMapper extends BaseMapper {
+ /**
+ * 根据标签组名查询
+ *
+ * @param labelGroupName 标签组名
+ * @return int 标签组数量
+ */
+ @Select("select count(1) from data_label_group where name = #{labelGroupName} and deleted = 0")
+ int selectByLabelGroupName(@Param("labelGroupName") String labelGroupName);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelMapper.java
new file mode 100644
index 0000000..1040584
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelMapper.java
@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.entity.DataLabel;
+
+import java.util.List;
+
+/**
+ * @description 数据集标签
+ * @date 2020-10-14
+ */
+public interface DataLabelMapper extends BaseMapper {
+ /**
+ * 批量保存数据集标签
+ *
+ * @param listDataLabel 标签数据
+ */
+ void saveBatchDataLabel(@Param("listDataLabel") List listDataLabel);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataSequenceMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataSequenceMapper.java
new file mode 100644
index 0000000..abf5d00
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataSequenceMapper.java
@@ -0,0 +1,66 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+import org.apache.ibatis.annotations.Update;
+import org.dubhe.datasetutil.domain.entity.DataSequence;
+
+/**
+ * @description 序列Mapper
+ * @date 2020-09-23
+ */
+public interface DataSequenceMapper extends BaseMapper {
+ /**
+ * 根据业务编码查询序列
+ *
+ * @param businessCode 业务信息
+ * @return DataSequence 根据业务编码得到的序列
+ */
+ @Select("select id, business_code ,start, step from data_sequence where business_code = #{businessCode}")
+ DataSequence selectByBusiness(String businessCode);
+
+ /**
+ * 根据业务编码更新序列起始值
+ *
+ * @param businessCode 业务信息
+ * @return DataSequence 根据业务编码更新序列起始值
+ */
+ @Update("update data_sequence set start = start + step where business_code = #{businessCode} ")
+ int updateStartByBusinessCode(String businessCode);
+
+ /**
+ * 查询存在表的记录数
+ *
+ * @param tableName 类型名称
+ * @return int 根据类型查在表的记录数量
+ */
+ @Select("select count(1) from ${tableName}")
+ int checkTableExist(@Param("tableName") String tableName);
+
+ /**
+ * 执行创建表
+ *
+ * @param tableName 类型名称
+ * @param oldTableName 旧类型名称
+ */
+ @Update({"CREATE TABLE ${tableName} AS select * from ${oldTableName} "})
+ void createNewTable(@Param("tableName") String tableName, @Param("oldTableName") String oldTableName);
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataVersionFileMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataVersionFileMapper.java
new file mode 100644
index 0000000..d8c107d
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataVersionFileMapper.java
@@ -0,0 +1,37 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+
+import java.util.List;
+
+/**
+ * @description 数据集文件中间表 Mapper接口
+ * @date 2020-9-17
+ */
+public interface DataVersionFileMapper extends BaseMapper {
+
+ /**
+ * 插入数据集文件中间表数据
+ *
+ * @param listDataVersionFile 数据集文件中间表数据集合
+ */
+ void saveBatchDataFileVersion(@Param("listDataVersionFile") List listDataVersionFile);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetDataLabelMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetDataLabelMapper.java
new file mode 100644
index 0000000..cff8aa6
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetDataLabelMapper.java
@@ -0,0 +1,37 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.entity.DatasetDataLabel;
+
+import java.util.List;
+
+/**
+ * @description 数据集与标签关系
+ * @date 2020-10-14
+ */
+public interface DatasetDataLabelMapper extends BaseMapper {
+
+ /**
+ * 批量保存数据集与标签关系
+ *
+ * @param datasetDataLabelList 数据集与标签
+ */
+ void saveBatchDatasetDataLabel(@Param("datasetDataLabelList") List datasetDataLabelList);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetMapper.java
new file mode 100644
index 0000000..f21cc5b
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetMapper.java
@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+import org.dubhe.datasetutil.domain.entity.Dataset;
+
+/**
+ * @description 数据集 Mapper接口
+ * @date 2020-9-17
+ */
+public interface DatasetMapper extends BaseMapper {
+
+ /**
+ * 根据数据集ID查询数据集
+ *
+ * @param datasetId 数据集id
+ * @return Dataset 根据数据集ID得到数据集
+ */
+ @Select("select * from data_dataset where id = #{datasetId} and is_import = 1")
+ Dataset findDatasetById(@Param("datasetId") Long datasetId);
+
+ /**
+ * 查询数据集标签数量
+ *
+ * @param datasetId 数据集id
+ * @return int 数据集标签数量
+ */
+ @Select("select count(1) from data_dataset_label where dataset_id = #{datasetId}")
+ int findDataLabelById(@Param("datasetId") Long datasetId);
+
+ /**
+ * 查询数据集文件数量
+ *
+ * @param datasetId 数据集id
+ * @return int 数据集标签数量
+ */
+ @Select("select count(1) from data_file where dataset_id = #{datasetId}")
+ int findDataFileById(@Param("datasetId") Long datasetId);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/DataVersionFile.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/DataVersionFile.java
new file mode 100644
index 0000000..a7c89ec
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/DataVersionFile.java
@@ -0,0 +1,93 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.dto;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import lombok.Data;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+
+/**
+ * @description 数据集文件关系类
+ * @date 2020-9-17
+ */
+@Data
+public class DataVersionFile extends BaseEntity implements Serializable {
+
+ /**
+ * id
+ */
+ @TableId(type = IdType.AUTO)
+ private Long id;
+
+ /**
+ * 数据集id
+ */
+ private Long datasetId;
+
+ /**
+ * 版本号
+ */
+ private String versionName;
+
+ /**
+ * 文件id
+ */
+ private Long fileId;
+
+ /**
+ * 状态
+ */
+ private Integer status;
+
+ /**
+ * 数据集状态
+ */
+ private Integer annotationStatus;
+
+ /**
+ * 数据集状态备份
+ */
+ private Integer backupStatus;
+
+ /**
+ * 发布是否转换
+ */
+ private Integer changed;
+
+ public DataVersionFile() {
+ }
+
+ /**
+ * 插入数据集版本文件关系
+ *
+ * @param datasetId 数据集id
+ * @param fileId 文件id
+ * @param annotationStatus 数据集id
+ * @param status 状态
+ * @return DataVersionFile 数据集版本文件表
+ */
+ public DataVersionFile(Long datasetId, Long fileId,Integer annotationStatus,Integer status) {
+ this.datasetId = datasetId;
+ this.fileId = fileId;
+ this.annotationStatus = annotationStatus;
+ this.status = status;
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/IdAlloc.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/IdAlloc.java
new file mode 100644
index 0000000..87ae460
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/IdAlloc.java
@@ -0,0 +1,50 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.dto;
+
+import lombok.Data;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+
+/**
+ * @description ID策略实体
+ * @date 2020-10-16
+ */
+@Data
+public class IdAlloc {
+
+ /**
+ * 起始位置
+ */
+ private long startNumber;
+
+ /**
+ * 结束位置
+ */
+ private long endNumber;
+
+ /**
+ * 可用数量
+ */
+ private long usedNumber;
+
+ public IdAlloc() {
+ this.startNumber = MagicNumConstant.ZERO;
+ this.endNumber = MagicNumConstant.ZERO;
+ this.usedNumber = MagicNumConstant.ZERO;
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataFile.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataFile.java
new file mode 100644
index 0000000..19a553b
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataFile.java
@@ -0,0 +1,118 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import lombok.Data;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.awt.image.BufferedImage;
+import java.io.Serializable;
+
+/**
+ * @description 文件类
+ * @date 2020-09-17
+ */
+@Data
+public class DataFile extends BaseEntity implements Serializable {
+
+ /**
+ * id
+ */
+ @TableId(type = IdType.AUTO)
+ private Long id;
+
+ /**
+ * 名称
+ */
+ private String name;
+
+ /**
+ * 状态
+ */
+ private Integer status;
+
+ /**
+ * 数据集id
+ */
+ private Long datasetId;
+
+ /**
+ * url
+ */
+ private String url;
+
+ /**
+ * 文件类型
+ */
+ private Integer fileType;
+
+ /**
+ * 父id
+ */
+ private Long pid;
+
+ /**
+ * 帧间隔
+ */
+ private Integer frameInterval;
+
+ /**
+ * 增强类型
+ */
+ private Integer enhanceType;
+
+ /**
+ * 宽
+ */
+ private Integer width;
+
+ /**
+ * 高
+ */
+ private Integer height;
+
+ /**
+ * 拥有人id
+ */
+ private Long originUserId;
+
+ public DataFile() {}
+
+ /**
+ * 插入文件表
+ *
+ * @param name 文件名字
+ * @param datasetId 数据集id
+ * @param url 文件路径
+ * @param createUserId 创建人id
+ * @param read 文件宽高
+ * @return DataFile file对象
+ */
+ public DataFile(String name, Long datasetId, String url, Long createUserId, BufferedImage read) {
+ this.name = name.substring(0, name.lastIndexOf("."));
+ this.datasetId = datasetId;
+ this.url = url;
+ this.status = 101;
+ this.setDeleted(false);
+ this.originUserId = createUserId;
+ this.width = read.getWidth();
+ this.height = read.getHeight();
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataGroupLabel.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataGroupLabel.java
new file mode 100644
index 0000000..139f6fc
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataGroupLabel.java
@@ -0,0 +1,58 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+/**
+ * @description 标签组标签中间表
+ * @date 2020-10-21
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Builder
+@TableName("data_group_label")
+public class DataGroupLabel extends BaseEntity implements Serializable {
+
+ /**
+ * id
+ */
+ @TableId(value = "id", type = IdType.AUTO)
+ private Long id;
+
+ /**
+ * 标签id
+ */
+ @TableField(value = "label_id")
+ private Long labelId;
+
+ /**
+ * 标签组id
+ */
+ @TableField(value = "label_group_id")
+ private Long labelGroupId;
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabel.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabel.java
new file mode 100644
index 0000000..4296edb
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabel.java
@@ -0,0 +1,64 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+/**
+ * @description 数据集标签实体
+ * @date 2020-10-14
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Builder
+@TableName("data_label")
+public class DataLabel extends BaseEntity implements Serializable {
+
+ /**
+ * id
+ */
+ @TableId(value = "id", type = IdType.AUTO)
+ private Long id;
+
+ /**
+ * 名称
+ */
+ @TableField(value = "name")
+ private String name;
+
+ /**
+ * 颜色
+ */
+ @TableField(value = "color")
+ private String color;
+
+ /**
+ * 类型
+ */
+ @TableField(value = "type")
+ private Long type;
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabelGroup.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabelGroup.java
new file mode 100644
index 0000000..4bc6120
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabelGroup.java
@@ -0,0 +1,70 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+/**
+ * @description 标签组实体
+ * @date 2020-10-14
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Builder
+@TableName("data_label_group")
+public class DataLabelGroup extends BaseEntity implements Serializable {
+
+ /**
+ * id
+ */
+ @TableId(value = "id", type = IdType.AUTO)
+ private Long id;
+
+ /**
+ * 名称
+ */
+ @TableField(value = "name")
+ private String name;
+
+ /**
+ * 描述
+ */
+ @TableField(value = "remark")
+ private String remark;
+
+ /**
+ * 类型
+ */
+ @TableField(value = "type")
+ private Long type;
+
+ /**
+ * 拥有人id
+ */
+ @TableField(value = "origin_user_id")
+ private Long originUserId;
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataSequence.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataSequence.java
new file mode 100644
index 0000000..52fa0e0
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataSequence.java
@@ -0,0 +1,57 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+/**
+ * @description 序列表
+ * @date 2020-09-23
+ */
+@Data
+@TableName("data_sequence")
+public class DataSequence {
+
+ /**
+ * id
+ */
+ @TableId(value = "id", type = IdType.AUTO)
+ private Long id;
+
+ /**
+ * 业务code
+ */
+ @TableField(value = "business_code")
+ private String businessCode;
+
+ /**
+ * 启动
+ */
+ @TableField(value = "start")
+ private Long start;
+
+ /**
+ * 步数
+ */
+ @TableField(value = "step")
+ private Long step;
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/Dataset.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/Dataset.java
new file mode 100644
index 0000000..5abc7bc
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/Dataset.java
@@ -0,0 +1,145 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.*;
+import lombok.Data;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+import java.sql.Timestamp;
+
+/**
+ * @description 数据集类
+ * @date 2020-9-17
+ */
+@Data
+@TableName("data_dataset")
+public class Dataset extends BaseEntity implements Serializable {
+
+ /**
+ * 删除标识
+ */
+ @TableField("deleted")
+ private Boolean deleted = false;
+
+ /**
+ * 数据集名称
+ */
+ private String name;
+
+ /**
+ * 数据集备注
+ */
+ private String remark;
+
+ /**
+ * 类型
+ */
+ private Integer type;
+
+ /**
+ * 数据集类型
+ */
+ private Integer dataType;
+
+ /**
+ * 数据集状态
+ */
+ private Integer annotateType;
+
+ /**
+ * 任务id
+ */
+ private Long teamId;
+
+ /**
+ * id
+ */
+ @TableId(type = IdType.AUTO)
+ private Long id;
+
+ /**
+ * url
+ */
+ private String uri;
+
+ /**
+ * 状态
+ */
+ private Integer status;
+
+ /**
+ * 当前版本号
+ */
+ private String currentVersionName;
+
+ /**
+ * 是否为导入
+ */
+ @TableField(value = "is_import")
+ private boolean isImport;
+
+ /**
+ * 用户导入数据集压缩包地址
+ */
+ private String archiveUrl;
+
+ /**
+ * 解压状态
+ */
+ private Integer decompressState;
+
+ /**
+ * 解压失败原因
+ */
+ private String decompressFailReason;
+
+ /**
+ * 是否置顶
+ */
+ @TableField(value = "is_top")
+ private boolean isTop;
+
+ /**
+ * 创建时间
+ */
+ private Timestamp createTime;
+
+ /**
+ * 修改时间
+ */
+ private Timestamp updateTime;
+
+ /**
+ * 创建人id
+ */
+ private Long createUserId;
+
+ /**
+ * 修改人id
+ */
+ private Long updateUserId;
+
+ /**
+ * 拥有人id
+ */
+ private Long originUserId;
+
+ public Dataset() {}
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DatasetDataLabel.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DatasetDataLabel.java
new file mode 100644
index 0000000..da6b84d
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DatasetDataLabel.java
@@ -0,0 +1,58 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+/**
+ * @description 数据集与标签关系实体
+ * @date 2020-10-14
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Builder
+@TableName("data_label")
+public class DatasetDataLabel extends BaseEntity implements Serializable {
+
+ /**
+ * id
+ */
+ @TableId(value = "id", type = IdType.AUTO)
+ private Long id;
+
+ /**
+ * 数据集id
+ */
+ @TableField(value = "dataset_id")
+ private Long datasetId;
+
+ /**
+ * 标签id
+ */
+ @TableField(value = "label_id")
+ private Long labelId;
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/LogInfo.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/LogInfo.java
new file mode 100644
index 0000000..69924c5
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/LogInfo.java
@@ -0,0 +1,74 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.domain.entity;
+
+import com.alibaba.fastjson.annotation.JSONField;
+import com.xiaoleilu.hutool.date.DateUtil;
+import lombok.Data;
+import lombok.experimental.Accessors;
+
+import java.io.Serializable;
+
+/**
+ * @description 日志对象封装类
+ * @date 2020-06-29
+ */
+@Data
+@Accessors(chain = true)
+public class LogInfo implements Serializable {
+
+ /**
+ * id
+ */
+ @JSONField(ordinal = 1)
+ private String traceId;
+
+ /**
+ * 类型
+ */
+ @JSONField(ordinal = 2)
+ private String type;
+
+ /**
+ * 等级
+ */
+ @JSONField(ordinal = 3)
+ private String level;
+
+ /**
+ * 位置
+ */
+ @JSONField(ordinal = 4)
+ private String location;
+
+ /**
+ * 时间
+ */
+ @JSONField(ordinal = 5)
+ private String time = DateUtil.now();
+
+ /**
+ * 描述
+ */
+ @JSONField(ordinal = 6)
+ private Object info;
+
+ public void setInfo(Object info) {
+ this.info = info;
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImageUploadHandle.java b/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImageUploadHandle.java
new file mode 100644
index 0000000..312d5e5
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImageUploadHandle.java
@@ -0,0 +1,140 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.handle;
+
+import com.xiaoleilu.hutool.io.FileUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.common.util.DateUtil;
+import org.dubhe.datasetutil.common.util.GeneratorKeyUtil;
+import org.dubhe.datasetutil.common.util.MinioUtil;
+import org.dubhe.datasetutil.common.util.ThreadUtils;
+import org.dubhe.datasetutil.domain.entity.DataFile;
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+import org.dubhe.datasetutil.domain.entity.Dataset;
+import org.dubhe.datasetutil.service.DataFileService;
+import org.dubhe.datasetutil.service.DataVersionFileService;
+import org.dubhe.datasetutil.service.DatasetService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * @description 上传图片工具类
+ * @date 2020-09-17
+ */
+@Slf4j
+@Component
+public class DatasetImageUploadHandle {
+
+ @Autowired
+ private MinioUtil minioUtil;
+
+ @Autowired
+ private DataFileService dataFileService;
+
+ @Autowired
+ private DataVersionFileService dataVersionFileService;
+
+ @Autowired
+ private DatasetService datasetService;
+
+ @Autowired
+ private GeneratorKeyUtil generatorKeyUtil;
+
+ /**
+ * 启动线程
+ *
+ * @param imagePath 本地文件地址
+ * @param datasetId 数据集Id
+ */
+ public void execute(String imagePath, Long datasetId) throws Exception {
+ log.info("#-------------开始处理,时间[" + DateUtil.getNowStr() + "]-------------#");
+ List fileNames = FileUtil.listFileNames(imagePath);
+ log.info("#-------------文件数量[" + fileNames.size() + "]------------------------");
+ String fileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + datasetId
+ + BusinessConstant.FILE_SEPARATOR + BusinessConstant.IMAGE_ORIGIN + BusinessConstant.FILE_SEPARATOR;
+ List> partitions = new ArrayList<>();
+ int oneSize = ThreadUtils.createThread(fileNames.size());
+ List need = new ArrayList<>();
+ AtomicInteger atomicInteger = new AtomicInteger(0);
+ for (String fileName : fileNames) {
+ need.add(fileName);
+ if (need.size() == oneSize || atomicInteger.intValue() == fileNames.size() - 1) {
+ List now = new ArrayList<>(need);
+ need.clear();
+ partitions.add(() -> run(datasetId, now, fileBaseDir, imagePath));
+ }
+ atomicInteger.getAndIncrement();
+ }
+ ThreadUtils.runMultiThread(partitions);
+ }
+
+ /**
+ * 插入数据库数据
+ *
+ * @param datasetId 数据集Id
+ * @param fileNames 文件Name
+ * @param fileBaseDir 文件路径
+ * @param imagePath 文件地址
+ * @return java.lang.Integer 成功数量
+ */
+ public Integer run(Long datasetId, List fileNames, String fileBaseDir, String imagePath) {
+ Integer success = 0;
+ Dataset dataset = datasetService.findCreateUserIdById(datasetId);
+ List dataFiles = new ArrayList<>();
+ List dataVersionFiles = new ArrayList<>();
+ for (int i = 0; i < fileNames.size(); i++) {
+ try {
+ minioUtil.upLoadFile(fileBaseDir + fileNames.get(i), FileUtil.getInputStream(imagePath + BusinessConstant.FILE_SEPARATOR + fileNames.get(i)));
+ BufferedImage read = ImageIO.read(new File(imagePath + BusinessConstant.FILE_SEPARATOR + fileNames.get(i)));
+ success++;
+ dataFiles.add(new DataFile(fileNames.get(i), datasetId, minioUtil.getUrl(fileBaseDir + fileNames.get(i)), dataset.getCreateUserId(), read));
+
+ if (dataFiles.size() % 500 == 0 || i == fileNames.size() - 1) {
+ long startDataFileIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_FILE, dataFiles.size());
+ for (DataFile dataFileEntity : dataFiles) {
+ dataFileEntity.setId(startDataFileIndex++);
+ }
+
+ dataFileService.saveBatchDataFile(dataFiles);
+ for (DataFile file : dataFiles) {
+ dataVersionFiles.add(new DataVersionFile(datasetId, file.getId(), 101, 0));
+ }
+ long startDataFileVersionIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_VERSION_FILE, dataVersionFiles.size());
+ for (DataVersionFile dataVersionFile : dataVersionFiles) {
+ dataVersionFile.setId(startDataFileVersionIndex++);
+ }
+ dataVersionFileService.saveBatchDataFileVersion(dataVersionFiles);
+ dataVersionFiles.clear();
+ dataFiles.clear();
+ }
+ } catch (Exception e) {
+ log.error("{}", e);
+ }
+ }
+ return success;
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImportHandle.java b/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImportHandle.java
new file mode 100644
index 0000000..f456043
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImportHandle.java
@@ -0,0 +1,522 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.handle;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
+import com.xiaoleilu.hutool.io.FileUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+import org.dubhe.datasetutil.common.config.MinioConfig;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.common.constant.FileStateCodeConstant;
+import org.dubhe.datasetutil.common.exception.ImportDatasetException;
+import org.dubhe.datasetutil.common.util.*;
+import org.dubhe.datasetutil.domain.entity.*;
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+import org.dubhe.datasetutil.service.*;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+/**
+ * @description 导入数据集工具类
+ * @date 2020-10-12
+ */
+@Slf4j
+@Component
+public class DatasetImportHandle {
+
+ @Autowired
+ private DatasetService datasetService;
+
+ @Autowired
+ private DataLabelGroupService dataLabelGroupService;
+
+ @Autowired
+ private DataGroupLabelService dataGroupLabelService;
+
+ @Autowired
+ private DataLabelService dataLabelService;
+
+ @Autowired
+ private DataFileService dataFileService;
+
+ @Autowired
+ private DatasetDataLabelService datasetDataLabelService;
+
+ @Autowired
+ private DataVersionFileService dataVersionFileService;
+
+ @Autowired
+ private MinioUtil minioUtil;
+
+ @Autowired
+ private MinioConfig minioConfig;
+
+ @Autowired
+ private GeneratorKeyUtil generatorKeyUtil;
+
+ /**
+ * 可支持的图片格式集合
+ */
+ private static final List SUFFIX_LIST = new ArrayList<>();
+
+ /**
+ * 标注文件中JSON的key
+ */
+ static List annotationFileContextKey = new ArrayList<>();
+
+ /**
+ * 加载静态集合数据
+ */
+ static {
+ SUFFIX_LIST.add(".jpg");
+ SUFFIX_LIST.add(".png");
+ SUFFIX_LIST.add(".bmp");
+ SUFFIX_LIST.add(".jpeg");
+
+ annotationFileContextKey.add("score");
+ annotationFileContextKey.add("area");
+ annotationFileContextKey.add("name");
+ annotationFileContextKey.add("bbox");
+ annotationFileContextKey.add("segmentation");
+ annotationFileContextKey.add("iscrowd");
+ }
+
+ /**
+ * 导入数据集
+ *
+ * @param scanner 输入
+ */
+ public void importDataset(Scanner scanner) throws Exception {
+ Dataset dataset = verificationDatasetId(scanner);
+ String filePath = verificationFilePath(scanner);
+ File labelJsonFile = verificationFile(filePath);
+ DataLabelGroup dataLabelGroup = saveDataLabelGroup(HandleFileUtil.getLabelGroupName(labelJsonFile.getName()), dataset);
+ List dataLabelList = readLabelContext(labelJsonFile);
+ saveDataLabel(dataset, dataLabelList, dataLabelGroup.getId());
+ executeUploadAndSave(dataLabelList, filePath, dataset);
+ datasetService.updateDatasetStatus(dataset);
+ log.warn("");
+ PrintUtils.printLine(" Success: 执行成功 ", PrintUtils.GREEN);
+ log.warn("");
+ log.warn("# 是否结束? Y / N #");
+ Scanner scannerExit = new Scanner(System.in);
+ if (BusinessConstant.Y.toLowerCase().equals(scannerExit.nextLine().toLowerCase())) {
+ System.exit(0);
+ }
+ }
+
+ /**
+ * 检查文件结构 、类型
+ *
+ * @param globalFilePath 文件路径
+ * @return file 标签文件
+ */
+ public File verificationFile(String globalFilePath) throws IOException {
+ File labelRootFiles = new File(globalFilePath);
+ File imageRootFiles = new File(globalFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN));
+ File annotationRootFiles = new File(globalFilePath + HandleFileUtil.generateFilePath(BusinessConstant.ANNOTATION));
+ if (imageRootFiles.list() == null || annotationRootFiles.listFiles() == null) {
+ throw new ImportDatasetException(" 【" + globalFilePath + "】目录中的图片目录(origin)或者标注文件目录(annotation)的文件夹为空 ");
+ }
+ File labelJsonFile = null;
+ for (File file : Objects.requireNonNull(labelRootFiles.listFiles())) {
+ if (file.isFile() && file.getName().toLowerCase().startsWith(BusinessConstant.LABEL.toLowerCase())
+ && file.getName().contains(BusinessConstant.UNDERLINE)
+ && file.getName().toLowerCase().endsWith(BusinessConstant.SUFFIX_JSON.toLowerCase())) {
+ labelJsonFile = file;
+ }
+ }
+ if (labelJsonFile == null) {
+ throw new ImportDatasetException(" 【" + globalFilePath + "】目录中未找到标签组文件");
+ }
+ dealLabelGroup(labelJsonFile.getName());
+ List dataLabelList = readLabelContext(labelJsonFile);
+ Map> dataLabelMap = dataLabelList.stream().collect(Collectors.groupingBy(DataLabel::getName));
+ for (Map.Entry> entry : dataLabelMap.entrySet()) {
+ if (entry.getValue().size() > 1) {
+ throw new ImportDatasetException(" 标签组中标签存在重复标签:【" + entry.getKey() + "】");
+ }
+ }
+ File[] imageFiles = imageRootFiles.listFiles();
+ if (imageFiles == null || imageFiles.length == 0) {
+ throw new ImportDatasetException(" 图片文件下不存在图片文件 ");
+ }
+ for (File imageFile : imageFiles) {
+ String suffixFileName = imageFile.getName().substring(imageFile.getName().lastIndexOf(BusinessConstant.SPOT));
+ if (!SUFFIX_LIST.contains(suffixFileName.toLowerCase())) {
+ throw new ImportDatasetException(" 图片文件文件夹中存在非法格式 ");
+ }
+ }
+ File[] annotationFiles = annotationRootFiles.listFiles();
+ if (annotationFiles == null || annotationFiles.length == 0) {
+ throw new ImportDatasetException(" 图片文件下不存在标注文件 ");
+ }
+ for (File annotationFile : annotationFiles) {
+ if (!annotationFile.getName().toLowerCase().endsWith(BusinessConstant.SUFFIX_JSON.toLowerCase())) {
+ throw new ImportDatasetException(" 标注文件文件夹中存在非法格式 ");
+ }
+ }
+ return labelJsonFile;
+ }
+
+ /**
+ * 启动线程
+ *
+ * @param filePath 本地根目录
+ * @param dataset 数据集
+ */
+ public void executeUploadAndSave(List dataLabelList, String filePath, Dataset dataset) throws Exception {
+ String localImageFilePath = filePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN);
+ List imageFileNameList = FileUtil.listFileNames(localImageFilePath);
+ log.info("需要处理: 【" + imageFileNameList.size() + "】张图片");
+ int oneSize = ThreadUtils.createThread(imageFileNameList.size());
+ log.info("需要创建线程数: 【" + oneSize + "】 条");
+ List> partitions = new ArrayList<>();
+ List need = new ArrayList<>();
+ AtomicInteger atomicInteger = new AtomicInteger(MagicNumConstant.ZERO);
+ for (String fileName : imageFileNameList) {
+ need.add(fileName);
+ if (need.size() == oneSize || atomicInteger.intValue() == imageFileNameList.size() - MagicNumConstant.ONE) {
+ List fileNameList = new ArrayList<>(need);
+ need.clear();
+ partitions.add(() -> runTask(dataLabelList, dataset, fileNameList, filePath));
+ }
+ atomicInteger.getAndIncrement();
+ }
+ ThreadUtils.runMultiThread(partitions);
+ }
+
+ /**
+ * 实际执行任务
+ *
+ * @param dataset 数据集
+ * @param fileNameList 文件名字集合
+ * @param dataSetRootFilePath 文件路径
+ * @return Integer 执行次数
+ */
+ private Integer runTask(List dataLabelList, Dataset dataset, List fileNameList, String dataSetRootFilePath) throws Exception {
+ List dataFilesList = new ArrayList<>();
+ AtomicInteger atomicInteger = new AtomicInteger(MagicNumConstant.ZERO);
+ String imageFileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + dataset.getId()
+ + BusinessConstant.FILE_SEPARATOR + BusinessConstant.IMAGE_ORIGIN + BusinessConstant.FILE_SEPARATOR;
+ String annotationFileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + dataset.getId()
+ + BusinessConstant.FILE_SEPARATOR + BusinessConstant.ANNOTATION + BusinessConstant.FILE_SEPARATOR;
+ for (String fileName : fileNameList) {
+ String imageUploadFile = imageFileBaseDir + fileName;
+ String annotationFileName = HandleFileUtil.readFileName(fileName);
+ File annotationFile = new File(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.ANNOTATION) + BusinessConstant.FILE_SEPARATOR + annotationFileName + BusinessConstant.SUFFIX_JSON.toLowerCase());
+ JSONArray jsonArray = replaceJsonNode(annotationFile, dataLabelList);
+ minioUtil.upLoadFile(imageUploadFile, FileUtil.getInputStream(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN) + BusinessConstant.FILE_SEPARATOR + fileName));
+ minioUtil.upLoadFile(annotationFileBaseDir + annotationFileName, IOUtils.toInputStream(jsonArray.toString(), StandardCharsets.UTF_8.name()));
+ DataFile dataFile = new DataFile();
+ dataFile.setName(annotationFileName);
+ dataFile.setUrl(minioConfig.getBucketName() + BusinessConstant.FILE_SEPARATOR + imageUploadFile);
+ dataFile.setStatus(FileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE);
+ dataFile.setDatasetId(dataset.getId());
+ dataFile.setFileType(MagicNumConstant.ZERO);
+ dataFile.setPid(MagicNumConstant.ZERO_LONG);
+ dataFile.setCreateUserId(dataset.getCreateUserId());
+ try {
+ BufferedImage image = ImageIO.read(new File(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN) + BusinessConstant.FILE_SEPARATOR + fileName));
+ dataFile.setWidth(image.getWidth());
+ dataFile.setHeight(image.getHeight());
+ } catch (IOException e) {
+ throw new ImportDatasetException(" 读取图片高和宽失败 ");
+ }
+ dataFile.setOriginUserId(dataset.getCreateUserId());
+ dataFilesList.add(dataFile);
+ if (dataFilesList.size() % MagicNumConstant.FIVE_HUNDRED == MagicNumConstant.ZERO || atomicInteger.intValue() == fileNameList.size() - MagicNumConstant.ONE) {
+ long startDataFileIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_FILE, dataFilesList.size());
+ for (DataFile dataFileEntity : dataFilesList) {
+ dataFileEntity.setId(startDataFileIndex++);
+ }
+ saveDataFile(dataFilesList);
+ List dataVersionFileList = new ArrayList<>();
+ for (DataFile file : dataFilesList) {
+ DataVersionFile dataVersionFile = new DataVersionFile();
+ dataVersionFile.setDatasetId(dataset.getId());
+ dataVersionFile.setFileId(file.getId());
+ dataVersionFile.setStatus(MagicNumConstant.ZERO);
+ dataVersionFile.setAnnotationStatus(FileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE);
+ dataVersionFileList.add(dataVersionFile);
+ }
+ long startDataFileVersionIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_VERSION_FILE, dataVersionFileList.size());
+ for (DataVersionFile dataVersionFile : dataVersionFileList) {
+ dataVersionFile.setId(startDataFileVersionIndex++);
+ }
+ saveDataVersionFile(dataVersionFileList);
+ dataVersionFileList.clear();
+ dataFilesList.clear();
+ }
+ atomicInteger.getAndIncrement();
+ }
+ return atomicInteger.getAndIncrement();
+
+ }
+
+ /**
+ * 检查并且替换JSON中的节点
+ *
+ * @param annotationFile 标注文件
+ * @param dataLabelList 数据集集合
+ */
+ public JSONArray replaceJsonNode(File annotationFile, List dataLabelList) throws IOException {
+ JSONArray jsonArray = new JSONArray();
+ if (annotationFile.exists()) {
+ String annotationFileContext = HandleFileUtil.readFile(annotationFile);
+ jsonArray = JSONArray.parseArray(annotationFileContext);
+ if (!jsonArray.isEmpty()) {
+ replaceAllNode(jsonArray, dataLabelList);
+ }
+ }
+ return jsonArray;
+ }
+
+ /**
+ * 替换节点值
+ *
+ * @param jsonArray 标注文件集合
+ * @param dataLabelList 标签集合
+ */
+ public void replaceAllNode(JSONArray jsonArray, List dataLabelList) {
+ for (int i = MagicNumConstant.ZERO; i < jsonArray.size(); i++) {
+ JSONObject jsonObject = jsonArray.getJSONObject(i);
+ jsonObject.put("category_id", findDataLabelId(dataLabelList, jsonObject.get("name").toString()));
+ jsonObject.remove("name");
+ }
+ }
+
+ /**
+ * 查询需要替换的节点
+ *
+ * @param dataLabelList 标签集合
+ * @param objectValue 替换的节点值
+ * @return long 替换标签的Id
+ */
+ public long findDataLabelId(List dataLabelList, String objectValue) {
+ Optional matchedDataLabel = dataLabelList.stream().filter(dataLabel -> objectValue.equals(dataLabel.getName())).findAny();
+ if (!matchedDataLabel.isPresent()) {
+ throw new ImportDatasetException(" 标注文件中name的值不存在于标签中!");
+ }
+ return matchedDataLabel.get().getId();
+ }
+
+ /**
+ * 获取JSON中所有的key
+ *
+ * @param jsonArray json集合
+ * @return JSON中所有的key
+ */
+ public List getJsonKeyList(JSONArray jsonArray) {
+ List listKey = new ArrayList<>();
+ for (Object object : jsonArray) {
+ LinkedHashMap jsonMap = JSON.parseObject(object.toString(), new TypeReference>() {
+ });
+ for (Map.Entry entry : jsonMap.entrySet()) {
+ listKey.add(entry.getKey());
+ }
+ }
+ return listKey.stream().distinct().collect(Collectors.toList());
+ }
+
+ /**
+ * 读取标签文件中标签数据
+ *
+ * @param file 标签文件
+ * @return List 标签数据集合
+ */
+ public List readLabelContext(File file) throws IOException {
+ String fileContext = HandleFileUtil.readFile(file);
+ List dataLabelList = JSONArray.parseArray(fileContext, DataLabel.class);
+ for (DataLabel dataLabel : dataLabelList) {
+ if (StringUtils.isEmpty(dataLabel.getName()) || StringUtils.isEmpty(dataLabel.getColor())) {
+ throw new ImportDatasetException(" 标签文件不规范,未能读到 'name' 或者 'color' ");
+ }
+ }
+ return dataLabelList;
+ }
+
+ /**
+ * 检查标签组是否有重名
+ *
+ * @param labelGroupName 标签组名称
+ */
+ public void dealLabelGroup(String labelGroupName) {
+ String groupName = HandleFileUtil.getLabelGroupName(labelGroupName);
+ int count = dataLabelGroupService.selectByLabelGroupName(groupName);
+ if (count > MagicNumConstant.ZERO) {
+ throw new ImportDatasetException(" 标签组名称【" + groupName + "】已存在,请修改label_xxx.json文件名 ");
+ }
+ }
+
+ /**
+ * 保存标签组名称
+ *
+ * @param labelGroupName 标签组名称
+ * @param dataset 数据集
+ * @return DataLabelGroup 数据集标签组
+ */
+ public DataLabelGroup saveDataLabelGroup(String labelGroupName, Dataset dataset) {
+ long timeStamp = System.currentTimeMillis();
+ DataLabelGroup dataLabelGroup = new DataLabelGroup();
+ dataLabelGroup.setName(labelGroupName);
+ dataLabelGroup.setOriginUserId(dataset.getCreateUserId());
+ dataLabelGroup.setType(MagicNumConstant.ZERO_LONG);
+ dataLabelGroup.setCreateUserId(dataset.getCreateUserId());
+ dataLabelGroup.setCreateTime(new Timestamp(timeStamp));
+ dataLabelGroup.setUpdateTime(new Timestamp(timeStamp));
+ dataLabelGroupService.saveDataGroupLabel(dataLabelGroup);
+ return dataLabelGroup;
+ }
+
+ /**
+ * 批量保存标签
+ *
+ * @param dataset 数据集
+ * @param listDataLabel 标签集合
+ * @param dataLabelGroupId 标签组Id
+ */
+ public void saveDataLabel(Dataset dataset, List listDataLabel, Long dataLabelGroupId) {
+ listDataLabel.forEach(dataLabel -> dataLabel.setCreateUserId(dataset.getCreateUserId()));
+ dataLabelService.saveBatchDataLabel(listDataLabel);
+ List listDatasetDataLabel = new ArrayList<>();
+ for (DataLabel dataLabel : listDataLabel) {
+ DatasetDataLabel datasetDataLabel = new DatasetDataLabel();
+ datasetDataLabel.setLabelId(dataLabel.getId());
+ datasetDataLabel.setDatasetId(dataset.getId());
+ listDatasetDataLabel.add(datasetDataLabel);
+ }
+ saveDatasetDataLabel(listDatasetDataLabel);
+
+ List listDataGroupLabel = new ArrayList<>();
+ for (DatasetDataLabel datasetDataLabel : listDatasetDataLabel) {
+ DataGroupLabel dataGroupLabel = new DataGroupLabel();
+ dataGroupLabel.setLabelId(datasetDataLabel.getId());
+ dataGroupLabel.setLabelGroupId(dataLabelGroupId);
+ listDataGroupLabel.add(dataGroupLabel);
+ }
+ saveDatasetDataGroupLabel(listDataGroupLabel);
+
+ }
+
+ /**
+ * 批量保存文件
+ *
+ * @param listDataFile file集合
+ */
+ public void saveDataFile(List listDataFile) {
+ dataFileService.saveBatchDataFile(listDataFile);
+ }
+
+ /**
+ * 批量保存文件版本数据
+ *
+ * @param listDataVersionFile 文件版本数据
+ */
+ public void saveDataVersionFile(List listDataVersionFile) {
+ dataVersionFileService.saveBatchDataFileVersion(listDataVersionFile);
+ }
+
+
+ /**
+ * 批量保存标签与数据集关系表
+ *
+ * @param listDatasetDataLabel 标签与数据集关系表
+ */
+ public void saveDatasetDataLabel(List listDatasetDataLabel) {
+ datasetDataLabelService.saveBatchDatasetDataLabel(listDatasetDataLabel);
+ }
+
+ /**
+ * 批量保存标签与标签组的关系
+ *
+ * @param listDataGroupLabel 标签与标签组集合
+ */
+ public void saveDatasetDataGroupLabel(List listDataGroupLabel) {
+ dataGroupLabelService.saveDataGroupLabel(listDataGroupLabel);
+ }
+
+ /**
+ * 查询数据集
+ *
+ * @param datasetId 数据集Id
+ * @return Dataset 根据数据集ID查询返回的数据集
+ */
+ private Dataset findDataset(Long datasetId) {
+ return datasetService.findDatasetById(datasetId);
+ }
+
+ /**
+ * 校验数据集ID
+ *
+ * @param scanner 控制台输入参数
+ * @return Dataset 数据集
+ */
+ public Dataset verificationDatasetId(Scanner scanner) {
+ log.warn("# 请输入数据集ID #");
+ String datasetIdStr = scanner.nextLine();
+ long datasetId;
+ try {
+ datasetId = Long.parseLong(datasetIdStr.trim());
+ } catch (Exception e) {
+ throw new ImportDatasetException(" 数据集ID非法,请重新输入 ");
+ }
+ Dataset dataset = findDataset(datasetId);
+ if (dataset == null) {
+ throw new ImportDatasetException(" 数据集ID不存在,请重新输入 ");
+ }
+ int countDataLabel = datasetService.findDataLabelById(dataset.getId());
+ int countDataFile = datasetService.findDataFileById(dataset.getId());
+ if (countDataLabel > MagicNumConstant.ZERO || countDataFile > MagicNumConstant.ZERO) {
+ throw new ImportDatasetException(" 当前数据集文件已存在,请勿重新导入 ");
+ }
+ return dataset;
+ }
+
+ /**
+ * 校验文件路径
+ *
+ * @param scanner 输入控制台
+ * @return String 字符串
+ */
+ public String verificationFilePath(Scanner scanner) {
+ log.warn("# 请输入待导入数据集绝对路径地址 #");
+ String filePath = scanner.nextLine();
+ File file = new File(filePath.trim());
+ if (!file.exists()) {
+ throw new ImportDatasetException(" 【" + filePath + "】 文件路径不存在,请重新输入");
+ }
+ return filePath;
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataFileService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataFileService.java
new file mode 100644
index 0000000..c100a35
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataFileService.java
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DataFile;
+import java.util.List;
+
+/**
+ * @description 文件服务
+ * @date 2020-09-17
+ */
+public interface DataFileService {
+
+ /**
+ * 批量写入文件内容
+ *
+ * @param dataFiles 文件数据集合
+ */
+ void saveBatchDataFile(List dataFiles);
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataGroupLabelService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataGroupLabelService.java
new file mode 100644
index 0000000..5c5c0ce
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataGroupLabelService.java
@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DataGroupLabel;
+
+import java.util.List;
+
+/**
+ * @description 标签与标签组的服务接口
+ * @date 2020-10-21
+ */
+public interface DataGroupLabelService {
+ /**
+ * 保存标签与标签组的关系
+ *
+ * @param listDataGroupLabel 标签与标签组数据
+ */
+ void saveDataGroupLabel(List listDataGroupLabel);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelGroupService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelGroupService.java
new file mode 100644
index 0000000..9f2f1c4
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelGroupService.java
@@ -0,0 +1,40 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DataLabelGroup;
+
+/**
+ * @description 数据集标签组服务接口
+ * @date 2020-10-14
+ */
+public interface DataLabelGroupService {
+ /**
+ * 根据标签组名查询
+ *
+ * @param labelGroupName 标签组名称
+ * @return int 数量
+ */
+ int selectByLabelGroupName(String labelGroupName);
+
+ /**
+ * 保存数据集标签组标签
+ *
+ * @param dataLabelGroup 标签组名称
+ */
+ void saveDataGroupLabel(DataLabelGroup dataLabelGroup);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelService.java
new file mode 100644
index 0000000..a4a98fa
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelService.java
@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DataLabel;
+
+import java.util.List;
+
+/**
+ * @description 数据集标签服务接口
+ * @date 2020-10-14
+ */
+public interface DataLabelService {
+ /**
+ * 批量保存数据集标签
+ *
+ * @param listDataLabel 数据集标签集合
+ */
+ void saveBatchDataLabel(List listDataLabel);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataSequenceService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataSequenceService.java
new file mode 100644
index 0000000..5199174
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataSequenceService.java
@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+
+import org.dubhe.datasetutil.domain.entity.DataSequence;
+
+/**
+ * @description 获取序列服务接口
+ * @date 2020-09-23
+ */
+public interface DataSequenceService {
+ /**
+ * 根据业务编码获取序列号
+ *
+ * @param businessCode 业务编码
+ * @return DataSequence 数据索引
+ */
+ DataSequence getSequence(String businessCode);
+ /**
+ * 根据业务编码更新起点
+ *
+ * @param businessCode 业务编码
+ * @return int 数量
+ */
+ int updateSequenceStart(String businessCode);
+
+ /**
+ * 检查表是否存在
+ *
+ * @param tableName 表名
+ * @return boolean 存在结果
+ */
+ boolean checkTableExist(String tableName);
+
+ /**
+ * 执行存储过程创建表
+ *
+ * @param tableId 表ID
+ */
+ void createTable(String tableId);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataVersionFileService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataVersionFileService.java
new file mode 100644
index 0000000..5b3d5a9
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataVersionFileService.java
@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+
+import java.util.List;
+
+/**
+ * @description 数据集文件关系服务
+ * @date 2020-9-17
+ */
+public interface DataVersionFileService {
+ /**
+ * 插入数据集文件数据
+ *
+ * @param dataVersionFiles 数据集文件数据集合
+ */
+ void saveBatchDataFileVersion(List dataVersionFiles);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetDataLabelService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetDataLabelService.java
new file mode 100644
index 0000000..6752391
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetDataLabelService.java
@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DatasetDataLabel;
+
+import java.util.List;
+
+/**
+ * @description 数据集与标签关系服务接口
+ * @date 2020-10-14
+ */
+public interface DatasetDataLabelService {
+ /**
+ * 批量保存数据集与标签关系
+ *
+ * @param listDatasetDataLabel 数据集标签集合
+ */
+ void saveBatchDatasetDataLabel(List listDatasetDataLabel);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetService.java
new file mode 100644
index 0000000..06d2b8b
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetService.java
@@ -0,0 +1,65 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.Dataset;
+
+/**
+ * @description 数据集服务
+ * @date 2020-9-17
+ */
+public interface DatasetService {
+ /**
+ * 根据数据集Id查询创建人Id
+ *
+ * @param datasetId 数据集Id
+ * @return Dataset 数据集
+ */
+ Dataset findCreateUserIdById(Long datasetId);
+
+ /**
+ * 根据ID查询数据集
+ *
+ * @param datasetId 数据集Id
+ * @return Dataset 数据集
+ */
+ Dataset findDatasetById(Long datasetId);
+
+ /**
+ * 更新数据集状态
+ *
+ * @param dataset 数据集
+ * @return int 数量
+ */
+ int updateDatasetStatus(Dataset dataset);
+
+ /**
+ * 查询数据集标签数量
+ *
+ * @param datasetId 数据集ID
+ * @return int 数量
+ */
+ int findDataLabelById(Long datasetId);
+
+ /**
+ * 查询数据集文件数量
+ *
+ * @param datasetId 数据集ID
+ * @return int 数量
+ */
+ int findDataFileById(Long datasetId);
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataFileServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataFileServiceImpl.java
new file mode 100644
index 0000000..1186efe
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataFileServiceImpl.java
@@ -0,0 +1,60 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.dao.DataFileMapper;
+import org.dubhe.datasetutil.domain.entity.DataFile;
+import org.dubhe.datasetutil.service.DataFileService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description 数据集文件 服务实现类
+ * @date 2020-09-17
+ */
+@Slf4j
+@Service
+public class DataFileServiceImpl extends ServiceImpl implements DataFileService {
+
+ @Autowired
+ private DataFileMapper dataFileMapper;
+
+ /**
+ * 批量写入文件内容
+ *
+ * @param listDataFile 文件数据集合
+ */
+ @Override
+ public void saveBatchDataFile(List listDataFile) {
+ int listSize = listDataFile.size();
+ if (listSize > BusinessConstant.SUB_LENGTH) {
+ List> partitionList = Lists.partition(listDataFile, BusinessConstant.SUB_LENGTH);
+ for (List subPartitionList : partitionList) {
+ dataFileMapper.saveBatchDataFile(subPartitionList);
+ }
+ } else {
+ dataFileMapper.saveBatchDataFile(listDataFile);
+ }
+ }
+
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataGroupLabelServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataGroupLabelServiceImpl.java
new file mode 100644
index 0000000..dd9dde0
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataGroupLabelServiceImpl.java
@@ -0,0 +1,57 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.google.common.collect.Lists;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.dao.DataGroupLabelMapper;
+import org.dubhe.datasetutil.domain.entity.DataGroupLabel;
+import org.dubhe.datasetutil.service.DataGroupLabelService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description 标签与标签组的服务接口实现
+ * @date 2020-10-21
+ */
+@Service
+public class DataGroupLabelServiceImpl extends ServiceImpl implements DataGroupLabelService {
+
+ @Autowired
+ private DataGroupLabelMapper dataGroupLabelMapper;
+
+ /**
+ * 插入标签组
+ *
+ * @param listDataGroupLabel 标签组数据
+ */
+ @Override
+ public void saveDataGroupLabel(List listDataGroupLabel) {
+ int listSize = listDataGroupLabel.size();
+ if (listSize > BusinessConstant.SUB_LENGTH) {
+ List> partitionList = Lists.partition(listDataGroupLabel, BusinessConstant.SUB_LENGTH);
+ for (List subPartitionList : partitionList) {
+ dataGroupLabelMapper.saveDataGroupLabel(subPartitionList);
+ }
+ } else {
+ dataGroupLabelMapper.saveDataGroupLabel(listDataGroupLabel);
+ }
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelGroupServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelGroupServiceImpl.java
new file mode 100644
index 0000000..47b936b
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelGroupServiceImpl.java
@@ -0,0 +1,60 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.dubhe.datasetutil.dao.DataGroupLabelMapper;
+import org.dubhe.datasetutil.dao.DataLabelGroupMapper;
+import org.dubhe.datasetutil.domain.entity.DataGroupLabel;
+import org.dubhe.datasetutil.domain.entity.DataLabelGroup;
+import org.dubhe.datasetutil.service.DataLabelGroupService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+/**
+ * @description 数据集标签组服务实现接口
+ * @date 2020-10-14
+ */
+@Service
+public class DataLabelGroupServiceImpl extends ServiceImpl implements DataLabelGroupService {
+
+ @Autowired
+ private DataLabelGroupMapper dataLabelGroupMapper;
+
+ /**
+ * 根据标签组名查询
+ *
+ * @param labelGroupName 标签组名称
+ * @return int 数量
+ */
+ @Override
+ public int selectByLabelGroupName(String labelGroupName) {
+ return dataLabelGroupMapper.selectByLabelGroupName(labelGroupName);
+ }
+
+ /**
+ * 保存标签组名称
+ *
+ * @param dataLabelGroup 标签组名称
+ */
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void saveDataGroupLabel(DataLabelGroup dataLabelGroup) {
+ dataLabelGroupMapper.insert(dataLabelGroup);
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelServiceImpl.java
new file mode 100644
index 0000000..0d2fa0b
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelServiceImpl.java
@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.google.common.collect.Lists;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.dao.DataLabelMapper;
+import org.dubhe.datasetutil.domain.entity.DataLabel;
+import org.dubhe.datasetutil.service.DataLabelService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description 数据集标签服务接口实现
+ * @date 2020-10-14
+ */
+@Service
+public class DataLabelServiceImpl implements DataLabelService {
+
+ @Autowired
+ private DataLabelMapper dataLabelMapper;
+
+ /**
+ * 批量保存数据集标签 (分批)
+ *
+ * @param listDataLabel 数据集标签
+ */
+ @Override
+ public void saveBatchDataLabel(List listDataLabel) {
+ int listSize = listDataLabel.size();
+ if(listSize > BusinessConstant.SUB_LENGTH){
+ List> partitionList = Lists.partition(listDataLabel,BusinessConstant.SUB_LENGTH);
+ for (List subPartitionList: partitionList) {
+ dataLabelMapper.saveBatchDataLabel(subPartitionList);
+ }
+ }else{
+ dataLabelMapper.saveBatchDataLabel(listDataLabel);
+ }
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataSequenceServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataSequenceServiceImpl.java
new file mode 100644
index 0000000..22f1dbb
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataSequenceServiceImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import org.dubhe.datasetutil.dao.DataSequenceMapper;
+import org.dubhe.datasetutil.domain.entity.DataSequence;
+import org.dubhe.datasetutil.service.DataSequenceService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+
+/**
+ * @description 获取序列服务接口实现
+ * @date 2020-09-23
+ */
+@Service
+public class DataSequenceServiceImpl implements DataSequenceService {
+
+ @Autowired
+ private DataSequenceMapper dataSequenceMapper;
+
+ @Override
+ public DataSequence getSequence(String businessCode) {
+ return dataSequenceMapper.selectByBusiness(businessCode);
+ }
+
+ /**
+ * 更新数据索引
+ *
+ * @param businessCode 业务编码
+ * @return int 数量
+ */
+ @Override
+ public int updateSequenceStart(String businessCode) {
+ return dataSequenceMapper.updateStartByBusinessCode(businessCode);
+ }
+
+ /**
+ * 检查表名
+ *
+ * @param tableName 表名
+ * @return true/false
+ */
+ @Override
+ public boolean checkTableExist(String tableName) {
+ try {
+ dataSequenceMapper.checkTableExist(tableName);
+ return true;
+ }catch (Exception e){
+ return false;
+ }
+ }
+
+ /**
+ * 创建表
+ *
+ * @param tableName 类型名称
+ */
+ @Override
+ public void createTable(String tableName) {
+ String oldTableName = tableName.substring(0,tableName.lastIndexOf("_"));
+ dataSequenceMapper.createNewTable(tableName,oldTableName);
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataVersionFileServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataVersionFileServiceImpl.java
new file mode 100644
index 0000000..a032328
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataVersionFileServiceImpl.java
@@ -0,0 +1,44 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.dubhe.datasetutil.dao.DataVersionFileMapper;
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+import org.dubhe.datasetutil.service.DataVersionFileService;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description 数据集文件 服务实现类
+ * @date 2020-09-17
+ */
+@Service
+public class DataVersionFileServiceImpl extends ServiceImpl implements DataVersionFileService {
+
+
+ /**
+ * 插入数据集文件数据
+ *
+ * @param listDataVersionFile 数据集文件数据集合
+ */
+ @Override
+ public void saveBatchDataFileVersion(List listDataVersionFile) {
+ baseMapper.saveBatchDataFileVersion(listDataVersionFile);
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetDataLabelServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetDataLabelServiceImpl.java
new file mode 100644
index 0000000..6f0cf52
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetDataLabelServiceImpl.java
@@ -0,0 +1,57 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.google.common.collect.Lists;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+import org.dubhe.datasetutil.dao.DatasetDataLabelMapper;
+import org.dubhe.datasetutil.domain.entity.DatasetDataLabel;
+import org.dubhe.datasetutil.service.DatasetDataLabelService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description 数据集和标签服务实现类
+ * @date 2020-10-14
+ */
+@Service
+public class DatasetDataLabelServiceImpl implements DatasetDataLabelService {
+
+ @Autowired
+ private DatasetDataLabelMapper datasetDataLabelMapper;
+
+ private static final int SUB_LENGTH = MagicNumConstant.FIVE_THOUSAND;
+
+ /**
+ * 批量保存数据集标签
+ *
+ * @param listDatasetDataLabel 数据集标签集合
+ */
+ @Override
+ public void saveBatchDatasetDataLabel(List listDatasetDataLabel) {
+ if(listDatasetDataLabel.size() > SUB_LENGTH){
+ List> partitionList = Lists.partition(listDatasetDataLabel,SUB_LENGTH);
+ for (List subPartitionList: partitionList) {
+ datasetDataLabelMapper.saveBatchDatasetDataLabel(subPartitionList);
+ }
+ }else{
+ datasetDataLabelMapper.saveBatchDatasetDataLabel(listDatasetDataLabel);
+ }
+ }
+}
diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetServiceImpl.java
new file mode 100644
index 0000000..10ab8cc
--- /dev/null
+++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetServiceImpl.java
@@ -0,0 +1,92 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.dubhe.datasetutil.common.constant.DataStateCodeConstant;
+import org.dubhe.datasetutil.dao.DatasetMapper;
+import org.dubhe.datasetutil.domain.entity.Dataset;
+import org.dubhe.datasetutil.service.DatasetService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * @description 数据集 服务实现类
+ * @date 2020-09-17
+ */
+@Service
+public class DatasetServiceImpl extends ServiceImpl implements DatasetService {
+
+ @Autowired
+ private DatasetMapper datasetMapper;
+
+ /**
+ * 根据数据集Id查询创建人Id
+ *
+ * @param datasetId 数据集Id
+ * @return dataset 数据集合
+ */
+ @Override
+ public Dataset findCreateUserIdById(Long datasetId) {
+ return baseMapper.selectById(datasetId);
+ }
+
+ /**
+ * 根据ID 查询数据集
+ *
+ * @param datasetId 数据集ID
+ * @return Dataset 数据集
+ */
+ @Override
+ public Dataset findDatasetById(Long datasetId) {
+ return datasetMapper.findDatasetById(datasetId);
+ }
+
+ /**
+ * 更新数据集状态
+ *
+ * @param dataset 数据集
+ * @return int 数量
+ */
+ @Override
+ public int updateDatasetStatus(Dataset dataset) {
+ dataset.setStatus(DataStateCodeConstant.ANNOTATION_COMPLETE_STATE);
+ return datasetMapper.updateById(dataset);
+ }
+
+ /**
+ * 查询数据集数据标签数量
+ *
+ * @param datasetId 数据集ID
+ * @return int 数量
+ */
+ @Override
+ public int findDataLabelById(Long datasetId) {
+ return datasetMapper.findDataLabelById(datasetId);
+ }
+
+ /**
+ * 查询数据集文件数量
+ *
+ * @param datasetId 数据集ID
+ * @return int 数量
+ */
+ @Override
+ public int findDataFileById(Long datasetId) {
+ return datasetMapper.findDataFileById(datasetId);
+ }
+}
diff --git a/dataset-util/src/main/resources/banner.txt b/dataset-util/src/main/resources/banner.txt
new file mode 100644
index 0000000..e69de29
diff --git a/dataset-util/src/main/resources/logback-spring-dev.xml b/dataset-util/src/main/resources/logback-spring-dev.xml
new file mode 100644
index 0000000..90e2344
--- /dev/null
+++ b/dataset-util/src/main/resources/logback-spring-dev.xml
@@ -0,0 +1,254 @@
+
+
+ dubhe
+
+
+
+
+
+
+
+ ${log.pattern}
+
+
+ WARN
+ INFO
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/info/dubhe-info.log
+
+ logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ INFO
+ INFO,K8S_CALLBACK
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/debug/dubhe-debug.log
+
+ logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ DEBUG
+ DEBUG
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/error/dubhe-error.log
+
+ logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ ERROR
+ ERROR
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/warn/dubhe-warn.log
+
+ logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+
+ WARN
+ WARN
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/trace/dubhe-trace.log
+
+ logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ TRACE
+ TRACE
+ ACCEPT
+ DENY
+
+
+
+
+
+
+ logs/${log.path}/info/dubhe-schedule.log
+
+ logs/${log.path}/info/dubhe-${app.active}-schedule-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ INFO
+ SCHEDULE
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/info/dubhe-request.log
+
+ logs/${log.path}/info/dubhe-${app.active}-request-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ INFO
+
+ GLOBAL_REQUEST
+ ACCEPT
+ DENY
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dataset-util/src/main/resources/logback-spring-test.xml b/dataset-util/src/main/resources/logback-spring-test.xml
new file mode 100644
index 0000000..038359c
--- /dev/null
+++ b/dataset-util/src/main/resources/logback-spring-test.xml
@@ -0,0 +1,248 @@
+
+
+ dubhe
+
+
+
+
+
+
+
+
+ ${log.pattern}
+
+
+
+
+
+ logs/${log.path}/info/dubhe-info.log
+
+ logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ INFO
+ INFO,K8S_CALLBACK
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/debug/dubhe-debug.log
+
+ logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ DEBUG
+ DEBUG
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/error/dubhe-error.log
+
+ logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ ERROR
+ ERROR
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/warn/dubhe-warn.log
+
+ logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+
+ WARN
+ WARN
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/trace/dubhe-trace.log
+
+ logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ TRACE
+ TRACE
+ ACCEPT
+ DENY
+
+
+
+
+
+
+ logs/${log.path}/info/dubhe-schedule.log
+
+ logs/${log.path}/info/dubhe-${app.active}-schedule-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ INFO
+ SCHEDULE
+ ACCEPT
+ DENY
+
+
+
+
+
+ logs/${log.path}/info/dubhe-request.log
+
+ logs/${log.path}/info/dubhe-${app.active}-request-%d{yyyy-MM-dd}.%i.log
+
+
+ 50MB
+ 7
+ 250MB
+
+
+ %m%n
+
+
+ true
+
+ INFO
+
+ GLOBAL_REQUEST
+ ACCEPT
+ DENY
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dataset-util/src/main/resources/mapper/DataFileMapper.xml b/dataset-util/src/main/resources/mapper/DataFileMapper.xml
new file mode 100644
index 0000000..1f7b10f
--- /dev/null
+++ b/dataset-util/src/main/resources/mapper/DataFileMapper.xml
@@ -0,0 +1,22 @@
+
+
+
+
+
+ insert into data_file (id,`name`,dataset_id,status,url,enhance_type,width,height,origin_user_id,create_user_id,pid)
+ values
+
+ (#{file.id},#{file.name},#{file.datasetId},#{file.status},
+ #{file.url},#{file.enhanceType},#{file.width},#{file.height},#{file.originUserId},#{file.createUserId}
+
+
+ ,#{file.pid}
+
+
+ ,0
+
+
+ )
+
+
+
\ No newline at end of file
diff --git a/dataset-util/src/main/resources/mapper/DataGroupLabelMapper.xml b/dataset-util/src/main/resources/mapper/DataGroupLabelMapper.xml
new file mode 100644
index 0000000..91059a9
--- /dev/null
+++ b/dataset-util/src/main/resources/mapper/DataGroupLabelMapper.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+ insert into data_group_label (label_id,label_group_id)
+ values
+
+ (#{dataGroupLabel.labelId},#{dataGroupLabel.labelGroupId})
+
+
+
\ No newline at end of file
diff --git a/dataset-util/src/main/resources/mapper/DataLabelMapper.xml b/dataset-util/src/main/resources/mapper/DataLabelMapper.xml
new file mode 100644
index 0000000..20d80cb
--- /dev/null
+++ b/dataset-util/src/main/resources/mapper/DataLabelMapper.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+ insert into data_label (`name`,color,create_user_id)
+ values
+
+ ( #{dataLabel.name},#{dataLabel.color},#{dataLabel.createUserId})
+
+
+
\ No newline at end of file
diff --git a/dataset-util/src/main/resources/mapper/DataVersionFileMapper.xml b/dataset-util/src/main/resources/mapper/DataVersionFileMapper.xml
new file mode 100644
index 0000000..5eb4245
--- /dev/null
+++ b/dataset-util/src/main/resources/mapper/DataVersionFileMapper.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+ insert into data_dataset_version_file (id,dataset_id,file_id,annotation_status,status)
+ values
+
+ (#{temp.id},#{temp.datasetId},#{temp.fileId},#{temp.annotationStatus},#{temp.status})
+
+
+
\ No newline at end of file
diff --git a/dataset-util/src/main/resources/mapper/DatasetDataLabelMapper.xml b/dataset-util/src/main/resources/mapper/DatasetDataLabelMapper.xml
new file mode 100644
index 0000000..59c4e56
--- /dev/null
+++ b/dataset-util/src/main/resources/mapper/DatasetDataLabelMapper.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+ insert into data_dataset_label(dataset_id,label_id )
+ values
+
+ (#{datasetDataLabel.datasetId},#{datasetDataLabel.labelId})
+
+
+
\ No newline at end of file
diff --git a/dataset-util/src/main/resources/mapper/DatasetMapper.xml b/dataset-util/src/main/resources/mapper/DatasetMapper.xml
new file mode 100644
index 0000000..7e7617f
--- /dev/null
+++ b/dataset-util/src/main/resources/mapper/DatasetMapper.xml
@@ -0,0 +1,5 @@
+
+
+
+
+
\ No newline at end of file