update dataset-util

dataset-util/.gitignore

@@ -0,0 +1,6 @@
+# Created by .ignore support plugin (hsz.mobi)
+.idea
+*.iml
+logs
+HELP.md
+target/

dataset-util/README.md

@@ -0,0 +1,94 @@
+# 之江天枢-数据集导入脚本
+
+**之江天枢一站式人工智能开源平台**（简称：**之江天枢**）,为了实现其他平台已标注完成的数据集在「一站式开发平台」上进行开发，我们增加了数据集导入的功能，实现对数据集的全流程功能操作。
+
+## 源码部署
+
+安装如下软件环境。
+- OpenJDK：1.8+
+
+## 下载脚本
+
+- 数据集模板：http://tianshu.org.cn/static/upload/file/dubhe-dataset-template.zip 
+- 上传数据集脚本：http://tianshu.org.cn/static/upload/file/upload_dataset.zip
+
+
+## 脚本使用说明：
+
+- 登录天枢系统深度学习平台，在数据管理菜单下的数据集管理中创建数据集。获取数据集ID
+- 需要自行准备图片文件、标注文件、标签文件
+   
+## 运行脚本：
+
+1.解压下载的zip文件，需要自行配置数据源、MinIO相关配置
+
+2.运行脚本Windows 运行 run.bat; macOS/Linux 系统运行 run.sh 
+
+注：可自行配置'application-{env}.xml'文件，执行命令后面添加 'run.bat {env}'即可执行对应的application-{env}.xml；然后按提示进行操作
+
+3.输入数据集ID 
+
+4.输入待导入数据集绝对路径
+
+   
+## 目录结构：
+
+```
+标签文件： label_{name}.json ({name} 代表标签组名，可自行定义; 只读标签组文件夹下的第一个标签组文件，标签文件仅支持:.json 支持大小写;文件内容为JSON字符串)
+图片文件目录： origin (图片文件需要有后缀名，支持四种格式：.jpg,.png,.bmp,.jpeg 支持大小写)
+标注文件目录： annotation (标注文件需要有后缀名，仅支持格式：.json 支持大小写; 文件内容为JSON字符串) 
+```    
+## 文件格式
+
+- 标签文件内容样例:
+```
+    name: 名称
+    color: 颜色(16进制编码)
+```     
+详细示例：   
+```
+[{
+	"name": "行人",
+	"color": "#ffbb96"
+}, 
+{
+	"name": "自行车",
+	"color": "#fcffe6"
+}, 
+{
+	"name": "汽车",
+	"color": "#f4ffb8"
+}]
+```
+
+- 标注文件内容样例：
+```
+    name: 名称
+    bbox: 标注位置
+    score：分数
+```
+详细示例： 
+```
+[{
+	"name": "行人",
+	"bbox": [321.6755762696266, 171.32076993584633, 185.67924201488495, 145.02639323472977],
+	"score": 0.6922634840011597
+}, 
+{
+	"name": "自行车",
+	"bbox": [40.88740050792694, 22.707078605890274, 451.21362805366516, 326.0102793574333],
+	"score": 0.6069411635398865
+}]
+```
+  
+## 了解更多 
+  
+http://docs.dubhe.ai/docs/module/dataset/import-dataset
+   
+
+
+
+
+
+
+

dataset-util/pom.xml

@@ -0,0 +1,142 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.dubhe</groupId>
+    <artifactId>dataset-util</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <name>dataset-util</name>
+    <description>数据处理模块工具</description>
+
+    <properties>
+        <java.version>1.8</java.version>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <spring-boot.version>2.3.0.RELEASE</spring-boot.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>druid</artifactId>
+            <version>1.1.6</version>
+        </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>8.0.17</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>com.baomidou</groupId>
+            <artifactId>mybatis-plus-boot-starter</artifactId>
+            <version>3.2.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>fastjson</artifactId>
+            <version>1.2.47</version>
+        </dependency>
+        <dependency>
+            <groupId>io.minio</groupId>
+            <artifactId>minio</artifactId>
+            <version>7.0.2</version>
+        </dependency>
+        <dependency>
+            <groupId>com.xiaoleilu</groupId>
+            <artifactId>hutool-all</artifactId>
+            <version>3.0.1</version>
+        </dependency>
+
+        <!-- for spring boot -->
+        <dependency>
+            <groupId>org.apache.shardingsphere</groupId>
+            <artifactId>sharding-jdbc-spring-boot-starter</artifactId>
+            <version>4.0.0-RC1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>1.3.2</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.aspectj</groupId>
+            <artifactId>aspectjweaver</artifactId>
+            <version>1.8.9</version>
+        </dependency>
+
+        <!--监控sql日志-->
+        <dependency>
+            <groupId>org.bgee.log4jdbc-log4j2</groupId>
+            <artifactId>log4jdbc-log4j2-jdbc4.1</artifactId>
+            <version>1.16</version>
+        </dependency>
+
+        <!--工具包-->
+        <dependency>
+            <groupId>cn.hutool</groupId>
+            <artifactId>hutool-all</artifactId>
+            <version>5.0.6</version>
+        </dependency>
+
+    </dependencies>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-dependencies</artifactId>
+                <version>${spring-boot.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                    <encoding>UTF-8</encoding>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <version>2.3.0.RELEASE</version>
+                <configuration>
+                    <mainClass>org.dubhe.datasetutil.DatasetUtilApplication</mainClass>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>repackage</id>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

dataset-util/src/main/java/org/dubhe/datasetutil/DatasetUtilApplication.java

@@ -0,0 +1,119 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil;
+
+import lombok.extern.slf4j.Slf4j;
+import org.dubhe.datasetutil.common.util.SpringContextHolder;
+import org.dubhe.datasetutil.handle.DatasetImageUploadHandle;
+import org.dubhe.datasetutil.handle.DatasetImportHandle;
+import org.dubhe.datasetutil.common.util.PrintUtils;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ApplicationContext;
+
+import java.util.Scanner;
+
+/**
+ * @description 文档导入工程启动类
+ * @date 2020-09-17
+ */
+@Slf4j
+@SpringBootApplication
+@MapperScan("org.dubhe.datasetutil.dao")
+public class DatasetUtilApplication {
+
+    /**
+     * 主函数
+     *
+     * @param args 入参
+     */
+    public static void main(String[] args) {
+        ApplicationContext applicationContext = SpringApplication.run(DatasetUtilApplication.class, args);
+        SpringContextHolder springContextHolder = new SpringContextHolder();
+        springContextHolder.setApplicationContext(applicationContext);
+        execute(applicationContext);
+    }
+
+    /**
+     * 执行脚本
+     *
+     * @param applicationContext 请求上下文
+     */
+    public static void execute(ApplicationContext applicationContext) {
+        while (true) {
+            Scanner scanner = new Scanner(System.in);
+            log.warn("###################请输入需要执行的任务#############");
+            log.warn("#  输入1.执行上传图片          ");
+            log.warn("#  输入2.执行导入数据集        ");
+            log.warn("#  输入命令 ：exit 退出        ");
+            log.warn("################################################");
+            String a = scanner.nextLine();
+            switch (a) {
+                case "1":
+                    uploadDatasetImage(scanner, applicationContext);
+                    break;
+                case "2":
+                    importDataset(scanner, applicationContext);
+                    break;
+                case "exit":
+                default:
+                    System.exit(0);
+                    break;
+            }
+        }
+    }
+
+    /**
+     * 导入图片
+     *
+     * @param scanner            输入控制台
+     * @param applicationContext 请求上下文
+     */
+    public static void uploadDatasetImage(Scanner scanner, ApplicationContext applicationContext) {
+        log.warn("# 请输入数据集ID #");
+        String datasetIdStr = scanner.nextLine();
+        Long datasetId = Long.parseLong(datasetIdStr);
+        log.warn("# 请输入要上传的图片地址 #");
+        String filePath = scanner.nextLine();
+        DatasetImageUploadHandle datasetImageUploadHandle = (DatasetImageUploadHandle) applicationContext.getBean("datasetImageUploadHandle");
+        try {
+            datasetImageUploadHandle.execute(filePath, datasetId);
+        } catch (Exception e) {
+            log.error("", e);
+            log.error("# 数据集上传失败，请重新尝试.....");
+        }
+    }
+
+    /**
+     * 导入数据集
+     *
+     * @param scanner            输入控制台
+     * @param applicationContext 请求上下文
+     */
+    public static void importDataset(Scanner scanner, ApplicationContext applicationContext) {
+        DatasetImportHandle datasetImportHandle = (DatasetImportHandle) applicationContext.getBean("datasetImportHandle");
+        try{
+            datasetImportHandle.importDataset(scanner);
+        } catch (Exception e) {
+            log.error("");
+            PrintUtils.printLine("  Error：" + e.getMessage(), PrintUtils.RED);
+            log.error("");
+        }
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/aspect/LogAspect.java

@@ -0,0 +1,76 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.dubhe.datasetutil.common.aspect;
+
+import lombok.extern.slf4j.Slf4j;
+import org.aspectj.lang.JoinPoint;
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Pointcut;
+import org.dubhe.datasetutil.common.enums.LogEnum;
+import org.dubhe.datasetutil.common.util.LogUtil;
+import org.slf4j.MDC;
+import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
+
+import java.util.UUID;
+
+/**
+ * @description 日志切面
+ * @date 2020-04-10
+ */
+@Component
+@Aspect
+@Slf4j
+public class LogAspect {
+
+	public static final String TRACE_ID = "traceId";
+
+	@Pointcut("execution(* org.dubhe..service..*.*(..))) ")
+	public void serviceAspect() {
+	}
+
+	@Pointcut(" serviceAspect() ")
+	public void aroundAspect() {
+	}
+
+	@Around("aroundAspect()")
+	public Object around(JoinPoint joinPoint) throws Throwable {
+		if (StringUtils.isEmpty(MDC.get(TRACE_ID))) {
+			MDC.put(TRACE_ID, UUID.randomUUID().toString());
+		}
+		return combineLogInfo(joinPoint);
+	}
+
+    /**
+     * 根据连接点返回结果
+     *
+     * @param joinPoint 连接点
+     * @return Object 返回结果
+     */
+	private Object combineLogInfo(JoinPoint joinPoint) throws Throwable {
+		Object[] param = joinPoint.getArgs();
+		LogUtil.info(LogEnum.REST_REQ, "uri:{},input:{},==>begin", joinPoint.getSignature(), param);
+		long start = System.currentTimeMillis();
+		Object result = ((ProceedingJoinPoint) joinPoint).proceed();
+		long end = System.currentTimeMillis();
+		LogUtil.info(LogEnum.REST_REQ, "uri:{},output:{},proc_time:{},<==end", joinPoint.getSignature().toString(),
+				result, end - start);
+		return result;
+	}
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/base/BaseEntity.java

@@ -0,0 +1,66 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.base;
+
+import com.baomidou.mybatisplus.annotation.FieldFill;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableLogic;
+import lombok.Data;
+import java.io.Serializable;
+import java.sql.Timestamp;
+
+/**
+ * @description  Entity基础类
+ * @date 2020-03-15
+ */
+@Data
+public class BaseEntity implements Serializable {
+
+    private static final long serialVersionUID = 4936056317364745513L;
+
+    /**
+     * 删除标识
+     **/
+    @TableField(value = "deleted",fill = FieldFill.INSERT)
+    @TableLogic
+    private Boolean deleted = false;
+
+    /**
+     * 创建人id
+     **/
+    @TableField(value = "create_user_id",fill = FieldFill.INSERT)
+    private Long createUserId;
+
+    /**
+     * 修改人id
+     **/
+    @TableField(value = "update_user_id",fill = FieldFill.INSERT_UPDATE)
+    private Long updateUserId;
+
+    /**
+     * 创建时间
+     **/
+    @TableField(value = "create_time",fill = FieldFill.INSERT)
+    private Timestamp createTime;
+
+    /**
+     * 修改时间
+     **/
+    @TableField(value = "update_time",fill = FieldFill.INSERT_UPDATE)
+    private Timestamp updateTime;
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/base/DataResponseBody.java

@@ -0,0 +1,69 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.base;
+
+
+import lombok.Data;
+import org.slf4j.MDC;
+
+import java.io.Serializable;
+
+/**
+ * @description 统一的公共响应体
+ * @date 2020-03-16
+ */
+@Data
+public class DataResponseBody<T> implements Serializable {
+
+    /**
+     * 返回状态码
+     */
+    private Integer code;
+    /**
+     * 返回信息
+     */
+    private String msg;
+    /**
+     * 泛型数据
+     */
+    private T data;
+    /**
+     * 链路追踪ID
+     */
+    private String traceId;
+
+    public DataResponseBody() {
+        this(ResponseCode.SUCCESS, null);
+    }
+
+    public DataResponseBody(T data) {
+        this(ResponseCode.SUCCESS, null, data);
+    }
+
+    public DataResponseBody(Integer code, String msg) {
+        this(code, msg, null);
+    }
+
+    public DataResponseBody(Integer code, String msg, T data) {
+        this.code = code;
+        this.msg = msg;
+        this.data = data;
+        this.traceId = MDC.get("traceId");
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/base/MagicNumConstant.java

@@ -0,0 +1,103 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.base;
+
+/**
+ * @description 魔数数值类
+ * @date 2020-09-17
+ */
+public final class MagicNumConstant {
+
+    public static final int NEGATIVE_ONE = -1;
+    public static final int ZERO = 0;
+    public static final int ONE = 1;
+    public static final int TWO = 2;
+    public static final int THREE = 3;
+    public static final int FOUR = 4;
+    public static final int FIVE = 5;
+    public static final int SIX = 6;
+    public static final int SEVEN = 7;
+    public static final int EIGHT = 8;
+    public static final int NINE = 9;
+    public static final int TEN = 10;
+
+    public static final int ELEVEN = 11;
+    public static final int SIXTEEN = 16;
+    public static final int TWENTY = 20;
+    public static final int FIFTY = 50;
+    public static final int SIXTY = 60;
+    public static final int SIXTY_TWO = 62;
+    public static final int SIXTY_FOUR = 64;
+    public static final int INTEGER_TWO_HUNDRED_AND_FIFTY_FIVE = 255;
+    public static final int ONE_HUNDRED = 100;
+    public static final int ONE_HUNDRED_TWENTY_EIGHT = 128;
+    public static final int TWO_HUNDRED = 200;
+    public static final int FIVE_HUNDRED = 500;
+    public static final int FIVE_HUNDRED_AND_SIXTEEN = 516;
+    public static final int ONE_THOUSAND = 1000;
+    public static final int BINARY_TEN_EXP = 1024;
+    public static final int ONE_THOUSAND_ONE_HUNDRED = 1100;
+    public static final int ONE_THOUSAND_ONE_HUNDRED_ONE = 1101;
+    public static final int ONE_THOUSAND_TWO_HUNDRED = 1200;
+    public static final int ONE_THOUSAND_TWO_HUNDRED_ONE = 1201;
+    public static final int ONE_THOUSAND_TWENTY_FOUR = 1024;
+    public static final int ONE_THOUSAND_THREE_HUNDRED = 1300;
+    public static final int ONE_THOUSAND_THREE_HUNDRED_ONE = 1301;
+    public static final int ONE_THOUSAND_THREE_HUNDRED_NINE = 1309;
+    public static final int ONE_THOUSAND_FIVE_HUNDRED = 1500;
+    public static final int TWO_THOUSAND = 2000;
+    public static final int TWO_THOUSAND_TWENTY_EIGHT = 2048;
+    public static final int THREE_THOUSAND = 3000;
+    public static final int FOUR_THOUSAND = 4000;
+    public static final int FIVE_THOUSAND = 5000;
+    public static final int NINE_THOUSAND = 9000;
+    public static final int NINE_THOUSAND_NINE_HUNDRED_NINTY_NINE = 9999;
+    public static final int TEN_THOUSAND = 10000;
+    public static final int FIFTEEN_THOUSAND = 15000;
+    public static final int HUNDRED_THOUSAND = 100000;
+    public static final int MILLION = 1000000;
+    public static final int ONE_MINUTE = 60000;
+
+    public static final long NEGATIVE_ONE__LONG = -1L;
+    public static final long ZERO_LONG = 0L;
+    public static final long ONE_LONG = 1L;
+    public static final long TWO_LONG = 2L;
+    public static final long THREE_LONG = 3L;
+    public static final long FOUR_LONG = 4L;
+    public static final long FIVE_LONG = 5L;
+    public static final long SIX_LONG = 6L;
+    public static final long SEVEN_LONG = 7L;
+    public static final long EIGHT_LONG = 8L;
+    public static final long NINE_LONG = 9L;
+    public static final long TEN_LONG = 10L;
+
+    public static final long TWELVE_LONG = 12L;
+    public static final long SIXTY_LONG = 60L;
+    public static final long FIFTY_LONG = 50L;
+    public static final long THOUSAND_LONG = 1000L;
+    public static final long TEN_THOUSAND_LONG = 10000L;
+    public static final long ONE_ZERO_ONE_ZERO_ONE_ZERO_LONG = 101010L;
+    public static final long NINE_ZERO_NINE_ZERO_NINE_ZERO_LONG = 909090L;
+    public static final long ONE_YEAR_BEFORE_LONG = 1552579200000L;
+
+    public static final int SIXITY_0XFF = 0xFF;
+
+
+    private MagicNumConstant() {
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/base/ResponseCode.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.base;
+
+/**
+ * @description  返回状态码
+ * @date 2020-02-23
+ */
+public class ResponseCode {
+
+    public static Integer SUCCESS = 200;
+    public static Integer UNAUTHORIZED = 401;
+    public static Integer ERROR = 10000;
+    public static Integer ENTITY_NOT_EXIST = 10001;
+    public static Integer BADREQUEST = 10002;
+    public static Integer SERVICE_ERROR = 10003;
+    public static Integer DOCKER_ERROR = 10004;
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MinioConfig.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.config;
+
+import io.minio.MinioClient;
+import io.minio.errors.InvalidEndpointException;
+import io.minio.errors.InvalidPortException;
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+/**
+ * @description MinIO 配置类
+ * @date 2020-09-17
+ */
+@Data
+@Component
+@ConfigurationProperties(prefix = "minio")
+public class MinioConfig {
+
+    private String endpoint;
+
+    private int port;
+
+    private String accessKey;
+
+    private String secretKey;
+
+    private Boolean secure;
+
+    private String bucketName;
+
+    /**
+     * 获取Minio客户端信息
+     *
+     * @return Minio客户端信息
+     */       
+    @Bean
+    public MinioClient getMinioClient() throws InvalidEndpointException, InvalidPortException {
+        return new MinioClient(endpoint, port, accessKey, secretKey,secure);
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MybatisPlusConfig.java

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.config;
+
+import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @description 配置分页插件
+ * @date 2020-09-17
+ */
+@Configuration
+public class MybatisPlusConfig {
+
+    /**
+     * 分页插件
+     */
+    @Bean
+    public PaginationInterceptor paginationInterceptor() {
+        return new PaginationInterceptor();
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/BusinessConstant.java

@@ -0,0 +1,86 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.constant;
+
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+
+/**
+ * @description 常量
+ * @date 2020-10-19
+ */
+public class BusinessConstant {
+
+    private BusinessConstant(){}
+
+    /**
+     * 分表业务编码 - 文件表
+     */
+    public static final String DATA_FILE = "DATA_FILE";
+
+    /**
+     * 分表业务编码 - 文件版本关系表
+     */
+    public static final String DATA_VERSION_FILE = "DATA_VERSION_FILE";
+    /**
+     * 图片文件路径
+     */
+    public static final String IMAGE_ORIGIN = "origin";
+    /**
+     * 标注文件路径
+     */
+    public static final String ANNOTATION = "annotation";
+    /**
+     * 标签文件路径
+     */
+    public static final String LABEL = "label";
+    /**
+     * 分隔符
+     */
+    public static final String FILE_SEPARATOR = "/";
+    /**
+     * 后缀.
+     */
+    public static final String SPOT = ".";
+
+    /**
+     * JSON后缀名
+     */
+    public static final String SUFFIX_JSON = ".JSON";
+    /**
+     * minio根目录
+     */
+    public static final String MINIO_ROOT_PATH = "dataset";
+    /**
+     * 下划线
+     */
+    public static final String UNDERLINE = "_";
+
+    /**
+     * 分段ID范围区间 50表示 50间隔ID存一张表
+     */
+    public static final long INTERVAL_NUMBER = MagicNumConstant.FIFTY_LONG;
+    /**
+     * 分批长度
+     */
+    public static final int SUB_LENGTH = MagicNumConstant.FIVE_THOUSAND;
+
+    /**
+     * 字母Y
+     */
+    public static final String Y = "Y";
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/DataStateCodeConstant.java

@@ -0,0 +1,84 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.constant;
+
+/**
+ * @description 数据集状态码
+ * @date 2020-09-03
+ */
+public class DataStateCodeConstant {
+
+    private DataStateCodeConstant() {
+    }
+
+    /**
+     * 未标注
+     */
+    public static final Integer NOT_ANNOTATION_STATE = 101;
+    /**
+     * 手动标注中
+     */
+    public static final Integer MANUAL_ANNOTATION_STATE = 102;
+    /**
+     * 自动标注中
+     */
+    public static final Integer AUTOMATIC_LABELING_STATE = 103;
+    /**
+     * 自动标注完成
+     */
+    public static final Integer AUTO_TAG_COMPLETE_STATE = 104;
+    /**
+     * 标注完成
+     */
+    public static final Integer ANNOTATION_COMPLETE_STATE = 105;
+
+    /**
+     * 目标跟踪中
+     */
+    public static final Integer TARGET_FOLLOW_STATE = 201;
+    /**
+     * 目标跟踪完成
+     */
+    public static final Integer TARGET_COMPLETE_STATE = 202;
+    /**
+     * 目标跟踪失败
+     */
+    public static final Integer TARGET_FAILURE_STATE = 203;
+
+    /**
+     * 未采样
+     */
+    public static final Integer NOT_SAMPLED_STATE = 301;
+    /**
+     * 采样中
+     */
+    public static final Integer SAMPLING_STATE = 302;
+    /**
+     * 采样失败
+     */
+    public static final Integer SAMPLED_FAILURE_STATE = 303;
+
+    /**
+     * 增强中
+     */
+    public static final Integer STRENGTHENING_STATE = 401;
+    /**
+     * 导入中
+     */
+    public static final Integer IN_THE_IMPORT_STATE = 402;
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/FileStateCodeConstant.java

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.constant;
+
+/**
+ * @description 数据集状态码
+ * @date 2020-09-03
+ */
+public class FileStateCodeConstant {
+
+    private FileStateCodeConstant(){
+
+    }
+
+    /**
+     * 未标注
+     */
+    public static final Integer NOT_ANNOTATION_FILE_STATE = 101;
+    /**
+     * 手动标注中
+     */
+    public static final Integer MANUAL_ANNOTATION_FILE_STATE = 102;
+    /**
+     * 自动标注完成
+     */
+    public static final Integer AUTO_TAG_COMPLETE_FILE_STATE = 103;
+    /**
+     * 标注完成
+     */
+    public static final Integer ANNOTATION_COMPLETE_FILE_STATE = 104;
+    /**
+     * 标注未识别
+     */
+    public static final Integer ANNOTATION_NOT_DISTINGUISH_FILE_STATE = 105;
+    /**
+     * 目标跟踪完成
+     */
+    public static final Integer TARGET_COMPLETE_FILE_STATE = 201;
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/enums/LogEnum.java

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.enums;
+
+import lombok.Getter;
+
+/**
+ * @description 日志类型枚举类
+ * @date 2020-06-23
+ */
+@Getter
+public enum LogEnum {
+
+    //系统报错日志
+    SYS_ERR,
+    //用户请求日志
+    REST_REQ,
+    //全局请求日志
+    GLOBAL_REQ,
+    //数据集模块
+    BIZ_DATASET,
+    //分表模块
+    DATA_SEQUENCE;
+
+    /**
+     * 判断日志类型不能为空
+     *
+     * @param  logType 日志类型
+     * @return boolean 返回类型
+     */
+    public static boolean isLogType(LogEnum logType) {
+        if (logType != null) {
+            return true;
+        }
+        return false;
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/BusinessException.java

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.exception;
+
+import lombok.Getter;
+import org.dubhe.datasetutil.common.base.DataResponseBody;
+import org.dubhe.datasetutil.common.base.ResponseCode;
+
+/**
+ * @description  业务异常
+ * @date 2020-03-13
+ */
+@Getter
+public class BusinessException extends RuntimeException {
+
+    private DataResponseBody responseBody;
+
+    public BusinessException(String msg) {
+        super(msg);
+        this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
+    }
+
+    public BusinessException(String msg, Throwable cause) {
+        super(msg,cause);
+        this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg);
+    }
+
+    public BusinessException(Throwable cause) {
+        super(cause);
+        this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST);
+    }
+
+    public BusinessException(Integer code, String msg, String info, Throwable cause) {
+        super(msg,cause);
+        if (info == null) {
+            this.responseBody = new DataResponseBody(code, msg);
+        } else {
+            this.responseBody = new DataResponseBody(code, msg + ":" + info);
+        }
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/DataSequenceException.java

@@ -0,0 +1,31 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.exception;
+
+import lombok.Getter;
+
+/**
+ * @description 获取序列异常
+ * @date 2020-09-23
+ */
+@Getter
+public class DataSequenceException extends BusinessException {
+
+    public DataSequenceException(String msg) {
+        super(msg);
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ErrorCode.java

@@ -0,0 +1,39 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.exception;
+
+
+/**
+ * @description 异常code
+ * @date 2020-03-26
+ */
+public interface ErrorCode {
+
+    /**
+     * 错误码
+     * @return code
+     */
+    Integer getCode();
+
+    /**
+     * error info
+     * @return
+     */
+    String getMsg();
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ImportDatasetException.java

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.exception;
+
+import lombok.Getter;
+
+/**
+ * @description 数据集导入异常处理
+ * @date 2020-10-14
+ */
+@Getter
+public class ImportDatasetException extends BusinessException {
+
+    /**
+     * 数据集导入异常处理
+     *
+     * @param msg 信息
+     */       
+    public ImportDatasetException(String msg) {
+        super(msg);
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/BaseLogFilter.java

@@ -0,0 +1,82 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.filter;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.filter.AbstractMatcherFilter;
+import ch.qos.logback.core.spi.FilterReply;
+import org.slf4j.Marker;
+
+/**
+ * @description 自定义日志过滤器
+ * @date 2020-07-21
+ */
+public class BaseLogFilter extends AbstractMatcherFilter<ILoggingEvent> {
+
+    Level level;
+
+    /**
+     * 重写decide方法
+     *
+     * @param iLoggingEvent 待决定的事件
+     * @return FilterReply  过滤器
+     */
+    @Override
+    public FilterReply decide(ILoggingEvent iLoggingEvent) {
+        if (!isStarted()) {
+            return FilterReply.NEUTRAL;
+        }
+        final String msg = iLoggingEvent.getMessage();
+        //自定义级别
+        if (checkLevel(iLoggingEvent) && msg != null && msg.startsWith("{") && msg.endsWith("}")) {
+            final Marker marker = iLoggingEvent.getMarker();
+            if (marker != null && this.getName() != null && this.getName().contains(marker.getName())) {
+                return onMatch;
+            }
+        }
+
+        return onMismatch;
+    }
+
+    /**
+     * 检查等级
+     *
+     * @param iLoggingEvent 待决定的事件
+     * @return boolean 检查结果
+     */       
+    protected boolean checkLevel(ILoggingEvent iLoggingEvent) {
+        return this.level != null
+                && iLoggingEvent.getLevel() != null
+                && iLoggingEvent.getLevel().toInt() == this.level.toInt();
+    }
+
+    public void setLevel(Level level) {
+        this.level = level;
+    }
+
+    /**
+     * 启动
+     */       
+    @Override
+    public void start() {
+        if (this.level != null) {
+            super.start();
+        }
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/ConsoleLogFilter.java

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.filter;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.spi.FilterReply;
+import org.dubhe.datasetutil.common.util.LogUtil;
+import org.slf4j.MarkerFactory;
+
+/**
+ * @description 自定义日志过滤器
+ * @date 2020-07-21
+ */
+public class ConsoleLogFilter extends BaseLogFilter {
+
+    /**
+     * 重写decide方法
+     *
+     * @param iLoggingEvent 待决定的事件
+     * @return FilterReply  过滤器
+     */
+    @Override
+    public FilterReply decide(ILoggingEvent iLoggingEvent) {
+        if (!isStarted()) {
+            return FilterReply.NEUTRAL;
+        }
+        return checkLevel(iLoggingEvent) ? onMatch : onMismatch;
+    }
+
+    /**
+     * 检查等级
+     *
+     * @param iLoggingEvent 待决定的事件
+     * @return boolean 检查结果
+     */
+    @Override
+    protected boolean checkLevel(ILoggingEvent iLoggingEvent) {
+        return this.level != null
+                && iLoggingEvent.getLevel() != null
+                && iLoggingEvent.getLevel().toInt() >= this.level.toInt()
+                && !MarkerFactory.getMarker(LogUtil.SCHEDULE_LEVEL).equals(iLoggingEvent.getMarker())
+                && !"log4jdbc.log4j2".equals(iLoggingEvent.getLoggerName());
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/GlobalRequestLogFilter.java

@@ -0,0 +1,40 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.filter;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+
+/**
+ * @description 全局请求 日志过滤器
+ * @date 2020-08-13
+ */
+public class GlobalRequestLogFilter extends BaseLogFilter {
+
+
+    /**
+     * 日志等级
+     *
+     * @param iLoggingEvent 日志等级
+     * @return boolean 执行结果
+     */       
+    @Override
+    public boolean checkLevel(ILoggingEvent iLoggingEvent) {
+        return this.level != null && true;
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/util/DateUtil.java

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+/**
+ * @description 时间 配置类
+ * @date 2020-09-17
+ */
+public class DateUtil {
+
+    private DateUtil(){
+    }
+
+    /**
+     * 时间格式
+     */
+    private static final String DATA_FORMAT_STR = "yyyy年MM月dd日 HH时mm分ss秒";
+
+    /**
+     * 获取开始时间
+     *
+     * @return String 返回控制台开始时间
+     */              
+    public static String getNowStr() {
+        return new SimpleDateFormat(DATA_FORMAT_STR).format(new Date());
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/util/GeneratorKeyUtil.java

@@ -0,0 +1,118 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import cn.hutool.core.util.ObjectUtil;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+import org.dubhe.datasetutil.common.enums.LogEnum;
+import org.dubhe.datasetutil.common.exception.DataSequenceException;
+import org.dubhe.datasetutil.domain.dto.IdAlloc;
+import org.dubhe.datasetutil.domain.entity.DataSequence;
+import org.dubhe.datasetutil.service.DataSequenceService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.StringUtils;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @description 生成ID工具类
+ * @date 2020-09-23
+ */
+@Component
+public class GeneratorKeyUtil {
+
+    @Autowired
+    private DataSequenceService dataSequenceService;
+
+    private ConcurrentHashMap<String, IdAlloc> idAllocConcurrentHashMap = new ConcurrentHashMap<>();
+
+    /**
+     * 根据业务编码，数量获取序列号
+     *
+     * @param businessCode 业务编码
+     * @param number       数量
+     * @return Long 起始位置
+     */
+    @Transactional(rollbackFor = Exception.class)
+    public synchronized Long getSequenceByBusinessCode(String businessCode, int number) {
+        if (StringUtils.isEmpty(businessCode)) {
+            throw new DataSequenceException("业务编码不可为空");
+        }
+        if (number == MagicNumConstant.ZERO) {
+            throw new DataSequenceException("需要获取的序列号长度不可为0或者空");
+        }
+        IdAlloc idAlloc = idAllocConcurrentHashMap.get(businessCode);
+        if (ObjectUtil.isNull(idAlloc)) {
+            idAlloc = new IdAlloc();
+            idAllocConcurrentHashMap.put(businessCode, idAlloc);
+        }
+
+        if (idAlloc.getUsedNumber() == MagicNumConstant.ZERO) {
+            DataSequence dataSequence = getDataSequence(businessCode);
+            updateDataSequence(businessCode);
+            idAlloc.setStartNumber(dataSequence.getStart());
+            idAlloc.setEndNumber(dataSequence.getStart() + dataSequence.getStep() - MagicNumConstant.ONE);
+            idAlloc.setUsedNumber(idAlloc.getEndNumber() - idAlloc.getStartNumber() + MagicNumConstant.ONE);
+        }
+        if (idAlloc.getUsedNumber() <= number) {
+            expansionUsedNumber(businessCode, number);
+        }
+        long returnStartNumber = idAlloc.getStartNumber();
+        idAlloc.setStartNumber(idAlloc.getStartNumber() + number);
+        idAlloc.setUsedNumber(idAlloc.getUsedNumber() - number);
+        return returnStartNumber;
+    }
+
+    /**
+     * 根据业务编码获取配置信息
+     * @param businessCode 业务编码
+     * @return DataSequence 数据索引
+     */
+    private DataSequence getDataSequence(String businessCode) {
+        DataSequence dataSequence = dataSequenceService.getSequence(businessCode);
+        if (dataSequence == null || dataSequence.getStart() == null || dataSequence.getStep() == null) {
+            throw new DataSequenceException("配置出错，请检查data_sequence表配置");
+        }
+        return dataSequence;
+    }
+
+    /**
+     * 根据业务编码更新起始值
+     * @param businessCode 业务编码
+     */
+    private void updateDataSequence(String businessCode) {
+        dataSequenceService.updateSequenceStart(businessCode);
+    }
+
+    /**
+     * 多次扩容
+     * @param businessCode 业务编码
+     * @param number 数量
+     */
+    private void expansionUsedNumber(String businessCode, int number) {
+        IdAlloc idAlloc = idAllocConcurrentHashMap.get(businessCode);
+        updateDataSequence(businessCode);
+        DataSequence dataSequenceNew = getDataSequence(businessCode);
+        idAlloc.setEndNumber(idAlloc.getEndNumber() + dataSequenceNew.getStep());
+        idAlloc.setUsedNumber(idAlloc.getEndNumber() - idAlloc.getStartNumber() + MagicNumConstant.ONE);
+        if (idAlloc.getUsedNumber() <= number) {
+            expansionUsedNumber(businessCode, number);
+        }
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/util/HandleFileUtil.java

@@ -0,0 +1,91 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.LineIterator;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * @description 文件工具类
+ * @date 2020-10-15
+ */
+public class HandleFileUtil {
+
+    /**
+     * 读取文件内容
+     *
+     * @param file 文件对象
+     * @return String 文件内容
+     */
+    public static String readFile(File file) throws IOException{
+        StringBuilder stringBuffer = new StringBuilder();
+        LineIterator fileContext =  FileUtils.lineIterator(file,"UTF-8");
+        while (fileContext.hasNext()) {
+            stringBuffer.append(fileContext.nextLine());
+        }
+        return stringBuffer.toString();
+    }
+
+
+    /**
+     * 获取文件名后缀名
+     *
+     * @param fileName 文件名
+     * @return String 文件后缀名
+     */
+    public static String readFileSuffixName(String fileName){
+        return fileName.substring(fileName.lastIndexOf("."));
+    }
+
+
+    /**
+     * 获取文件名(踢除后缀名)
+     *
+     * @param fileName 文件名
+     * @return String 文件名(踢除后缀名)
+     */
+    public static String readFileName(String fileName){
+        return fileName.substring(0,fileName.lastIndexOf("."));
+    }
+
+
+    /**
+     * 生成文件路径
+     *
+     * @param businessCode 业务类型
+     * @return String 文件路径
+     */
+    public static String generateFilePath(String businessCode){
+        return BusinessConstant.FILE_SEPARATOR + businessCode;
+    }
+
+
+    /**
+     * 获取标签组名称
+     *
+     * @param fileName 文件名称
+     * @return String 标签组名
+     */
+    public static String getLabelGroupName(String fileName){
+        return fileName.substring(fileName.indexOf(BusinessConstant.UNDERLINE) + 1,fileName.lastIndexOf(BusinessConstant.SPOT));
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/util/LogUtil.java

@@ -0,0 +1,300 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.util;
+
+import ch.qos.logback.classic.Level;
+import com.alibaba.fastjson.JSON;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.dubhe.datasetutil.common.aspect.LogAspect;
+import org.dubhe.datasetutil.common.enums.LogEnum;
+import org.dubhe.datasetutil.domain.entity.LogInfo;
+import org.slf4j.MDC;
+import org.slf4j.MarkerFactory;
+import org.slf4j.helpers.MessageFormatter;
+
+import java.util.Arrays;
+import java.util.UUID;
+
+/**
+ * @description 日志工具类
+ * @date 2020-06-29
+ */
+@Slf4j
+public class LogUtil {
+
+    private static final String TRACE_TYPE = "TRACE_TYPE";
+
+    public static final String SCHEDULE_LEVEL = "SCHEDULE";
+
+    private static final String GLOBAL_REQUEST_LEVEL = "GLOBAL_REQUEST";
+
+    private static final String TRACE_LEVEL = "TRACE";
+
+    private static final String DEBUG_LEVEL = "DEBUG";
+
+    private static final String INFO_LEVEL = "INFO";
+
+    private static final String WARN_LEVEL = "WARN";
+
+    private static final String ERROR_LEVEL = "ERROR";
+
+
+    public static void startScheduleTrace() {
+        MDC.put(TRACE_TYPE, SCHEDULE_LEVEL);
+    }
+
+    public static void cleanTrace() {
+        MDC.clear();
+    }
+
+    /**
+     * info级别的日志
+     *
+     * @param logType 日志类型
+     * @param object  打印的日志参数
+     */
+
+    public static void info(LogEnum logType, Object... object) {
+
+        logHandle(logType, Level.INFO, object);
+    }
+
+    /**
+     * debug级别的日志
+     *
+     * @param logType 日志类型
+     * @param object  打印的日志参数
+     */
+    public static void debug(LogEnum logType, Object... object) {
+        logHandle(logType, Level.DEBUG, object);
+    }
+
+    /**
+     * error级别的日志
+     *
+     * @param logType 日志类型
+     * @param object  打印的日志参数
+     */
+    public static void error(LogEnum logType, Object... object) {
+        errorObjectHandle(object);
+        logHandle(logType, Level.ERROR, object);
+    }
+
+    /**
+     * warn级别的日志
+     *
+     * @param logType 日志类型
+     * @param object  打印的日志参数
+     */
+    public static void warn(LogEnum logType, Object... object) {
+        logHandle(logType, Level.WARN, object);
+    }
+
+    /**
+     * trace级别的日志
+     *
+     * @param logType 日志类型
+     * @param object  打印的日志参数
+     */
+    public static void trace(LogEnum logType, Object... object) {
+        logHandle(logType, Level.TRACE, object);
+    }
+
+    /**
+     * 日志处理
+     *
+     * @param logType 日志类型
+     * @param level   日志级别
+     * @param object  打印的日志参数
+     */
+    private static void logHandle(LogEnum logType, Level level, Object[] object) {
+
+        LogInfo logInfo = generateLogInfo(logType, level, object);
+
+        switch (logInfo.getLevel()) {
+            case TRACE_LEVEL:
+                log.trace(MarkerFactory.getMarker(TRACE_LEVEL), logJsonStringLengthLimit(logInfo));
+                break;
+            case DEBUG_LEVEL:
+                log.debug(MarkerFactory.getMarker(DEBUG_LEVEL), logJsonStringLengthLimit(logInfo));
+                break;
+            case GLOBAL_REQUEST_LEVEL:
+                logInfo.setLevel(null);
+                logInfo.setType(null);
+                logInfo.setLocation(null);
+                log.info(MarkerFactory.getMarker(GLOBAL_REQUEST_LEVEL), logJsonStringLengthLimit(logInfo));
+                break;
+            case SCHEDULE_LEVEL:
+                log.info(MarkerFactory.getMarker(SCHEDULE_LEVEL), logJsonStringLengthLimit(logInfo));
+                break;
+            case INFO_LEVEL:
+                log.info(MarkerFactory.getMarker(INFO_LEVEL), logJsonStringLengthLimit(logInfo));
+                break;
+            case WARN_LEVEL:
+                log.warn(MarkerFactory.getMarker(WARN_LEVEL), logJsonStringLengthLimit(logInfo));
+                break;
+            case ERROR_LEVEL:
+                log.error(MarkerFactory.getMarker(ERROR_LEVEL), logJsonStringLengthLimit(logInfo));
+                break;
+            default:
+        }
+
+    }
+
+
+    /**
+     * 日志信息组装的内部方法
+     *
+     * @param logType 日志类型
+     * @param level   日志级别
+     * @param object  打印的日志参数
+     * @return LogInfo 日志对象
+     */
+    private static LogInfo generateLogInfo(LogEnum logType, Level level, Object[] object) {
+
+
+        LogInfo logInfo = new LogInfo();
+        // 日志类型检测
+        if (!LogEnum.isLogType(logType)) {
+            level = Level.ERROR;
+            object = new Object[1];
+            object[0] = "日志类型【".concat(String.valueOf(logType)).concat("】不正确！");
+            logType = LogEnum.SYS_ERR;
+        }
+
+        // 获取trace_id
+        if (StringUtils.isEmpty(MDC.get(LogAspect.TRACE_ID))) {
+            MDC.put(LogAspect.TRACE_ID, UUID.randomUUID().toString());
+        }
+        // 设置logInfo的level,type,traceId属性
+        logInfo.setLevel(level.levelStr)
+                .setType(logType.toString())
+                .setTraceId(MDC.get(LogAspect.TRACE_ID));
+
+
+        //自定义日志级别
+        //LogEnum、 MDC中的 TRACE_TYPE 做日志分流标识
+        if (Level.INFO.toInt() == level.toInt()) {
+            if (LogEnum.GLOBAL_REQ.equals(logType)) {
+                //info全局请求
+                logInfo.setLevel(GLOBAL_REQUEST_LEVEL);
+            } else {
+                //schedule定时等 链路记录
+                String traceType = MDC.get(TRACE_TYPE);
+                if (StringUtils.isNotBlank(traceType)) {
+                    logInfo.setLevel(traceType);
+                }
+            }
+        }
+
+        // 设置logInfo的堆栈信息
+        setLogStackInfo(logInfo);
+        // 设置logInfo的info信息
+        setLogInfo(logInfo, object);
+        // 截取logInfo的长度并转换成json字符串
+        return logInfo;
+    }
+
+    /**
+     * 设置loginfo的堆栈信息
+     *
+     * @param logInfo 日志对象
+     */
+    private static void setLogStackInfo(LogInfo logInfo) {
+        StackTraceElement[] elements = Thread.currentThread().getStackTrace();
+        if (elements.length >= 6) {
+            StackTraceElement element = elements[5];
+            logInfo.setLocation(String.format("%s#%s:%s", element.getClassName(), element.getMethodName(), element.getLineNumber()));
+        }
+    }
+
+    /**
+     * 限制log日志的长度并转换成json
+     *
+     * @param logInfo 日志对象
+     * @return String 转换的json
+     */
+    private static String logJsonStringLengthLimit(LogInfo logInfo) {
+        try {
+
+            String jsonString = JSON.toJSONString(logInfo);
+            if (StringUtils.isBlank(jsonString)) {
+                return "";
+            }
+            if (jsonString.length() > 10000) {
+                String trunk = logInfo.getInfo().toString().substring(0, 9000);
+                logInfo.setInfo(trunk);
+                jsonString = JSON.toJSONString(logInfo);
+            }
+            return jsonString;
+
+        } catch (Exception e) {
+            logInfo.setLevel(Level.ERROR.levelStr).setType(LogEnum.SYS_ERR.toString())
+                    .setInfo("cannot serialize exception: " + ExceptionUtils.getStackTrace(e));
+            return JSON.toJSONString(logInfo);
+        }
+    }
+
+    /**
+     * 设置日志对象的info信息
+     *
+     * @param logInfo 日志对象
+     * @param object  打印的日志参数
+     */
+    private static void setLogInfo(LogInfo logInfo, Object[] object) {
+
+        if (object.length > 1) {
+            logInfo.setInfo(MessageFormatter.arrayFormat(object[0].toString(),
+                    Arrays.copyOfRange(object, 1, object.length)).getMessage());
+
+        } else if (object.length == 1 && object[0] instanceof Exception) {
+            logInfo.setInfo((ExceptionUtils.getStackTrace((Exception) object[0])));
+            log.error((ExceptionUtils.getStackTrace((Exception) object[0])));
+        } else if (object.length == 1) {
+            logInfo.setInfo(object[0] == null ? "" : object[0]);
+        } else {
+            logInfo.setInfo("");
+        }
+
+    }
+
+    /**
+     * 处理Exception的情况
+     *
+     * @param object 打印的日志参数
+     */
+    private static void errorObjectHandle(Object[] object) {
+
+        if (object.length == 2 && object[1] instanceof Exception) {
+            log.error(String.valueOf(object[0]), (Exception) object[1]);
+            object[1] = ExceptionUtils.getStackTrace((Exception) object[1]);
+
+        } else if (object.length >= 3) {
+            log.error(String.valueOf(object[0]),
+                    Arrays.copyOfRange(object, 1, object.length));
+            for (int i = 0; i < object.length; i++) {
+                if (object[i] instanceof Exception) {
+                    object[i] = ExceptionUtils.getStackTrace((Exception) object[i]);
+                }
+
+            }
+        }
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MinioUtil.java

@@ -0,0 +1,64 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import io.minio.MinioClient;
+import io.minio.PutObjectOptions;
+import org.dubhe.datasetutil.common.config.MinioConfig;
+import org.dubhe.datasetutil.common.enums.LogEnum;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.io.InputStream;
+
+/**
+ * @description Minio工具类
+ * @date 2020-09-17
+ */
+@Component
+public class MinioUtil {
+
+    @Autowired
+    private MinioClient minioClient;
+
+    @Autowired
+    private MinioConfig minioConfig;
+
+    /**
+     * 上传文件
+     *
+     * @param objectName 对象名称
+     * @param inputStream 文件流
+     * @throws Exception 上传异常
+     */
+    public void upLoadFile(String objectName, InputStream inputStream) throws Exception {
+        LogUtil.info(LogEnum.BIZ_DATASET,"文件上传名称为: 【" + objectName + "】");
+        PutObjectOptions options = new PutObjectOptions(inputStream.available(), -1);
+        minioClient.putObject(minioConfig.getBucketName(), objectName, inputStream, options);
+    }
+
+    /**
+     * 获取文件URL
+     * 
+     * @param objectName 对象名称
+     * @return String 文件路径
+     */
+    public String getUrl(String objectName) {
+        return minioConfig.getBucketName() + "/" + objectName;
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MyPreciseShardingAlgorithm.java

@@ -0,0 +1,72 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm;
+import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.service.DataSequenceService;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Collection;
+
+/**
+ * @description 数据分片
+ * @date 2020-09-21
+ */
+public class MyPreciseShardingAlgorithm implements PreciseShardingAlgorithm<Long>{
+
+    @Autowired
+    private DataSequenceService dataSequenceService;
+
+    /**
+     * 数据表分片
+     *
+     * @param collection           集合
+     * @param preciseShardingValue 分片值
+     * @return 字符串
+     */
+    @Override
+    public String doSharding(Collection<String> collection, PreciseShardingValue<Long> preciseShardingValue) {
+        long startIndex = MagicNumConstant.ONE;
+        long endIndex = MagicNumConstant.FIFTY;
+        dataSequenceService =  SpringContextHolder.getBean(DataSequenceService.class);
+        String tableName = preciseShardingValue.getLogicTableName()+ BusinessConstant.UNDERLINE + preciseSharding(preciseShardingValue.getValue(),startIndex ,endIndex);
+        if(!dataSequenceService.checkTableExist(tableName)){
+            dataSequenceService.createTable(tableName);
+        }
+        return tableName;
+    }
+
+    /**
+     * 分片实现
+     *
+     * @param indexId    起始位置
+     * @param startIndex 起始值
+     * @param endIndex   结束值
+     * @return long 返回截止值
+     */
+    public long preciseSharding(long indexId,long startIndex , long endIndex){
+        if(indexId > endIndex){
+            startIndex = startIndex + BusinessConstant.INTERVAL_NUMBER;
+            endIndex = endIndex + BusinessConstant.INTERVAL_NUMBER;
+            return preciseSharding(indexId,startIndex,endIndex);
+        }
+        return endIndex / BusinessConstant.INTERVAL_NUMBER;
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/util/PrintUtils.java

@@ -0,0 +1,156 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+/**
+ * @description 控制台字体颜色
+ * @date 2020-10-22
+ */
+public class PrintUtils {
+    /**
+     * 白色
+     */
+    public static final int WHITE = 30;
+
+    /**
+     * 白色背景
+     */
+    public static final int WHITE_BACKGROUND = 40;
+
+    /**
+     * 红色
+     */
+    public static final int RED = 31;
+
+    /**
+     * 红色背景
+     */
+    public static final int RED_BACKGROUND = 41;
+
+    /**
+     * 绿色
+     */
+    public static final int GREEN = 32;
+    
+    /**
+     * 绿色背景
+     */
+    public static final int GREEN_BACKGROUND = 42;
+
+    /**
+     * 黄色
+     */
+    public static final int YELLOW = 33;
+
+    /**
+     * 黄色背景
+     */
+    public static final int YELLOW_BACKGROUND = 43;
+
+    /**
+     * 蓝色
+     */
+    public static final int BLUE = 34;
+
+    /**
+     * 蓝色背景
+     */
+    public static final int BLUE_BACKGROUND = 44;
+
+    /**
+     * 品红（洋红）
+     */
+    public static final int MAGENTA = 35;
+
+    /**
+     * 品红背景
+     */
+    public static final int MAGENTA_BACKGROUND = 45;
+
+    /**
+     * 蓝绿
+     */
+    public static final int CYAN = 36;
+
+    /**
+     * 蓝绿背景
+     */
+    public static final int CYAN_BACKGROUND = 46;
+
+    /**
+     * 黑色
+     */
+    public static final int BLACK = 37;
+
+    /**
+     * 黑色背景
+     */
+    public static final int BLACK_BACKGROUND = 47;
+
+    /**
+     * 粗体
+     */
+    public static final int BOLD = 1;
+
+    /**
+     * 斜体
+     */
+    public static final int ITATIC = 3;
+
+    /**
+     * 下划线
+     */
+    public static final int UNDERLINE = 4;
+
+    /**
+     * 反转
+     */
+    public static final int REVERSE = 7;
+
+    /**
+     * 格式化
+     *
+     * @param txt   文本
+     * @param codes 信息
+     * @return String 格式化后的内容
+     */       
+    private static String FMT(String txt, int... codes) {
+        StringBuffer sb = new StringBuffer();
+        for (int code : codes) {
+            sb.append(code + ";");
+        }
+        String _code = sb.toString();
+        if (_code.endsWith(";")) {
+            _code = _code.substring(0, _code.length() - 1);
+        }
+        return (char) 27 + "[" + _code + "m" + txt + (char) 27 + "[0m";
+    }
+
+    /**
+     * 打印并换行
+     */
+    public static void printLine(String txt, int... codes) {
+        System.out.println(FMT(txt, codes));
+    }
+
+    /**
+     * 默认打印红色文字
+     */
+    public static void PN(String txt) {
+        System.out.println(FMT(txt, new int[]{RED}));
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/util/SpringContextHolder.java

@@ -0,0 +1,95 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.common.util;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+
+/**
+ * @description 上下文工具类
+ * @date 2020-03-25
+ */
+@Slf4j
+public class SpringContextHolder implements ApplicationContextAware, DisposableBean {
+
+    private static ApplicationContext applicationContext = null;
+
+    /**
+     * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型.
+     *
+     * @param name bean名称
+     * @return T 根据Bean名称转型的对象
+     */
+    @SuppressWarnings("unchecked")
+    public static <T> T getBean(String name) {
+        assertContextInjected();
+        return (T) applicationContext.getBean(name);
+    }
+
+    /**
+     * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型.
+     *
+     * @param requiredType 要求类型
+     * @return T 根据Bean名称转型的对象
+     */
+    public static <T> T getBean(Class<T> requiredType) {
+        assertContextInjected();
+        return applicationContext.getBean(requiredType);
+    }
+
+    /**
+     * 检查ApplicationContext不为空.
+     */
+    private static void assertContextInjected() {
+        if (applicationContext == null) {
+            throw new IllegalStateException("applicaitonContext属性未注入, 请在applicationContext" +
+                    ".xml中定义SpringContextHolder或在SpringBoot启动类中注册SpringContextHolder.");
+        }
+    }
+
+    /**
+     * 清除SpringContextHolder中的ApplicationContext为Null.
+     */
+    private static void clearHolder() {
+        log.debug("清除SpringContextHolder中的ApplicationContext:"
+                + applicationContext);
+        applicationContext = null;
+    }
+
+    @Override
+    public void destroy() { SpringContextHolder.clearHolder();
+    }
+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+        if (SpringContextHolder.applicationContext != null) {
+            log.warn("SpringContextHolder中的ApplicationContext被覆盖, 原有ApplicationContext为:" + SpringContextHolder.applicationContext);
+        }
+        SpringContextHolder.applicationContext = applicationContext;
+    }
+
+    /**
+     * 获取当前环境
+     */
+    public static String getActiveProfile(){
+        return applicationContext.getEnvironment().getActiveProfiles()[0];
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/common/util/ThreadUtils.java

@@ -0,0 +1,72 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.common.util;
+
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.List;
+import java.util.concurrent.*;
+
+/**
+ * @description 线程工具
+ * @date 2020-10-19
+ */
+@Slf4j
+public class ThreadUtils {
+
+    private ThreadUtils(){}
+
+    /**
+     * 根据需要处理的数量创建线程数
+     *
+     * @param listSize 集合数量
+     * @return int 数量
+     */
+    public static int createThread(int listSize) {
+        return listSize / getNeedThreadNumber() == 0 ? 1 : listSize / getNeedThreadNumber();
+    }
+
+
+    /**
+     * 获取需要创建的线程数
+     *
+     * @return int 数量
+     */
+    public static int getNeedThreadNumber() {
+        final int numOfCores = Runtime.getRuntime().availableProcessors();
+        final double blockingCoefficient = 0.8;
+        return (int) (numOfCores / (1 - blockingCoefficient));
+    }
+
+    /**
+     * 按要求分多线程执行
+     *
+     * @param partitions 分线程集合
+     * @throws Exception 线程执行异常
+     */
+    public static void runMultiThread(List<Callable<Integer>> partitions) throws Exception {
+        final ExecutorService executorService = Executors.newFixedThreadPool(ThreadUtils.getNeedThreadNumber());
+        final List<Future<Integer>> valueOfStocks = executorService.invokeAll(partitions);
+        Integer endCount = 0;
+        for (final Future<Integer> value : valueOfStocks) {
+            endCount += value.get();
+        }
+        log.warn("#-------------处理结束,成功处理文件 【" + endCount + "】个-------------#");
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataFileMapper.java

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.entity.DataFile;
+import java.util.List;
+
+/**
+ * @description 数据集文件 Mapper接口
+ * @date 2020-09-17
+ */
+public interface DataFileMapper extends BaseMapper<DataFile> {
+
+    /**
+     * 插入文件数据
+     *
+     * @param listDataFile 文件数据集合
+     */
+    void saveBatchDataFile(@Param("listDataFile") List<DataFile> listDataFile);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataGroupLabelMapper.java

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.entity.DataGroupLabel;
+
+import java.util.List;
+
+/**
+ * @description 标签组与标签的关系MAPPER
+ * @date 2020-10-21
+ */
+public interface DataGroupLabelMapper extends BaseMapper<DataGroupLabel> {
+    /**
+     * 插入标签组与标签的关数据
+     *
+     * @param listDataGroupLabel 标签组与标签数据集合
+     */
+    void saveDataGroupLabel(@Param("list") List<DataGroupLabel> listDataGroupLabel);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelGroupMapper.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+import org.dubhe.datasetutil.domain.entity.DataLabelGroup;
+
+/**
+ * @description 数据集标签组
+ * @date 2020-10-14
+ */
+public interface DataLabelGroupMapper extends BaseMapper<DataLabelGroup> {
+    /**
+     * 根据标签组名查询
+     *
+     * @param labelGroupName 标签组名
+     * @return int 标签组数量
+     */
+    @Select("select count(1) from data_label_group where name = #{labelGroupName} and deleted = 0")
+    int selectByLabelGroupName(@Param("labelGroupName") String labelGroupName);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelMapper.java

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.entity.DataLabel;
+
+import java.util.List;
+
+/**
+ * @description 数据集标签
+ * @date 2020-10-14
+ */
+public interface DataLabelMapper extends BaseMapper<DataLabel> {
+    /**
+     * 批量保存数据集标签
+     *
+     * @param listDataLabel 标签数据
+     */
+    void saveBatchDataLabel(@Param("listDataLabel") List<DataLabel> listDataLabel);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataSequenceMapper.java

@@ -0,0 +1,66 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+import org.apache.ibatis.annotations.Update;
+import org.dubhe.datasetutil.domain.entity.DataSequence;
+
+/**
+ * @description  序列Mapper
+ * @date 2020-09-23
+ */
+public interface DataSequenceMapper extends BaseMapper<DataSequence> {
+    /**
+     * 根据业务编码查询序列
+     *
+     * @param businessCode 业务信息
+     * @return DataSequence 根据业务编码得到的序列
+     */
+    @Select("select id, business_code ,start, step from data_sequence where business_code = #{businessCode}")
+    DataSequence selectByBusiness(String businessCode);
+
+    /**
+     * 根据业务编码更新序列起始值
+     *
+     * @param businessCode 业务信息
+     * @return DataSequence 根据业务编码更新序列起始值
+     */
+    @Update("update data_sequence set start = start + step where business_code = #{businessCode} ")
+    int updateStartByBusinessCode(String businessCode);
+
+    /**
+     * 查询存在表的记录数
+     *
+     * @param tableName 类型名称
+     * @return int 根据类型查在表的记录数量
+     */
+    @Select("select count(1) from ${tableName}")
+    int checkTableExist(@Param("tableName") String tableName);
+
+    /**
+     * 执行创建表
+     *
+     * @param tableName    类型名称
+     * @param oldTableName 旧类型名称
+     */
+    @Update({"CREATE TABLE ${tableName} AS select * from  ${oldTableName} "})
+    void createNewTable(@Param("tableName") String tableName, @Param("oldTableName") String oldTableName);
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataVersionFileMapper.java

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+
+import java.util.List;
+
+/**
+ * @description 数据集文件中间表 Mapper接口
+ * @date 2020-9-17
+ */
+public interface DataVersionFileMapper extends BaseMapper<DataVersionFile> {
+
+    /**
+     * 插入数据集文件中间表数据
+     *
+     * @param listDataVersionFile 数据集文件中间表数据集合
+     */
+    void saveBatchDataFileVersion(@Param("listDataVersionFile") List<DataVersionFile> listDataVersionFile);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetDataLabelMapper.java

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.dubhe.datasetutil.domain.entity.DatasetDataLabel;
+
+import java.util.List;
+
+/**
+ * @description 数据集与标签关系
+ * @date 2020-10-14
+ */
+public interface DatasetDataLabelMapper extends BaseMapper<DatasetDataLabel> {
+
+    /**
+     * 批量保存数据集与标签关系
+     *
+     * @param datasetDataLabelList 数据集与标签
+     */
+    void saveBatchDatasetDataLabel(@Param("datasetDataLabelList") List<DatasetDataLabel> datasetDataLabelList);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetMapper.java

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.annotations.Select;
+import org.dubhe.datasetutil.domain.entity.Dataset;
+
+/**
+ * @description 数据集 Mapper接口
+ * @date 2020-9-17
+ */
+public interface DatasetMapper  extends BaseMapper<Dataset> {
+
+    /**
+     * 根据数据集ID查询数据集
+     *
+     * @param datasetId 数据集id
+     * @return Dataset  根据数据集ID得到数据集
+     */
+    @Select("select * from data_dataset where id = #{datasetId} and is_import = 1")
+    Dataset findDatasetById(@Param("datasetId") Long datasetId);
+
+    /**
+     * 查询数据集标签数量
+     *
+     * @param datasetId 数据集id
+     * @return int 数据集标签数量
+     */
+    @Select("select count(1) from data_dataset_label where dataset_id = #{datasetId}")
+    int findDataLabelById(@Param("datasetId") Long datasetId);
+
+    /**
+     * 查询数据集文件数量
+     *
+     * @param datasetId 数据集id
+     * @return int 数据集标签数量
+     */
+    @Select("select count(1) from data_file where dataset_id = #{datasetId}")
+    int findDataFileById(@Param("datasetId") Long datasetId);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/DataVersionFile.java

@@ -0,0 +1,93 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.dto;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import lombok.Data;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+
+/**
+ * @description 数据集文件关系类
+ * @date 2020-9-17
+ */
+@Data
+public class DataVersionFile extends BaseEntity implements Serializable  {
+
+    /**
+     * id
+     */
+    @TableId(type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * 数据集id
+     */
+    private Long datasetId;
+
+    /**
+     * 版本号
+     */
+    private String versionName;
+
+    /**
+     * 文件id
+     */
+    private Long fileId;
+
+    /**
+     * 状态
+     */
+    private Integer status;
+
+    /**
+     * 数据集状态
+     */
+    private Integer annotationStatus;
+
+    /**
+     * 数据集状态备份
+     */
+    private Integer backupStatus;
+
+    /**
+     * 发布是否转换
+     */
+    private Integer changed;
+
+    public DataVersionFile() {
+    }
+
+    /**
+     * 插入数据集版本文件关系
+     *
+     * @param datasetId         数据集id
+     * @param fileId            文件id
+     * @param annotationStatus  数据集id
+     * @param status            状态
+     * @return DataVersionFile  数据集版本文件表
+     */       
+    public DataVersionFile(Long datasetId, Long fileId,Integer annotationStatus,Integer status) {
+        this.datasetId = datasetId;
+        this.fileId = fileId;
+        this.annotationStatus = annotationStatus;
+        this.status = status;
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/IdAlloc.java

@@ -0,0 +1,50 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.dto;
+
+import lombok.Data;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+
+/**
+ * @description ID策略实体
+ * @date 2020-10-16
+ */
+@Data
+public class IdAlloc {
+
+    /**
+     * 起始位置
+     */
+    private long startNumber;
+
+    /**
+     * 结束位置
+     */
+    private long endNumber;
+
+    /**
+     * 可用数量
+     */
+    private long usedNumber;
+
+    public IdAlloc() {
+        this.startNumber = MagicNumConstant.ZERO;
+        this.endNumber = MagicNumConstant.ZERO;
+        this.usedNumber = MagicNumConstant.ZERO;
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataFile.java

@@ -0,0 +1,118 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import lombok.Data;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.awt.image.BufferedImage;
+import java.io.Serializable;
+
+/**
+ * @description 文件类
+ * @date 2020-09-17
+ */
+@Data
+public class DataFile extends BaseEntity implements Serializable {
+
+    /**
+     * id
+     */
+    @TableId(type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * 名称
+     */
+    private String name;
+
+    /**
+     * 状态
+     */
+    private Integer status;
+
+    /**
+     * 数据集id
+     */
+    private Long datasetId;
+
+    /**
+     * url
+     */
+    private String url;
+
+    /**
+     * 文件类型
+     */
+    private Integer fileType;
+
+    /**
+     * 父id
+     */
+    private Long pid;
+
+    /**
+     * 帧间隔
+     */
+    private Integer frameInterval;
+
+    /**
+     * 增强类型
+     */
+    private Integer enhanceType;
+
+    /**
+     * 宽
+     */
+    private Integer width;
+
+    /**
+     * 高
+     */
+    private Integer height;
+
+    /**
+     * 拥有人id
+     */
+    private Long originUserId;
+
+    public DataFile() {}
+
+    /**
+     * 插入文件表
+     *
+     * @param name         文件名字
+     * @param datasetId    数据集id
+     * @param url          文件路径
+     * @param createUserId 创建人id
+     * @param read         文件宽高
+     * @return DataFile   file对象
+     */       
+    public DataFile(String name, Long datasetId, String url, Long createUserId, BufferedImage read) {
+        this.name = name.substring(0, name.lastIndexOf("."));
+        this.datasetId = datasetId;
+        this.url = url;
+        this.status = 101;
+        this.setDeleted(false);
+        this.originUserId = createUserId;
+        this.width = read.getWidth();
+        this.height = read.getHeight();
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataGroupLabel.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+/**
+ * @description 标签组标签中间表
+ * @date 2020-10-21
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Builder
+@TableName("data_group_label")
+public class DataGroupLabel extends BaseEntity implements Serializable {
+
+    /**
+     * id
+     */
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * 标签id
+     */
+    @TableField(value = "label_id")
+    private Long labelId;
+
+    /**
+     * 标签组id
+     */
+    @TableField(value = "label_group_id")
+    private Long labelGroupId;
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabel.java

@@ -0,0 +1,64 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+/**
+ * @description 数据集标签实体
+ * @date 2020-10-14
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Builder
+@TableName("data_label")
+public class DataLabel extends BaseEntity implements Serializable {
+
+    /**
+     * id
+     */
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * 名称
+     */
+    @TableField(value = "name")
+    private String name;
+
+    /**
+     * 颜色
+     */
+    @TableField(value = "color")
+    private String color;
+
+    /**
+     * 类型
+     */
+    @TableField(value = "type")
+    private Long type;
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabelGroup.java

@@ -0,0 +1,70 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+/**
+ * @description 标签组实体
+ * @date 2020-10-14
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Builder
+@TableName("data_label_group")
+public class DataLabelGroup extends BaseEntity implements Serializable {
+
+    /**
+     * id
+     */
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * 名称
+     */
+    @TableField(value = "name")
+    private String name;
+
+    /**
+     * 描述
+     */
+    @TableField(value = "remark")
+    private String remark;
+
+    /**
+     * 类型
+     */
+    @TableField(value = "type")
+    private Long type;
+
+    /**
+     * 拥有人id
+     */
+    @TableField(value = "origin_user_id")
+    private Long originUserId;
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataSequence.java

@@ -0,0 +1,57 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+/**
+ * @description 序列表
+ * @date 2020-09-23
+ */
+@Data
+@TableName("data_sequence")
+public class DataSequence {
+
+    /**
+     * id
+     */
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * 业务code
+     */
+    @TableField(value = "business_code")
+    private String businessCode;
+
+    /**
+     * 启动
+     */
+    @TableField(value = "start")
+    private Long start;
+
+    /**
+     * 步数
+     */
+    @TableField(value = "step")
+    private Long step;
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/Dataset.java

@@ -0,0 +1,145 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.*;
+import lombok.Data;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+import java.sql.Timestamp;
+
+/**
+ * @description 数据集类
+ * @date 2020-9-17
+ */
+@Data
+@TableName("data_dataset")
+public class Dataset extends BaseEntity implements Serializable  {
+
+    /**
+     * 删除标识
+     */
+    @TableField("deleted")
+    private Boolean deleted = false;
+
+    /**
+     * 数据集名称
+     */
+    private String name;
+
+    /**
+     * 数据集备注
+     */
+    private String remark;
+
+    /**
+     * 类型
+     */
+    private Integer type;
+
+    /**
+     * 数据集类型
+     */
+    private Integer dataType;
+
+    /**
+     * 数据集状态
+     */
+    private Integer annotateType;
+
+    /**
+     * 任务id
+     */
+    private Long teamId;
+
+    /**
+     * id
+     */
+    @TableId(type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * url
+     */
+    private String uri;
+
+    /**
+     * 状态
+     */
+    private Integer status;
+
+    /**
+     * 当前版本号
+     */
+    private String currentVersionName;
+
+    /**
+     * 是否为导入
+     */
+    @TableField(value = "is_import")
+    private boolean isImport;
+
+    /**
+     * 用户导入数据集压缩包地址
+     */
+    private String archiveUrl;
+
+    /**
+     * 解压状态
+     */
+    private Integer decompressState;
+
+    /**
+     * 解压失败原因
+     */
+    private String decompressFailReason;
+
+    /**
+     * 是否置顶
+     */
+    @TableField(value = "is_top")
+    private boolean isTop;
+
+    /**
+     * 创建时间
+     */
+    private Timestamp createTime;
+
+    /**
+     * 修改时间
+     */
+    private Timestamp updateTime;
+
+    /**
+     * 创建人id
+     */
+    private Long createUserId;
+
+    /**
+     * 修改人id
+     */
+    private Long updateUserId;
+
+    /**
+     * 拥有人id
+     */
+    private Long originUserId;
+
+    public Dataset() {}
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DatasetDataLabel.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.domain.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.*;
+import org.dubhe.datasetutil.common.base.BaseEntity;
+
+import java.io.Serializable;
+
+/**
+ * @description 数据集与标签关系实体
+ * @date 2020-10-14
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = false)
+@Builder
+@TableName("data_label")
+public class DatasetDataLabel extends BaseEntity implements Serializable {
+
+    /**
+     * id
+     */
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+
+    /**
+     * 数据集id
+     */
+    @TableField(value = "dataset_id")
+    private Long datasetId;
+
+    /**
+     * 标签id
+     */
+    @TableField(value = "label_id")
+    private Long labelId;
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/LogInfo.java

@@ -0,0 +1,74 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+
+package org.dubhe.datasetutil.domain.entity;
+
+import com.alibaba.fastjson.annotation.JSONField;
+import com.xiaoleilu.hutool.date.DateUtil;
+import lombok.Data;
+import lombok.experimental.Accessors;
+
+import java.io.Serializable;
+
+/**
+ * @description 日志对象封装类
+ * @date 2020-06-29
+ */
+@Data
+@Accessors(chain = true)
+public class LogInfo implements Serializable {
+
+    /**
+     * id
+     */
+    @JSONField(ordinal = 1)
+    private String traceId;
+
+    /**
+     * 类型
+     */
+    @JSONField(ordinal = 2)
+    private String type;
+
+    /**
+     * 等级
+     */
+    @JSONField(ordinal = 3)
+    private String level;
+
+    /**
+     * 位置
+     */
+    @JSONField(ordinal = 4)
+    private String location;
+
+    /**
+     * 时间
+     */
+    @JSONField(ordinal = 5)
+    private String time = DateUtil.now();
+
+    /**
+     * 描述
+     */
+    @JSONField(ordinal = 6)
+    private Object info;
+
+    public void setInfo(Object info) {
+        this.info = info;
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImageUploadHandle.java

@@ -0,0 +1,140 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.handle;
+
+import com.xiaoleilu.hutool.io.FileUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.common.util.DateUtil;
+import org.dubhe.datasetutil.common.util.GeneratorKeyUtil;
+import org.dubhe.datasetutil.common.util.MinioUtil;
+import org.dubhe.datasetutil.common.util.ThreadUtils;
+import org.dubhe.datasetutil.domain.entity.DataFile;
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+import org.dubhe.datasetutil.domain.entity.Dataset;
+import org.dubhe.datasetutil.service.DataFileService;
+import org.dubhe.datasetutil.service.DataVersionFileService;
+import org.dubhe.datasetutil.service.DatasetService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * @description 上传图片工具类
+ * @date 2020-09-17
+ */
+@Slf4j
+@Component
+public class DatasetImageUploadHandle {
+
+    @Autowired
+    private MinioUtil minioUtil;
+
+    @Autowired
+    private DataFileService dataFileService;
+
+    @Autowired
+    private DataVersionFileService dataVersionFileService;
+
+    @Autowired
+    private DatasetService datasetService;
+
+    @Autowired
+    private GeneratorKeyUtil generatorKeyUtil;
+
+    /**
+     * 启动线程
+     *
+     * @param imagePath 本地文件地址
+     * @param datasetId 数据集Id
+     */
+    public void execute(String imagePath, Long datasetId) throws Exception {
+        log.info("#-------------开始处理,时间[" + DateUtil.getNowStr() + "]-------------#");
+        List<String> fileNames = FileUtil.listFileNames(imagePath);
+        log.info("#-------------文件数量[" + fileNames.size() + "]------------------------");
+        String fileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + datasetId
+                + BusinessConstant.FILE_SEPARATOR + BusinessConstant.IMAGE_ORIGIN + BusinessConstant.FILE_SEPARATOR;
+        List<Callable<Integer>> partitions = new ArrayList<>();
+        int oneSize = ThreadUtils.createThread(fileNames.size());
+        List<String> need = new ArrayList<>();
+        AtomicInteger atomicInteger = new AtomicInteger(0);
+        for (String fileName : fileNames) {
+            need.add(fileName);
+            if (need.size() == oneSize || atomicInteger.intValue() == fileNames.size() - 1) {
+                List<String> now = new ArrayList<>(need);
+                need.clear();
+                partitions.add(() -> run(datasetId, now, fileBaseDir, imagePath));
+            }
+            atomicInteger.getAndIncrement();
+        }
+        ThreadUtils.runMultiThread(partitions);
+    }
+
+    /**
+     * 插入数据库数据
+     *
+     * @param datasetId   数据集Id
+     * @param fileNames   文件Name
+     * @param fileBaseDir 文件路径
+     * @param imagePath   文件地址
+     * @return java.lang.Integer 成功数量
+     */
+    public Integer run(Long datasetId, List<String> fileNames, String fileBaseDir, String imagePath) {
+        Integer success = 0;
+        Dataset dataset = datasetService.findCreateUserIdById(datasetId);
+        List<DataFile> dataFiles = new ArrayList<>();
+        List<DataVersionFile> dataVersionFiles = new ArrayList<>();
+        for (int i = 0; i < fileNames.size(); i++) {
+            try {
+                minioUtil.upLoadFile(fileBaseDir + fileNames.get(i), FileUtil.getInputStream(imagePath + BusinessConstant.FILE_SEPARATOR  + fileNames.get(i)));
+                BufferedImage read = ImageIO.read(new File(imagePath + BusinessConstant.FILE_SEPARATOR  + fileNames.get(i)));
+                success++;
+                dataFiles.add(new DataFile(fileNames.get(i), datasetId, minioUtil.getUrl(fileBaseDir + fileNames.get(i)), dataset.getCreateUserId(), read));
+
+                if (dataFiles.size() % 500 == 0 || i == fileNames.size() - 1) {
+                    long startDataFileIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_FILE, dataFiles.size());
+                    for (DataFile dataFileEntity : dataFiles) {
+                        dataFileEntity.setId(startDataFileIndex++);
+                    }
+
+                    dataFileService.saveBatchDataFile(dataFiles);
+                    for (DataFile file : dataFiles) {
+                        dataVersionFiles.add(new DataVersionFile(datasetId, file.getId(), 101, 0));
+                    }
+                    long startDataFileVersionIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_VERSION_FILE, dataVersionFiles.size());
+                    for (DataVersionFile dataVersionFile : dataVersionFiles) {
+                        dataVersionFile.setId(startDataFileVersionIndex++);
+                    }
+                    dataVersionFileService.saveBatchDataFileVersion(dataVersionFiles);
+                    dataVersionFiles.clear();
+                    dataFiles.clear();
+                }
+            } catch (Exception e) {
+                log.error("{}", e);
+            }
+        }
+        return success;
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImportHandle.java

@@ -0,0 +1,522 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.handle;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
+import com.xiaoleilu.hutool.io.FileUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+import org.dubhe.datasetutil.common.config.MinioConfig;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.common.constant.FileStateCodeConstant;
+import org.dubhe.datasetutil.common.exception.ImportDatasetException;
+import org.dubhe.datasetutil.common.util.*;
+import org.dubhe.datasetutil.domain.entity.*;
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+import org.dubhe.datasetutil.service.*;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+/**
+ * @description 导入数据集工具类
+ * @date 2020-10-12
+ */
+@Slf4j
+@Component
+public class DatasetImportHandle {
+
+    @Autowired
+    private DatasetService datasetService;
+
+    @Autowired
+    private DataLabelGroupService dataLabelGroupService;
+
+    @Autowired
+    private DataGroupLabelService dataGroupLabelService;
+
+    @Autowired
+    private DataLabelService dataLabelService;
+
+    @Autowired
+    private DataFileService dataFileService;
+
+    @Autowired
+    private DatasetDataLabelService datasetDataLabelService;
+
+    @Autowired
+    private DataVersionFileService dataVersionFileService;
+
+    @Autowired
+    private MinioUtil minioUtil;
+
+    @Autowired
+    private MinioConfig minioConfig;
+
+    @Autowired
+    private GeneratorKeyUtil generatorKeyUtil;
+
+    /**
+     * 可支持的图片格式集合
+     */
+    private static final List<String> SUFFIX_LIST = new ArrayList<>();
+
+    /**
+     * 标注文件中JSON的key
+     */
+    static List<String> annotationFileContextKey = new ArrayList<>();
+
+    /**
+     * 加载静态集合数据
+     */
+    static {
+        SUFFIX_LIST.add(".jpg");
+        SUFFIX_LIST.add(".png");
+        SUFFIX_LIST.add(".bmp");
+        SUFFIX_LIST.add(".jpeg");
+
+        annotationFileContextKey.add("score");
+        annotationFileContextKey.add("area");
+        annotationFileContextKey.add("name");
+        annotationFileContextKey.add("bbox");
+        annotationFileContextKey.add("segmentation");
+        annotationFileContextKey.add("iscrowd");
+    }
+
+    /**
+     * 导入数据集
+     *
+     * @param scanner 输入
+     */
+    public void importDataset(Scanner scanner) throws Exception {
+        Dataset dataset = verificationDatasetId(scanner);
+        String filePath = verificationFilePath(scanner);
+        File labelJsonFile = verificationFile(filePath);
+        DataLabelGroup dataLabelGroup = saveDataLabelGroup(HandleFileUtil.getLabelGroupName(labelJsonFile.getName()), dataset);
+        List<DataLabel> dataLabelList = readLabelContext(labelJsonFile);
+        saveDataLabel(dataset, dataLabelList, dataLabelGroup.getId());
+        executeUploadAndSave(dataLabelList, filePath, dataset);
+        datasetService.updateDatasetStatus(dataset);
+        log.warn("");
+        PrintUtils.printLine("  Success: 执行成功  ", PrintUtils.GREEN);
+        log.warn("");
+        log.warn("#   是否结束? Y / N  #");
+        Scanner scannerExit = new Scanner(System.in);
+        if (BusinessConstant.Y.toLowerCase().equals(scannerExit.nextLine().toLowerCase())) {
+            System.exit(0);
+        }
+    }
+
+    /**
+     * 检查文件结构 、类型
+     *
+     * @param globalFilePath 文件路径
+     * @return file 标签文件
+     */
+    public File verificationFile(String globalFilePath) throws IOException {
+        File labelRootFiles = new File(globalFilePath);
+        File imageRootFiles = new File(globalFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN));
+        File annotationRootFiles = new File(globalFilePath + HandleFileUtil.generateFilePath(BusinessConstant.ANNOTATION));
+        if (imageRootFiles.list() == null || annotationRootFiles.listFiles() == null) {
+            throw new ImportDatasetException(" 【" + globalFilePath + "】目录中的图片目录(origin)或者标注文件目录(annotation)的文件夹为空 ");
+        }
+        File labelJsonFile = null;
+        for (File file : Objects.requireNonNull(labelRootFiles.listFiles())) {
+            if (file.isFile() && file.getName().toLowerCase().startsWith(BusinessConstant.LABEL.toLowerCase())
+                    && file.getName().contains(BusinessConstant.UNDERLINE)
+                    && file.getName().toLowerCase().endsWith(BusinessConstant.SUFFIX_JSON.toLowerCase())) {
+                labelJsonFile = file;
+            }
+        }
+        if (labelJsonFile == null) {
+            throw new ImportDatasetException(" 【" + globalFilePath + "】目录中未找到标签组文件");
+        }
+        dealLabelGroup(labelJsonFile.getName());
+        List<DataLabel> dataLabelList = readLabelContext(labelJsonFile);
+        Map<String, List<DataLabel>> dataLabelMap = dataLabelList.stream().collect(Collectors.groupingBy(DataLabel::getName));
+        for (Map.Entry<String, List<DataLabel>> entry : dataLabelMap.entrySet()) {
+            if (entry.getValue().size() > 1) {
+                throw new ImportDatasetException(" 标签组中标签存在重复标签：【" + entry.getKey() + "】");
+            }
+        }
+        File[] imageFiles = imageRootFiles.listFiles();
+        if (imageFiles == null || imageFiles.length == 0) {
+            throw new ImportDatasetException(" 图片文件下不存在图片文件 ");
+        }
+        for (File imageFile : imageFiles) {
+            String suffixFileName = imageFile.getName().substring(imageFile.getName().lastIndexOf(BusinessConstant.SPOT));
+            if (!SUFFIX_LIST.contains(suffixFileName.toLowerCase())) {
+                throw new ImportDatasetException(" 图片文件文件夹中存在非法格式 ");
+            }
+        }
+        File[] annotationFiles = annotationRootFiles.listFiles();
+        if (annotationFiles == null || annotationFiles.length == 0) {
+            throw new ImportDatasetException(" 图片文件下不存在标注文件 ");
+        }
+        for (File annotationFile : annotationFiles) {
+            if (!annotationFile.getName().toLowerCase().endsWith(BusinessConstant.SUFFIX_JSON.toLowerCase())) {
+                throw new ImportDatasetException(" 标注文件文件夹中存在非法格式 ");
+            }
+        }
+        return labelJsonFile;
+    }
+
+    /**
+     * 启动线程
+     *
+     * @param filePath 本地根目录
+     * @param dataset  数据集
+     */
+    public void executeUploadAndSave(List<DataLabel> dataLabelList, String filePath, Dataset dataset) throws Exception {
+        String localImageFilePath = filePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN);
+        List<String> imageFileNameList = FileUtil.listFileNames(localImageFilePath);
+        log.info("需要处理： 【" + imageFileNameList.size() + "】张图片");
+        int oneSize = ThreadUtils.createThread(imageFileNameList.size());
+        log.info("需要创建线程数： 【" + oneSize + "】 条");
+        List<Callable<Integer>> partitions = new ArrayList<>();
+        List<String> need = new ArrayList<>();
+        AtomicInteger atomicInteger = new AtomicInteger(MagicNumConstant.ZERO);
+        for (String fileName : imageFileNameList) {
+            need.add(fileName);
+            if (need.size() == oneSize || atomicInteger.intValue() == imageFileNameList.size() - MagicNumConstant.ONE) {
+                List<String> fileNameList = new ArrayList<>(need);
+                need.clear();
+                partitions.add(() -> runTask(dataLabelList, dataset, fileNameList, filePath));
+            }
+            atomicInteger.getAndIncrement();
+        }
+        ThreadUtils.runMultiThread(partitions);
+    }
+
+    /**
+     * 实际执行任务
+     *
+     * @param dataset              数据集
+     * @param fileNameList         文件名字集合
+     * @param dataSetRootFilePath  文件路径
+     * @return Integer 执行次数
+     */
+    private Integer runTask(List<DataLabel> dataLabelList, Dataset dataset, List<String> fileNameList, String dataSetRootFilePath) throws Exception {
+        List<DataFile> dataFilesList = new ArrayList<>();
+        AtomicInteger atomicInteger = new AtomicInteger(MagicNumConstant.ZERO);
+        String imageFileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + dataset.getId()
+                + BusinessConstant.FILE_SEPARATOR + BusinessConstant.IMAGE_ORIGIN + BusinessConstant.FILE_SEPARATOR;
+        String annotationFileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + dataset.getId()
+                + BusinessConstant.FILE_SEPARATOR + BusinessConstant.ANNOTATION + BusinessConstant.FILE_SEPARATOR;
+        for (String fileName : fileNameList) {
+            String imageUploadFile = imageFileBaseDir + fileName;
+            String annotationFileName = HandleFileUtil.readFileName(fileName);
+            File annotationFile = new File(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.ANNOTATION) + BusinessConstant.FILE_SEPARATOR + annotationFileName + BusinessConstant.SUFFIX_JSON.toLowerCase());
+            JSONArray jsonArray = replaceJsonNode(annotationFile, dataLabelList);
+            minioUtil.upLoadFile(imageUploadFile, FileUtil.getInputStream(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN) + BusinessConstant.FILE_SEPARATOR + fileName));
+            minioUtil.upLoadFile(annotationFileBaseDir + annotationFileName, IOUtils.toInputStream(jsonArray.toString(), StandardCharsets.UTF_8.name()));
+            DataFile dataFile = new DataFile();
+            dataFile.setName(annotationFileName);
+            dataFile.setUrl(minioConfig.getBucketName() + BusinessConstant.FILE_SEPARATOR + imageUploadFile);
+            dataFile.setStatus(FileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE);
+            dataFile.setDatasetId(dataset.getId());
+            dataFile.setFileType(MagicNumConstant.ZERO);
+            dataFile.setPid(MagicNumConstant.ZERO_LONG);
+            dataFile.setCreateUserId(dataset.getCreateUserId());
+            try {
+                BufferedImage image = ImageIO.read(new File(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN) + BusinessConstant.FILE_SEPARATOR + fileName));
+                dataFile.setWidth(image.getWidth());
+                dataFile.setHeight(image.getHeight());
+            } catch (IOException e) {
+                throw new ImportDatasetException(" 读取图片高和宽失败 ");
+            }
+            dataFile.setOriginUserId(dataset.getCreateUserId());
+            dataFilesList.add(dataFile);
+            if (dataFilesList.size() % MagicNumConstant.FIVE_HUNDRED == MagicNumConstant.ZERO || atomicInteger.intValue() == fileNameList.size() - MagicNumConstant.ONE) {
+                long startDataFileIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_FILE, dataFilesList.size());
+                for (DataFile dataFileEntity : dataFilesList) {
+                    dataFileEntity.setId(startDataFileIndex++);
+                }
+                saveDataFile(dataFilesList);
+                List<DataVersionFile> dataVersionFileList = new ArrayList<>();
+                for (DataFile file : dataFilesList) {
+                    DataVersionFile dataVersionFile = new DataVersionFile();
+                    dataVersionFile.setDatasetId(dataset.getId());
+                    dataVersionFile.setFileId(file.getId());
+                    dataVersionFile.setStatus(MagicNumConstant.ZERO);
+                    dataVersionFile.setAnnotationStatus(FileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE);
+                    dataVersionFileList.add(dataVersionFile);
+                }
+                long startDataFileVersionIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_VERSION_FILE, dataVersionFileList.size());
+                for (DataVersionFile dataVersionFile : dataVersionFileList) {
+                    dataVersionFile.setId(startDataFileVersionIndex++);
+                }
+                saveDataVersionFile(dataVersionFileList);
+                dataVersionFileList.clear();
+                dataFilesList.clear();
+            }
+            atomicInteger.getAndIncrement();
+        }
+        return atomicInteger.getAndIncrement();
+
+    }
+
+    /**
+     * 检查并且替换JSON中的节点
+     *
+     * @param annotationFile 标注文件
+     * @param dataLabelList  数据集集合
+     */
+    public JSONArray replaceJsonNode(File annotationFile, List<DataLabel> dataLabelList) throws IOException {
+        JSONArray jsonArray = new JSONArray();
+        if (annotationFile.exists()) {
+            String annotationFileContext = HandleFileUtil.readFile(annotationFile);
+            jsonArray = JSONArray.parseArray(annotationFileContext);
+            if (!jsonArray.isEmpty()) {
+                replaceAllNode(jsonArray, dataLabelList);
+            }
+        }
+        return jsonArray;
+    }
+
+    /**
+     * 替换节点值
+     *
+     * @param jsonArray     标注文件集合
+     * @param dataLabelList 标签集合
+     */
+    public void replaceAllNode(JSONArray jsonArray, List<DataLabel> dataLabelList) {
+        for (int i = MagicNumConstant.ZERO; i < jsonArray.size(); i++) {
+            JSONObject jsonObject = jsonArray.getJSONObject(i);
+            jsonObject.put("category_id", findDataLabelId(dataLabelList, jsonObject.get("name").toString()));
+            jsonObject.remove("name");
+        }
+    }
+
+    /**
+     * 查询需要替换的节点
+     *
+     * @param dataLabelList 标签集合
+     * @param objectValue   替换的节点值
+     * @return long 替换标签的Id
+     */
+    public long findDataLabelId(List<DataLabel> dataLabelList, String objectValue) {
+        Optional<DataLabel> matchedDataLabel = dataLabelList.stream().filter(dataLabel -> objectValue.equals(dataLabel.getName())).findAny();
+        if (!matchedDataLabel.isPresent()) {
+            throw new ImportDatasetException(" 标注文件中name的值不存在于标签中!");
+        }
+        return matchedDataLabel.get().getId();
+    }
+
+    /**
+     * 获取JSON中所有的key
+     *
+     * @param jsonArray json集合
+     * @return JSON中所有的key
+     */
+    public List<String> getJsonKeyList(JSONArray jsonArray) {
+        List<String> listKey = new ArrayList<>();
+        for (Object object : jsonArray) {
+            LinkedHashMap<String, String> jsonMap = JSON.parseObject(object.toString(), new TypeReference<LinkedHashMap<String, String>>() {
+            });
+            for (Map.Entry<String, String> entry : jsonMap.entrySet()) {
+                listKey.add(entry.getKey());
+            }
+        }
+        return listKey.stream().distinct().collect(Collectors.toList());
+    }
+
+    /**
+     * 读取标签文件中标签数据
+     *
+     * @param file 标签文件
+     * @return List<DataLabel> 标签数据集合
+     */
+    public List<DataLabel> readLabelContext(File file) throws IOException {
+        String fileContext = HandleFileUtil.readFile(file);
+        List<DataLabel> dataLabelList = JSONArray.parseArray(fileContext, DataLabel.class);
+        for (DataLabel dataLabel : dataLabelList) {
+            if (StringUtils.isEmpty(dataLabel.getName()) || StringUtils.isEmpty(dataLabel.getColor())) {
+                throw new ImportDatasetException(" 标签文件不规范,未能读到 'name' 或者 'color' ");
+            }
+        }
+        return dataLabelList;
+    }
+
+    /**
+     * 检查标签组是否有重名
+     *
+     * @param labelGroupName 标签组名称
+     */
+    public void dealLabelGroup(String labelGroupName) {
+        String groupName = HandleFileUtil.getLabelGroupName(labelGroupName);
+        int count = dataLabelGroupService.selectByLabelGroupName(groupName);
+        if (count > MagicNumConstant.ZERO) {
+            throw new ImportDatasetException(" 标签组名称【" + groupName + "】已存在，请修改label_xxx.json文件名 ");
+        }
+    }
+
+    /**
+     * 保存标签组名称
+     *
+     * @param labelGroupName 标签组名称
+     * @param dataset        数据集
+     * @return DataLabelGroup 数据集标签组
+     */
+    public DataLabelGroup saveDataLabelGroup(String labelGroupName, Dataset dataset) {
+        long timeStamp = System.currentTimeMillis();
+        DataLabelGroup dataLabelGroup = new DataLabelGroup();
+        dataLabelGroup.setName(labelGroupName);
+        dataLabelGroup.setOriginUserId(dataset.getCreateUserId());
+        dataLabelGroup.setType(MagicNumConstant.ZERO_LONG);
+        dataLabelGroup.setCreateUserId(dataset.getCreateUserId());
+        dataLabelGroup.setCreateTime(new Timestamp(timeStamp));
+        dataLabelGroup.setUpdateTime(new Timestamp(timeStamp));
+        dataLabelGroupService.saveDataGroupLabel(dataLabelGroup);
+        return dataLabelGroup;
+    }
+
+    /**
+     * 批量保存标签
+     *
+     * @param dataset          数据集
+     * @param listDataLabel    标签集合
+     * @param dataLabelGroupId 标签组Id
+     */
+    public void saveDataLabel(Dataset dataset, List<DataLabel> listDataLabel, Long dataLabelGroupId) {
+        listDataLabel.forEach(dataLabel -> dataLabel.setCreateUserId(dataset.getCreateUserId()));
+        dataLabelService.saveBatchDataLabel(listDataLabel);
+        List<DatasetDataLabel> listDatasetDataLabel = new ArrayList<>();
+        for (DataLabel dataLabel : listDataLabel) {
+            DatasetDataLabel datasetDataLabel = new DatasetDataLabel();
+            datasetDataLabel.setLabelId(dataLabel.getId());
+            datasetDataLabel.setDatasetId(dataset.getId());
+            listDatasetDataLabel.add(datasetDataLabel);
+        }
+        saveDatasetDataLabel(listDatasetDataLabel);
+
+        List<DataGroupLabel> listDataGroupLabel = new ArrayList<>();
+        for (DatasetDataLabel datasetDataLabel : listDatasetDataLabel) {
+            DataGroupLabel dataGroupLabel = new DataGroupLabel();
+            dataGroupLabel.setLabelId(datasetDataLabel.getId());
+            dataGroupLabel.setLabelGroupId(dataLabelGroupId);
+            listDataGroupLabel.add(dataGroupLabel);
+        }
+        saveDatasetDataGroupLabel(listDataGroupLabel);
+
+    }
+
+    /**
+     * 批量保存文件
+     *
+     * @param listDataFile file集合
+     */
+    public void saveDataFile(List<DataFile> listDataFile) {
+        dataFileService.saveBatchDataFile(listDataFile);
+    }
+
+    /**
+     * 批量保存文件版本数据
+     *
+     * @param listDataVersionFile 文件版本数据
+     */
+    public void saveDataVersionFile(List<DataVersionFile> listDataVersionFile) {
+        dataVersionFileService.saveBatchDataFileVersion(listDataVersionFile);
+    }
+
+
+    /**
+     * 批量保存标签与数据集关系表
+     *
+     * @param listDatasetDataLabel 标签与数据集关系表
+     */
+    public void saveDatasetDataLabel(List<DatasetDataLabel> listDatasetDataLabel) {
+        datasetDataLabelService.saveBatchDatasetDataLabel(listDatasetDataLabel);
+    }
+
+    /**
+     * 批量保存标签与标签组的关系
+     *
+     * @param listDataGroupLabel  标签与标签组集合
+     */
+    public void saveDatasetDataGroupLabel(List<DataGroupLabel> listDataGroupLabel) {
+        dataGroupLabelService.saveDataGroupLabel(listDataGroupLabel);
+    }
+
+    /**
+     * 查询数据集
+     *
+     * @param datasetId 数据集Id
+     * @return Dataset  根据数据集ID查询返回的数据集
+     */
+    private Dataset findDataset(Long datasetId) {
+        return datasetService.findDatasetById(datasetId);
+    }
+
+    /**
+     * 校验数据集ID
+     *
+     * @param scanner  控制台输入参数
+     * @return Dataset 数据集
+     */
+    public Dataset verificationDatasetId(Scanner scanner) {
+        log.warn("# 请输入数据集ID #");
+        String datasetIdStr = scanner.nextLine();
+        long datasetId;
+        try {
+            datasetId = Long.parseLong(datasetIdStr.trim());
+        } catch (Exception e) {
+            throw new ImportDatasetException(" 数据集ID非法,请重新输入 ");
+        }
+        Dataset dataset = findDataset(datasetId);
+        if (dataset == null) {
+            throw new ImportDatasetException(" 数据集ID不存在，请重新输入 ");
+        }
+        int countDataLabel = datasetService.findDataLabelById(dataset.getId());
+        int countDataFile = datasetService.findDataFileById(dataset.getId());
+        if (countDataLabel > MagicNumConstant.ZERO || countDataFile > MagicNumConstant.ZERO) {
+            throw new ImportDatasetException(" 当前数据集文件已存在，请勿重新导入 ");
+        }
+        return dataset;
+    }
+
+    /**
+     * 校验文件路径
+     *
+     * @param scanner 输入控制台
+     * @return String 字符串
+     */
+    public String verificationFilePath(Scanner scanner) {
+        log.warn("# 请输入待导入数据集绝对路径地址 #");
+        String filePath = scanner.nextLine();
+        File file = new File(filePath.trim());
+        if (!file.exists()) {
+            throw new ImportDatasetException("  【" + filePath + "】 文件路径不存在,请重新输入");
+        }
+        return filePath;
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/DataFileService.java

@@ -0,0 +1,35 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DataFile;
+import java.util.List;
+
+/**
+ * @description 文件服务
+ * @date 2020-09-17
+ */
+public interface DataFileService {
+
+    /**
+     * 批量写入文件内容
+     *
+     * @param dataFiles 文件数据集合
+     */
+    void saveBatchDataFile(List<DataFile> dataFiles);
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/DataGroupLabelService.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DataGroupLabel;
+
+import java.util.List;
+
+/**
+ * @description 标签与标签组的服务接口
+ * @date 2020-10-21
+ */
+public interface DataGroupLabelService {
+    /**
+     * 保存标签与标签组的关系
+     *
+     * @param listDataGroupLabel 标签与标签组数据
+     */
+    void saveDataGroupLabel(List<DataGroupLabel> listDataGroupLabel);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelGroupService.java

@@ -0,0 +1,40 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DataLabelGroup;
+
+/**
+ * @description 数据集标签组服务接口
+ * @date 2020-10-14
+ */
+public interface DataLabelGroupService {
+    /**
+     * 根据标签组名查询
+     *
+     * @param labelGroupName 标签组名称
+     * @return int 数量
+     */
+    int selectByLabelGroupName(String labelGroupName);
+
+    /**
+     * 保存数据集标签组标签
+     *
+     * @param dataLabelGroup 标签组名称
+     */
+    void saveDataGroupLabel(DataLabelGroup dataLabelGroup);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelService.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DataLabel;
+
+import java.util.List;
+
+/**
+ * @description 数据集标签服务接口
+ * @date 2020-10-14
+ */
+public interface DataLabelService {
+    /**
+     * 批量保存数据集标签
+     *
+     * @param listDataLabel 数据集标签集合
+     */
+    void saveBatchDataLabel(List<DataLabel> listDataLabel);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/DataSequenceService.java

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+
+import org.dubhe.datasetutil.domain.entity.DataSequence;
+
+/**
+ * @description 获取序列服务接口
+ * @date 2020-09-23
+ */
+public interface DataSequenceService {
+    /**
+     * 根据业务编码获取序列号
+     *
+     * @param businessCode 业务编码
+     * @return DataSequence 数据索引
+     */
+    DataSequence getSequence(String businessCode);
+    /**
+     * 根据业务编码更新起点
+     *
+     * @param businessCode 业务编码
+     * @return int 数量
+     */
+    int updateSequenceStart(String businessCode);
+
+    /**
+     * 检查表是否存在
+     *
+     * @param tableName 表名
+     * @return boolean 存在结果
+     */
+    boolean checkTableExist(String tableName);
+
+    /**
+     * 执行存储过程创建表
+     *
+     * @param tableId 表ID
+     */
+    void createTable(String tableId);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/DataVersionFileService.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+
+import java.util.List;
+
+/**
+ * @description 数据集文件关系服务
+ * @date 2020-9-17
+ */
+public interface DataVersionFileService {
+    /**
+     * 插入数据集文件数据
+     *
+     * @param dataVersionFiles 数据集文件数据集合
+     */
+    void saveBatchDataFileVersion(List<DataVersionFile> dataVersionFiles);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetDataLabelService.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.DatasetDataLabel;
+
+import java.util.List;
+
+/**
+ * @description 数据集与标签关系服务接口
+ * @date 2020-10-14
+ */
+public interface DatasetDataLabelService {
+    /**
+     * 批量保存数据集与标签关系
+     *
+     * @param listDatasetDataLabel 数据集标签集合
+     */
+    void saveBatchDatasetDataLabel(List<DatasetDataLabel> listDatasetDataLabel);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetService.java

@@ -0,0 +1,65 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service;
+
+import org.dubhe.datasetutil.domain.entity.Dataset;
+
+/**
+ * @description 数据集服务
+ * @date 2020-9-17
+ */
+public interface DatasetService {
+    /**
+     * 根据数据集Id查询创建人Id
+     *
+     * @param datasetId 数据集Id
+     * @return Dataset 数据集
+     */
+    Dataset findCreateUserIdById(Long datasetId);
+
+    /**
+     * 根据ID查询数据集
+     *
+     * @param datasetId 数据集Id
+     * @return Dataset 数据集
+     */
+    Dataset findDatasetById(Long datasetId);
+
+    /**
+     * 更新数据集状态
+     *
+     * @param dataset 数据集
+     * @return int 数量
+     */
+    int updateDatasetStatus(Dataset dataset);
+
+    /**
+     * 查询数据集标签数量
+     *
+     * @param datasetId 数据集ID
+     * @return int 数量
+     */
+    int findDataLabelById(Long datasetId);
+
+    /**
+     * 查询数据集文件数量
+     *
+     * @param datasetId 数据集ID
+     * @return int 数量
+     */
+    int findDataFileById(Long datasetId);
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataFileServiceImpl.java

@@ -0,0 +1,60 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.dao.DataFileMapper;
+import org.dubhe.datasetutil.domain.entity.DataFile;
+import org.dubhe.datasetutil.service.DataFileService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description 数据集文件 服务实现类
+ * @date 2020-09-17
+ */
+@Slf4j
+@Service
+public class DataFileServiceImpl extends ServiceImpl<DataFileMapper, DataFile> implements DataFileService {
+
+    @Autowired
+    private DataFileMapper dataFileMapper;
+
+    /**
+     * 批量写入文件内容
+     *
+     * @param listDataFile 文件数据集合
+     */
+    @Override
+    public void saveBatchDataFile(List<DataFile> listDataFile) {
+        int listSize = listDataFile.size();
+        if (listSize > BusinessConstant.SUB_LENGTH) {
+            List<List<DataFile>> partitionList = Lists.partition(listDataFile, BusinessConstant.SUB_LENGTH);
+            for (List<DataFile> subPartitionList : partitionList) {
+                dataFileMapper.saveBatchDataFile(subPartitionList);
+            }
+        } else {
+            dataFileMapper.saveBatchDataFile(listDataFile);
+        }
+    }
+
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataGroupLabelServiceImpl.java

@@ -0,0 +1,57 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.google.common.collect.Lists;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.dao.DataGroupLabelMapper;
+import org.dubhe.datasetutil.domain.entity.DataGroupLabel;
+import org.dubhe.datasetutil.service.DataGroupLabelService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description 标签与标签组的服务接口实现
+ * @date 2020-10-21
+ */
+@Service
+public class DataGroupLabelServiceImpl extends ServiceImpl<DataGroupLabelMapper, DataGroupLabel> implements DataGroupLabelService {
+
+    @Autowired
+    private DataGroupLabelMapper dataGroupLabelMapper;
+
+    /**
+     * 插入标签组
+     *
+     * @param listDataGroupLabel 标签组数据
+     */
+    @Override
+    public void saveDataGroupLabel(List<DataGroupLabel> listDataGroupLabel) {
+        int listSize = listDataGroupLabel.size();
+        if (listSize > BusinessConstant.SUB_LENGTH) {
+            List<List<DataGroupLabel>> partitionList = Lists.partition(listDataGroupLabel, BusinessConstant.SUB_LENGTH);
+            for (List<DataGroupLabel> subPartitionList : partitionList) {
+                dataGroupLabelMapper.saveDataGroupLabel(subPartitionList);
+            }
+        } else {
+            dataGroupLabelMapper.saveDataGroupLabel(listDataGroupLabel);
+        }
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelGroupServiceImpl.java

@@ -0,0 +1,60 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.dubhe.datasetutil.dao.DataGroupLabelMapper;
+import org.dubhe.datasetutil.dao.DataLabelGroupMapper;
+import org.dubhe.datasetutil.domain.entity.DataGroupLabel;
+import org.dubhe.datasetutil.domain.entity.DataLabelGroup;
+import org.dubhe.datasetutil.service.DataLabelGroupService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+/**
+ * @description 数据集标签组服务实现接口
+ * @date 2020-10-14
+ */
+@Service
+public class DataLabelGroupServiceImpl extends ServiceImpl<DataGroupLabelMapper, DataGroupLabel> implements DataLabelGroupService {
+
+    @Autowired
+    private DataLabelGroupMapper dataLabelGroupMapper;
+
+    /**
+     * 根据标签组名查询
+     *
+     * @param labelGroupName 标签组名称
+     * @return int 数量
+     */
+    @Override
+    public int selectByLabelGroupName(String labelGroupName) {
+        return dataLabelGroupMapper.selectByLabelGroupName(labelGroupName);
+    }
+
+    /**
+     * 保存标签组名称
+     *
+     * @param dataLabelGroup 标签组名称
+     */
+    @Override
+    @Transactional(rollbackFor = Exception.class)
+    public void saveDataGroupLabel(DataLabelGroup dataLabelGroup) {
+        dataLabelGroupMapper.insert(dataLabelGroup);
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelServiceImpl.java

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.google.common.collect.Lists;
+import org.dubhe.datasetutil.common.constant.BusinessConstant;
+import org.dubhe.datasetutil.dao.DataLabelMapper;
+import org.dubhe.datasetutil.domain.entity.DataLabel;
+import org.dubhe.datasetutil.service.DataLabelService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description 数据集标签服务接口实现
+ * @date 2020-10-14
+ */
+@Service
+public class DataLabelServiceImpl implements DataLabelService {
+
+    @Autowired
+    private DataLabelMapper dataLabelMapper;
+
+    /**
+     * 批量保存数据集标签 (分批)
+     *
+     * @param listDataLabel 数据集标签
+     */
+    @Override
+    public void saveBatchDataLabel(List<DataLabel> listDataLabel) {
+        int listSize = listDataLabel.size();
+        if(listSize > BusinessConstant.SUB_LENGTH){
+           List<List<DataLabel>> partitionList =  Lists.partition(listDataLabel,BusinessConstant.SUB_LENGTH);
+            for (List<DataLabel> subPartitionList: partitionList) {
+                dataLabelMapper.saveBatchDataLabel(subPartitionList);
+            }
+        }else{
+            dataLabelMapper.saveBatchDataLabel(listDataLabel);
+        }
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataSequenceServiceImpl.java

@@ -0,0 +1,78 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import org.dubhe.datasetutil.dao.DataSequenceMapper;
+import org.dubhe.datasetutil.domain.entity.DataSequence;
+import org.dubhe.datasetutil.service.DataSequenceService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+
+/**
+ * @description 获取序列服务接口实现
+ * @date 2020-09-23
+ */
+@Service
+public class DataSequenceServiceImpl implements DataSequenceService {
+
+    @Autowired
+    private DataSequenceMapper dataSequenceMapper;
+
+    @Override
+    public DataSequence getSequence(String businessCode) {
+        return dataSequenceMapper.selectByBusiness(businessCode);
+    }
+
+    /**
+     * 更新数据索引
+     *
+     * @param businessCode 业务编码
+     * @return int 数量
+     */
+    @Override
+    public int updateSequenceStart(String businessCode) {
+        return dataSequenceMapper.updateStartByBusinessCode(businessCode);
+    }
+
+    /**
+     * 检查表名
+     *
+     * @param tableName 表名
+     * @return true/false
+     */
+    @Override
+    public boolean checkTableExist(String tableName) {
+        try {
+            dataSequenceMapper.checkTableExist(tableName);
+            return true;
+        }catch (Exception e){
+            return false;
+        }
+    }
+
+    /**
+     * 创建表
+     *
+     * @param tableName 类型名称
+     */
+    @Override
+    public void createTable(String tableName) {
+        String oldTableName = tableName.substring(0,tableName.lastIndexOf("_"));
+        dataSequenceMapper.createNewTable(tableName,oldTableName);
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataVersionFileServiceImpl.java

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.dubhe.datasetutil.dao.DataVersionFileMapper;
+import org.dubhe.datasetutil.domain.dto.DataVersionFile;
+import org.dubhe.datasetutil.service.DataVersionFileService;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description 数据集文件 服务实现类
+ * @date 2020-09-17
+ */
+@Service
+public class DataVersionFileServiceImpl  extends ServiceImpl <DataVersionFileMapper, DataVersionFile> implements DataVersionFileService {
+
+
+    /**
+     * 插入数据集文件数据
+     *
+     * @param listDataVersionFile 数据集文件数据集合
+     */
+    @Override
+    public void saveBatchDataFileVersion(List<DataVersionFile> listDataVersionFile) {
+       baseMapper.saveBatchDataFileVersion(listDataVersionFile);
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetDataLabelServiceImpl.java

@@ -0,0 +1,57 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.google.common.collect.Lists;
+import org.dubhe.datasetutil.common.base.MagicNumConstant;
+import org.dubhe.datasetutil.dao.DatasetDataLabelMapper;
+import org.dubhe.datasetutil.domain.entity.DatasetDataLabel;
+import org.dubhe.datasetutil.service.DatasetDataLabelService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @description  数据集和标签服务实现类
+ * @date 2020-10-14
+ */
+@Service
+public class DatasetDataLabelServiceImpl implements DatasetDataLabelService {
+
+    @Autowired
+    private DatasetDataLabelMapper datasetDataLabelMapper;
+
+    private static final int SUB_LENGTH = MagicNumConstant.FIVE_THOUSAND;
+
+    /**
+     * 批量保存数据集标签
+     *
+     * @param listDatasetDataLabel 数据集标签集合
+     */
+    @Override
+    public void saveBatchDatasetDataLabel(List<DatasetDataLabel> listDatasetDataLabel) {
+        if(listDatasetDataLabel.size() > SUB_LENGTH){
+            List<List<DatasetDataLabel>> partitionList =  Lists.partition(listDatasetDataLabel,SUB_LENGTH);
+            for (List<DatasetDataLabel> subPartitionList: partitionList) {
+                datasetDataLabelMapper.saveBatchDatasetDataLabel(subPartitionList);
+            }
+        }else{
+            datasetDataLabelMapper.saveBatchDatasetDataLabel(listDatasetDataLabel);
+        }
+    }
+}

dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetServiceImpl.java

@@ -0,0 +1,92 @@
+/**
+ * Copyright 2020 Zhejiang Lab. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================
+ */
+package org.dubhe.datasetutil.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.dubhe.datasetutil.common.constant.DataStateCodeConstant;
+import org.dubhe.datasetutil.dao.DatasetMapper;
+import org.dubhe.datasetutil.domain.entity.Dataset;
+import org.dubhe.datasetutil.service.DatasetService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * @description 数据集 服务实现类
+ * @date 2020-09-17
+ */
+@Service
+public class DatasetServiceImpl extends ServiceImpl<DatasetMapper, Dataset> implements DatasetService {
+
+    @Autowired
+    private DatasetMapper datasetMapper;
+
+    /**
+     * 根据数据集Id查询创建人Id
+     *
+     * @param datasetId 数据集Id
+     * @return dataset 数据集合
+     */
+    @Override
+    public Dataset findCreateUserIdById(Long datasetId) {
+        return baseMapper.selectById(datasetId);
+    }
+
+    /**
+     * 根据ID 查询数据集
+     *
+     * @param datasetId 数据集ID
+     * @return Dataset 数据集
+     */
+    @Override
+    public Dataset findDatasetById(Long datasetId) {
+        return datasetMapper.findDatasetById(datasetId);
+    }
+
+    /**
+     * 更新数据集状态
+     *
+     * @param dataset 数据集
+     * @return int 数量
+     */
+    @Override
+    public int updateDatasetStatus(Dataset dataset) {
+        dataset.setStatus(DataStateCodeConstant.ANNOTATION_COMPLETE_STATE);
+        return datasetMapper.updateById(dataset);
+    }
+
+    /**
+     * 查询数据集数据标签数量
+     *
+     * @param datasetId 数据集ID
+     * @return int 数量
+     */
+    @Override
+    public int findDataLabelById(Long datasetId) {
+        return datasetMapper.findDataLabelById(datasetId);
+    }
+
+    /**
+     * 查询数据集文件数量
+     *
+     * @param datasetId 数据集ID
+     * @return int 数量
+     */
+    @Override
+    public int findDataFileById(Long datasetId) {
+        return datasetMapper.findDataFileById(datasetId);
+    }
+}

dataset-util/src/main/resources/logback-spring-dev.xml

@@ -0,0 +1,254 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration scan="true" scanPeriod="30 seconds"
+	debug="false">
+	<contextName>dubhe</contextName>
+	<springProperty scope="context" name="app.active"
+		source="spring.profiles.active" />
+	<property name="log.path" value="dubhe-${app.active}" />
+	<property name="log.pattern"
+		value=" %gray(%msg%n)" />
+
+	<!--输出到控制台 -->
+	<appender name="console"
+		class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern>${log.pattern}</pattern>
+		</encoder>
+		<filter class="org.dubhe.datasetutil.common.filter.ConsoleLogFilter">
+			<level>WARN</level>
+			<name>INFO</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="info_file"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/info/dubhe-info.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>INFO</level>
+			<name>INFO,K8S_CALLBACK</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="debug_info"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/debug/dubhe-debug.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>DEBUG</level>
+			<name>DEBUG</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="error_file"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/error/dubhe-error.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>ERROR</level>
+			<name>ERROR</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="warn_file"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/warn/dubhe-warn.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>WARN</level>
+			<name>WARN</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="trace_file"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/trace/dubhe-trace.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>TRACE</level>
+			<name>TRACE</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="schedule_file"
+			  class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/info/dubhe-schedule.log</file>
+		<rollingPolicy
+				class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-schedule-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>INFO</level>
+			<name>SCHEDULE</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="global_request_file"
+			  class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/info/dubhe-request.log</file>
+		<rollingPolicy
+				class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-request-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.GlobalRequestLogFilter">
+			<level>INFO</level>
+			<!-- name必须配置，不然没有日志，注意看GlobalLogFilter -->
+			<name>GLOBAL_REQUEST</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+
+	<!--普通日志输出到控制台 -->
+	<root level="info">
+		<appender-ref ref="console" />
+		<appender-ref ref="info_file" />
+		<appender-ref ref="schedule_file"/>
+		<appender-ref ref="global_request_file"/>
+		<appender-ref ref="debug_info" />
+		<appender-ref ref="error_file" />
+		<appender-ref ref="warn_file" />
+		<appender-ref ref="trace_file" />
+	</root>
+
+	<!--监控sql日志输出 -->
+	<!-- <logger name="jdbc.sqlonly" level="OFF" additivity="false"> -->
+	<logger name="jdbc.sqlonly" level="INFO" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="jdbc.resultset" level="ERROR" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<!-- 如想看到表格数据，将OFF改为INFO -->
+	<logger name="jdbc.resultsettable" level="OFF"
+		additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="jdbc.connection" level="OFF" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="jdbc.sqltiming" level="OFF" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="jdbc.audit" level="OFF" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="org.dubhe.k8s" level="DEBUG" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+</configuration>

dataset-util/src/main/resources/logback-spring-test.xml

@@ -0,0 +1,248 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration scan="true" scanPeriod="30 seconds"
+	debug="false">
+	<contextName>dubhe</contextName>
+	<property name="log.charset" value="utf-8" />
+	<springProperty scope="context" name="app.active"
+		source="spring.profiles.active" />
+	<property name="log.path" value="dubhe-${app.active}" />
+	<property name="log.pattern"
+		value="%black(%contextName-) %red(%d{yyyy-MM-dd HH:mm:ss}) %green([%thread]) %highlight(%-5level) %boldMagenta(%logger{36}) [%X{traceId}] - %gray(%msg%n)" />
+
+	<!--输出到控制台 -->
+	<appender name="console"
+		class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern>${log.pattern}</pattern>
+		</encoder>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="info_file"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/info/dubhe-info.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>INFO</level>
+			<name>INFO,K8S_CALLBACK</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="debug_info"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/debug/dubhe-debug.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>DEBUG</level>
+			<name>DEBUG</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="error_file"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/error/dubhe-error.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>ERROR</level>
+			<name>ERROR</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="warn_file"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/warn/dubhe-warn.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>WARN</level>
+			<name>WARN</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="trace_file"
+		class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/trace/dubhe-trace.log</file>
+		<rollingPolicy
+			class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>TRACE</level>
+			<name>TRACE</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="schedule_file"
+			  class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/info/dubhe-schedule.log</file>
+		<rollingPolicy
+				class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-schedule-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter">
+			<level>INFO</level>
+			<name>SCHEDULE</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!-- 滚动记录文件，先将日志记录到指定文件，复合条件后日志将记录到其他文件 -->
+	<appender name="global_request_file"
+			  class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<file>logs/${log.path}/info/dubhe-request.log</file>
+		<rollingPolicy
+				class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-request-%d{yyyy-MM-dd}.%i.log
+			</fileNamePattern>
+			<!-- 单个日志文件最多50MB, 14天的日志周期，最大不能超过250MB -->
+			<maxFileSize>50MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>250MB</totalSizeCap>
+		</rollingPolicy>
+		<encoder>
+			<pattern>%m%n</pattern>
+		</encoder>
+		<!-- 默认true日志追加到文件结尾，false清空现存文件 -->
+		<append>true</append>
+		<filter class="org.dubhe.datasetutil.common.filter.GlobalRequestLogFilter">
+			<level>INFO</level>
+			<!-- name必须配置，不然没有日志，注意看GlobalLogFilter -->
+			<name>GLOBAL_REQUEST</name>
+			<onMatch>ACCEPT</onMatch>
+			<onMismatch>DENY</onMismatch>
+		</filter>
+	</appender>
+
+	<!--普通日志输出到控制台 -->
+	<root level="info">
+		<appender-ref ref="console" />
+		<appender-ref ref="info_file" />
+		<appender-ref ref="schedule_file"/>
+		<appender-ref ref="global_request_file"/>
+		<appender-ref ref="debug_info" />
+		<appender-ref ref="error_file" />
+		<appender-ref ref="warn_file" />
+		<appender-ref ref="trace_file" />
+	</root>
+
+	<!--监控sql日志输出 -->
+	<!-- <logger name="jdbc.sqlonly" level="OFF" additivity="false"> -->
+	<logger name="jdbc.sqlonly" level="INFO" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="jdbc.resultset" level="ERROR" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<!-- 如想看到表格数据，将OFF改为INFO -->
+	<logger name="jdbc.resultsettable" level="OFF"
+		additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="jdbc.connection" level="OFF" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="jdbc.sqltiming" level="OFF" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="jdbc.audit" level="OFF" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+
+	<logger name="org.dubhe.k8s" level="DEBUG" additivity="false">
+		<appender-ref ref="console" />
+	</logger>
+</configuration>

dataset-util/src/main/resources/mapper/DataFileMapper.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+<mapper namespace="org.dubhe.datasetutil.dao.DataFileMapper">
+    <insert id="saveBatchDataFile" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
+        insert into data_file (id,`name`,dataset_id,status,url,enhance_type,width,height,origin_user_id,create_user_id,pid)
+        values
+        <foreach collection="listDataFile" item="file" separator=",">
+            (#{file.id},#{file.name},#{file.datasetId},#{file.status},
+            #{file.url},#{file.enhanceType},#{file.width},#{file.height},#{file.originUserId},#{file.createUserId}
+            <choose>
+                <when test="file.pid != null">
+                    ,#{file.pid}
+                </when>
+                <otherwise>
+                    ,0
+                </otherwise>
+            </choose>
+            )
+        </foreach>
+    </insert>
+</mapper>

dataset-util/src/main/resources/mapper/DataGroupLabelMapper.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+<mapper namespace="org.dubhe.datasetutil.dao.DataGroupLabelMapper">
+    <insert id="saveDataGroupLabel" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
+        insert into data_group_label (label_id,label_group_id)
+        values
+        <foreach collection="list" item="dataGroupLabel" separator=",">
+            (#{dataGroupLabel.labelId},#{dataGroupLabel.labelGroupId})
+        </foreach>
+    </insert>
+</mapper>

dataset-util/src/main/resources/mapper/DataLabelMapper.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+<mapper namespace="org.dubhe.datasetutil.dao.DataLabelMapper">
+    <insert id="saveBatchDataLabel" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
+        insert into data_label (`name`,color,create_user_id)
+        values
+        <foreach collection="listDataLabel" item="dataLabel" separator=",">
+            ( #{dataLabel.name},#{dataLabel.color},#{dataLabel.createUserId})
+        </foreach>
+    </insert>
+</mapper>

dataset-util/src/main/resources/mapper/DataVersionFileMapper.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+<mapper namespace="org.dubhe.datasetutil.dao.DataVersionFileMapper">
+    <insert id="saveBatchDataFileVersion" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
+        insert into data_dataset_version_file (id,dataset_id,file_id,annotation_status,status)
+        values
+        <foreach collection="listDataVersionFile" item="temp" separator=",">
+            (#{temp.id},#{temp.datasetId},#{temp.fileId},#{temp.annotationStatus},#{temp.status})
+        </foreach>
+    </insert>
+</mapper>

dataset-util/src/main/resources/mapper/DatasetDataLabelMapper.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+<mapper namespace="org.dubhe.datasetutil.dao.DatasetDataLabelMapper">
+    <insert id="saveBatchDatasetDataLabel" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id">
+        insert into data_dataset_label(dataset_id,label_id )
+        values
+        <foreach collection="datasetDataLabelList" item="datasetDataLabel" separator=",">
+            (#{datasetDataLabel.datasetId},#{datasetDataLabel.labelId})
+        </foreach>
+    </insert>
+</mapper>

dataset-util/src/main/resources/mapper/DatasetMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+<mapper namespace="org.dubhe.datasetutil.dao.DatasetMapper">
+</mapper>