@@ -0,0 +1,6 @@ | |||
# Created by .ignore support plugin (hsz.mobi) | |||
.idea | |||
*.iml | |||
logs | |||
HELP.md | |||
target/ |
@@ -0,0 +1,94 @@ | |||
# 之江天枢-数据集导入脚本 | |||
**之江天枢一站式人工智能开源平台**(简称:**之江天枢**),为了实现其他平台已标注完成的数据集在「一站式开发平台」上进行开发,我们增加了数据集导入的功能,实现对数据集的全流程功能操作。 | |||
## 源码部署 | |||
安装如下软件环境。 | |||
- OpenJDK:1.8+ | |||
## 下载脚本 | |||
- 数据集模板:http://tianshu.org.cn/static/upload/file/dubhe-dataset-template.zip | |||
- 上传数据集脚本:http://tianshu.org.cn/static/upload/file/upload_dataset.zip | |||
## 脚本使用说明: | |||
- 登录天枢系统深度学习平台,在数据管理菜单下的数据集管理中创建数据集。获取数据集ID | |||
- 需要自行准备图片文件、标注文件、标签文件 | |||
## 运行脚本: | |||
1.解压下载的zip文件,需要自行配置数据源、MinIO相关配置 | |||
2.运行脚本Windows 运行 run.bat; macOS/Linux 系统运行 run.sh | |||
注:可自行配置'application-{env}.xml'文件,执行命令后面添加 'run.bat {env}'即可执行对应的application-{env}.xml;然后按提示进行操作 | |||
3.输入数据集ID | |||
4.输入待导入数据集绝对路径 | |||
## 目录结构: | |||
``` | |||
标签文件: label_{name}.json ({name} 代表标签组名,可自行定义; 只读标签组文件夹下的第一个标签组文件,标签文件仅支持:.json 支持大小写;文件内容为JSON字符串) | |||
图片文件目录: origin (图片文件需要有后缀名,支持四种格式:.jpg,.png,.bmp,.jpeg 支持大小写) | |||
标注文件目录: annotation (标注文件需要有后缀名,仅支持格式:.json 支持大小写; 文件内容为JSON字符串) | |||
``` | |||
## 文件格式 | |||
- 标签文件内容样例: | |||
``` | |||
name: 名称 | |||
color: 颜色(16进制编码) | |||
``` | |||
详细示例: | |||
``` | |||
[{ | |||
"name": "行人", | |||
"color": "#ffbb96" | |||
}, | |||
{ | |||
"name": "自行车", | |||
"color": "#fcffe6" | |||
}, | |||
{ | |||
"name": "汽车", | |||
"color": "#f4ffb8" | |||
}] | |||
``` | |||
- 标注文件内容样例: | |||
``` | |||
name: 名称 | |||
bbox: 标注位置 | |||
score:分数 | |||
``` | |||
详细示例: | |||
``` | |||
[{ | |||
"name": "行人", | |||
"bbox": [321.6755762696266, 171.32076993584633, 185.67924201488495, 145.02639323472977], | |||
"score": 0.6922634840011597 | |||
}, | |||
{ | |||
"name": "自行车", | |||
"bbox": [40.88740050792694, 22.707078605890274, 451.21362805366516, 326.0102793574333], | |||
"score": 0.6069411635398865 | |||
}] | |||
``` | |||
## 了解更多 | |||
http://docs.dubhe.ai/docs/module/dataset/import-dataset | |||
@@ -0,0 +1,142 @@ | |||
<?xml version="1.0" encoding="UTF-8"?> | |||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | |||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> | |||
<modelVersion>4.0.0</modelVersion> | |||
<groupId>org.dubhe</groupId> | |||
<artifactId>dataset-util</artifactId> | |||
<version>0.0.1-SNAPSHOT</version> | |||
<name>dataset-util</name> | |||
<description>数据处理模块工具</description> | |||
<properties> | |||
<java.version>1.8</java.version> | |||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> | |||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> | |||
<spring-boot.version>2.3.0.RELEASE</spring-boot.version> | |||
</properties> | |||
<dependencies> | |||
<dependency> | |||
<groupId>org.springframework.boot</groupId> | |||
<artifactId>spring-boot-starter</artifactId> | |||
</dependency> | |||
<dependency> | |||
<groupId>com.alibaba</groupId> | |||
<artifactId>druid</artifactId> | |||
<version>1.1.6</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>mysql</groupId> | |||
<artifactId>mysql-connector-java</artifactId> | |||
<version>8.0.17</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.projectlombok</groupId> | |||
<artifactId>lombok</artifactId> | |||
<optional>true</optional> | |||
</dependency> | |||
<dependency> | |||
<groupId>com.baomidou</groupId> | |||
<artifactId>mybatis-plus-boot-starter</artifactId> | |||
<version>3.2.0</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>com.alibaba</groupId> | |||
<artifactId>fastjson</artifactId> | |||
<version>1.2.47</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>io.minio</groupId> | |||
<artifactId>minio</artifactId> | |||
<version>7.0.2</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>com.xiaoleilu</groupId> | |||
<artifactId>hutool-all</artifactId> | |||
<version>3.0.1</version> | |||
</dependency> | |||
<!-- for spring boot --> | |||
<dependency> | |||
<groupId>org.apache.shardingsphere</groupId> | |||
<artifactId>sharding-jdbc-spring-boot-starter</artifactId> | |||
<version>4.0.0-RC1</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.apache.commons</groupId> | |||
<artifactId>commons-lang3</artifactId> | |||
</dependency> | |||
<dependency> | |||
<groupId>commons-io</groupId> | |||
<artifactId>commons-io</artifactId> | |||
<version>1.3.2</version> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.aspectj</groupId> | |||
<artifactId>aspectjweaver</artifactId> | |||
<version>1.8.9</version> | |||
</dependency> | |||
<!--监控sql日志--> | |||
<dependency> | |||
<groupId>org.bgee.log4jdbc-log4j2</groupId> | |||
<artifactId>log4jdbc-log4j2-jdbc4.1</artifactId> | |||
<version>1.16</version> | |||
</dependency> | |||
<!--工具包--> | |||
<dependency> | |||
<groupId>cn.hutool</groupId> | |||
<artifactId>hutool-all</artifactId> | |||
<version>5.0.6</version> | |||
</dependency> | |||
</dependencies> | |||
<dependencyManagement> | |||
<dependencies> | |||
<dependency> | |||
<groupId>org.springframework.boot</groupId> | |||
<artifactId>spring-boot-dependencies</artifactId> | |||
<version>${spring-boot.version}</version> | |||
<type>pom</type> | |||
<scope>import</scope> | |||
</dependency> | |||
</dependencies> | |||
</dependencyManagement> | |||
<build> | |||
<plugins> | |||
<plugin> | |||
<groupId>org.apache.maven.plugins</groupId> | |||
<artifactId>maven-compiler-plugin</artifactId> | |||
<version>3.8.1</version> | |||
<configuration> | |||
<source>1.8</source> | |||
<target>1.8</target> | |||
<encoding>UTF-8</encoding> | |||
</configuration> | |||
</plugin> | |||
<plugin> | |||
<groupId>org.springframework.boot</groupId> | |||
<artifactId>spring-boot-maven-plugin</artifactId> | |||
<version>2.3.0.RELEASE</version> | |||
<configuration> | |||
<mainClass>org.dubhe.datasetutil.DatasetUtilApplication</mainClass> | |||
</configuration> | |||
<executions> | |||
<execution> | |||
<id>repackage</id> | |||
<goals> | |||
<goal>repackage</goal> | |||
</goals> | |||
</execution> | |||
</executions> | |||
</plugin> | |||
</plugins> | |||
</build> | |||
</project> |
@@ -0,0 +1,119 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.dubhe.datasetutil.common.util.SpringContextHolder; | |||
import org.dubhe.datasetutil.handle.DatasetImageUploadHandle; | |||
import org.dubhe.datasetutil.handle.DatasetImportHandle; | |||
import org.dubhe.datasetutil.common.util.PrintUtils; | |||
import org.mybatis.spring.annotation.MapperScan; | |||
import org.springframework.boot.SpringApplication; | |||
import org.springframework.boot.autoconfigure.SpringBootApplication; | |||
import org.springframework.context.ApplicationContext; | |||
import java.util.Scanner; | |||
/** | |||
* @description 文档导入工程启动类 | |||
* @date 2020-09-17 | |||
*/ | |||
@Slf4j | |||
@SpringBootApplication | |||
@MapperScan("org.dubhe.datasetutil.dao") | |||
public class DatasetUtilApplication { | |||
/** | |||
* 主函数 | |||
* | |||
* @param args 入参 | |||
*/ | |||
public static void main(String[] args) { | |||
ApplicationContext applicationContext = SpringApplication.run(DatasetUtilApplication.class, args); | |||
SpringContextHolder springContextHolder = new SpringContextHolder(); | |||
springContextHolder.setApplicationContext(applicationContext); | |||
execute(applicationContext); | |||
} | |||
/** | |||
* 执行脚本 | |||
* | |||
* @param applicationContext 请求上下文 | |||
*/ | |||
public static void execute(ApplicationContext applicationContext) { | |||
while (true) { | |||
Scanner scanner = new Scanner(System.in); | |||
log.warn("###################请输入需要执行的任务#############"); | |||
log.warn("# 输入1.执行上传图片 "); | |||
log.warn("# 输入2.执行导入数据集 "); | |||
log.warn("# 输入命令 :exit 退出 "); | |||
log.warn("################################################"); | |||
String a = scanner.nextLine(); | |||
switch (a) { | |||
case "1": | |||
uploadDatasetImage(scanner, applicationContext); | |||
break; | |||
case "2": | |||
importDataset(scanner, applicationContext); | |||
break; | |||
case "exit": | |||
default: | |||
System.exit(0); | |||
break; | |||
} | |||
} | |||
} | |||
/** | |||
* 导入图片 | |||
* | |||
* @param scanner 输入控制台 | |||
* @param applicationContext 请求上下文 | |||
*/ | |||
public static void uploadDatasetImage(Scanner scanner, ApplicationContext applicationContext) { | |||
log.warn("# 请输入数据集ID #"); | |||
String datasetIdStr = scanner.nextLine(); | |||
Long datasetId = Long.parseLong(datasetIdStr); | |||
log.warn("# 请输入要上传的图片地址 #"); | |||
String filePath = scanner.nextLine(); | |||
DatasetImageUploadHandle datasetImageUploadHandle = (DatasetImageUploadHandle) applicationContext.getBean("datasetImageUploadHandle"); | |||
try { | |||
datasetImageUploadHandle.execute(filePath, datasetId); | |||
} catch (Exception e) { | |||
log.error("", e); | |||
log.error("# 数据集上传失败,请重新尝试....."); | |||
} | |||
} | |||
/** | |||
* 导入数据集 | |||
* | |||
* @param scanner 输入控制台 | |||
* @param applicationContext 请求上下文 | |||
*/ | |||
public static void importDataset(Scanner scanner, ApplicationContext applicationContext) { | |||
DatasetImportHandle datasetImportHandle = (DatasetImportHandle) applicationContext.getBean("datasetImportHandle"); | |||
try{ | |||
datasetImportHandle.importDataset(scanner); | |||
} catch (Exception e) { | |||
log.error(""); | |||
PrintUtils.printLine(" Error:" + e.getMessage(), PrintUtils.RED); | |||
log.error(""); | |||
} | |||
} | |||
} |
@@ -0,0 +1,76 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
*/ | |||
package org.dubhe.datasetutil.common.aspect; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.aspectj.lang.JoinPoint; | |||
import org.aspectj.lang.ProceedingJoinPoint; | |||
import org.aspectj.lang.annotation.Around; | |||
import org.aspectj.lang.annotation.Aspect; | |||
import org.aspectj.lang.annotation.Pointcut; | |||
import org.dubhe.datasetutil.common.enums.LogEnum; | |||
import org.dubhe.datasetutil.common.util.LogUtil; | |||
import org.slf4j.MDC; | |||
import org.springframework.stereotype.Component; | |||
import org.springframework.util.StringUtils; | |||
import java.util.UUID; | |||
/** | |||
* @description 日志切面 | |||
* @date 2020-04-10 | |||
*/ | |||
@Component | |||
@Aspect | |||
@Slf4j | |||
public class LogAspect { | |||
public static final String TRACE_ID = "traceId"; | |||
@Pointcut("execution(* org.dubhe..service..*.*(..))) ") | |||
public void serviceAspect() { | |||
} | |||
@Pointcut(" serviceAspect() ") | |||
public void aroundAspect() { | |||
} | |||
@Around("aroundAspect()") | |||
public Object around(JoinPoint joinPoint) throws Throwable { | |||
if (StringUtils.isEmpty(MDC.get(TRACE_ID))) { | |||
MDC.put(TRACE_ID, UUID.randomUUID().toString()); | |||
} | |||
return combineLogInfo(joinPoint); | |||
} | |||
/** | |||
* 根据连接点返回结果 | |||
* | |||
* @param joinPoint 连接点 | |||
* @return Object 返回结果 | |||
*/ | |||
private Object combineLogInfo(JoinPoint joinPoint) throws Throwable { | |||
Object[] param = joinPoint.getArgs(); | |||
LogUtil.info(LogEnum.REST_REQ, "uri:{},input:{},==>begin", joinPoint.getSignature(), param); | |||
long start = System.currentTimeMillis(); | |||
Object result = ((ProceedingJoinPoint) joinPoint).proceed(); | |||
long end = System.currentTimeMillis(); | |||
LogUtil.info(LogEnum.REST_REQ, "uri:{},output:{},proc_time:{},<==end", joinPoint.getSignature().toString(), | |||
result, end - start); | |||
return result; | |||
} | |||
} |
@@ -0,0 +1,66 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.base; | |||
import com.baomidou.mybatisplus.annotation.FieldFill; | |||
import com.baomidou.mybatisplus.annotation.TableField; | |||
import com.baomidou.mybatisplus.annotation.TableLogic; | |||
import lombok.Data; | |||
import java.io.Serializable; | |||
import java.sql.Timestamp; | |||
/** | |||
* @description Entity基础类 | |||
* @date 2020-03-15 | |||
*/ | |||
@Data | |||
public class BaseEntity implements Serializable { | |||
private static final long serialVersionUID = 4936056317364745513L; | |||
/** | |||
* 删除标识 | |||
**/ | |||
@TableField(value = "deleted",fill = FieldFill.INSERT) | |||
@TableLogic | |||
private Boolean deleted = false; | |||
/** | |||
* 创建人id | |||
**/ | |||
@TableField(value = "create_user_id",fill = FieldFill.INSERT) | |||
private Long createUserId; | |||
/** | |||
* 修改人id | |||
**/ | |||
@TableField(value = "update_user_id",fill = FieldFill.INSERT_UPDATE) | |||
private Long updateUserId; | |||
/** | |||
* 创建时间 | |||
**/ | |||
@TableField(value = "create_time",fill = FieldFill.INSERT) | |||
private Timestamp createTime; | |||
/** | |||
* 修改时间 | |||
**/ | |||
@TableField(value = "update_time",fill = FieldFill.INSERT_UPDATE) | |||
private Timestamp updateTime; | |||
} |
@@ -0,0 +1,69 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.base; | |||
import lombok.Data; | |||
import org.slf4j.MDC; | |||
import java.io.Serializable; | |||
/** | |||
* @description 统一的公共响应体 | |||
* @date 2020-03-16 | |||
*/ | |||
@Data | |||
public class DataResponseBody<T> implements Serializable { | |||
/** | |||
* 返回状态码 | |||
*/ | |||
private Integer code; | |||
/** | |||
* 返回信息 | |||
*/ | |||
private String msg; | |||
/** | |||
* 泛型数据 | |||
*/ | |||
private T data; | |||
/** | |||
* 链路追踪ID | |||
*/ | |||
private String traceId; | |||
public DataResponseBody() { | |||
this(ResponseCode.SUCCESS, null); | |||
} | |||
public DataResponseBody(T data) { | |||
this(ResponseCode.SUCCESS, null, data); | |||
} | |||
public DataResponseBody(Integer code, String msg) { | |||
this(code, msg, null); | |||
} | |||
public DataResponseBody(Integer code, String msg, T data) { | |||
this.code = code; | |||
this.msg = msg; | |||
this.data = data; | |||
this.traceId = MDC.get("traceId"); | |||
} | |||
} |
@@ -0,0 +1,103 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.base; | |||
/** | |||
* @description 魔数数值类 | |||
* @date 2020-09-17 | |||
*/ | |||
public final class MagicNumConstant { | |||
public static final int NEGATIVE_ONE = -1; | |||
public static final int ZERO = 0; | |||
public static final int ONE = 1; | |||
public static final int TWO = 2; | |||
public static final int THREE = 3; | |||
public static final int FOUR = 4; | |||
public static final int FIVE = 5; | |||
public static final int SIX = 6; | |||
public static final int SEVEN = 7; | |||
public static final int EIGHT = 8; | |||
public static final int NINE = 9; | |||
public static final int TEN = 10; | |||
public static final int ELEVEN = 11; | |||
public static final int SIXTEEN = 16; | |||
public static final int TWENTY = 20; | |||
public static final int FIFTY = 50; | |||
public static final int SIXTY = 60; | |||
public static final int SIXTY_TWO = 62; | |||
public static final int SIXTY_FOUR = 64; | |||
public static final int INTEGER_TWO_HUNDRED_AND_FIFTY_FIVE = 255; | |||
public static final int ONE_HUNDRED = 100; | |||
public static final int ONE_HUNDRED_TWENTY_EIGHT = 128; | |||
public static final int TWO_HUNDRED = 200; | |||
public static final int FIVE_HUNDRED = 500; | |||
public static final int FIVE_HUNDRED_AND_SIXTEEN = 516; | |||
public static final int ONE_THOUSAND = 1000; | |||
public static final int BINARY_TEN_EXP = 1024; | |||
public static final int ONE_THOUSAND_ONE_HUNDRED = 1100; | |||
public static final int ONE_THOUSAND_ONE_HUNDRED_ONE = 1101; | |||
public static final int ONE_THOUSAND_TWO_HUNDRED = 1200; | |||
public static final int ONE_THOUSAND_TWO_HUNDRED_ONE = 1201; | |||
public static final int ONE_THOUSAND_TWENTY_FOUR = 1024; | |||
public static final int ONE_THOUSAND_THREE_HUNDRED = 1300; | |||
public static final int ONE_THOUSAND_THREE_HUNDRED_ONE = 1301; | |||
public static final int ONE_THOUSAND_THREE_HUNDRED_NINE = 1309; | |||
public static final int ONE_THOUSAND_FIVE_HUNDRED = 1500; | |||
public static final int TWO_THOUSAND = 2000; | |||
public static final int TWO_THOUSAND_TWENTY_EIGHT = 2048; | |||
public static final int THREE_THOUSAND = 3000; | |||
public static final int FOUR_THOUSAND = 4000; | |||
public static final int FIVE_THOUSAND = 5000; | |||
public static final int NINE_THOUSAND = 9000; | |||
public static final int NINE_THOUSAND_NINE_HUNDRED_NINTY_NINE = 9999; | |||
public static final int TEN_THOUSAND = 10000; | |||
public static final int FIFTEEN_THOUSAND = 15000; | |||
public static final int HUNDRED_THOUSAND = 100000; | |||
public static final int MILLION = 1000000; | |||
public static final int ONE_MINUTE = 60000; | |||
public static final long NEGATIVE_ONE__LONG = -1L; | |||
public static final long ZERO_LONG = 0L; | |||
public static final long ONE_LONG = 1L; | |||
public static final long TWO_LONG = 2L; | |||
public static final long THREE_LONG = 3L; | |||
public static final long FOUR_LONG = 4L; | |||
public static final long FIVE_LONG = 5L; | |||
public static final long SIX_LONG = 6L; | |||
public static final long SEVEN_LONG = 7L; | |||
public static final long EIGHT_LONG = 8L; | |||
public static final long NINE_LONG = 9L; | |||
public static final long TEN_LONG = 10L; | |||
public static final long TWELVE_LONG = 12L; | |||
public static final long SIXTY_LONG = 60L; | |||
public static final long FIFTY_LONG = 50L; | |||
public static final long THOUSAND_LONG = 1000L; | |||
public static final long TEN_THOUSAND_LONG = 10000L; | |||
public static final long ONE_ZERO_ONE_ZERO_ONE_ZERO_LONG = 101010L; | |||
public static final long NINE_ZERO_NINE_ZERO_NINE_ZERO_LONG = 909090L; | |||
public static final long ONE_YEAR_BEFORE_LONG = 1552579200000L; | |||
public static final int SIXITY_0XFF = 0xFF; | |||
private MagicNumConstant() { | |||
} | |||
} |
@@ -0,0 +1,34 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.base; | |||
/** | |||
* @description 返回状态码 | |||
* @date 2020-02-23 | |||
*/ | |||
public class ResponseCode { | |||
public static Integer SUCCESS = 200; | |||
public static Integer UNAUTHORIZED = 401; | |||
public static Integer ERROR = 10000; | |||
public static Integer ENTITY_NOT_EXIST = 10001; | |||
public static Integer BADREQUEST = 10002; | |||
public static Integer SERVICE_ERROR = 10003; | |||
public static Integer DOCKER_ERROR = 10004; | |||
} |
@@ -0,0 +1,58 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.config; | |||
import io.minio.MinioClient; | |||
import io.minio.errors.InvalidEndpointException; | |||
import io.minio.errors.InvalidPortException; | |||
import lombok.Data; | |||
import org.springframework.boot.context.properties.ConfigurationProperties; | |||
import org.springframework.context.annotation.Bean; | |||
import org.springframework.stereotype.Component; | |||
/** | |||
* @description MinIO 配置类 | |||
* @date 2020-09-17 | |||
*/ | |||
@Data | |||
@Component | |||
@ConfigurationProperties(prefix = "minio") | |||
public class MinioConfig { | |||
private String endpoint; | |||
private int port; | |||
private String accessKey; | |||
private String secretKey; | |||
private Boolean secure; | |||
private String bucketName; | |||
/** | |||
* 获取Minio客户端信息 | |||
* | |||
* @return Minio客户端信息 | |||
*/ | |||
@Bean | |||
public MinioClient getMinioClient() throws InvalidEndpointException, InvalidPortException { | |||
return new MinioClient(endpoint, port, accessKey, secretKey,secure); | |||
} | |||
} |
@@ -0,0 +1,37 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.config; | |||
import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; | |||
import org.springframework.context.annotation.Bean; | |||
import org.springframework.context.annotation.Configuration; | |||
/** | |||
* @description 配置分页插件 | |||
* @date 2020-09-17 | |||
*/ | |||
@Configuration | |||
public class MybatisPlusConfig { | |||
/** | |||
* 分页插件 | |||
*/ | |||
@Bean | |||
public PaginationInterceptor paginationInterceptor() { | |||
return new PaginationInterceptor(); | |||
} | |||
} |
@@ -0,0 +1,86 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.constant; | |||
import org.dubhe.datasetutil.common.base.MagicNumConstant; | |||
/** | |||
* @description 常量 | |||
* @date 2020-10-19 | |||
*/ | |||
public class BusinessConstant { | |||
private BusinessConstant(){} | |||
/** | |||
* 分表业务编码 - 文件表 | |||
*/ | |||
public static final String DATA_FILE = "DATA_FILE"; | |||
/** | |||
* 分表业务编码 - 文件版本关系表 | |||
*/ | |||
public static final String DATA_VERSION_FILE = "DATA_VERSION_FILE"; | |||
/** | |||
* 图片文件路径 | |||
*/ | |||
public static final String IMAGE_ORIGIN = "origin"; | |||
/** | |||
* 标注文件路径 | |||
*/ | |||
public static final String ANNOTATION = "annotation"; | |||
/** | |||
* 标签文件路径 | |||
*/ | |||
public static final String LABEL = "label"; | |||
/** | |||
* 分隔符 | |||
*/ | |||
public static final String FILE_SEPARATOR = "/"; | |||
/** | |||
* 后缀. | |||
*/ | |||
public static final String SPOT = "."; | |||
/** | |||
* JSON后缀名 | |||
*/ | |||
public static final String SUFFIX_JSON = ".JSON"; | |||
/** | |||
* minio根目录 | |||
*/ | |||
public static final String MINIO_ROOT_PATH = "dataset"; | |||
/** | |||
* 下划线 | |||
*/ | |||
public static final String UNDERLINE = "_"; | |||
/** | |||
* 分段ID范围区间 50表示 50间隔ID存一张表 | |||
*/ | |||
public static final long INTERVAL_NUMBER = MagicNumConstant.FIFTY_LONG; | |||
/** | |||
* 分批长度 | |||
*/ | |||
public static final int SUB_LENGTH = MagicNumConstant.FIVE_THOUSAND; | |||
/** | |||
* 字母Y | |||
*/ | |||
public static final String Y = "Y"; | |||
} |
@@ -0,0 +1,84 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.constant; | |||
/** | |||
* @description 数据集状态码 | |||
* @date 2020-09-03 | |||
*/ | |||
public class DataStateCodeConstant { | |||
private DataStateCodeConstant() { | |||
} | |||
/** | |||
* 未标注 | |||
*/ | |||
public static final Integer NOT_ANNOTATION_STATE = 101; | |||
/** | |||
* 手动标注中 | |||
*/ | |||
public static final Integer MANUAL_ANNOTATION_STATE = 102; | |||
/** | |||
* 自动标注中 | |||
*/ | |||
public static final Integer AUTOMATIC_LABELING_STATE = 103; | |||
/** | |||
* 自动标注完成 | |||
*/ | |||
public static final Integer AUTO_TAG_COMPLETE_STATE = 104; | |||
/** | |||
* 标注完成 | |||
*/ | |||
public static final Integer ANNOTATION_COMPLETE_STATE = 105; | |||
/** | |||
* 目标跟踪中 | |||
*/ | |||
public static final Integer TARGET_FOLLOW_STATE = 201; | |||
/** | |||
* 目标跟踪完成 | |||
*/ | |||
public static final Integer TARGET_COMPLETE_STATE = 202; | |||
/** | |||
* 目标跟踪失败 | |||
*/ | |||
public static final Integer TARGET_FAILURE_STATE = 203; | |||
/** | |||
* 未采样 | |||
*/ | |||
public static final Integer NOT_SAMPLED_STATE = 301; | |||
/** | |||
* 采样中 | |||
*/ | |||
public static final Integer SAMPLING_STATE = 302; | |||
/** | |||
* 采样失败 | |||
*/ | |||
public static final Integer SAMPLED_FAILURE_STATE = 303; | |||
/** | |||
* 增强中 | |||
*/ | |||
public static final Integer STRENGTHENING_STATE = 401; | |||
/** | |||
* 导入中 | |||
*/ | |||
public static final Integer IN_THE_IMPORT_STATE = 402; | |||
} |
@@ -0,0 +1,54 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.constant; | |||
/** | |||
* @description 数据集状态码 | |||
* @date 2020-09-03 | |||
*/ | |||
public class FileStateCodeConstant { | |||
private FileStateCodeConstant(){ | |||
} | |||
/** | |||
* 未标注 | |||
*/ | |||
public static final Integer NOT_ANNOTATION_FILE_STATE = 101; | |||
/** | |||
* 手动标注中 | |||
*/ | |||
public static final Integer MANUAL_ANNOTATION_FILE_STATE = 102; | |||
/** | |||
* 自动标注完成 | |||
*/ | |||
public static final Integer AUTO_TAG_COMPLETE_FILE_STATE = 103; | |||
/** | |||
* 标注完成 | |||
*/ | |||
public static final Integer ANNOTATION_COMPLETE_FILE_STATE = 104; | |||
/** | |||
* 标注未识别 | |||
*/ | |||
public static final Integer ANNOTATION_NOT_DISTINGUISH_FILE_STATE = 105; | |||
/** | |||
* 目标跟踪完成 | |||
*/ | |||
public static final Integer TARGET_COMPLETE_FILE_STATE = 201; | |||
} |
@@ -0,0 +1,52 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.enums; | |||
import lombok.Getter; | |||
/** | |||
* @description 日志类型枚举类 | |||
* @date 2020-06-23 | |||
*/ | |||
@Getter | |||
public enum LogEnum { | |||
//系统报错日志 | |||
SYS_ERR, | |||
//用户请求日志 | |||
REST_REQ, | |||
//全局请求日志 | |||
GLOBAL_REQ, | |||
//数据集模块 | |||
BIZ_DATASET, | |||
//分表模块 | |||
DATA_SEQUENCE; | |||
/** | |||
* 判断日志类型不能为空 | |||
* | |||
* @param logType 日志类型 | |||
* @return boolean 返回类型 | |||
*/ | |||
public static boolean isLogType(LogEnum logType) { | |||
if (logType != null) { | |||
return true; | |||
} | |||
return false; | |||
} | |||
} |
@@ -0,0 +1,56 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.exception; | |||
import lombok.Getter; | |||
import org.dubhe.datasetutil.common.base.DataResponseBody; | |||
import org.dubhe.datasetutil.common.base.ResponseCode; | |||
/** | |||
* @description 业务异常 | |||
* @date 2020-03-13 | |||
*/ | |||
@Getter | |||
public class BusinessException extends RuntimeException { | |||
private DataResponseBody responseBody; | |||
public BusinessException(String msg) { | |||
super(msg); | |||
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); | |||
} | |||
public BusinessException(String msg, Throwable cause) { | |||
super(msg,cause); | |||
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); | |||
} | |||
public BusinessException(Throwable cause) { | |||
super(cause); | |||
this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST); | |||
} | |||
public BusinessException(Integer code, String msg, String info, Throwable cause) { | |||
super(msg,cause); | |||
if (info == null) { | |||
this.responseBody = new DataResponseBody(code, msg); | |||
} else { | |||
this.responseBody = new DataResponseBody(code, msg + ":" + info); | |||
} | |||
} | |||
} |
@@ -0,0 +1,31 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.exception; | |||
import lombok.Getter; | |||
/** | |||
* @description 获取序列异常 | |||
* @date 2020-09-23 | |||
*/ | |||
@Getter | |||
public class DataSequenceException extends BusinessException { | |||
public DataSequenceException(String msg) { | |||
super(msg); | |||
} | |||
} |
@@ -0,0 +1,39 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.exception; | |||
/** | |||
* @description 异常code | |||
* @date 2020-03-26 | |||
*/ | |||
public interface ErrorCode { | |||
/** | |||
* 错误码 | |||
* @return code | |||
*/ | |||
Integer getCode(); | |||
/** | |||
* error info | |||
* @return | |||
*/ | |||
String getMsg(); | |||
} |
@@ -0,0 +1,36 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.exception; | |||
import lombok.Getter; | |||
/** | |||
* @description 数据集导入异常处理 | |||
* @date 2020-10-14 | |||
*/ | |||
@Getter | |||
public class ImportDatasetException extends BusinessException { | |||
/** | |||
* 数据集导入异常处理 | |||
* | |||
* @param msg 信息 | |||
*/ | |||
public ImportDatasetException(String msg) { | |||
super(msg); | |||
} | |||
} |
@@ -0,0 +1,82 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.filter; | |||
import ch.qos.logback.classic.Level; | |||
import ch.qos.logback.classic.spi.ILoggingEvent; | |||
import ch.qos.logback.core.filter.AbstractMatcherFilter; | |||
import ch.qos.logback.core.spi.FilterReply; | |||
import org.slf4j.Marker; | |||
/** | |||
* @description 自定义日志过滤器 | |||
* @date 2020-07-21 | |||
*/ | |||
public class BaseLogFilter extends AbstractMatcherFilter<ILoggingEvent> { | |||
Level level; | |||
/** | |||
* 重写decide方法 | |||
* | |||
* @param iLoggingEvent 待决定的事件 | |||
* @return FilterReply 过滤器 | |||
*/ | |||
@Override | |||
public FilterReply decide(ILoggingEvent iLoggingEvent) { | |||
if (!isStarted()) { | |||
return FilterReply.NEUTRAL; | |||
} | |||
final String msg = iLoggingEvent.getMessage(); | |||
//自定义级别 | |||
if (checkLevel(iLoggingEvent) && msg != null && msg.startsWith("{") && msg.endsWith("}")) { | |||
final Marker marker = iLoggingEvent.getMarker(); | |||
if (marker != null && this.getName() != null && this.getName().contains(marker.getName())) { | |||
return onMatch; | |||
} | |||
} | |||
return onMismatch; | |||
} | |||
/** | |||
* 检查等级 | |||
* | |||
* @param iLoggingEvent 待决定的事件 | |||
* @return boolean 检查结果 | |||
*/ | |||
protected boolean checkLevel(ILoggingEvent iLoggingEvent) { | |||
return this.level != null | |||
&& iLoggingEvent.getLevel() != null | |||
&& iLoggingEvent.getLevel().toInt() == this.level.toInt(); | |||
} | |||
public void setLevel(Level level) { | |||
this.level = level; | |||
} | |||
/** | |||
* 启动 | |||
*/ | |||
@Override | |||
public void start() { | |||
if (this.level != null) { | |||
super.start(); | |||
} | |||
} | |||
} |
@@ -0,0 +1,59 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.filter; | |||
import ch.qos.logback.classic.spi.ILoggingEvent; | |||
import ch.qos.logback.core.spi.FilterReply; | |||
import org.dubhe.datasetutil.common.util.LogUtil; | |||
import org.slf4j.MarkerFactory; | |||
/** | |||
* @description 自定义日志过滤器 | |||
* @date 2020-07-21 | |||
*/ | |||
public class ConsoleLogFilter extends BaseLogFilter { | |||
/** | |||
* 重写decide方法 | |||
* | |||
* @param iLoggingEvent 待决定的事件 | |||
* @return FilterReply 过滤器 | |||
*/ | |||
@Override | |||
public FilterReply decide(ILoggingEvent iLoggingEvent) { | |||
if (!isStarted()) { | |||
return FilterReply.NEUTRAL; | |||
} | |||
return checkLevel(iLoggingEvent) ? onMatch : onMismatch; | |||
} | |||
/** | |||
* 检查等级 | |||
* | |||
* @param iLoggingEvent 待决定的事件 | |||
* @return boolean 检查结果 | |||
*/ | |||
@Override | |||
protected boolean checkLevel(ILoggingEvent iLoggingEvent) { | |||
return this.level != null | |||
&& iLoggingEvent.getLevel() != null | |||
&& iLoggingEvent.getLevel().toInt() >= this.level.toInt() | |||
&& !MarkerFactory.getMarker(LogUtil.SCHEDULE_LEVEL).equals(iLoggingEvent.getMarker()) | |||
&& !"log4jdbc.log4j2".equals(iLoggingEvent.getLoggerName()); | |||
} | |||
} |
@@ -0,0 +1,40 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.filter; | |||
import ch.qos.logback.classic.spi.ILoggingEvent; | |||
/** | |||
* @description 全局请求 日志过滤器 | |||
* @date 2020-08-13 | |||
*/ | |||
public class GlobalRequestLogFilter extends BaseLogFilter { | |||
/** | |||
* 日志等级 | |||
* | |||
* @param iLoggingEvent 日志等级 | |||
* @return boolean 执行结果 | |||
*/ | |||
@Override | |||
public boolean checkLevel(ILoggingEvent iLoggingEvent) { | |||
return this.level != null && true; | |||
} | |||
} |
@@ -0,0 +1,45 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.util; | |||
import java.text.SimpleDateFormat; | |||
import java.util.Date; | |||
/** | |||
* @description 时间 配置类 | |||
* @date 2020-09-17 | |||
*/ | |||
public class DateUtil { | |||
private DateUtil(){ | |||
} | |||
/** | |||
* 时间格式 | |||
*/ | |||
private static final String DATA_FORMAT_STR = "yyyy年MM月dd日 HH时mm分ss秒"; | |||
/** | |||
* 获取开始时间 | |||
* | |||
* @return String 返回控制台开始时间 | |||
*/ | |||
public static String getNowStr() { | |||
return new SimpleDateFormat(DATA_FORMAT_STR).format(new Date()); | |||
} | |||
} |
@@ -0,0 +1,118 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.util; | |||
import cn.hutool.core.util.ObjectUtil; | |||
import org.dubhe.datasetutil.common.base.MagicNumConstant; | |||
import org.dubhe.datasetutil.common.enums.LogEnum; | |||
import org.dubhe.datasetutil.common.exception.DataSequenceException; | |||
import org.dubhe.datasetutil.domain.dto.IdAlloc; | |||
import org.dubhe.datasetutil.domain.entity.DataSequence; | |||
import org.dubhe.datasetutil.service.DataSequenceService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Component; | |||
import org.springframework.transaction.annotation.Transactional; | |||
import org.springframework.util.StringUtils; | |||
import java.util.concurrent.ConcurrentHashMap; | |||
/** | |||
* @description 生成ID工具类 | |||
* @date 2020-09-23 | |||
*/ | |||
@Component | |||
public class GeneratorKeyUtil { | |||
@Autowired | |||
private DataSequenceService dataSequenceService; | |||
private ConcurrentHashMap<String, IdAlloc> idAllocConcurrentHashMap = new ConcurrentHashMap<>(); | |||
/** | |||
* 根据业务编码,数量获取序列号 | |||
* | |||
* @param businessCode 业务编码 | |||
* @param number 数量 | |||
* @return Long 起始位置 | |||
*/ | |||
@Transactional(rollbackFor = Exception.class) | |||
public synchronized Long getSequenceByBusinessCode(String businessCode, int number) { | |||
if (StringUtils.isEmpty(businessCode)) { | |||
throw new DataSequenceException("业务编码不可为空"); | |||
} | |||
if (number == MagicNumConstant.ZERO) { | |||
throw new DataSequenceException("需要获取的序列号长度不可为0或者空"); | |||
} | |||
IdAlloc idAlloc = idAllocConcurrentHashMap.get(businessCode); | |||
if (ObjectUtil.isNull(idAlloc)) { | |||
idAlloc = new IdAlloc(); | |||
idAllocConcurrentHashMap.put(businessCode, idAlloc); | |||
} | |||
if (idAlloc.getUsedNumber() == MagicNumConstant.ZERO) { | |||
DataSequence dataSequence = getDataSequence(businessCode); | |||
updateDataSequence(businessCode); | |||
idAlloc.setStartNumber(dataSequence.getStart()); | |||
idAlloc.setEndNumber(dataSequence.getStart() + dataSequence.getStep() - MagicNumConstant.ONE); | |||
idAlloc.setUsedNumber(idAlloc.getEndNumber() - idAlloc.getStartNumber() + MagicNumConstant.ONE); | |||
} | |||
if (idAlloc.getUsedNumber() <= number) { | |||
expansionUsedNumber(businessCode, number); | |||
} | |||
long returnStartNumber = idAlloc.getStartNumber(); | |||
idAlloc.setStartNumber(idAlloc.getStartNumber() + number); | |||
idAlloc.setUsedNumber(idAlloc.getUsedNumber() - number); | |||
return returnStartNumber; | |||
} | |||
/** | |||
* 根据业务编码获取配置信息 | |||
* @param businessCode 业务编码 | |||
* @return DataSequence 数据索引 | |||
*/ | |||
private DataSequence getDataSequence(String businessCode) { | |||
DataSequence dataSequence = dataSequenceService.getSequence(businessCode); | |||
if (dataSequence == null || dataSequence.getStart() == null || dataSequence.getStep() == null) { | |||
throw new DataSequenceException("配置出错,请检查data_sequence表配置"); | |||
} | |||
return dataSequence; | |||
} | |||
/** | |||
* 根据业务编码更新起始值 | |||
* @param businessCode 业务编码 | |||
*/ | |||
private void updateDataSequence(String businessCode) { | |||
dataSequenceService.updateSequenceStart(businessCode); | |||
} | |||
/** | |||
* 多次扩容 | |||
* @param businessCode 业务编码 | |||
* @param number 数量 | |||
*/ | |||
private void expansionUsedNumber(String businessCode, int number) { | |||
IdAlloc idAlloc = idAllocConcurrentHashMap.get(businessCode); | |||
updateDataSequence(businessCode); | |||
DataSequence dataSequenceNew = getDataSequence(businessCode); | |||
idAlloc.setEndNumber(idAlloc.getEndNumber() + dataSequenceNew.getStep()); | |||
idAlloc.setUsedNumber(idAlloc.getEndNumber() - idAlloc.getStartNumber() + MagicNumConstant.ONE); | |||
if (idAlloc.getUsedNumber() <= number) { | |||
expansionUsedNumber(businessCode, number); | |||
} | |||
} | |||
} |
@@ -0,0 +1,91 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.util; | |||
import org.apache.commons.io.FileUtils; | |||
import org.apache.commons.io.LineIterator; | |||
import org.dubhe.datasetutil.common.constant.BusinessConstant; | |||
import java.io.File; | |||
import java.io.IOException; | |||
/** | |||
* @description 文件工具类 | |||
* @date 2020-10-15 | |||
*/ | |||
public class HandleFileUtil { | |||
/** | |||
* 读取文件内容 | |||
* | |||
* @param file 文件对象 | |||
* @return String 文件内容 | |||
*/ | |||
public static String readFile(File file) throws IOException{ | |||
StringBuilder stringBuffer = new StringBuilder(); | |||
LineIterator fileContext = FileUtils.lineIterator(file,"UTF-8"); | |||
while (fileContext.hasNext()) { | |||
stringBuffer.append(fileContext.nextLine()); | |||
} | |||
return stringBuffer.toString(); | |||
} | |||
/** | |||
* 获取文件名后缀名 | |||
* | |||
* @param fileName 文件名 | |||
* @return String 文件后缀名 | |||
*/ | |||
public static String readFileSuffixName(String fileName){ | |||
return fileName.substring(fileName.lastIndexOf(".")); | |||
} | |||
/** | |||
* 获取文件名(踢除后缀名) | |||
* | |||
* @param fileName 文件名 | |||
* @return String 文件名(踢除后缀名) | |||
*/ | |||
public static String readFileName(String fileName){ | |||
return fileName.substring(0,fileName.lastIndexOf(".")); | |||
} | |||
/** | |||
* 生成文件路径 | |||
* | |||
* @param businessCode 业务类型 | |||
* @return String 文件路径 | |||
*/ | |||
public static String generateFilePath(String businessCode){ | |||
return BusinessConstant.FILE_SEPARATOR + businessCode; | |||
} | |||
/** | |||
* 获取标签组名称 | |||
* | |||
* @param fileName 文件名称 | |||
* @return String 标签组名 | |||
*/ | |||
public static String getLabelGroupName(String fileName){ | |||
return fileName.substring(fileName.indexOf(BusinessConstant.UNDERLINE) + 1,fileName.lastIndexOf(BusinessConstant.SPOT)); | |||
} | |||
} |
@@ -0,0 +1,300 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.util; | |||
import ch.qos.logback.classic.Level; | |||
import com.alibaba.fastjson.JSON; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.apache.commons.lang3.StringUtils; | |||
import org.apache.commons.lang3.exception.ExceptionUtils; | |||
import org.dubhe.datasetutil.common.aspect.LogAspect; | |||
import org.dubhe.datasetutil.common.enums.LogEnum; | |||
import org.dubhe.datasetutil.domain.entity.LogInfo; | |||
import org.slf4j.MDC; | |||
import org.slf4j.MarkerFactory; | |||
import org.slf4j.helpers.MessageFormatter; | |||
import java.util.Arrays; | |||
import java.util.UUID; | |||
/** | |||
* @description 日志工具类 | |||
* @date 2020-06-29 | |||
*/ | |||
@Slf4j | |||
public class LogUtil { | |||
private static final String TRACE_TYPE = "TRACE_TYPE"; | |||
public static final String SCHEDULE_LEVEL = "SCHEDULE"; | |||
private static final String GLOBAL_REQUEST_LEVEL = "GLOBAL_REQUEST"; | |||
private static final String TRACE_LEVEL = "TRACE"; | |||
private static final String DEBUG_LEVEL = "DEBUG"; | |||
private static final String INFO_LEVEL = "INFO"; | |||
private static final String WARN_LEVEL = "WARN"; | |||
private static final String ERROR_LEVEL = "ERROR"; | |||
public static void startScheduleTrace() { | |||
MDC.put(TRACE_TYPE, SCHEDULE_LEVEL); | |||
} | |||
public static void cleanTrace() { | |||
MDC.clear(); | |||
} | |||
/** | |||
* info级别的日志 | |||
* | |||
* @param logType 日志类型 | |||
* @param object 打印的日志参数 | |||
*/ | |||
public static void info(LogEnum logType, Object... object) { | |||
logHandle(logType, Level.INFO, object); | |||
} | |||
/** | |||
* debug级别的日志 | |||
* | |||
* @param logType 日志类型 | |||
* @param object 打印的日志参数 | |||
*/ | |||
public static void debug(LogEnum logType, Object... object) { | |||
logHandle(logType, Level.DEBUG, object); | |||
} | |||
/** | |||
* error级别的日志 | |||
* | |||
* @param logType 日志类型 | |||
* @param object 打印的日志参数 | |||
*/ | |||
public static void error(LogEnum logType, Object... object) { | |||
errorObjectHandle(object); | |||
logHandle(logType, Level.ERROR, object); | |||
} | |||
/** | |||
* warn级别的日志 | |||
* | |||
* @param logType 日志类型 | |||
* @param object 打印的日志参数 | |||
*/ | |||
public static void warn(LogEnum logType, Object... object) { | |||
logHandle(logType, Level.WARN, object); | |||
} | |||
/** | |||
* trace级别的日志 | |||
* | |||
* @param logType 日志类型 | |||
* @param object 打印的日志参数 | |||
*/ | |||
public static void trace(LogEnum logType, Object... object) { | |||
logHandle(logType, Level.TRACE, object); | |||
} | |||
/** | |||
* 日志处理 | |||
* | |||
* @param logType 日志类型 | |||
* @param level 日志级别 | |||
* @param object 打印的日志参数 | |||
*/ | |||
private static void logHandle(LogEnum logType, Level level, Object[] object) { | |||
LogInfo logInfo = generateLogInfo(logType, level, object); | |||
switch (logInfo.getLevel()) { | |||
case TRACE_LEVEL: | |||
log.trace(MarkerFactory.getMarker(TRACE_LEVEL), logJsonStringLengthLimit(logInfo)); | |||
break; | |||
case DEBUG_LEVEL: | |||
log.debug(MarkerFactory.getMarker(DEBUG_LEVEL), logJsonStringLengthLimit(logInfo)); | |||
break; | |||
case GLOBAL_REQUEST_LEVEL: | |||
logInfo.setLevel(null); | |||
logInfo.setType(null); | |||
logInfo.setLocation(null); | |||
log.info(MarkerFactory.getMarker(GLOBAL_REQUEST_LEVEL), logJsonStringLengthLimit(logInfo)); | |||
break; | |||
case SCHEDULE_LEVEL: | |||
log.info(MarkerFactory.getMarker(SCHEDULE_LEVEL), logJsonStringLengthLimit(logInfo)); | |||
break; | |||
case INFO_LEVEL: | |||
log.info(MarkerFactory.getMarker(INFO_LEVEL), logJsonStringLengthLimit(logInfo)); | |||
break; | |||
case WARN_LEVEL: | |||
log.warn(MarkerFactory.getMarker(WARN_LEVEL), logJsonStringLengthLimit(logInfo)); | |||
break; | |||
case ERROR_LEVEL: | |||
log.error(MarkerFactory.getMarker(ERROR_LEVEL), logJsonStringLengthLimit(logInfo)); | |||
break; | |||
default: | |||
} | |||
} | |||
/** | |||
* 日志信息组装的内部方法 | |||
* | |||
* @param logType 日志类型 | |||
* @param level 日志级别 | |||
* @param object 打印的日志参数 | |||
* @return LogInfo 日志对象 | |||
*/ | |||
private static LogInfo generateLogInfo(LogEnum logType, Level level, Object[] object) { | |||
LogInfo logInfo = new LogInfo(); | |||
// 日志类型检测 | |||
if (!LogEnum.isLogType(logType)) { | |||
level = Level.ERROR; | |||
object = new Object[1]; | |||
object[0] = "日志类型【".concat(String.valueOf(logType)).concat("】不正确!"); | |||
logType = LogEnum.SYS_ERR; | |||
} | |||
// 获取trace_id | |||
if (StringUtils.isEmpty(MDC.get(LogAspect.TRACE_ID))) { | |||
MDC.put(LogAspect.TRACE_ID, UUID.randomUUID().toString()); | |||
} | |||
// 设置logInfo的level,type,traceId属性 | |||
logInfo.setLevel(level.levelStr) | |||
.setType(logType.toString()) | |||
.setTraceId(MDC.get(LogAspect.TRACE_ID)); | |||
//自定义日志级别 | |||
//LogEnum、 MDC中的 TRACE_TYPE 做日志分流标识 | |||
if (Level.INFO.toInt() == level.toInt()) { | |||
if (LogEnum.GLOBAL_REQ.equals(logType)) { | |||
//info全局请求 | |||
logInfo.setLevel(GLOBAL_REQUEST_LEVEL); | |||
} else { | |||
//schedule定时等 链路记录 | |||
String traceType = MDC.get(TRACE_TYPE); | |||
if (StringUtils.isNotBlank(traceType)) { | |||
logInfo.setLevel(traceType); | |||
} | |||
} | |||
} | |||
// 设置logInfo的堆栈信息 | |||
setLogStackInfo(logInfo); | |||
// 设置logInfo的info信息 | |||
setLogInfo(logInfo, object); | |||
// 截取logInfo的长度并转换成json字符串 | |||
return logInfo; | |||
} | |||
/** | |||
* 设置loginfo的堆栈信息 | |||
* | |||
* @param logInfo 日志对象 | |||
*/ | |||
private static void setLogStackInfo(LogInfo logInfo) { | |||
StackTraceElement[] elements = Thread.currentThread().getStackTrace(); | |||
if (elements.length >= 6) { | |||
StackTraceElement element = elements[5]; | |||
logInfo.setLocation(String.format("%s#%s:%s", element.getClassName(), element.getMethodName(), element.getLineNumber())); | |||
} | |||
} | |||
/** | |||
* 限制log日志的长度并转换成json | |||
* | |||
* @param logInfo 日志对象 | |||
* @return String 转换的json | |||
*/ | |||
private static String logJsonStringLengthLimit(LogInfo logInfo) { | |||
try { | |||
String jsonString = JSON.toJSONString(logInfo); | |||
if (StringUtils.isBlank(jsonString)) { | |||
return ""; | |||
} | |||
if (jsonString.length() > 10000) { | |||
String trunk = logInfo.getInfo().toString().substring(0, 9000); | |||
logInfo.setInfo(trunk); | |||
jsonString = JSON.toJSONString(logInfo); | |||
} | |||
return jsonString; | |||
} catch (Exception e) { | |||
logInfo.setLevel(Level.ERROR.levelStr).setType(LogEnum.SYS_ERR.toString()) | |||
.setInfo("cannot serialize exception: " + ExceptionUtils.getStackTrace(e)); | |||
return JSON.toJSONString(logInfo); | |||
} | |||
} | |||
/** | |||
* 设置日志对象的info信息 | |||
* | |||
* @param logInfo 日志对象 | |||
* @param object 打印的日志参数 | |||
*/ | |||
private static void setLogInfo(LogInfo logInfo, Object[] object) { | |||
if (object.length > 1) { | |||
logInfo.setInfo(MessageFormatter.arrayFormat(object[0].toString(), | |||
Arrays.copyOfRange(object, 1, object.length)).getMessage()); | |||
} else if (object.length == 1 && object[0] instanceof Exception) { | |||
logInfo.setInfo((ExceptionUtils.getStackTrace((Exception) object[0]))); | |||
log.error((ExceptionUtils.getStackTrace((Exception) object[0]))); | |||
} else if (object.length == 1) { | |||
logInfo.setInfo(object[0] == null ? "" : object[0]); | |||
} else { | |||
logInfo.setInfo(""); | |||
} | |||
} | |||
/** | |||
* 处理Exception的情况 | |||
* | |||
* @param object 打印的日志参数 | |||
*/ | |||
private static void errorObjectHandle(Object[] object) { | |||
if (object.length == 2 && object[1] instanceof Exception) { | |||
log.error(String.valueOf(object[0]), (Exception) object[1]); | |||
object[1] = ExceptionUtils.getStackTrace((Exception) object[1]); | |||
} else if (object.length >= 3) { | |||
log.error(String.valueOf(object[0]), | |||
Arrays.copyOfRange(object, 1, object.length)); | |||
for (int i = 0; i < object.length; i++) { | |||
if (object[i] instanceof Exception) { | |||
object[i] = ExceptionUtils.getStackTrace((Exception) object[i]); | |||
} | |||
} | |||
} | |||
} | |||
} |
@@ -0,0 +1,64 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.util; | |||
import io.minio.MinioClient; | |||
import io.minio.PutObjectOptions; | |||
import org.dubhe.datasetutil.common.config.MinioConfig; | |||
import org.dubhe.datasetutil.common.enums.LogEnum; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Component; | |||
import java.io.InputStream; | |||
/** | |||
* @description Minio工具类 | |||
* @date 2020-09-17 | |||
*/ | |||
@Component | |||
public class MinioUtil { | |||
@Autowired | |||
private MinioClient minioClient; | |||
@Autowired | |||
private MinioConfig minioConfig; | |||
/** | |||
* 上传文件 | |||
* | |||
* @param objectName 对象名称 | |||
* @param inputStream 文件流 | |||
* @throws Exception 上传异常 | |||
*/ | |||
public void upLoadFile(String objectName, InputStream inputStream) throws Exception { | |||
LogUtil.info(LogEnum.BIZ_DATASET,"文件上传名称为: 【" + objectName + "】"); | |||
PutObjectOptions options = new PutObjectOptions(inputStream.available(), -1); | |||
minioClient.putObject(minioConfig.getBucketName(), objectName, inputStream, options); | |||
} | |||
/** | |||
* 获取文件URL | |||
* | |||
* @param objectName 对象名称 | |||
* @return String 文件路径 | |||
*/ | |||
public String getUrl(String objectName) { | |||
return minioConfig.getBucketName() + "/" + objectName; | |||
} | |||
} |
@@ -0,0 +1,72 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.util; | |||
import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm; | |||
import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue; | |||
import org.dubhe.datasetutil.common.base.MagicNumConstant; | |||
import org.dubhe.datasetutil.common.constant.BusinessConstant; | |||
import org.dubhe.datasetutil.service.DataSequenceService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import java.util.Collection; | |||
/** | |||
* @description 数据分片 | |||
* @date 2020-09-21 | |||
*/ | |||
public class MyPreciseShardingAlgorithm implements PreciseShardingAlgorithm<Long>{ | |||
@Autowired | |||
private DataSequenceService dataSequenceService; | |||
/** | |||
* 数据表分片 | |||
* | |||
* @param collection 集合 | |||
* @param preciseShardingValue 分片值 | |||
* @return 字符串 | |||
*/ | |||
@Override | |||
public String doSharding(Collection<String> collection, PreciseShardingValue<Long> preciseShardingValue) { | |||
long startIndex = MagicNumConstant.ONE; | |||
long endIndex = MagicNumConstant.FIFTY; | |||
dataSequenceService = SpringContextHolder.getBean(DataSequenceService.class); | |||
String tableName = preciseShardingValue.getLogicTableName()+ BusinessConstant.UNDERLINE + preciseSharding(preciseShardingValue.getValue(),startIndex ,endIndex); | |||
if(!dataSequenceService.checkTableExist(tableName)){ | |||
dataSequenceService.createTable(tableName); | |||
} | |||
return tableName; | |||
} | |||
/** | |||
* 分片实现 | |||
* | |||
* @param indexId 起始位置 | |||
* @param startIndex 起始值 | |||
* @param endIndex 结束值 | |||
* @return long 返回截止值 | |||
*/ | |||
public long preciseSharding(long indexId,long startIndex , long endIndex){ | |||
if(indexId > endIndex){ | |||
startIndex = startIndex + BusinessConstant.INTERVAL_NUMBER; | |||
endIndex = endIndex + BusinessConstant.INTERVAL_NUMBER; | |||
return preciseSharding(indexId,startIndex,endIndex); | |||
} | |||
return endIndex / BusinessConstant.INTERVAL_NUMBER; | |||
} | |||
} |
@@ -0,0 +1,156 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.util; | |||
/** | |||
* @description 控制台字体颜色 | |||
* @date 2020-10-22 | |||
*/ | |||
public class PrintUtils { | |||
/** | |||
* 白色 | |||
*/ | |||
public static final int WHITE = 30; | |||
/** | |||
* 白色背景 | |||
*/ | |||
public static final int WHITE_BACKGROUND = 40; | |||
/** | |||
* 红色 | |||
*/ | |||
public static final int RED = 31; | |||
/** | |||
* 红色背景 | |||
*/ | |||
public static final int RED_BACKGROUND = 41; | |||
/** | |||
* 绿色 | |||
*/ | |||
public static final int GREEN = 32; | |||
/** | |||
* 绿色背景 | |||
*/ | |||
public static final int GREEN_BACKGROUND = 42; | |||
/** | |||
* 黄色 | |||
*/ | |||
public static final int YELLOW = 33; | |||
/** | |||
* 黄色背景 | |||
*/ | |||
public static final int YELLOW_BACKGROUND = 43; | |||
/** | |||
* 蓝色 | |||
*/ | |||
public static final int BLUE = 34; | |||
/** | |||
* 蓝色背景 | |||
*/ | |||
public static final int BLUE_BACKGROUND = 44; | |||
/** | |||
* 品红(洋红) | |||
*/ | |||
public static final int MAGENTA = 35; | |||
/** | |||
* 品红背景 | |||
*/ | |||
public static final int MAGENTA_BACKGROUND = 45; | |||
/** | |||
* 蓝绿 | |||
*/ | |||
public static final int CYAN = 36; | |||
/** | |||
* 蓝绿背景 | |||
*/ | |||
public static final int CYAN_BACKGROUND = 46; | |||
/** | |||
* 黑色 | |||
*/ | |||
public static final int BLACK = 37; | |||
/** | |||
* 黑色背景 | |||
*/ | |||
public static final int BLACK_BACKGROUND = 47; | |||
/** | |||
* 粗体 | |||
*/ | |||
public static final int BOLD = 1; | |||
/** | |||
* 斜体 | |||
*/ | |||
public static final int ITATIC = 3; | |||
/** | |||
* 下划线 | |||
*/ | |||
public static final int UNDERLINE = 4; | |||
/** | |||
* 反转 | |||
*/ | |||
public static final int REVERSE = 7; | |||
/** | |||
* 格式化 | |||
* | |||
* @param txt 文本 | |||
* @param codes 信息 | |||
* @return String 格式化后的内容 | |||
*/ | |||
private static String FMT(String txt, int... codes) { | |||
StringBuffer sb = new StringBuffer(); | |||
for (int code : codes) { | |||
sb.append(code + ";"); | |||
} | |||
String _code = sb.toString(); | |||
if (_code.endsWith(";")) { | |||
_code = _code.substring(0, _code.length() - 1); | |||
} | |||
return (char) 27 + "[" + _code + "m" + txt + (char) 27 + "[0m"; | |||
} | |||
/** | |||
* 打印并换行 | |||
*/ | |||
public static void printLine(String txt, int... codes) { | |||
System.out.println(FMT(txt, codes)); | |||
} | |||
/** | |||
* 默认打印红色文字 | |||
*/ | |||
public static void PN(String txt) { | |||
System.out.println(FMT(txt, new int[]{RED})); | |||
} | |||
} |
@@ -0,0 +1,95 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.util; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.springframework.beans.BeansException; | |||
import org.springframework.beans.factory.DisposableBean; | |||
import org.springframework.context.ApplicationContext; | |||
import org.springframework.context.ApplicationContextAware; | |||
/** | |||
* @description 上下文工具类 | |||
* @date 2020-03-25 | |||
*/ | |||
@Slf4j | |||
public class SpringContextHolder implements ApplicationContextAware, DisposableBean { | |||
private static ApplicationContext applicationContext = null; | |||
/** | |||
* 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型. | |||
* | |||
* @param name bean名称 | |||
* @return T 根据Bean名称转型的对象 | |||
*/ | |||
@SuppressWarnings("unchecked") | |||
public static <T> T getBean(String name) { | |||
assertContextInjected(); | |||
return (T) applicationContext.getBean(name); | |||
} | |||
/** | |||
* 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型. | |||
* | |||
* @param requiredType 要求类型 | |||
* @return T 根据Bean名称转型的对象 | |||
*/ | |||
public static <T> T getBean(Class<T> requiredType) { | |||
assertContextInjected(); | |||
return applicationContext.getBean(requiredType); | |||
} | |||
/** | |||
* 检查ApplicationContext不为空. | |||
*/ | |||
private static void assertContextInjected() { | |||
if (applicationContext == null) { | |||
throw new IllegalStateException("applicaitonContext属性未注入, 请在applicationContext" + | |||
".xml中定义SpringContextHolder或在SpringBoot启动类中注册SpringContextHolder."); | |||
} | |||
} | |||
/** | |||
* 清除SpringContextHolder中的ApplicationContext为Null. | |||
*/ | |||
private static void clearHolder() { | |||
log.debug("清除SpringContextHolder中的ApplicationContext:" | |||
+ applicationContext); | |||
applicationContext = null; | |||
} | |||
@Override | |||
public void destroy() { SpringContextHolder.clearHolder(); | |||
} | |||
@Override | |||
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { | |||
if (SpringContextHolder.applicationContext != null) { | |||
log.warn("SpringContextHolder中的ApplicationContext被覆盖, 原有ApplicationContext为:" + SpringContextHolder.applicationContext); | |||
} | |||
SpringContextHolder.applicationContext = applicationContext; | |||
} | |||
/** | |||
* 获取当前环境 | |||
*/ | |||
public static String getActiveProfile(){ | |||
return applicationContext.getEnvironment().getActiveProfiles()[0]; | |||
} | |||
} |
@@ -0,0 +1,72 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.common.util; | |||
import lombok.extern.slf4j.Slf4j; | |||
import java.util.List; | |||
import java.util.concurrent.*; | |||
/** | |||
* @description 线程工具 | |||
* @date 2020-10-19 | |||
*/ | |||
@Slf4j | |||
public class ThreadUtils { | |||
private ThreadUtils(){} | |||
/** | |||
* 根据需要处理的数量创建线程数 | |||
* | |||
* @param listSize 集合数量 | |||
* @return int 数量 | |||
*/ | |||
public static int createThread(int listSize) { | |||
return listSize / getNeedThreadNumber() == 0 ? 1 : listSize / getNeedThreadNumber(); | |||
} | |||
/** | |||
* 获取需要创建的线程数 | |||
* | |||
* @return int 数量 | |||
*/ | |||
public static int getNeedThreadNumber() { | |||
final int numOfCores = Runtime.getRuntime().availableProcessors(); | |||
final double blockingCoefficient = 0.8; | |||
return (int) (numOfCores / (1 - blockingCoefficient)); | |||
} | |||
/** | |||
* 按要求分多线程执行 | |||
* | |||
* @param partitions 分线程集合 | |||
* @throws Exception 线程执行异常 | |||
*/ | |||
public static void runMultiThread(List<Callable<Integer>> partitions) throws Exception { | |||
final ExecutorService executorService = Executors.newFixedThreadPool(ThreadUtils.getNeedThreadNumber()); | |||
final List<Future<Integer>> valueOfStocks = executorService.invokeAll(partitions); | |||
Integer endCount = 0; | |||
for (final Future<Integer> value : valueOfStocks) { | |||
endCount += value.get(); | |||
} | |||
log.warn("#-------------处理结束,成功处理文件 【" + endCount + "】个-------------#"); | |||
} | |||
} |
@@ -0,0 +1,36 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.dao; | |||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; | |||
import org.apache.ibatis.annotations.Param; | |||
import org.dubhe.datasetutil.domain.entity.DataFile; | |||
import java.util.List; | |||
/** | |||
* @description 数据集文件 Mapper接口 | |||
* @date 2020-09-17 | |||
*/ | |||
public interface DataFileMapper extends BaseMapper<DataFile> { | |||
/** | |||
* 插入文件数据 | |||
* | |||
* @param listDataFile 文件数据集合 | |||
*/ | |||
void saveBatchDataFile(@Param("listDataFile") List<DataFile> listDataFile); | |||
} |
@@ -0,0 +1,36 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.dao; | |||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; | |||
import org.apache.ibatis.annotations.Param; | |||
import org.dubhe.datasetutil.domain.entity.DataGroupLabel; | |||
import java.util.List; | |||
/** | |||
* @description 标签组与标签的关系MAPPER | |||
* @date 2020-10-21 | |||
*/ | |||
public interface DataGroupLabelMapper extends BaseMapper<DataGroupLabel> { | |||
/** | |||
* 插入标签组与标签的关数据 | |||
* | |||
* @param listDataGroupLabel 标签组与标签数据集合 | |||
*/ | |||
void saveDataGroupLabel(@Param("list") List<DataGroupLabel> listDataGroupLabel); | |||
} |
@@ -0,0 +1,38 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.dao; | |||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; | |||
import org.apache.ibatis.annotations.Param; | |||
import org.apache.ibatis.annotations.Select; | |||
import org.dubhe.datasetutil.domain.entity.DataLabelGroup; | |||
/** | |||
* @description 数据集标签组 | |||
* @date 2020-10-14 | |||
*/ | |||
public interface DataLabelGroupMapper extends BaseMapper<DataLabelGroup> { | |||
/** | |||
* 根据标签组名查询 | |||
* | |||
* @param labelGroupName 标签组名 | |||
* @return int 标签组数量 | |||
*/ | |||
@Select("select count(1) from data_label_group where name = #{labelGroupName} and deleted = 0") | |||
int selectByLabelGroupName(@Param("labelGroupName") String labelGroupName); | |||
} |
@@ -0,0 +1,36 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.dao; | |||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; | |||
import org.apache.ibatis.annotations.Param; | |||
import org.dubhe.datasetutil.domain.entity.DataLabel; | |||
import java.util.List; | |||
/** | |||
* @description 数据集标签 | |||
* @date 2020-10-14 | |||
*/ | |||
public interface DataLabelMapper extends BaseMapper<DataLabel> { | |||
/** | |||
* 批量保存数据集标签 | |||
* | |||
* @param listDataLabel 标签数据 | |||
*/ | |||
void saveBatchDataLabel(@Param("listDataLabel") List<DataLabel> listDataLabel); | |||
} |
@@ -0,0 +1,66 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.dao; | |||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; | |||
import org.apache.ibatis.annotations.Param; | |||
import org.apache.ibatis.annotations.Select; | |||
import org.apache.ibatis.annotations.Update; | |||
import org.dubhe.datasetutil.domain.entity.DataSequence; | |||
/** | |||
* @description 序列Mapper | |||
* @date 2020-09-23 | |||
*/ | |||
public interface DataSequenceMapper extends BaseMapper<DataSequence> { | |||
/** | |||
* 根据业务编码查询序列 | |||
* | |||
* @param businessCode 业务信息 | |||
* @return DataSequence 根据业务编码得到的序列 | |||
*/ | |||
@Select("select id, business_code ,start, step from data_sequence where business_code = #{businessCode}") | |||
DataSequence selectByBusiness(String businessCode); | |||
/** | |||
* 根据业务编码更新序列起始值 | |||
* | |||
* @param businessCode 业务信息 | |||
* @return DataSequence 根据业务编码更新序列起始值 | |||
*/ | |||
@Update("update data_sequence set start = start + step where business_code = #{businessCode} ") | |||
int updateStartByBusinessCode(String businessCode); | |||
/** | |||
* 查询存在表的记录数 | |||
* | |||
* @param tableName 类型名称 | |||
* @return int 根据类型查在表的记录数量 | |||
*/ | |||
@Select("select count(1) from ${tableName}") | |||
int checkTableExist(@Param("tableName") String tableName); | |||
/** | |||
* 执行创建表 | |||
* | |||
* @param tableName 类型名称 | |||
* @param oldTableName 旧类型名称 | |||
*/ | |||
@Update({"CREATE TABLE ${tableName} AS select * from ${oldTableName} "}) | |||
void createNewTable(@Param("tableName") String tableName, @Param("oldTableName") String oldTableName); | |||
} |
@@ -0,0 +1,37 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.dao; | |||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; | |||
import org.apache.ibatis.annotations.Param; | |||
import org.dubhe.datasetutil.domain.dto.DataVersionFile; | |||
import java.util.List; | |||
/** | |||
* @description 数据集文件中间表 Mapper接口 | |||
* @date 2020-9-17 | |||
*/ | |||
public interface DataVersionFileMapper extends BaseMapper<DataVersionFile> { | |||
/** | |||
* 插入数据集文件中间表数据 | |||
* | |||
* @param listDataVersionFile 数据集文件中间表数据集合 | |||
*/ | |||
void saveBatchDataFileVersion(@Param("listDataVersionFile") List<DataVersionFile> listDataVersionFile); | |||
} |
@@ -0,0 +1,37 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.dao; | |||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; | |||
import org.apache.ibatis.annotations.Param; | |||
import org.dubhe.datasetutil.domain.entity.DatasetDataLabel; | |||
import java.util.List; | |||
/** | |||
* @description 数据集与标签关系 | |||
* @date 2020-10-14 | |||
*/ | |||
public interface DatasetDataLabelMapper extends BaseMapper<DatasetDataLabel> { | |||
/** | |||
* 批量保存数据集与标签关系 | |||
* | |||
* @param datasetDataLabelList 数据集与标签 | |||
*/ | |||
void saveBatchDatasetDataLabel(@Param("datasetDataLabelList") List<DatasetDataLabel> datasetDataLabelList); | |||
} |
@@ -0,0 +1,56 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.dao; | |||
import com.baomidou.mybatisplus.core.mapper.BaseMapper; | |||
import org.apache.ibatis.annotations.Param; | |||
import org.apache.ibatis.annotations.Select; | |||
import org.dubhe.datasetutil.domain.entity.Dataset; | |||
/** | |||
* @description 数据集 Mapper接口 | |||
* @date 2020-9-17 | |||
*/ | |||
public interface DatasetMapper extends BaseMapper<Dataset> { | |||
/** | |||
* 根据数据集ID查询数据集 | |||
* | |||
* @param datasetId 数据集id | |||
* @return Dataset 根据数据集ID得到数据集 | |||
*/ | |||
@Select("select * from data_dataset where id = #{datasetId} and is_import = 1") | |||
Dataset findDatasetById(@Param("datasetId") Long datasetId); | |||
/** | |||
* 查询数据集标签数量 | |||
* | |||
* @param datasetId 数据集id | |||
* @return int 数据集标签数量 | |||
*/ | |||
@Select("select count(1) from data_dataset_label where dataset_id = #{datasetId}") | |||
int findDataLabelById(@Param("datasetId") Long datasetId); | |||
/** | |||
* 查询数据集文件数量 | |||
* | |||
* @param datasetId 数据集id | |||
* @return int 数据集标签数量 | |||
*/ | |||
@Select("select count(1) from data_file where dataset_id = #{datasetId}") | |||
int findDataFileById(@Param("datasetId") Long datasetId); | |||
} |
@@ -0,0 +1,93 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.dto; | |||
import com.baomidou.mybatisplus.annotation.IdType; | |||
import com.baomidou.mybatisplus.annotation.TableId; | |||
import lombok.Data; | |||
import org.dubhe.datasetutil.common.base.BaseEntity; | |||
import java.io.Serializable; | |||
/** | |||
* @description 数据集文件关系类 | |||
* @date 2020-9-17 | |||
*/ | |||
@Data | |||
public class DataVersionFile extends BaseEntity implements Serializable { | |||
/** | |||
* id | |||
*/ | |||
@TableId(type = IdType.AUTO) | |||
private Long id; | |||
/** | |||
* 数据集id | |||
*/ | |||
private Long datasetId; | |||
/** | |||
* 版本号 | |||
*/ | |||
private String versionName; | |||
/** | |||
* 文件id | |||
*/ | |||
private Long fileId; | |||
/** | |||
* 状态 | |||
*/ | |||
private Integer status; | |||
/** | |||
* 数据集状态 | |||
*/ | |||
private Integer annotationStatus; | |||
/** | |||
* 数据集状态备份 | |||
*/ | |||
private Integer backupStatus; | |||
/** | |||
* 发布是否转换 | |||
*/ | |||
private Integer changed; | |||
public DataVersionFile() { | |||
} | |||
/** | |||
* 插入数据集版本文件关系 | |||
* | |||
* @param datasetId 数据集id | |||
* @param fileId 文件id | |||
* @param annotationStatus 数据集id | |||
* @param status 状态 | |||
* @return DataVersionFile 数据集版本文件表 | |||
*/ | |||
public DataVersionFile(Long datasetId, Long fileId,Integer annotationStatus,Integer status) { | |||
this.datasetId = datasetId; | |||
this.fileId = fileId; | |||
this.annotationStatus = annotationStatus; | |||
this.status = status; | |||
} | |||
} |
@@ -0,0 +1,50 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.dto; | |||
import lombok.Data; | |||
import org.dubhe.datasetutil.common.base.MagicNumConstant; | |||
/** | |||
* @description ID策略实体 | |||
* @date 2020-10-16 | |||
*/ | |||
@Data | |||
public class IdAlloc { | |||
/** | |||
* 起始位置 | |||
*/ | |||
private long startNumber; | |||
/** | |||
* 结束位置 | |||
*/ | |||
private long endNumber; | |||
/** | |||
* 可用数量 | |||
*/ | |||
private long usedNumber; | |||
public IdAlloc() { | |||
this.startNumber = MagicNumConstant.ZERO; | |||
this.endNumber = MagicNumConstant.ZERO; | |||
this.usedNumber = MagicNumConstant.ZERO; | |||
} | |||
} |
@@ -0,0 +1,118 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.entity; | |||
import com.baomidou.mybatisplus.annotation.IdType; | |||
import com.baomidou.mybatisplus.annotation.TableId; | |||
import lombok.Data; | |||
import org.dubhe.datasetutil.common.base.BaseEntity; | |||
import java.awt.image.BufferedImage; | |||
import java.io.Serializable; | |||
/** | |||
* @description 文件类 | |||
* @date 2020-09-17 | |||
*/ | |||
@Data | |||
public class DataFile extends BaseEntity implements Serializable { | |||
/** | |||
* id | |||
*/ | |||
@TableId(type = IdType.AUTO) | |||
private Long id; | |||
/** | |||
* 名称 | |||
*/ | |||
private String name; | |||
/** | |||
* 状态 | |||
*/ | |||
private Integer status; | |||
/** | |||
* 数据集id | |||
*/ | |||
private Long datasetId; | |||
/** | |||
* url | |||
*/ | |||
private String url; | |||
/** | |||
* 文件类型 | |||
*/ | |||
private Integer fileType; | |||
/** | |||
* 父id | |||
*/ | |||
private Long pid; | |||
/** | |||
* 帧间隔 | |||
*/ | |||
private Integer frameInterval; | |||
/** | |||
* 增强类型 | |||
*/ | |||
private Integer enhanceType; | |||
/** | |||
* 宽 | |||
*/ | |||
private Integer width; | |||
/** | |||
* 高 | |||
*/ | |||
private Integer height; | |||
/** | |||
* 拥有人id | |||
*/ | |||
private Long originUserId; | |||
public DataFile() {} | |||
/** | |||
* 插入文件表 | |||
* | |||
* @param name 文件名字 | |||
* @param datasetId 数据集id | |||
* @param url 文件路径 | |||
* @param createUserId 创建人id | |||
* @param read 文件宽高 | |||
* @return DataFile file对象 | |||
*/ | |||
public DataFile(String name, Long datasetId, String url, Long createUserId, BufferedImage read) { | |||
this.name = name.substring(0, name.lastIndexOf(".")); | |||
this.datasetId = datasetId; | |||
this.url = url; | |||
this.status = 101; | |||
this.setDeleted(false); | |||
this.originUserId = createUserId; | |||
this.width = read.getWidth(); | |||
this.height = read.getHeight(); | |||
} | |||
} |
@@ -0,0 +1,58 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.entity; | |||
import com.baomidou.mybatisplus.annotation.IdType; | |||
import com.baomidou.mybatisplus.annotation.TableField; | |||
import com.baomidou.mybatisplus.annotation.TableId; | |||
import com.baomidou.mybatisplus.annotation.TableName; | |||
import lombok.*; | |||
import org.dubhe.datasetutil.common.base.BaseEntity; | |||
import java.io.Serializable; | |||
/** | |||
* @description 标签组标签中间表 | |||
* @date 2020-10-21 | |||
*/ | |||
@Data | |||
@NoArgsConstructor | |||
@AllArgsConstructor | |||
@EqualsAndHashCode(callSuper = false) | |||
@Builder | |||
@TableName("data_group_label") | |||
public class DataGroupLabel extends BaseEntity implements Serializable { | |||
/** | |||
* id | |||
*/ | |||
@TableId(value = "id", type = IdType.AUTO) | |||
private Long id; | |||
/** | |||
* 标签id | |||
*/ | |||
@TableField(value = "label_id") | |||
private Long labelId; | |||
/** | |||
* 标签组id | |||
*/ | |||
@TableField(value = "label_group_id") | |||
private Long labelGroupId; | |||
} |
@@ -0,0 +1,64 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.entity; | |||
import com.baomidou.mybatisplus.annotation.IdType; | |||
import com.baomidou.mybatisplus.annotation.TableField; | |||
import com.baomidou.mybatisplus.annotation.TableId; | |||
import com.baomidou.mybatisplus.annotation.TableName; | |||
import lombok.*; | |||
import org.dubhe.datasetutil.common.base.BaseEntity; | |||
import java.io.Serializable; | |||
/** | |||
* @description 数据集标签实体 | |||
* @date 2020-10-14 | |||
*/ | |||
@Data | |||
@NoArgsConstructor | |||
@AllArgsConstructor | |||
@EqualsAndHashCode(callSuper = false) | |||
@Builder | |||
@TableName("data_label") | |||
public class DataLabel extends BaseEntity implements Serializable { | |||
/** | |||
* id | |||
*/ | |||
@TableId(value = "id", type = IdType.AUTO) | |||
private Long id; | |||
/** | |||
* 名称 | |||
*/ | |||
@TableField(value = "name") | |||
private String name; | |||
/** | |||
* 颜色 | |||
*/ | |||
@TableField(value = "color") | |||
private String color; | |||
/** | |||
* 类型 | |||
*/ | |||
@TableField(value = "type") | |||
private Long type; | |||
} |
@@ -0,0 +1,70 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.entity; | |||
import com.baomidou.mybatisplus.annotation.IdType; | |||
import com.baomidou.mybatisplus.annotation.TableField; | |||
import com.baomidou.mybatisplus.annotation.TableId; | |||
import com.baomidou.mybatisplus.annotation.TableName; | |||
import lombok.*; | |||
import org.dubhe.datasetutil.common.base.BaseEntity; | |||
import java.io.Serializable; | |||
/** | |||
* @description 标签组实体 | |||
* @date 2020-10-14 | |||
*/ | |||
@Data | |||
@NoArgsConstructor | |||
@AllArgsConstructor | |||
@EqualsAndHashCode(callSuper = false) | |||
@Builder | |||
@TableName("data_label_group") | |||
public class DataLabelGroup extends BaseEntity implements Serializable { | |||
/** | |||
* id | |||
*/ | |||
@TableId(value = "id", type = IdType.AUTO) | |||
private Long id; | |||
/** | |||
* 名称 | |||
*/ | |||
@TableField(value = "name") | |||
private String name; | |||
/** | |||
* 描述 | |||
*/ | |||
@TableField(value = "remark") | |||
private String remark; | |||
/** | |||
* 类型 | |||
*/ | |||
@TableField(value = "type") | |||
private Long type; | |||
/** | |||
* 拥有人id | |||
*/ | |||
@TableField(value = "origin_user_id") | |||
private Long originUserId; | |||
} |
@@ -0,0 +1,57 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.entity; | |||
import com.baomidou.mybatisplus.annotation.IdType; | |||
import com.baomidou.mybatisplus.annotation.TableField; | |||
import com.baomidou.mybatisplus.annotation.TableId; | |||
import com.baomidou.mybatisplus.annotation.TableName; | |||
import lombok.Data; | |||
/** | |||
* @description 序列表 | |||
* @date 2020-09-23 | |||
*/ | |||
@Data | |||
@TableName("data_sequence") | |||
public class DataSequence { | |||
/** | |||
* id | |||
*/ | |||
@TableId(value = "id", type = IdType.AUTO) | |||
private Long id; | |||
/** | |||
* 业务code | |||
*/ | |||
@TableField(value = "business_code") | |||
private String businessCode; | |||
/** | |||
* 启动 | |||
*/ | |||
@TableField(value = "start") | |||
private Long start; | |||
/** | |||
* 步数 | |||
*/ | |||
@TableField(value = "step") | |||
private Long step; | |||
} |
@@ -0,0 +1,145 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.entity; | |||
import com.baomidou.mybatisplus.annotation.*; | |||
import lombok.Data; | |||
import org.dubhe.datasetutil.common.base.BaseEntity; | |||
import java.io.Serializable; | |||
import java.sql.Timestamp; | |||
/** | |||
* @description 数据集类 | |||
* @date 2020-9-17 | |||
*/ | |||
@Data | |||
@TableName("data_dataset") | |||
public class Dataset extends BaseEntity implements Serializable { | |||
/** | |||
* 删除标识 | |||
*/ | |||
@TableField("deleted") | |||
private Boolean deleted = false; | |||
/** | |||
* 数据集名称 | |||
*/ | |||
private String name; | |||
/** | |||
* 数据集备注 | |||
*/ | |||
private String remark; | |||
/** | |||
* 类型 | |||
*/ | |||
private Integer type; | |||
/** | |||
* 数据集类型 | |||
*/ | |||
private Integer dataType; | |||
/** | |||
* 数据集状态 | |||
*/ | |||
private Integer annotateType; | |||
/** | |||
* 任务id | |||
*/ | |||
private Long teamId; | |||
/** | |||
* id | |||
*/ | |||
@TableId(type = IdType.AUTO) | |||
private Long id; | |||
/** | |||
* url | |||
*/ | |||
private String uri; | |||
/** | |||
* 状态 | |||
*/ | |||
private Integer status; | |||
/** | |||
* 当前版本号 | |||
*/ | |||
private String currentVersionName; | |||
/** | |||
* 是否为导入 | |||
*/ | |||
@TableField(value = "is_import") | |||
private boolean isImport; | |||
/** | |||
* 用户导入数据集压缩包地址 | |||
*/ | |||
private String archiveUrl; | |||
/** | |||
* 解压状态 | |||
*/ | |||
private Integer decompressState; | |||
/** | |||
* 解压失败原因 | |||
*/ | |||
private String decompressFailReason; | |||
/** | |||
* 是否置顶 | |||
*/ | |||
@TableField(value = "is_top") | |||
private boolean isTop; | |||
/** | |||
* 创建时间 | |||
*/ | |||
private Timestamp createTime; | |||
/** | |||
* 修改时间 | |||
*/ | |||
private Timestamp updateTime; | |||
/** | |||
* 创建人id | |||
*/ | |||
private Long createUserId; | |||
/** | |||
* 修改人id | |||
*/ | |||
private Long updateUserId; | |||
/** | |||
* 拥有人id | |||
*/ | |||
private Long originUserId; | |||
public Dataset() {} | |||
} |
@@ -0,0 +1,58 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.entity; | |||
import com.baomidou.mybatisplus.annotation.IdType; | |||
import com.baomidou.mybatisplus.annotation.TableField; | |||
import com.baomidou.mybatisplus.annotation.TableId; | |||
import com.baomidou.mybatisplus.annotation.TableName; | |||
import lombok.*; | |||
import org.dubhe.datasetutil.common.base.BaseEntity; | |||
import java.io.Serializable; | |||
/** | |||
* @description 数据集与标签关系实体 | |||
* @date 2020-10-14 | |||
*/ | |||
@Data | |||
@NoArgsConstructor | |||
@AllArgsConstructor | |||
@EqualsAndHashCode(callSuper = false) | |||
@Builder | |||
@TableName("data_label") | |||
public class DatasetDataLabel extends BaseEntity implements Serializable { | |||
/** | |||
* id | |||
*/ | |||
@TableId(value = "id", type = IdType.AUTO) | |||
private Long id; | |||
/** | |||
* 数据集id | |||
*/ | |||
@TableField(value = "dataset_id") | |||
private Long datasetId; | |||
/** | |||
* 标签id | |||
*/ | |||
@TableField(value = "label_id") | |||
private Long labelId; | |||
} |
@@ -0,0 +1,74 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.domain.entity; | |||
import com.alibaba.fastjson.annotation.JSONField; | |||
import com.xiaoleilu.hutool.date.DateUtil; | |||
import lombok.Data; | |||
import lombok.experimental.Accessors; | |||
import java.io.Serializable; | |||
/** | |||
* @description 日志对象封装类 | |||
* @date 2020-06-29 | |||
*/ | |||
@Data | |||
@Accessors(chain = true) | |||
public class LogInfo implements Serializable { | |||
/** | |||
* id | |||
*/ | |||
@JSONField(ordinal = 1) | |||
private String traceId; | |||
/** | |||
* 类型 | |||
*/ | |||
@JSONField(ordinal = 2) | |||
private String type; | |||
/** | |||
* 等级 | |||
*/ | |||
@JSONField(ordinal = 3) | |||
private String level; | |||
/** | |||
* 位置 | |||
*/ | |||
@JSONField(ordinal = 4) | |||
private String location; | |||
/** | |||
* 时间 | |||
*/ | |||
@JSONField(ordinal = 5) | |||
private String time = DateUtil.now(); | |||
/** | |||
* 描述 | |||
*/ | |||
@JSONField(ordinal = 6) | |||
private Object info; | |||
public void setInfo(Object info) { | |||
this.info = info; | |||
} | |||
} |
@@ -0,0 +1,140 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.handle; | |||
import com.xiaoleilu.hutool.io.FileUtil; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.dubhe.datasetutil.common.constant.BusinessConstant; | |||
import org.dubhe.datasetutil.common.util.DateUtil; | |||
import org.dubhe.datasetutil.common.util.GeneratorKeyUtil; | |||
import org.dubhe.datasetutil.common.util.MinioUtil; | |||
import org.dubhe.datasetutil.common.util.ThreadUtils; | |||
import org.dubhe.datasetutil.domain.entity.DataFile; | |||
import org.dubhe.datasetutil.domain.dto.DataVersionFile; | |||
import org.dubhe.datasetutil.domain.entity.Dataset; | |||
import org.dubhe.datasetutil.service.DataFileService; | |||
import org.dubhe.datasetutil.service.DataVersionFileService; | |||
import org.dubhe.datasetutil.service.DatasetService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Component; | |||
import javax.imageio.ImageIO; | |||
import java.awt.image.BufferedImage; | |||
import java.io.File; | |||
import java.util.ArrayList; | |||
import java.util.List; | |||
import java.util.concurrent.Callable; | |||
import java.util.concurrent.atomic.AtomicInteger; | |||
/** | |||
* @description 上传图片工具类 | |||
* @date 2020-09-17 | |||
*/ | |||
@Slf4j | |||
@Component | |||
public class DatasetImageUploadHandle { | |||
@Autowired | |||
private MinioUtil minioUtil; | |||
@Autowired | |||
private DataFileService dataFileService; | |||
@Autowired | |||
private DataVersionFileService dataVersionFileService; | |||
@Autowired | |||
private DatasetService datasetService; | |||
@Autowired | |||
private GeneratorKeyUtil generatorKeyUtil; | |||
/** | |||
* 启动线程 | |||
* | |||
* @param imagePath 本地文件地址 | |||
* @param datasetId 数据集Id | |||
*/ | |||
public void execute(String imagePath, Long datasetId) throws Exception { | |||
log.info("#-------------开始处理,时间[" + DateUtil.getNowStr() + "]-------------#"); | |||
List<String> fileNames = FileUtil.listFileNames(imagePath); | |||
log.info("#-------------文件数量[" + fileNames.size() + "]------------------------"); | |||
String fileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + datasetId | |||
+ BusinessConstant.FILE_SEPARATOR + BusinessConstant.IMAGE_ORIGIN + BusinessConstant.FILE_SEPARATOR; | |||
List<Callable<Integer>> partitions = new ArrayList<>(); | |||
int oneSize = ThreadUtils.createThread(fileNames.size()); | |||
List<String> need = new ArrayList<>(); | |||
AtomicInteger atomicInteger = new AtomicInteger(0); | |||
for (String fileName : fileNames) { | |||
need.add(fileName); | |||
if (need.size() == oneSize || atomicInteger.intValue() == fileNames.size() - 1) { | |||
List<String> now = new ArrayList<>(need); | |||
need.clear(); | |||
partitions.add(() -> run(datasetId, now, fileBaseDir, imagePath)); | |||
} | |||
atomicInteger.getAndIncrement(); | |||
} | |||
ThreadUtils.runMultiThread(partitions); | |||
} | |||
/** | |||
* 插入数据库数据 | |||
* | |||
* @param datasetId 数据集Id | |||
* @param fileNames 文件Name | |||
* @param fileBaseDir 文件路径 | |||
* @param imagePath 文件地址 | |||
* @return java.lang.Integer 成功数量 | |||
*/ | |||
public Integer run(Long datasetId, List<String> fileNames, String fileBaseDir, String imagePath) { | |||
Integer success = 0; | |||
Dataset dataset = datasetService.findCreateUserIdById(datasetId); | |||
List<DataFile> dataFiles = new ArrayList<>(); | |||
List<DataVersionFile> dataVersionFiles = new ArrayList<>(); | |||
for (int i = 0; i < fileNames.size(); i++) { | |||
try { | |||
minioUtil.upLoadFile(fileBaseDir + fileNames.get(i), FileUtil.getInputStream(imagePath + BusinessConstant.FILE_SEPARATOR + fileNames.get(i))); | |||
BufferedImage read = ImageIO.read(new File(imagePath + BusinessConstant.FILE_SEPARATOR + fileNames.get(i))); | |||
success++; | |||
dataFiles.add(new DataFile(fileNames.get(i), datasetId, minioUtil.getUrl(fileBaseDir + fileNames.get(i)), dataset.getCreateUserId(), read)); | |||
if (dataFiles.size() % 500 == 0 || i == fileNames.size() - 1) { | |||
long startDataFileIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_FILE, dataFiles.size()); | |||
for (DataFile dataFileEntity : dataFiles) { | |||
dataFileEntity.setId(startDataFileIndex++); | |||
} | |||
dataFileService.saveBatchDataFile(dataFiles); | |||
for (DataFile file : dataFiles) { | |||
dataVersionFiles.add(new DataVersionFile(datasetId, file.getId(), 101, 0)); | |||
} | |||
long startDataFileVersionIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_VERSION_FILE, dataVersionFiles.size()); | |||
for (DataVersionFile dataVersionFile : dataVersionFiles) { | |||
dataVersionFile.setId(startDataFileVersionIndex++); | |||
} | |||
dataVersionFileService.saveBatchDataFileVersion(dataVersionFiles); | |||
dataVersionFiles.clear(); | |||
dataFiles.clear(); | |||
} | |||
} catch (Exception e) { | |||
log.error("{}", e); | |||
} | |||
} | |||
return success; | |||
} | |||
} |
@@ -0,0 +1,522 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.handle; | |||
import com.alibaba.fastjson.JSON; | |||
import com.alibaba.fastjson.JSONArray; | |||
import com.alibaba.fastjson.JSONObject; | |||
import com.alibaba.fastjson.TypeReference; | |||
import com.xiaoleilu.hutool.io.FileUtil; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.apache.commons.io.IOUtils; | |||
import org.apache.commons.lang3.StringUtils; | |||
import org.dubhe.datasetutil.common.base.MagicNumConstant; | |||
import org.dubhe.datasetutil.common.config.MinioConfig; | |||
import org.dubhe.datasetutil.common.constant.BusinessConstant; | |||
import org.dubhe.datasetutil.common.constant.FileStateCodeConstant; | |||
import org.dubhe.datasetutil.common.exception.ImportDatasetException; | |||
import org.dubhe.datasetutil.common.util.*; | |||
import org.dubhe.datasetutil.domain.entity.*; | |||
import org.dubhe.datasetutil.domain.dto.DataVersionFile; | |||
import org.dubhe.datasetutil.service.*; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Component; | |||
import javax.imageio.ImageIO; | |||
import java.awt.image.BufferedImage; | |||
import java.io.File; | |||
import java.io.IOException; | |||
import java.nio.charset.StandardCharsets; | |||
import java.sql.Timestamp; | |||
import java.util.*; | |||
import java.util.concurrent.Callable; | |||
import java.util.concurrent.atomic.AtomicInteger; | |||
import java.util.stream.Collectors; | |||
/** | |||
* @description 导入数据集工具类 | |||
* @date 2020-10-12 | |||
*/ | |||
@Slf4j | |||
@Component | |||
public class DatasetImportHandle { | |||
@Autowired | |||
private DatasetService datasetService; | |||
@Autowired | |||
private DataLabelGroupService dataLabelGroupService; | |||
@Autowired | |||
private DataGroupLabelService dataGroupLabelService; | |||
@Autowired | |||
private DataLabelService dataLabelService; | |||
@Autowired | |||
private DataFileService dataFileService; | |||
@Autowired | |||
private DatasetDataLabelService datasetDataLabelService; | |||
@Autowired | |||
private DataVersionFileService dataVersionFileService; | |||
@Autowired | |||
private MinioUtil minioUtil; | |||
@Autowired | |||
private MinioConfig minioConfig; | |||
@Autowired | |||
private GeneratorKeyUtil generatorKeyUtil; | |||
/** | |||
* 可支持的图片格式集合 | |||
*/ | |||
private static final List<String> SUFFIX_LIST = new ArrayList<>(); | |||
/** | |||
* 标注文件中JSON的key | |||
*/ | |||
static List<String> annotationFileContextKey = new ArrayList<>(); | |||
/** | |||
* 加载静态集合数据 | |||
*/ | |||
static { | |||
SUFFIX_LIST.add(".jpg"); | |||
SUFFIX_LIST.add(".png"); | |||
SUFFIX_LIST.add(".bmp"); | |||
SUFFIX_LIST.add(".jpeg"); | |||
annotationFileContextKey.add("score"); | |||
annotationFileContextKey.add("area"); | |||
annotationFileContextKey.add("name"); | |||
annotationFileContextKey.add("bbox"); | |||
annotationFileContextKey.add("segmentation"); | |||
annotationFileContextKey.add("iscrowd"); | |||
} | |||
/** | |||
* 导入数据集 | |||
* | |||
* @param scanner 输入 | |||
*/ | |||
public void importDataset(Scanner scanner) throws Exception { | |||
Dataset dataset = verificationDatasetId(scanner); | |||
String filePath = verificationFilePath(scanner); | |||
File labelJsonFile = verificationFile(filePath); | |||
DataLabelGroup dataLabelGroup = saveDataLabelGroup(HandleFileUtil.getLabelGroupName(labelJsonFile.getName()), dataset); | |||
List<DataLabel> dataLabelList = readLabelContext(labelJsonFile); | |||
saveDataLabel(dataset, dataLabelList, dataLabelGroup.getId()); | |||
executeUploadAndSave(dataLabelList, filePath, dataset); | |||
datasetService.updateDatasetStatus(dataset); | |||
log.warn(""); | |||
PrintUtils.printLine(" Success: 执行成功 ", PrintUtils.GREEN); | |||
log.warn(""); | |||
log.warn("# 是否结束? Y / N #"); | |||
Scanner scannerExit = new Scanner(System.in); | |||
if (BusinessConstant.Y.toLowerCase().equals(scannerExit.nextLine().toLowerCase())) { | |||
System.exit(0); | |||
} | |||
} | |||
/** | |||
* 检查文件结构 、类型 | |||
* | |||
* @param globalFilePath 文件路径 | |||
* @return file 标签文件 | |||
*/ | |||
public File verificationFile(String globalFilePath) throws IOException { | |||
File labelRootFiles = new File(globalFilePath); | |||
File imageRootFiles = new File(globalFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN)); | |||
File annotationRootFiles = new File(globalFilePath + HandleFileUtil.generateFilePath(BusinessConstant.ANNOTATION)); | |||
if (imageRootFiles.list() == null || annotationRootFiles.listFiles() == null) { | |||
throw new ImportDatasetException(" 【" + globalFilePath + "】目录中的图片目录(origin)或者标注文件目录(annotation)的文件夹为空 "); | |||
} | |||
File labelJsonFile = null; | |||
for (File file : Objects.requireNonNull(labelRootFiles.listFiles())) { | |||
if (file.isFile() && file.getName().toLowerCase().startsWith(BusinessConstant.LABEL.toLowerCase()) | |||
&& file.getName().contains(BusinessConstant.UNDERLINE) | |||
&& file.getName().toLowerCase().endsWith(BusinessConstant.SUFFIX_JSON.toLowerCase())) { | |||
labelJsonFile = file; | |||
} | |||
} | |||
if (labelJsonFile == null) { | |||
throw new ImportDatasetException(" 【" + globalFilePath + "】目录中未找到标签组文件"); | |||
} | |||
dealLabelGroup(labelJsonFile.getName()); | |||
List<DataLabel> dataLabelList = readLabelContext(labelJsonFile); | |||
Map<String, List<DataLabel>> dataLabelMap = dataLabelList.stream().collect(Collectors.groupingBy(DataLabel::getName)); | |||
for (Map.Entry<String, List<DataLabel>> entry : dataLabelMap.entrySet()) { | |||
if (entry.getValue().size() > 1) { | |||
throw new ImportDatasetException(" 标签组中标签存在重复标签:【" + entry.getKey() + "】"); | |||
} | |||
} | |||
File[] imageFiles = imageRootFiles.listFiles(); | |||
if (imageFiles == null || imageFiles.length == 0) { | |||
throw new ImportDatasetException(" 图片文件下不存在图片文件 "); | |||
} | |||
for (File imageFile : imageFiles) { | |||
String suffixFileName = imageFile.getName().substring(imageFile.getName().lastIndexOf(BusinessConstant.SPOT)); | |||
if (!SUFFIX_LIST.contains(suffixFileName.toLowerCase())) { | |||
throw new ImportDatasetException(" 图片文件文件夹中存在非法格式 "); | |||
} | |||
} | |||
File[] annotationFiles = annotationRootFiles.listFiles(); | |||
if (annotationFiles == null || annotationFiles.length == 0) { | |||
throw new ImportDatasetException(" 图片文件下不存在标注文件 "); | |||
} | |||
for (File annotationFile : annotationFiles) { | |||
if (!annotationFile.getName().toLowerCase().endsWith(BusinessConstant.SUFFIX_JSON.toLowerCase())) { | |||
throw new ImportDatasetException(" 标注文件文件夹中存在非法格式 "); | |||
} | |||
} | |||
return labelJsonFile; | |||
} | |||
/** | |||
* 启动线程 | |||
* | |||
* @param filePath 本地根目录 | |||
* @param dataset 数据集 | |||
*/ | |||
public void executeUploadAndSave(List<DataLabel> dataLabelList, String filePath, Dataset dataset) throws Exception { | |||
String localImageFilePath = filePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN); | |||
List<String> imageFileNameList = FileUtil.listFileNames(localImageFilePath); | |||
log.info("需要处理: 【" + imageFileNameList.size() + "】张图片"); | |||
int oneSize = ThreadUtils.createThread(imageFileNameList.size()); | |||
log.info("需要创建线程数: 【" + oneSize + "】 条"); | |||
List<Callable<Integer>> partitions = new ArrayList<>(); | |||
List<String> need = new ArrayList<>(); | |||
AtomicInteger atomicInteger = new AtomicInteger(MagicNumConstant.ZERO); | |||
for (String fileName : imageFileNameList) { | |||
need.add(fileName); | |||
if (need.size() == oneSize || atomicInteger.intValue() == imageFileNameList.size() - MagicNumConstant.ONE) { | |||
List<String> fileNameList = new ArrayList<>(need); | |||
need.clear(); | |||
partitions.add(() -> runTask(dataLabelList, dataset, fileNameList, filePath)); | |||
} | |||
atomicInteger.getAndIncrement(); | |||
} | |||
ThreadUtils.runMultiThread(partitions); | |||
} | |||
/** | |||
* 实际执行任务 | |||
* | |||
* @param dataset 数据集 | |||
* @param fileNameList 文件名字集合 | |||
* @param dataSetRootFilePath 文件路径 | |||
* @return Integer 执行次数 | |||
*/ | |||
private Integer runTask(List<DataLabel> dataLabelList, Dataset dataset, List<String> fileNameList, String dataSetRootFilePath) throws Exception { | |||
List<DataFile> dataFilesList = new ArrayList<>(); | |||
AtomicInteger atomicInteger = new AtomicInteger(MagicNumConstant.ZERO); | |||
String imageFileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + dataset.getId() | |||
+ BusinessConstant.FILE_SEPARATOR + BusinessConstant.IMAGE_ORIGIN + BusinessConstant.FILE_SEPARATOR; | |||
String annotationFileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + dataset.getId() | |||
+ BusinessConstant.FILE_SEPARATOR + BusinessConstant.ANNOTATION + BusinessConstant.FILE_SEPARATOR; | |||
for (String fileName : fileNameList) { | |||
String imageUploadFile = imageFileBaseDir + fileName; | |||
String annotationFileName = HandleFileUtil.readFileName(fileName); | |||
File annotationFile = new File(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.ANNOTATION) + BusinessConstant.FILE_SEPARATOR + annotationFileName + BusinessConstant.SUFFIX_JSON.toLowerCase()); | |||
JSONArray jsonArray = replaceJsonNode(annotationFile, dataLabelList); | |||
minioUtil.upLoadFile(imageUploadFile, FileUtil.getInputStream(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN) + BusinessConstant.FILE_SEPARATOR + fileName)); | |||
minioUtil.upLoadFile(annotationFileBaseDir + annotationFileName, IOUtils.toInputStream(jsonArray.toString(), StandardCharsets.UTF_8.name())); | |||
DataFile dataFile = new DataFile(); | |||
dataFile.setName(annotationFileName); | |||
dataFile.setUrl(minioConfig.getBucketName() + BusinessConstant.FILE_SEPARATOR + imageUploadFile); | |||
dataFile.setStatus(FileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE); | |||
dataFile.setDatasetId(dataset.getId()); | |||
dataFile.setFileType(MagicNumConstant.ZERO); | |||
dataFile.setPid(MagicNumConstant.ZERO_LONG); | |||
dataFile.setCreateUserId(dataset.getCreateUserId()); | |||
try { | |||
BufferedImage image = ImageIO.read(new File(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN) + BusinessConstant.FILE_SEPARATOR + fileName)); | |||
dataFile.setWidth(image.getWidth()); | |||
dataFile.setHeight(image.getHeight()); | |||
} catch (IOException e) { | |||
throw new ImportDatasetException(" 读取图片高和宽失败 "); | |||
} | |||
dataFile.setOriginUserId(dataset.getCreateUserId()); | |||
dataFilesList.add(dataFile); | |||
if (dataFilesList.size() % MagicNumConstant.FIVE_HUNDRED == MagicNumConstant.ZERO || atomicInteger.intValue() == fileNameList.size() - MagicNumConstant.ONE) { | |||
long startDataFileIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_FILE, dataFilesList.size()); | |||
for (DataFile dataFileEntity : dataFilesList) { | |||
dataFileEntity.setId(startDataFileIndex++); | |||
} | |||
saveDataFile(dataFilesList); | |||
List<DataVersionFile> dataVersionFileList = new ArrayList<>(); | |||
for (DataFile file : dataFilesList) { | |||
DataVersionFile dataVersionFile = new DataVersionFile(); | |||
dataVersionFile.setDatasetId(dataset.getId()); | |||
dataVersionFile.setFileId(file.getId()); | |||
dataVersionFile.setStatus(MagicNumConstant.ZERO); | |||
dataVersionFile.setAnnotationStatus(FileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE); | |||
dataVersionFileList.add(dataVersionFile); | |||
} | |||
long startDataFileVersionIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_VERSION_FILE, dataVersionFileList.size()); | |||
for (DataVersionFile dataVersionFile : dataVersionFileList) { | |||
dataVersionFile.setId(startDataFileVersionIndex++); | |||
} | |||
saveDataVersionFile(dataVersionFileList); | |||
dataVersionFileList.clear(); | |||
dataFilesList.clear(); | |||
} | |||
atomicInteger.getAndIncrement(); | |||
} | |||
return atomicInteger.getAndIncrement(); | |||
} | |||
/** | |||
* 检查并且替换JSON中的节点 | |||
* | |||
* @param annotationFile 标注文件 | |||
* @param dataLabelList 数据集集合 | |||
*/ | |||
public JSONArray replaceJsonNode(File annotationFile, List<DataLabel> dataLabelList) throws IOException { | |||
JSONArray jsonArray = new JSONArray(); | |||
if (annotationFile.exists()) { | |||
String annotationFileContext = HandleFileUtil.readFile(annotationFile); | |||
jsonArray = JSONArray.parseArray(annotationFileContext); | |||
if (!jsonArray.isEmpty()) { | |||
replaceAllNode(jsonArray, dataLabelList); | |||
} | |||
} | |||
return jsonArray; | |||
} | |||
/** | |||
* 替换节点值 | |||
* | |||
* @param jsonArray 标注文件集合 | |||
* @param dataLabelList 标签集合 | |||
*/ | |||
public void replaceAllNode(JSONArray jsonArray, List<DataLabel> dataLabelList) { | |||
for (int i = MagicNumConstant.ZERO; i < jsonArray.size(); i++) { | |||
JSONObject jsonObject = jsonArray.getJSONObject(i); | |||
jsonObject.put("category_id", findDataLabelId(dataLabelList, jsonObject.get("name").toString())); | |||
jsonObject.remove("name"); | |||
} | |||
} | |||
/** | |||
* 查询需要替换的节点 | |||
* | |||
* @param dataLabelList 标签集合 | |||
* @param objectValue 替换的节点值 | |||
* @return long 替换标签的Id | |||
*/ | |||
public long findDataLabelId(List<DataLabel> dataLabelList, String objectValue) { | |||
Optional<DataLabel> matchedDataLabel = dataLabelList.stream().filter(dataLabel -> objectValue.equals(dataLabel.getName())).findAny(); | |||
if (!matchedDataLabel.isPresent()) { | |||
throw new ImportDatasetException(" 标注文件中name的值不存在于标签中!"); | |||
} | |||
return matchedDataLabel.get().getId(); | |||
} | |||
/** | |||
* 获取JSON中所有的key | |||
* | |||
* @param jsonArray json集合 | |||
* @return JSON中所有的key | |||
*/ | |||
public List<String> getJsonKeyList(JSONArray jsonArray) { | |||
List<String> listKey = new ArrayList<>(); | |||
for (Object object : jsonArray) { | |||
LinkedHashMap<String, String> jsonMap = JSON.parseObject(object.toString(), new TypeReference<LinkedHashMap<String, String>>() { | |||
}); | |||
for (Map.Entry<String, String> entry : jsonMap.entrySet()) { | |||
listKey.add(entry.getKey()); | |||
} | |||
} | |||
return listKey.stream().distinct().collect(Collectors.toList()); | |||
} | |||
/** | |||
* 读取标签文件中标签数据 | |||
* | |||
* @param file 标签文件 | |||
* @return List<DataLabel> 标签数据集合 | |||
*/ | |||
public List<DataLabel> readLabelContext(File file) throws IOException { | |||
String fileContext = HandleFileUtil.readFile(file); | |||
List<DataLabel> dataLabelList = JSONArray.parseArray(fileContext, DataLabel.class); | |||
for (DataLabel dataLabel : dataLabelList) { | |||
if (StringUtils.isEmpty(dataLabel.getName()) || StringUtils.isEmpty(dataLabel.getColor())) { | |||
throw new ImportDatasetException(" 标签文件不规范,未能读到 'name' 或者 'color' "); | |||
} | |||
} | |||
return dataLabelList; | |||
} | |||
/** | |||
* 检查标签组是否有重名 | |||
* | |||
* @param labelGroupName 标签组名称 | |||
*/ | |||
public void dealLabelGroup(String labelGroupName) { | |||
String groupName = HandleFileUtil.getLabelGroupName(labelGroupName); | |||
int count = dataLabelGroupService.selectByLabelGroupName(groupName); | |||
if (count > MagicNumConstant.ZERO) { | |||
throw new ImportDatasetException(" 标签组名称【" + groupName + "】已存在,请修改label_xxx.json文件名 "); | |||
} | |||
} | |||
/** | |||
* 保存标签组名称 | |||
* | |||
* @param labelGroupName 标签组名称 | |||
* @param dataset 数据集 | |||
* @return DataLabelGroup 数据集标签组 | |||
*/ | |||
public DataLabelGroup saveDataLabelGroup(String labelGroupName, Dataset dataset) { | |||
long timeStamp = System.currentTimeMillis(); | |||
DataLabelGroup dataLabelGroup = new DataLabelGroup(); | |||
dataLabelGroup.setName(labelGroupName); | |||
dataLabelGroup.setOriginUserId(dataset.getCreateUserId()); | |||
dataLabelGroup.setType(MagicNumConstant.ZERO_LONG); | |||
dataLabelGroup.setCreateUserId(dataset.getCreateUserId()); | |||
dataLabelGroup.setCreateTime(new Timestamp(timeStamp)); | |||
dataLabelGroup.setUpdateTime(new Timestamp(timeStamp)); | |||
dataLabelGroupService.saveDataGroupLabel(dataLabelGroup); | |||
return dataLabelGroup; | |||
} | |||
/** | |||
* 批量保存标签 | |||
* | |||
* @param dataset 数据集 | |||
* @param listDataLabel 标签集合 | |||
* @param dataLabelGroupId 标签组Id | |||
*/ | |||
public void saveDataLabel(Dataset dataset, List<DataLabel> listDataLabel, Long dataLabelGroupId) { | |||
listDataLabel.forEach(dataLabel -> dataLabel.setCreateUserId(dataset.getCreateUserId())); | |||
dataLabelService.saveBatchDataLabel(listDataLabel); | |||
List<DatasetDataLabel> listDatasetDataLabel = new ArrayList<>(); | |||
for (DataLabel dataLabel : listDataLabel) { | |||
DatasetDataLabel datasetDataLabel = new DatasetDataLabel(); | |||
datasetDataLabel.setLabelId(dataLabel.getId()); | |||
datasetDataLabel.setDatasetId(dataset.getId()); | |||
listDatasetDataLabel.add(datasetDataLabel); | |||
} | |||
saveDatasetDataLabel(listDatasetDataLabel); | |||
List<DataGroupLabel> listDataGroupLabel = new ArrayList<>(); | |||
for (DatasetDataLabel datasetDataLabel : listDatasetDataLabel) { | |||
DataGroupLabel dataGroupLabel = new DataGroupLabel(); | |||
dataGroupLabel.setLabelId(datasetDataLabel.getId()); | |||
dataGroupLabel.setLabelGroupId(dataLabelGroupId); | |||
listDataGroupLabel.add(dataGroupLabel); | |||
} | |||
saveDatasetDataGroupLabel(listDataGroupLabel); | |||
} | |||
/** | |||
* 批量保存文件 | |||
* | |||
* @param listDataFile file集合 | |||
*/ | |||
public void saveDataFile(List<DataFile> listDataFile) { | |||
dataFileService.saveBatchDataFile(listDataFile); | |||
} | |||
/** | |||
* 批量保存文件版本数据 | |||
* | |||
* @param listDataVersionFile 文件版本数据 | |||
*/ | |||
public void saveDataVersionFile(List<DataVersionFile> listDataVersionFile) { | |||
dataVersionFileService.saveBatchDataFileVersion(listDataVersionFile); | |||
} | |||
/** | |||
* 批量保存标签与数据集关系表 | |||
* | |||
* @param listDatasetDataLabel 标签与数据集关系表 | |||
*/ | |||
public void saveDatasetDataLabel(List<DatasetDataLabel> listDatasetDataLabel) { | |||
datasetDataLabelService.saveBatchDatasetDataLabel(listDatasetDataLabel); | |||
} | |||
/** | |||
* 批量保存标签与标签组的关系 | |||
* | |||
* @param listDataGroupLabel 标签与标签组集合 | |||
*/ | |||
public void saveDatasetDataGroupLabel(List<DataGroupLabel> listDataGroupLabel) { | |||
dataGroupLabelService.saveDataGroupLabel(listDataGroupLabel); | |||
} | |||
/** | |||
* 查询数据集 | |||
* | |||
* @param datasetId 数据集Id | |||
* @return Dataset 根据数据集ID查询返回的数据集 | |||
*/ | |||
private Dataset findDataset(Long datasetId) { | |||
return datasetService.findDatasetById(datasetId); | |||
} | |||
/** | |||
* 校验数据集ID | |||
* | |||
* @param scanner 控制台输入参数 | |||
* @return Dataset 数据集 | |||
*/ | |||
public Dataset verificationDatasetId(Scanner scanner) { | |||
log.warn("# 请输入数据集ID #"); | |||
String datasetIdStr = scanner.nextLine(); | |||
long datasetId; | |||
try { | |||
datasetId = Long.parseLong(datasetIdStr.trim()); | |||
} catch (Exception e) { | |||
throw new ImportDatasetException(" 数据集ID非法,请重新输入 "); | |||
} | |||
Dataset dataset = findDataset(datasetId); | |||
if (dataset == null) { | |||
throw new ImportDatasetException(" 数据集ID不存在,请重新输入 "); | |||
} | |||
int countDataLabel = datasetService.findDataLabelById(dataset.getId()); | |||
int countDataFile = datasetService.findDataFileById(dataset.getId()); | |||
if (countDataLabel > MagicNumConstant.ZERO || countDataFile > MagicNumConstant.ZERO) { | |||
throw new ImportDatasetException(" 当前数据集文件已存在,请勿重新导入 "); | |||
} | |||
return dataset; | |||
} | |||
/** | |||
* 校验文件路径 | |||
* | |||
* @param scanner 输入控制台 | |||
* @return String 字符串 | |||
*/ | |||
public String verificationFilePath(Scanner scanner) { | |||
log.warn("# 请输入待导入数据集绝对路径地址 #"); | |||
String filePath = scanner.nextLine(); | |||
File file = new File(filePath.trim()); | |||
if (!file.exists()) { | |||
throw new ImportDatasetException(" 【" + filePath + "】 文件路径不存在,请重新输入"); | |||
} | |||
return filePath; | |||
} | |||
} |
@@ -0,0 +1,35 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service; | |||
import org.dubhe.datasetutil.domain.entity.DataFile; | |||
import java.util.List; | |||
/** | |||
* @description 文件服务 | |||
* @date 2020-09-17 | |||
*/ | |||
public interface DataFileService { | |||
/** | |||
* 批量写入文件内容 | |||
* | |||
* @param dataFiles 文件数据集合 | |||
*/ | |||
void saveBatchDataFile(List<DataFile> dataFiles); | |||
} |
@@ -0,0 +1,34 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service; | |||
import org.dubhe.datasetutil.domain.entity.DataGroupLabel; | |||
import java.util.List; | |||
/** | |||
* @description 标签与标签组的服务接口 | |||
* @date 2020-10-21 | |||
*/ | |||
public interface DataGroupLabelService { | |||
/** | |||
* 保存标签与标签组的关系 | |||
* | |||
* @param listDataGroupLabel 标签与标签组数据 | |||
*/ | |||
void saveDataGroupLabel(List<DataGroupLabel> listDataGroupLabel); | |||
} |
@@ -0,0 +1,40 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service; | |||
import org.dubhe.datasetutil.domain.entity.DataLabelGroup; | |||
/** | |||
* @description 数据集标签组服务接口 | |||
* @date 2020-10-14 | |||
*/ | |||
public interface DataLabelGroupService { | |||
/** | |||
* 根据标签组名查询 | |||
* | |||
* @param labelGroupName 标签组名称 | |||
* @return int 数量 | |||
*/ | |||
int selectByLabelGroupName(String labelGroupName); | |||
/** | |||
* 保存数据集标签组标签 | |||
* | |||
* @param dataLabelGroup 标签组名称 | |||
*/ | |||
void saveDataGroupLabel(DataLabelGroup dataLabelGroup); | |||
} |
@@ -0,0 +1,34 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service; | |||
import org.dubhe.datasetutil.domain.entity.DataLabel; | |||
import java.util.List; | |||
/** | |||
* @description 数据集标签服务接口 | |||
* @date 2020-10-14 | |||
*/ | |||
public interface DataLabelService { | |||
/** | |||
* 批量保存数据集标签 | |||
* | |||
* @param listDataLabel 数据集标签集合 | |||
*/ | |||
void saveBatchDataLabel(List<DataLabel> listDataLabel); | |||
} |
@@ -0,0 +1,56 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service; | |||
import org.dubhe.datasetutil.domain.entity.DataSequence; | |||
/** | |||
* @description 获取序列服务接口 | |||
* @date 2020-09-23 | |||
*/ | |||
public interface DataSequenceService { | |||
/** | |||
* 根据业务编码获取序列号 | |||
* | |||
* @param businessCode 业务编码 | |||
* @return DataSequence 数据索引 | |||
*/ | |||
DataSequence getSequence(String businessCode); | |||
/** | |||
* 根据业务编码更新起点 | |||
* | |||
* @param businessCode 业务编码 | |||
* @return int 数量 | |||
*/ | |||
int updateSequenceStart(String businessCode); | |||
/** | |||
* 检查表是否存在 | |||
* | |||
* @param tableName 表名 | |||
* @return boolean 存在结果 | |||
*/ | |||
boolean checkTableExist(String tableName); | |||
/** | |||
* 执行存储过程创建表 | |||
* | |||
* @param tableId 表ID | |||
*/ | |||
void createTable(String tableId); | |||
} |
@@ -0,0 +1,34 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service; | |||
import org.dubhe.datasetutil.domain.dto.DataVersionFile; | |||
import java.util.List; | |||
/** | |||
* @description 数据集文件关系服务 | |||
* @date 2020-9-17 | |||
*/ | |||
public interface DataVersionFileService { | |||
/** | |||
* 插入数据集文件数据 | |||
* | |||
* @param dataVersionFiles 数据集文件数据集合 | |||
*/ | |||
void saveBatchDataFileVersion(List<DataVersionFile> dataVersionFiles); | |||
} |
@@ -0,0 +1,34 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service; | |||
import org.dubhe.datasetutil.domain.entity.DatasetDataLabel; | |||
import java.util.List; | |||
/** | |||
* @description 数据集与标签关系服务接口 | |||
* @date 2020-10-14 | |||
*/ | |||
public interface DatasetDataLabelService { | |||
/** | |||
* 批量保存数据集与标签关系 | |||
* | |||
* @param listDatasetDataLabel 数据集标签集合 | |||
*/ | |||
void saveBatchDatasetDataLabel(List<DatasetDataLabel> listDatasetDataLabel); | |||
} |
@@ -0,0 +1,65 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service; | |||
import org.dubhe.datasetutil.domain.entity.Dataset; | |||
/** | |||
* @description 数据集服务 | |||
* @date 2020-9-17 | |||
*/ | |||
public interface DatasetService { | |||
/** | |||
* 根据数据集Id查询创建人Id | |||
* | |||
* @param datasetId 数据集Id | |||
* @return Dataset 数据集 | |||
*/ | |||
Dataset findCreateUserIdById(Long datasetId); | |||
/** | |||
* 根据ID查询数据集 | |||
* | |||
* @param datasetId 数据集Id | |||
* @return Dataset 数据集 | |||
*/ | |||
Dataset findDatasetById(Long datasetId); | |||
/** | |||
* 更新数据集状态 | |||
* | |||
* @param dataset 数据集 | |||
* @return int 数量 | |||
*/ | |||
int updateDatasetStatus(Dataset dataset); | |||
/** | |||
* 查询数据集标签数量 | |||
* | |||
* @param datasetId 数据集ID | |||
* @return int 数量 | |||
*/ | |||
int findDataLabelById(Long datasetId); | |||
/** | |||
* 查询数据集文件数量 | |||
* | |||
* @param datasetId 数据集ID | |||
* @return int 数量 | |||
*/ | |||
int findDataFileById(Long datasetId); | |||
} |
@@ -0,0 +1,60 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service.impl; | |||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; | |||
import com.google.common.collect.Lists; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.dubhe.datasetutil.common.constant.BusinessConstant; | |||
import org.dubhe.datasetutil.dao.DataFileMapper; | |||
import org.dubhe.datasetutil.domain.entity.DataFile; | |||
import org.dubhe.datasetutil.service.DataFileService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
import java.util.List; | |||
/** | |||
* @description 数据集文件 服务实现类 | |||
* @date 2020-09-17 | |||
*/ | |||
@Slf4j | |||
@Service | |||
public class DataFileServiceImpl extends ServiceImpl<DataFileMapper, DataFile> implements DataFileService { | |||
@Autowired | |||
private DataFileMapper dataFileMapper; | |||
/** | |||
* 批量写入文件内容 | |||
* | |||
* @param listDataFile 文件数据集合 | |||
*/ | |||
@Override | |||
public void saveBatchDataFile(List<DataFile> listDataFile) { | |||
int listSize = listDataFile.size(); | |||
if (listSize > BusinessConstant.SUB_LENGTH) { | |||
List<List<DataFile>> partitionList = Lists.partition(listDataFile, BusinessConstant.SUB_LENGTH); | |||
for (List<DataFile> subPartitionList : partitionList) { | |||
dataFileMapper.saveBatchDataFile(subPartitionList); | |||
} | |||
} else { | |||
dataFileMapper.saveBatchDataFile(listDataFile); | |||
} | |||
} | |||
} |
@@ -0,0 +1,57 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service.impl; | |||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; | |||
import com.google.common.collect.Lists; | |||
import org.dubhe.datasetutil.common.constant.BusinessConstant; | |||
import org.dubhe.datasetutil.dao.DataGroupLabelMapper; | |||
import org.dubhe.datasetutil.domain.entity.DataGroupLabel; | |||
import org.dubhe.datasetutil.service.DataGroupLabelService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
import java.util.List; | |||
/** | |||
* @description 标签与标签组的服务接口实现 | |||
* @date 2020-10-21 | |||
*/ | |||
@Service | |||
public class DataGroupLabelServiceImpl extends ServiceImpl<DataGroupLabelMapper, DataGroupLabel> implements DataGroupLabelService { | |||
@Autowired | |||
private DataGroupLabelMapper dataGroupLabelMapper; | |||
/** | |||
* 插入标签组 | |||
* | |||
* @param listDataGroupLabel 标签组数据 | |||
*/ | |||
@Override | |||
public void saveDataGroupLabel(List<DataGroupLabel> listDataGroupLabel) { | |||
int listSize = listDataGroupLabel.size(); | |||
if (listSize > BusinessConstant.SUB_LENGTH) { | |||
List<List<DataGroupLabel>> partitionList = Lists.partition(listDataGroupLabel, BusinessConstant.SUB_LENGTH); | |||
for (List<DataGroupLabel> subPartitionList : partitionList) { | |||
dataGroupLabelMapper.saveDataGroupLabel(subPartitionList); | |||
} | |||
} else { | |||
dataGroupLabelMapper.saveDataGroupLabel(listDataGroupLabel); | |||
} | |||
} | |||
} |
@@ -0,0 +1,60 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service.impl; | |||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; | |||
import org.dubhe.datasetutil.dao.DataGroupLabelMapper; | |||
import org.dubhe.datasetutil.dao.DataLabelGroupMapper; | |||
import org.dubhe.datasetutil.domain.entity.DataGroupLabel; | |||
import org.dubhe.datasetutil.domain.entity.DataLabelGroup; | |||
import org.dubhe.datasetutil.service.DataLabelGroupService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
import org.springframework.transaction.annotation.Transactional; | |||
/** | |||
* @description 数据集标签组服务实现接口 | |||
* @date 2020-10-14 | |||
*/ | |||
@Service | |||
public class DataLabelGroupServiceImpl extends ServiceImpl<DataGroupLabelMapper, DataGroupLabel> implements DataLabelGroupService { | |||
@Autowired | |||
private DataLabelGroupMapper dataLabelGroupMapper; | |||
/** | |||
* 根据标签组名查询 | |||
* | |||
* @param labelGroupName 标签组名称 | |||
* @return int 数量 | |||
*/ | |||
@Override | |||
public int selectByLabelGroupName(String labelGroupName) { | |||
return dataLabelGroupMapper.selectByLabelGroupName(labelGroupName); | |||
} | |||
/** | |||
* 保存标签组名称 | |||
* | |||
* @param dataLabelGroup 标签组名称 | |||
*/ | |||
@Override | |||
@Transactional(rollbackFor = Exception.class) | |||
public void saveDataGroupLabel(DataLabelGroup dataLabelGroup) { | |||
dataLabelGroupMapper.insert(dataLabelGroup); | |||
} | |||
} |
@@ -0,0 +1,56 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service.impl; | |||
import com.google.common.collect.Lists; | |||
import org.dubhe.datasetutil.common.constant.BusinessConstant; | |||
import org.dubhe.datasetutil.dao.DataLabelMapper; | |||
import org.dubhe.datasetutil.domain.entity.DataLabel; | |||
import org.dubhe.datasetutil.service.DataLabelService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
import java.util.List; | |||
/** | |||
* @description 数据集标签服务接口实现 | |||
* @date 2020-10-14 | |||
*/ | |||
@Service | |||
public class DataLabelServiceImpl implements DataLabelService { | |||
@Autowired | |||
private DataLabelMapper dataLabelMapper; | |||
/** | |||
* 批量保存数据集标签 (分批) | |||
* | |||
* @param listDataLabel 数据集标签 | |||
*/ | |||
@Override | |||
public void saveBatchDataLabel(List<DataLabel> listDataLabel) { | |||
int listSize = listDataLabel.size(); | |||
if(listSize > BusinessConstant.SUB_LENGTH){ | |||
List<List<DataLabel>> partitionList = Lists.partition(listDataLabel,BusinessConstant.SUB_LENGTH); | |||
for (List<DataLabel> subPartitionList: partitionList) { | |||
dataLabelMapper.saveBatchDataLabel(subPartitionList); | |||
} | |||
}else{ | |||
dataLabelMapper.saveBatchDataLabel(listDataLabel); | |||
} | |||
} | |||
} |
@@ -0,0 +1,78 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service.impl; | |||
import org.dubhe.datasetutil.dao.DataSequenceMapper; | |||
import org.dubhe.datasetutil.domain.entity.DataSequence; | |||
import org.dubhe.datasetutil.service.DataSequenceService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
/** | |||
* @description 获取序列服务接口实现 | |||
* @date 2020-09-23 | |||
*/ | |||
@Service | |||
public class DataSequenceServiceImpl implements DataSequenceService { | |||
@Autowired | |||
private DataSequenceMapper dataSequenceMapper; | |||
@Override | |||
public DataSequence getSequence(String businessCode) { | |||
return dataSequenceMapper.selectByBusiness(businessCode); | |||
} | |||
/** | |||
* 更新数据索引 | |||
* | |||
* @param businessCode 业务编码 | |||
* @return int 数量 | |||
*/ | |||
@Override | |||
public int updateSequenceStart(String businessCode) { | |||
return dataSequenceMapper.updateStartByBusinessCode(businessCode); | |||
} | |||
/** | |||
* 检查表名 | |||
* | |||
* @param tableName 表名 | |||
* @return true/false | |||
*/ | |||
@Override | |||
public boolean checkTableExist(String tableName) { | |||
try { | |||
dataSequenceMapper.checkTableExist(tableName); | |||
return true; | |||
}catch (Exception e){ | |||
return false; | |||
} | |||
} | |||
/** | |||
* 创建表 | |||
* | |||
* @param tableName 类型名称 | |||
*/ | |||
@Override | |||
public void createTable(String tableName) { | |||
String oldTableName = tableName.substring(0,tableName.lastIndexOf("_")); | |||
dataSequenceMapper.createNewTable(tableName,oldTableName); | |||
} | |||
} |
@@ -0,0 +1,44 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service.impl; | |||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; | |||
import org.dubhe.datasetutil.dao.DataVersionFileMapper; | |||
import org.dubhe.datasetutil.domain.dto.DataVersionFile; | |||
import org.dubhe.datasetutil.service.DataVersionFileService; | |||
import org.springframework.stereotype.Service; | |||
import java.util.List; | |||
/** | |||
* @description 数据集文件 服务实现类 | |||
* @date 2020-09-17 | |||
*/ | |||
@Service | |||
public class DataVersionFileServiceImpl extends ServiceImpl <DataVersionFileMapper, DataVersionFile> implements DataVersionFileService { | |||
/** | |||
* 插入数据集文件数据 | |||
* | |||
* @param listDataVersionFile 数据集文件数据集合 | |||
*/ | |||
@Override | |||
public void saveBatchDataFileVersion(List<DataVersionFile> listDataVersionFile) { | |||
baseMapper.saveBatchDataFileVersion(listDataVersionFile); | |||
} | |||
} |
@@ -0,0 +1,57 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service.impl; | |||
import com.google.common.collect.Lists; | |||
import org.dubhe.datasetutil.common.base.MagicNumConstant; | |||
import org.dubhe.datasetutil.dao.DatasetDataLabelMapper; | |||
import org.dubhe.datasetutil.domain.entity.DatasetDataLabel; | |||
import org.dubhe.datasetutil.service.DatasetDataLabelService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
import java.util.List; | |||
/** | |||
* @description 数据集和标签服务实现类 | |||
* @date 2020-10-14 | |||
*/ | |||
@Service | |||
public class DatasetDataLabelServiceImpl implements DatasetDataLabelService { | |||
@Autowired | |||
private DatasetDataLabelMapper datasetDataLabelMapper; | |||
private static final int SUB_LENGTH = MagicNumConstant.FIVE_THOUSAND; | |||
/** | |||
* 批量保存数据集标签 | |||
* | |||
* @param listDatasetDataLabel 数据集标签集合 | |||
*/ | |||
@Override | |||
public void saveBatchDatasetDataLabel(List<DatasetDataLabel> listDatasetDataLabel) { | |||
if(listDatasetDataLabel.size() > SUB_LENGTH){ | |||
List<List<DatasetDataLabel>> partitionList = Lists.partition(listDatasetDataLabel,SUB_LENGTH); | |||
for (List<DatasetDataLabel> subPartitionList: partitionList) { | |||
datasetDataLabelMapper.saveBatchDatasetDataLabel(subPartitionList); | |||
} | |||
}else{ | |||
datasetDataLabelMapper.saveBatchDatasetDataLabel(listDatasetDataLabel); | |||
} | |||
} | |||
} |
@@ -0,0 +1,92 @@ | |||
/** | |||
* Copyright 2020 Zhejiang Lab. All Rights Reserved. | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
* ============================================================= | |||
*/ | |||
package org.dubhe.datasetutil.service.impl; | |||
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; | |||
import org.dubhe.datasetutil.common.constant.DataStateCodeConstant; | |||
import org.dubhe.datasetutil.dao.DatasetMapper; | |||
import org.dubhe.datasetutil.domain.entity.Dataset; | |||
import org.dubhe.datasetutil.service.DatasetService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
/** | |||
* @description 数据集 服务实现类 | |||
* @date 2020-09-17 | |||
*/ | |||
@Service | |||
public class DatasetServiceImpl extends ServiceImpl<DatasetMapper, Dataset> implements DatasetService { | |||
@Autowired | |||
private DatasetMapper datasetMapper; | |||
/** | |||
* 根据数据集Id查询创建人Id | |||
* | |||
* @param datasetId 数据集Id | |||
* @return dataset 数据集合 | |||
*/ | |||
@Override | |||
public Dataset findCreateUserIdById(Long datasetId) { | |||
return baseMapper.selectById(datasetId); | |||
} | |||
/** | |||
* 根据ID 查询数据集 | |||
* | |||
* @param datasetId 数据集ID | |||
* @return Dataset 数据集 | |||
*/ | |||
@Override | |||
public Dataset findDatasetById(Long datasetId) { | |||
return datasetMapper.findDatasetById(datasetId); | |||
} | |||
/** | |||
* 更新数据集状态 | |||
* | |||
* @param dataset 数据集 | |||
* @return int 数量 | |||
*/ | |||
@Override | |||
public int updateDatasetStatus(Dataset dataset) { | |||
dataset.setStatus(DataStateCodeConstant.ANNOTATION_COMPLETE_STATE); | |||
return datasetMapper.updateById(dataset); | |||
} | |||
/** | |||
* 查询数据集数据标签数量 | |||
* | |||
* @param datasetId 数据集ID | |||
* @return int 数量 | |||
*/ | |||
@Override | |||
public int findDataLabelById(Long datasetId) { | |||
return datasetMapper.findDataLabelById(datasetId); | |||
} | |||
/** | |||
* 查询数据集文件数量 | |||
* | |||
* @param datasetId 数据集ID | |||
* @return int 数量 | |||
*/ | |||
@Override | |||
public int findDataFileById(Long datasetId) { | |||
return datasetMapper.findDataFileById(datasetId); | |||
} | |||
} |
@@ -0,0 +1,254 @@ | |||
<?xml version="1.0" encoding="UTF-8"?> | |||
<configuration scan="true" scanPeriod="30 seconds" | |||
debug="false"> | |||
<contextName>dubhe</contextName> | |||
<springProperty scope="context" name="app.active" | |||
source="spring.profiles.active" /> | |||
<property name="log.path" value="dubhe-${app.active}" /> | |||
<property name="log.pattern" | |||
value=" %gray(%msg%n)" /> | |||
<!--输出到控制台 --> | |||
<appender name="console" | |||
class="ch.qos.logback.core.ConsoleAppender"> | |||
<encoder> | |||
<pattern>${log.pattern}</pattern> | |||
</encoder> | |||
<filter class="org.dubhe.datasetutil.common.filter.ConsoleLogFilter"> | |||
<level>WARN</level> | |||
<name>INFO</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="info_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/info/dubhe-info.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>INFO</level> | |||
<name>INFO,K8S_CALLBACK</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="debug_info" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/debug/dubhe-debug.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>DEBUG</level> | |||
<name>DEBUG</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="error_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/error/dubhe-error.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>ERROR</level> | |||
<name>ERROR</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="warn_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/warn/dubhe-warn.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>WARN</level> | |||
<name>WARN</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="trace_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/trace/dubhe-trace.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>TRACE</level> | |||
<name>TRACE</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="schedule_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/info/dubhe-schedule.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-schedule-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>INFO</level> | |||
<name>SCHEDULE</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="global_request_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/info/dubhe-request.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-request-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.GlobalRequestLogFilter"> | |||
<level>INFO</level> | |||
<!-- name必须配置,不然没有日志,注意看GlobalLogFilter --> | |||
<name>GLOBAL_REQUEST</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!--普通日志输出到控制台 --> | |||
<root level="info"> | |||
<appender-ref ref="console" /> | |||
<appender-ref ref="info_file" /> | |||
<appender-ref ref="schedule_file"/> | |||
<appender-ref ref="global_request_file"/> | |||
<appender-ref ref="debug_info" /> | |||
<appender-ref ref="error_file" /> | |||
<appender-ref ref="warn_file" /> | |||
<appender-ref ref="trace_file" /> | |||
</root> | |||
<!--监控sql日志输出 --> | |||
<!-- <logger name="jdbc.sqlonly" level="OFF" additivity="false"> --> | |||
<logger name="jdbc.sqlonly" level="INFO" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="jdbc.resultset" level="ERROR" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<!-- 如想看到表格数据,将OFF改为INFO --> | |||
<logger name="jdbc.resultsettable" level="OFF" | |||
additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="jdbc.connection" level="OFF" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="jdbc.sqltiming" level="OFF" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="jdbc.audit" level="OFF" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="org.dubhe.k8s" level="DEBUG" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
</configuration> |
@@ -0,0 +1,248 @@ | |||
<?xml version="1.0" encoding="UTF-8"?> | |||
<configuration scan="true" scanPeriod="30 seconds" | |||
debug="false"> | |||
<contextName>dubhe</contextName> | |||
<property name="log.charset" value="utf-8" /> | |||
<springProperty scope="context" name="app.active" | |||
source="spring.profiles.active" /> | |||
<property name="log.path" value="dubhe-${app.active}" /> | |||
<property name="log.pattern" | |||
value="%black(%contextName-) %red(%d{yyyy-MM-dd HH:mm:ss}) %green([%thread]) %highlight(%-5level) %boldMagenta(%logger{36}) [%X{traceId}] - %gray(%msg%n)" /> | |||
<!--输出到控制台 --> | |||
<appender name="console" | |||
class="ch.qos.logback.core.ConsoleAppender"> | |||
<encoder> | |||
<pattern>${log.pattern}</pattern> | |||
</encoder> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="info_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/info/dubhe-info.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>INFO</level> | |||
<name>INFO,K8S_CALLBACK</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="debug_info" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/debug/dubhe-debug.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>DEBUG</level> | |||
<name>DEBUG</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="error_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/error/dubhe-error.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>ERROR</level> | |||
<name>ERROR</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="warn_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/warn/dubhe-warn.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>WARN</level> | |||
<name>WARN</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="trace_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/trace/dubhe-trace.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>TRACE</level> | |||
<name>TRACE</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="schedule_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/info/dubhe-schedule.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-schedule-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.BaseLogFilter"> | |||
<level>INFO</level> | |||
<name>SCHEDULE</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!-- 滚动记录文件,先将日志记录到指定文件,复合条件后日志将记录到其他文件 --> | |||
<appender name="global_request_file" | |||
class="ch.qos.logback.core.rolling.RollingFileAppender"> | |||
<file>logs/${log.path}/info/dubhe-request.log</file> | |||
<rollingPolicy | |||
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> | |||
<fileNamePattern>logs/${log.path}/info/dubhe-${app.active}-request-%d{yyyy-MM-dd}.%i.log | |||
</fileNamePattern> | |||
<!-- 单个日志文件最多50MB, 14天的日志周期,最大不能超过250MB --> | |||
<maxFileSize>50MB</maxFileSize> | |||
<maxHistory>7</maxHistory> | |||
<totalSizeCap>250MB</totalSizeCap> | |||
</rollingPolicy> | |||
<encoder> | |||
<pattern>%m%n</pattern> | |||
</encoder> | |||
<!-- 默认true日志追加到文件结尾,false清空现存文件 --> | |||
<append>true</append> | |||
<filter class="org.dubhe.datasetutil.common.filter.GlobalRequestLogFilter"> | |||
<level>INFO</level> | |||
<!-- name必须配置,不然没有日志,注意看GlobalLogFilter --> | |||
<name>GLOBAL_REQUEST</name> | |||
<onMatch>ACCEPT</onMatch> | |||
<onMismatch>DENY</onMismatch> | |||
</filter> | |||
</appender> | |||
<!--普通日志输出到控制台 --> | |||
<root level="info"> | |||
<appender-ref ref="console" /> | |||
<appender-ref ref="info_file" /> | |||
<appender-ref ref="schedule_file"/> | |||
<appender-ref ref="global_request_file"/> | |||
<appender-ref ref="debug_info" /> | |||
<appender-ref ref="error_file" /> | |||
<appender-ref ref="warn_file" /> | |||
<appender-ref ref="trace_file" /> | |||
</root> | |||
<!--监控sql日志输出 --> | |||
<!-- <logger name="jdbc.sqlonly" level="OFF" additivity="false"> --> | |||
<logger name="jdbc.sqlonly" level="INFO" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="jdbc.resultset" level="ERROR" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<!-- 如想看到表格数据,将OFF改为INFO --> | |||
<logger name="jdbc.resultsettable" level="OFF" | |||
additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="jdbc.connection" level="OFF" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="jdbc.sqltiming" level="OFF" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="jdbc.audit" level="OFF" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
<logger name="org.dubhe.k8s" level="DEBUG" additivity="false"> | |||
<appender-ref ref="console" /> | |||
</logger> | |||
</configuration> |
@@ -0,0 +1,22 @@ | |||
<?xml version="1.0" encoding="UTF-8" ?> | |||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > | |||
<mapper namespace="org.dubhe.datasetutil.dao.DataFileMapper"> | |||
<insert id="saveBatchDataFile" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id"> | |||
insert into data_file (id,`name`,dataset_id,status,url,enhance_type,width,height,origin_user_id,create_user_id,pid) | |||
values | |||
<foreach collection="listDataFile" item="file" separator=","> | |||
(#{file.id},#{file.name},#{file.datasetId},#{file.status}, | |||
#{file.url},#{file.enhanceType},#{file.width},#{file.height},#{file.originUserId},#{file.createUserId} | |||
<choose> | |||
<when test="file.pid != null"> | |||
,#{file.pid} | |||
</when> | |||
<otherwise> | |||
,0 | |||
</otherwise> | |||
</choose> | |||
) | |||
</foreach> | |||
</insert> | |||
</mapper> |
@@ -0,0 +1,12 @@ | |||
<?xml version="1.0" encoding="UTF-8" ?> | |||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > | |||
<mapper namespace="org.dubhe.datasetutil.dao.DataGroupLabelMapper"> | |||
<insert id="saveDataGroupLabel" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id"> | |||
insert into data_group_label (label_id,label_group_id) | |||
values | |||
<foreach collection="list" item="dataGroupLabel" separator=","> | |||
(#{dataGroupLabel.labelId},#{dataGroupLabel.labelGroupId}) | |||
</foreach> | |||
</insert> | |||
</mapper> |
@@ -0,0 +1,12 @@ | |||
<?xml version="1.0" encoding="UTF-8" ?> | |||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > | |||
<mapper namespace="org.dubhe.datasetutil.dao.DataLabelMapper"> | |||
<insert id="saveBatchDataLabel" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id"> | |||
insert into data_label (`name`,color,create_user_id) | |||
values | |||
<foreach collection="listDataLabel" item="dataLabel" separator=","> | |||
( #{dataLabel.name},#{dataLabel.color},#{dataLabel.createUserId}) | |||
</foreach> | |||
</insert> | |||
</mapper> |
@@ -0,0 +1,12 @@ | |||
<?xml version="1.0" encoding="UTF-8" ?> | |||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > | |||
<mapper namespace="org.dubhe.datasetutil.dao.DataVersionFileMapper"> | |||
<insert id="saveBatchDataFileVersion" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id"> | |||
insert into data_dataset_version_file (id,dataset_id,file_id,annotation_status,status) | |||
values | |||
<foreach collection="listDataVersionFile" item="temp" separator=","> | |||
(#{temp.id},#{temp.datasetId},#{temp.fileId},#{temp.annotationStatus},#{temp.status}) | |||
</foreach> | |||
</insert> | |||
</mapper> |
@@ -0,0 +1,12 @@ | |||
<?xml version="1.0" encoding="UTF-8" ?> | |||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > | |||
<mapper namespace="org.dubhe.datasetutil.dao.DatasetDataLabelMapper"> | |||
<insert id="saveBatchDatasetDataLabel" parameterType="java.util.List" useGeneratedKeys="true" keyProperty="id"> | |||
insert into data_dataset_label(dataset_id,label_id ) | |||
values | |||
<foreach collection="datasetDataLabelList" item="datasetDataLabel" separator=","> | |||
(#{datasetDataLabel.datasetId},#{datasetDataLabel.labelId}) | |||
</foreach> | |||
</insert> | |||
</mapper> |
@@ -0,0 +1,5 @@ | |||
<?xml version="1.0" encoding="UTF-8" ?> | |||
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > | |||
<mapper namespace="org.dubhe.datasetutil.dao.DatasetMapper"> | |||
</mapper> |