From 91d7963c97a37e702182bf2d750d4a16de98cb88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=8B=E6=B1=9F=E5=AE=9E=E9=AA=8C=E5=AE=A4?= Date: Mon, 26 Oct 2020 16:34:36 +0800 Subject: [PATCH] update dataset-util --- dataset-util/.gitignore | 6 + dataset-util/README.md | 94 ++++ dataset-util/pom.xml | 142 +++++ .../datasetutil/DatasetUtilApplication.java | 119 ++++ .../datasetutil/common/aspect/LogAspect.java | 76 +++ .../datasetutil/common/base/BaseEntity.java | 66 +++ .../common/base/DataResponseBody.java | 69 +++ .../common/base/MagicNumConstant.java | 103 ++++ .../datasetutil/common/base/ResponseCode.java | 34 ++ .../common/config/MinioConfig.java | 58 ++ .../common/config/MybatisPlusConfig.java | 37 ++ .../common/constant/BusinessConstant.java | 86 +++ .../constant/DataStateCodeConstant.java | 84 +++ .../constant/FileStateCodeConstant.java | 54 ++ .../datasetutil/common/enums/LogEnum.java | 52 ++ .../common/exception/BusinessException.java | 56 ++ .../exception/DataSequenceException.java | 31 ++ .../common/exception/ErrorCode.java | 39 ++ .../exception/ImportDatasetException.java | 36 ++ .../common/filter/BaseLogFilter.java | 82 +++ .../common/filter/ConsoleLogFilter.java | 59 ++ .../common/filter/GlobalRequestLogFilter.java | 40 ++ .../datasetutil/common/util/DateUtil.java | 45 ++ .../common/util/GeneratorKeyUtil.java | 118 ++++ .../common/util/HandleFileUtil.java | 91 +++ .../datasetutil/common/util/LogUtil.java | 300 ++++++++++ .../datasetutil/common/util/MinioUtil.java | 64 +++ .../util/MyPreciseShardingAlgorithm.java | 72 +++ .../datasetutil/common/util/PrintUtils.java | 156 ++++++ .../common/util/SpringContextHolder.java | 95 ++++ .../datasetutil/common/util/ThreadUtils.java | 72 +++ .../dubhe/datasetutil/dao/DataFileMapper.java | 36 ++ .../datasetutil/dao/DataGroupLabelMapper.java | 36 ++ .../datasetutil/dao/DataLabelGroupMapper.java | 38 ++ .../datasetutil/dao/DataLabelMapper.java | 36 ++ .../datasetutil/dao/DataSequenceMapper.java | 66 +++ .../dao/DataVersionFileMapper.java | 37 ++ .../dao/DatasetDataLabelMapper.java | 37 ++ .../dubhe/datasetutil/dao/DatasetMapper.java | 56 ++ .../domain/dto/DataVersionFile.java | 93 ++++ .../dubhe/datasetutil/domain/dto/IdAlloc.java | 50 ++ .../datasetutil/domain/entity/DataFile.java | 118 ++++ .../domain/entity/DataGroupLabel.java | 58 ++ .../datasetutil/domain/entity/DataLabel.java | 64 +++ .../domain/entity/DataLabelGroup.java | 70 +++ .../domain/entity/DataSequence.java | 57 ++ .../datasetutil/domain/entity/Dataset.java | 145 +++++ .../domain/entity/DatasetDataLabel.java | 58 ++ .../datasetutil/domain/entity/LogInfo.java | 74 +++ .../handle/DatasetImageUploadHandle.java | 140 +++++ .../handle/DatasetImportHandle.java | 522 ++++++++++++++++++ .../datasetutil/service/DataFileService.java | 35 ++ .../service/DataGroupLabelService.java | 34 ++ .../service/DataLabelGroupService.java | 40 ++ .../datasetutil/service/DataLabelService.java | 34 ++ .../service/DataSequenceService.java | 56 ++ .../service/DataVersionFileService.java | 34 ++ .../service/DatasetDataLabelService.java | 34 ++ .../datasetutil/service/DatasetService.java | 65 +++ .../service/impl/DataFileServiceImpl.java | 60 ++ .../impl/DataGroupLabelServiceImpl.java | 57 ++ .../impl/DataLabelGroupServiceImpl.java | 60 ++ .../service/impl/DataLabelServiceImpl.java | 56 ++ .../service/impl/DataSequenceServiceImpl.java | 78 +++ .../impl/DataVersionFileServiceImpl.java | 44 ++ .../impl/DatasetDataLabelServiceImpl.java | 57 ++ .../service/impl/DatasetServiceImpl.java | 92 +++ dataset-util/src/main/resources/banner.txt | 0 .../src/main/resources/logback-spring-dev.xml | 254 +++++++++ .../main/resources/logback-spring-test.xml | 248 +++++++++ .../main/resources/mapper/DataFileMapper.xml | 22 + .../resources/mapper/DataGroupLabelMapper.xml | 12 + .../main/resources/mapper/DataLabelMapper.xml | 12 + .../mapper/DataVersionFileMapper.xml | 12 + .../mapper/DatasetDataLabelMapper.xml | 12 + .../main/resources/mapper/DatasetMapper.xml | 5 + 76 files changed, 5640 insertions(+) create mode 100644 dataset-util/.gitignore create mode 100644 dataset-util/README.md create mode 100644 dataset-util/pom.xml create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/DatasetUtilApplication.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/aspect/LogAspect.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/base/BaseEntity.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/base/DataResponseBody.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/base/MagicNumConstant.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/base/ResponseCode.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MinioConfig.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MybatisPlusConfig.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/BusinessConstant.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/DataStateCodeConstant.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/FileStateCodeConstant.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/enums/LogEnum.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/BusinessException.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/DataSequenceException.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ErrorCode.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ImportDatasetException.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/BaseLogFilter.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/ConsoleLogFilter.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/GlobalRequestLogFilter.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/util/DateUtil.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/util/GeneratorKeyUtil.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/util/HandleFileUtil.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/util/LogUtil.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MinioUtil.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MyPreciseShardingAlgorithm.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/util/PrintUtils.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/util/SpringContextHolder.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/common/util/ThreadUtils.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataFileMapper.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataGroupLabelMapper.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelGroupMapper.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelMapper.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataSequenceMapper.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataVersionFileMapper.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetDataLabelMapper.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetMapper.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/DataVersionFile.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/IdAlloc.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataFile.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataGroupLabel.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabel.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabelGroup.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataSequence.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/Dataset.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DatasetDataLabel.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/LogInfo.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImageUploadHandle.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImportHandle.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/DataFileService.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/DataGroupLabelService.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelGroupService.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelService.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/DataSequenceService.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/DataVersionFileService.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetDataLabelService.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetService.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataFileServiceImpl.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataGroupLabelServiceImpl.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelGroupServiceImpl.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelServiceImpl.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataSequenceServiceImpl.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataVersionFileServiceImpl.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetDataLabelServiceImpl.java create mode 100644 dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetServiceImpl.java create mode 100644 dataset-util/src/main/resources/banner.txt create mode 100644 dataset-util/src/main/resources/logback-spring-dev.xml create mode 100644 dataset-util/src/main/resources/logback-spring-test.xml create mode 100644 dataset-util/src/main/resources/mapper/DataFileMapper.xml create mode 100644 dataset-util/src/main/resources/mapper/DataGroupLabelMapper.xml create mode 100644 dataset-util/src/main/resources/mapper/DataLabelMapper.xml create mode 100644 dataset-util/src/main/resources/mapper/DataVersionFileMapper.xml create mode 100644 dataset-util/src/main/resources/mapper/DatasetDataLabelMapper.xml create mode 100644 dataset-util/src/main/resources/mapper/DatasetMapper.xml diff --git a/dataset-util/.gitignore b/dataset-util/.gitignore new file mode 100644 index 0000000..df214c3 --- /dev/null +++ b/dataset-util/.gitignore @@ -0,0 +1,6 @@ +# Created by .ignore support plugin (hsz.mobi) +.idea +*.iml +logs +HELP.md +target/ diff --git a/dataset-util/README.md b/dataset-util/README.md new file mode 100644 index 0000000..124d4d9 --- /dev/null +++ b/dataset-util/README.md @@ -0,0 +1,94 @@ +# 之江天枢-数据集导入脚本 + +**之江天枢一站式人工智能开源平台**（简称：**之江天枢**）,为了实现其他平台已标注完成的数据集在「一站式开发平台」上进行开发，我们增加了数据集导入的功能，实现对数据集的全流程功能操作。 + +## 源码部署 + +安装如下软件环境。 +- OpenJDK：1.8+ + +## 下载脚本 + +- 数据集模板：http://tianshu.org.cn/static/upload/file/dubhe-dataset-template.zip +- 上传数据集脚本：http://tianshu.org.cn/static/upload/file/upload_dataset.zip + + +## 脚本使用说明： + +- 登录天枢系统深度学习平台，在数据管理菜单下的数据集管理中创建数据集。获取数据集ID +- 需要自行准备图片文件、标注文件、标签文件 + +## 运行脚本： + +1.解压下载的zip文件，需要自行配置数据源、MinIO相关配置 + +2.运行脚本Windows 运行 run.bat; macOS/Linux 系统运行 run.sh + +注：可自行配置'application-{env}.xml'文件，执行命令后面添加 'run.bat {env}'即可执行对应的application-{env}.xml；然后按提示进行操作 + +3.输入数据集ID + +4.输入待导入数据集绝对路径 + + +## 目录结构： + +``` +标签文件： label_{name}.json ({name} 代表标签组名，可自行定义; 只读标签组文件夹下的第一个标签组文件，标签文件仅支持:.json 支持大小写;文件内容为JSON字符串) +图片文件目录： origin (图片文件需要有后缀名，支持四种格式：.jpg,.png,.bmp,.jpeg 支持大小写) +标注文件目录： annotation (标注文件需要有后缀名，仅支持格式：.json 支持大小写; 文件内容为JSON字符串) +``` +## 文件格式 + +- 标签文件内容样例: +``` + name: 名称 + color: 颜色(16进制编码) +``` +详细示例： +``` +[{ + "name": "行人", + "color": "#ffbb96" +}, +{ + "name": "自行车", + "color": "#fcffe6" +}, +{ + "name": "汽车", + "color": "#f4ffb8" +}] +``` + +- 标注文件内容样例： +``` + name: 名称 + bbox: 标注位置 + score：分数 +``` +详细示例： +``` +[{ + "name": "行人", + "bbox": [321.6755762696266, 171.32076993584633, 185.67924201488495, 145.02639323472977], + "score": 0.6922634840011597 +}, +{ + "name": "自行车", + "bbox": [40.88740050792694, 22.707078605890274, 451.21362805366516, 326.0102793574333], + "score": 0.6069411635398865 +}] +``` + +## 了解更多 + +http://docs.dubhe.ai/docs/module/dataset/import-dataset + + + + + + + + diff --git a/dataset-util/pom.xml b/dataset-util/pom.xml new file mode 100644 index 0000000..171c082 --- /dev/null +++ b/dataset-util/pom.xml @@ -0,0 +1,142 @@ + + + 4.0.0 + org.dubhe + dataset-util + 0.0.1-SNAPSHOT + dataset-util + 数据处理模块工具 + + + 1.8 + UTF-8 + UTF-8 + 2.3.0.RELEASE + + + + + org.springframework.boot + spring-boot-starter + + + com.alibaba + druid + 1.1.6 + + + mysql + mysql-connector-java + 8.0.17 + + + org.projectlombok + lombok + true + + + com.baomidou + mybatis-plus-boot-starter + 3.2.0 + + + com.alibaba + fastjson + 1.2.47 + + + io.minio + minio + 7.0.2 + + + com.xiaoleilu + hutool-all + 3.0.1 + + + + + org.apache.shardingsphere + sharding-jdbc-spring-boot-starter + 4.0.0-RC1 + + + + org.apache.commons + commons-lang3 + + + + commons-io + commons-io + 1.3.2 + + + + org.aspectj + aspectjweaver + 1.8.9 + + + + + org.bgee.log4jdbc-log4j2 + log4jdbc-log4j2-jdbc4.1 + 1.16 + + + + + cn.hutool + hutool-all + 5.0.6 + + + + + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} + pom + import + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + UTF-8 + + + + org.springframework.boot + spring-boot-maven-plugin + 2.3.0.RELEASE + + org.dubhe.datasetutil.DatasetUtilApplication + + + + repackage + + repackage + + + + + + + + diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/DatasetUtilApplication.java b/dataset-util/src/main/java/org/dubhe/datasetutil/DatasetUtilApplication.java new file mode 100644 index 0000000..958b170 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/DatasetUtilApplication.java @@ -0,0 +1,119 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil; + +import lombok.extern.slf4j.Slf4j; +import org.dubhe.datasetutil.common.util.SpringContextHolder; +import org.dubhe.datasetutil.handle.DatasetImageUploadHandle; +import org.dubhe.datasetutil.handle.DatasetImportHandle; +import org.dubhe.datasetutil.common.util.PrintUtils; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.ApplicationContext; + +import java.util.Scanner; + +/** + * @description 文档导入工程启动类 + * @date 2020-09-17 + */ +@Slf4j +@SpringBootApplication +@MapperScan("org.dubhe.datasetutil.dao") +public class DatasetUtilApplication { + + /** + * 主函数 + * + * @param args 入参 + */ + public static void main(String[] args) { + ApplicationContext applicationContext = SpringApplication.run(DatasetUtilApplication.class, args); + SpringContextHolder springContextHolder = new SpringContextHolder(); + springContextHolder.setApplicationContext(applicationContext); + execute(applicationContext); + } + + /** + * 执行脚本 + * + * @param applicationContext 请求上下文 + */ + public static void execute(ApplicationContext applicationContext) { + while (true) { + Scanner scanner = new Scanner(System.in); + log.warn("###################请输入需要执行的任务#############"); + log.warn("# 输入1.执行上传图片 "); + log.warn("# 输入2.执行导入数据集 "); + log.warn("# 输入命令 ：exit 退出 "); + log.warn("################################################"); + String a = scanner.nextLine(); + switch (a) { + case "1": + uploadDatasetImage(scanner, applicationContext); + break; + case "2": + importDataset(scanner, applicationContext); + break; + case "exit": + default: + System.exit(0); + break; + } + } + } + + /** + * 导入图片 + * + * @param scanner 输入控制台 + * @param applicationContext 请求上下文 + */ + public static void uploadDatasetImage(Scanner scanner, ApplicationContext applicationContext) { + log.warn("# 请输入数据集ID #"); + String datasetIdStr = scanner.nextLine(); + Long datasetId = Long.parseLong(datasetIdStr); + log.warn("# 请输入要上传的图片地址 #"); + String filePath = scanner.nextLine(); + DatasetImageUploadHandle datasetImageUploadHandle = (DatasetImageUploadHandle) applicationContext.getBean("datasetImageUploadHandle"); + try { + datasetImageUploadHandle.execute(filePath, datasetId); + } catch (Exception e) { + log.error("", e); + log.error("# 数据集上传失败，请重新尝试....."); + } + } + + /** + * 导入数据集 + * + * @param scanner 输入控制台 + * @param applicationContext 请求上下文 + */ + public static void importDataset(Scanner scanner, ApplicationContext applicationContext) { + DatasetImportHandle datasetImportHandle = (DatasetImportHandle) applicationContext.getBean("datasetImportHandle"); + try{ + datasetImportHandle.importDataset(scanner); + } catch (Exception e) { + log.error(""); + PrintUtils.printLine(" Error：" + e.getMessage(), PrintUtils.RED); + log.error(""); + } + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/aspect/LogAspect.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/aspect/LogAspect.java new file mode 100644 index 0000000..60a3acc --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/aspect/LogAspect.java @@ -0,0 +1,76 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dubhe.datasetutil.common.aspect; + +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.dubhe.datasetutil.common.enums.LogEnum; +import org.dubhe.datasetutil.common.util.LogUtil; +import org.slf4j.MDC; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.util.UUID; + +/** + * @description 日志切面 + * @date 2020-04-10 + */ +@Component +@Aspect +@Slf4j +public class LogAspect { + + public static final String TRACE_ID = "traceId"; + + @Pointcut("execution(* org.dubhe..service..*.*(..))) ") + public void serviceAspect() { + } + + @Pointcut(" serviceAspect() ") + public void aroundAspect() { + } + + @Around("aroundAspect()") + public Object around(JoinPoint joinPoint) throws Throwable { + if (StringUtils.isEmpty(MDC.get(TRACE_ID))) { + MDC.put(TRACE_ID, UUID.randomUUID().toString()); + } + return combineLogInfo(joinPoint); + } + + /** + * 根据连接点返回结果 + * + * @param joinPoint 连接点 + * @return Object 返回结果 + */ + private Object combineLogInfo(JoinPoint joinPoint) throws Throwable { + Object[] param = joinPoint.getArgs(); + LogUtil.info(LogEnum.REST_REQ, "uri:{},input:{},==>begin", joinPoint.getSignature(), param); + long start = System.currentTimeMillis(); + Object result = ((ProceedingJoinPoint) joinPoint).proceed(); + long end = System.currentTimeMillis(); + LogUtil.info(LogEnum.REST_REQ, "uri:{},output:{},proc_time:{},<==end", joinPoint.getSignature().toString(), + result, end - start); + return result; + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/BaseEntity.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/BaseEntity.java new file mode 100644 index 0000000..c09fefe --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/BaseEntity.java @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.base; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableLogic; +import lombok.Data; +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description Entity基础类 + * @date 2020-03-15 + */ +@Data +public class BaseEntity implements Serializable { + + private static final long serialVersionUID = 4936056317364745513L; + + /** + * 删除标识 + **/ + @TableField(value = "deleted",fill = FieldFill.INSERT) + @TableLogic + private Boolean deleted = false; + + /** + * 创建人id + **/ + @TableField(value = "create_user_id",fill = FieldFill.INSERT) + private Long createUserId; + + /** + * 修改人id + **/ + @TableField(value = "update_user_id",fill = FieldFill.INSERT_UPDATE) + private Long updateUserId; + + /** + * 创建时间 + **/ + @TableField(value = "create_time",fill = FieldFill.INSERT) + private Timestamp createTime; + + /** + * 修改时间 + **/ + @TableField(value = "update_time",fill = FieldFill.INSERT_UPDATE) + private Timestamp updateTime; + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/DataResponseBody.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/DataResponseBody.java new file mode 100644 index 0000000..83ae2aa --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/DataResponseBody.java @@ -0,0 +1,69 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.base; + + +import lombok.Data; +import org.slf4j.MDC; + +import java.io.Serializable; + +/** + * @description 统一的公共响应体 + * @date 2020-03-16 + */ +@Data +public class DataResponseBody implements Serializable { + + /** + * 返回状态码 + */ + private Integer code; + /** + * 返回信息 + */ + private String msg; + /** + * 泛型数据 + */ + private T data; + /** + * 链路追踪ID + */ + private String traceId; + + public DataResponseBody() { + this(ResponseCode.SUCCESS, null); + } + + public DataResponseBody(T data) { + this(ResponseCode.SUCCESS, null, data); + } + + public DataResponseBody(Integer code, String msg) { + this(code, msg, null); + } + + public DataResponseBody(Integer code, String msg, T data) { + this.code = code; + this.msg = msg; + this.data = data; + this.traceId = MDC.get("traceId"); + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/MagicNumConstant.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/MagicNumConstant.java new file mode 100644 index 0000000..01472cb --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/MagicNumConstant.java @@ -0,0 +1,103 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.base; + +/** + * @description 魔数数值类 + * @date 2020-09-17 + */ +public final class MagicNumConstant { + + public static final int NEGATIVE_ONE = -1; + public static final int ZERO = 0; + public static final int ONE = 1; + public static final int TWO = 2; + public static final int THREE = 3; + public static final int FOUR = 4; + public static final int FIVE = 5; + public static final int SIX = 6; + public static final int SEVEN = 7; + public static final int EIGHT = 8; + public static final int NINE = 9; + public static final int TEN = 10; + + public static final int ELEVEN = 11; + public static final int SIXTEEN = 16; + public static final int TWENTY = 20; + public static final int FIFTY = 50; + public static final int SIXTY = 60; + public static final int SIXTY_TWO = 62; + public static final int SIXTY_FOUR = 64; + public static final int INTEGER_TWO_HUNDRED_AND_FIFTY_FIVE = 255; + public static final int ONE_HUNDRED = 100; + public static final int ONE_HUNDRED_TWENTY_EIGHT = 128; + public static final int TWO_HUNDRED = 200; + public static final int FIVE_HUNDRED = 500; + public static final int FIVE_HUNDRED_AND_SIXTEEN = 516; + public static final int ONE_THOUSAND = 1000; + public static final int BINARY_TEN_EXP = 1024; + public static final int ONE_THOUSAND_ONE_HUNDRED = 1100; + public static final int ONE_THOUSAND_ONE_HUNDRED_ONE = 1101; + public static final int ONE_THOUSAND_TWO_HUNDRED = 1200; + public static final int ONE_THOUSAND_TWO_HUNDRED_ONE = 1201; + public static final int ONE_THOUSAND_TWENTY_FOUR = 1024; + public static final int ONE_THOUSAND_THREE_HUNDRED = 1300; + public static final int ONE_THOUSAND_THREE_HUNDRED_ONE = 1301; + public static final int ONE_THOUSAND_THREE_HUNDRED_NINE = 1309; + public static final int ONE_THOUSAND_FIVE_HUNDRED = 1500; + public static final int TWO_THOUSAND = 2000; + public static final int TWO_THOUSAND_TWENTY_EIGHT = 2048; + public static final int THREE_THOUSAND = 3000; + public static final int FOUR_THOUSAND = 4000; + public static final int FIVE_THOUSAND = 5000; + public static final int NINE_THOUSAND = 9000; + public static final int NINE_THOUSAND_NINE_HUNDRED_NINTY_NINE = 9999; + public static final int TEN_THOUSAND = 10000; + public static final int FIFTEEN_THOUSAND = 15000; + public static final int HUNDRED_THOUSAND = 100000; + public static final int MILLION = 1000000; + public static final int ONE_MINUTE = 60000; + + public static final long NEGATIVE_ONE__LONG = -1L; + public static final long ZERO_LONG = 0L; + public static final long ONE_LONG = 1L; + public static final long TWO_LONG = 2L; + public static final long THREE_LONG = 3L; + public static final long FOUR_LONG = 4L; + public static final long FIVE_LONG = 5L; + public static final long SIX_LONG = 6L; + public static final long SEVEN_LONG = 7L; + public static final long EIGHT_LONG = 8L; + public static final long NINE_LONG = 9L; + public static final long TEN_LONG = 10L; + + public static final long TWELVE_LONG = 12L; + public static final long SIXTY_LONG = 60L; + public static final long FIFTY_LONG = 50L; + public static final long THOUSAND_LONG = 1000L; + public static final long TEN_THOUSAND_LONG = 10000L; + public static final long ONE_ZERO_ONE_ZERO_ONE_ZERO_LONG = 101010L; + public static final long NINE_ZERO_NINE_ZERO_NINE_ZERO_LONG = 909090L; + public static final long ONE_YEAR_BEFORE_LONG = 1552579200000L; + + public static final int SIXITY_0XFF = 0xFF; + + + private MagicNumConstant() { + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/ResponseCode.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/ResponseCode.java new file mode 100644 index 0000000..1985743 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/base/ResponseCode.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.base; + +/** + * @description 返回状态码 + * @date 2020-02-23 + */ +public class ResponseCode { + + public static Integer SUCCESS = 200; + public static Integer UNAUTHORIZED = 401; + public static Integer ERROR = 10000; + public static Integer ENTITY_NOT_EXIST = 10001; + public static Integer BADREQUEST = 10002; + public static Integer SERVICE_ERROR = 10003; + public static Integer DOCKER_ERROR = 10004; + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MinioConfig.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MinioConfig.java new file mode 100644 index 0000000..ac938ad --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MinioConfig.java @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.config; + +import io.minio.MinioClient; +import io.minio.errors.InvalidEndpointException; +import io.minio.errors.InvalidPortException; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +/** + * @description MinIO 配置类 + * @date 2020-09-17 + */ +@Data +@Component +@ConfigurationProperties(prefix = "minio") +public class MinioConfig { + + private String endpoint; + + private int port; + + private String accessKey; + + private String secretKey; + + private Boolean secure; + + private String bucketName; + + /** + * 获取Minio客户端信息 + * + * @return Minio客户端信息 + */ + @Bean + public MinioClient getMinioClient() throws InvalidEndpointException, InvalidPortException { + return new MinioClient(endpoint, port, accessKey, secretKey,secure); + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MybatisPlusConfig.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MybatisPlusConfig.java new file mode 100644 index 0000000..69857be --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/config/MybatisPlusConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.config; + +import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @description 配置分页插件 + * @date 2020-09-17 + */ +@Configuration +public class MybatisPlusConfig { + + /** + * 分页插件 + */ + @Bean + public PaginationInterceptor paginationInterceptor() { + return new PaginationInterceptor(); + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/BusinessConstant.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/BusinessConstant.java new file mode 100644 index 0000000..9c4d1d9 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/BusinessConstant.java @@ -0,0 +1,86 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.constant; + +import org.dubhe.datasetutil.common.base.MagicNumConstant; + +/** + * @description 常量 + * @date 2020-10-19 + */ +public class BusinessConstant { + + private BusinessConstant(){} + + /** + * 分表业务编码 - 文件表 + */ + public static final String DATA_FILE = "DATA_FILE"; + + /** + * 分表业务编码 - 文件版本关系表 + */ + public static final String DATA_VERSION_FILE = "DATA_VERSION_FILE"; + /** + * 图片文件路径 + */ + public static final String IMAGE_ORIGIN = "origin"; + /** + * 标注文件路径 + */ + public static final String ANNOTATION = "annotation"; + /** + * 标签文件路径 + */ + public static final String LABEL = "label"; + /** + * 分隔符 + */ + public static final String FILE_SEPARATOR = "/"; + /** + * 后缀. + */ + public static final String SPOT = "."; + + /** + * JSON后缀名 + */ + public static final String SUFFIX_JSON = ".JSON"; + /** + * minio根目录 + */ + public static final String MINIO_ROOT_PATH = "dataset"; + /** + * 下划线 + */ + public static final String UNDERLINE = "_"; + + /** + * 分段ID范围区间 50表示 50间隔ID存一张表 + */ + public static final long INTERVAL_NUMBER = MagicNumConstant.FIFTY_LONG; + /** + * 分批长度 + */ + public static final int SUB_LENGTH = MagicNumConstant.FIVE_THOUSAND; + + /** + * 字母Y + */ + public static final String Y = "Y"; + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/DataStateCodeConstant.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/DataStateCodeConstant.java new file mode 100644 index 0000000..b040eab --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/DataStateCodeConstant.java @@ -0,0 +1,84 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.constant; + +/** + * @description 数据集状态码 + * @date 2020-09-03 + */ +public class DataStateCodeConstant { + + private DataStateCodeConstant() { + } + + /** + * 未标注 + */ + public static final Integer NOT_ANNOTATION_STATE = 101; + /** + * 手动标注中 + */ + public static final Integer MANUAL_ANNOTATION_STATE = 102; + /** + * 自动标注中 + */ + public static final Integer AUTOMATIC_LABELING_STATE = 103; + /** + * 自动标注完成 + */ + public static final Integer AUTO_TAG_COMPLETE_STATE = 104; + /** + * 标注完成 + */ + public static final Integer ANNOTATION_COMPLETE_STATE = 105; + + /** + * 目标跟踪中 + */ + public static final Integer TARGET_FOLLOW_STATE = 201; + /** + * 目标跟踪完成 + */ + public static final Integer TARGET_COMPLETE_STATE = 202; + /** + * 目标跟踪失败 + */ + public static final Integer TARGET_FAILURE_STATE = 203; + + /** + * 未采样 + */ + public static final Integer NOT_SAMPLED_STATE = 301; + /** + * 采样中 + */ + public static final Integer SAMPLING_STATE = 302; + /** + * 采样失败 + */ + public static final Integer SAMPLED_FAILURE_STATE = 303; + + /** + * 增强中 + */ + public static final Integer STRENGTHENING_STATE = 401; + /** + * 导入中 + */ + public static final Integer IN_THE_IMPORT_STATE = 402; + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/FileStateCodeConstant.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/FileStateCodeConstant.java new file mode 100644 index 0000000..1662e85 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/constant/FileStateCodeConstant.java @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.constant; + +/** + * @description 数据集状态码 + * @date 2020-09-03 + */ +public class FileStateCodeConstant { + + private FileStateCodeConstant(){ + + } + + /** + * 未标注 + */ + public static final Integer NOT_ANNOTATION_FILE_STATE = 101; + /** + * 手动标注中 + */ + public static final Integer MANUAL_ANNOTATION_FILE_STATE = 102; + /** + * 自动标注完成 + */ + public static final Integer AUTO_TAG_COMPLETE_FILE_STATE = 103; + /** + * 标注完成 + */ + public static final Integer ANNOTATION_COMPLETE_FILE_STATE = 104; + /** + * 标注未识别 + */ + public static final Integer ANNOTATION_NOT_DISTINGUISH_FILE_STATE = 105; + /** + * 目标跟踪完成 + */ + public static final Integer TARGET_COMPLETE_FILE_STATE = 201; + +} \ No newline at end of file diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/enums/LogEnum.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/enums/LogEnum.java new file mode 100644 index 0000000..857d6eb --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/enums/LogEnum.java @@ -0,0 +1,52 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.enums; + +import lombok.Getter; + +/** + * @description 日志类型枚举类 + * @date 2020-06-23 + */ +@Getter +public enum LogEnum { + + //系统报错日志 + SYS_ERR, + //用户请求日志 + REST_REQ, + //全局请求日志 + GLOBAL_REQ, + //数据集模块 + BIZ_DATASET, + //分表模块 + DATA_SEQUENCE; + + /** + * 判断日志类型不能为空 + * + * @param logType 日志类型 + * @return boolean 返回类型 + */ + public static boolean isLogType(LogEnum logType) { + if (logType != null) { + return true; + } + return false; + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/BusinessException.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/BusinessException.java new file mode 100644 index 0000000..766ab02 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/BusinessException.java @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.exception; + +import lombok.Getter; +import org.dubhe.datasetutil.common.base.DataResponseBody; +import org.dubhe.datasetutil.common.base.ResponseCode; + +/** + * @description 业务异常 + * @date 2020-03-13 + */ +@Getter +public class BusinessException extends RuntimeException { + + private DataResponseBody responseBody; + + public BusinessException(String msg) { + super(msg); + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); + } + + public BusinessException(String msg, Throwable cause) { + super(msg,cause); + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); + } + + public BusinessException(Throwable cause) { + super(cause); + this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST); + } + + public BusinessException(Integer code, String msg, String info, Throwable cause) { + super(msg,cause); + if (info == null) { + this.responseBody = new DataResponseBody(code, msg); + } else { + this.responseBody = new DataResponseBody(code, msg + ":" + info); + } + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/DataSequenceException.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/DataSequenceException.java new file mode 100644 index 0000000..6ddf93c --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/DataSequenceException.java @@ -0,0 +1,31 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.exception; + +import lombok.Getter; + +/** + * @description 获取序列异常 + * @date 2020-09-23 + */ +@Getter +public class DataSequenceException extends BusinessException { + + public DataSequenceException(String msg) { + super(msg); + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ErrorCode.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ErrorCode.java new file mode 100644 index 0000000..28c7f60 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ErrorCode.java @@ -0,0 +1,39 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.exception; + + +/** + * @description 异常code + * @date 2020-03-26 + */ +public interface ErrorCode { + + /** + * 错误码 + * @return code + */ + Integer getCode(); + + /** + * error info + * @return + */ + String getMsg(); + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ImportDatasetException.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ImportDatasetException.java new file mode 100644 index 0000000..b8f884d --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/exception/ImportDatasetException.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.exception; + +import lombok.Getter; + +/** + * @description 数据集导入异常处理 + * @date 2020-10-14 + */ +@Getter +public class ImportDatasetException extends BusinessException { + + /** + * 数据集导入异常处理 + * + * @param msg 信息 + */ + public ImportDatasetException(String msg) { + super(msg); + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/BaseLogFilter.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/BaseLogFilter.java new file mode 100644 index 0000000..b315a2f --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/BaseLogFilter.java @@ -0,0 +1,82 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.filter; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.filter.AbstractMatcherFilter; +import ch.qos.logback.core.spi.FilterReply; +import org.slf4j.Marker; + +/** + * @description 自定义日志过滤器 + * @date 2020-07-21 + */ +public class BaseLogFilter extends AbstractMatcherFilter { + + Level level; + + /** + * 重写decide方法 + * + * @param iLoggingEvent 待决定的事件 + * @return FilterReply 过滤器 + */ + @Override + public FilterReply decide(ILoggingEvent iLoggingEvent) { + if (!isStarted()) { + return FilterReply.NEUTRAL; + } + final String msg = iLoggingEvent.getMessage(); + //自定义级别 + if (checkLevel(iLoggingEvent) && msg != null && msg.startsWith("{") && msg.endsWith("}")) { + final Marker marker = iLoggingEvent.getMarker(); + if (marker != null && this.getName() != null && this.getName().contains(marker.getName())) { + return onMatch; + } + } + + return onMismatch; + } + + /** + * 检查等级 + * + * @param iLoggingEvent 待决定的事件 + * @return boolean 检查结果 + */ + protected boolean checkLevel(ILoggingEvent iLoggingEvent) { + return this.level != null + && iLoggingEvent.getLevel() != null + && iLoggingEvent.getLevel().toInt() == this.level.toInt(); + } + + public void setLevel(Level level) { + this.level = level; + } + + /** + * 启动 + */ + @Override + public void start() { + if (this.level != null) { + super.start(); + } + } +} \ No newline at end of file diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/ConsoleLogFilter.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/ConsoleLogFilter.java new file mode 100644 index 0000000..70d52f6 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/ConsoleLogFilter.java @@ -0,0 +1,59 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.filter; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.spi.FilterReply; +import org.dubhe.datasetutil.common.util.LogUtil; +import org.slf4j.MarkerFactory; + +/** + * @description 自定义日志过滤器 + * @date 2020-07-21 + */ +public class ConsoleLogFilter extends BaseLogFilter { + + /** + * 重写decide方法 + * + * @param iLoggingEvent 待决定的事件 + * @return FilterReply 过滤器 + */ + @Override + public FilterReply decide(ILoggingEvent iLoggingEvent) { + if (!isStarted()) { + return FilterReply.NEUTRAL; + } + return checkLevel(iLoggingEvent) ? onMatch : onMismatch; + } + + /** + * 检查等级 + * + * @param iLoggingEvent 待决定的事件 + * @return boolean 检查结果 + */ + @Override + protected boolean checkLevel(ILoggingEvent iLoggingEvent) { + return this.level != null + && iLoggingEvent.getLevel() != null + && iLoggingEvent.getLevel().toInt() >= this.level.toInt() + && !MarkerFactory.getMarker(LogUtil.SCHEDULE_LEVEL).equals(iLoggingEvent.getMarker()) + && !"log4jdbc.log4j2".equals(iLoggingEvent.getLoggerName()); + } +} \ No newline at end of file diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/GlobalRequestLogFilter.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/GlobalRequestLogFilter.java new file mode 100644 index 0000000..50b4ded --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/filter/GlobalRequestLogFilter.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.filter; + +import ch.qos.logback.classic.spi.ILoggingEvent; + +/** + * @description 全局请求 日志过滤器 + * @date 2020-08-13 + */ +public class GlobalRequestLogFilter extends BaseLogFilter { + + + /** + * 日志等级 + * + * @param iLoggingEvent 日志等级 + * @return boolean 执行结果 + */ + @Override + public boolean checkLevel(ILoggingEvent iLoggingEvent) { + return this.level != null && true; + } + +} \ No newline at end of file diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/DateUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/DateUtil.java new file mode 100644 index 0000000..be7d8f9 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/DateUtil.java @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.util; + +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * @description 时间 配置类 + * @date 2020-09-17 + */ +public class DateUtil { + + private DateUtil(){ + } + + /** + * 时间格式 + */ + private static final String DATA_FORMAT_STR = "yyyy年MM月dd日 HH时mm分ss秒"; + + /** + * 获取开始时间 + * + * @return String 返回控制台开始时间 + */ + public static String getNowStr() { + return new SimpleDateFormat(DATA_FORMAT_STR).format(new Date()); + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/GeneratorKeyUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/GeneratorKeyUtil.java new file mode 100644 index 0000000..5aadb7f --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/GeneratorKeyUtil.java @@ -0,0 +1,118 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.util; + +import cn.hutool.core.util.ObjectUtil; +import org.dubhe.datasetutil.common.base.MagicNumConstant; +import org.dubhe.datasetutil.common.enums.LogEnum; +import org.dubhe.datasetutil.common.exception.DataSequenceException; +import org.dubhe.datasetutil.domain.dto.IdAlloc; +import org.dubhe.datasetutil.domain.entity.DataSequence; +import org.dubhe.datasetutil.service.DataSequenceService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.StringUtils; + +import java.util.concurrent.ConcurrentHashMap; + +/** + * @description 生成ID工具类 + * @date 2020-09-23 + */ +@Component +public class GeneratorKeyUtil { + + @Autowired + private DataSequenceService dataSequenceService; + + private ConcurrentHashMap idAllocConcurrentHashMap = new ConcurrentHashMap<>(); + + /** + * 根据业务编码，数量获取序列号 + * + * @param businessCode 业务编码 + * @param number 数量 + * @return Long 起始位置 + */ + @Transactional(rollbackFor = Exception.class) + public synchronized Long getSequenceByBusinessCode(String businessCode, int number) { + if (StringUtils.isEmpty(businessCode)) { + throw new DataSequenceException("业务编码不可为空"); + } + if (number == MagicNumConstant.ZERO) { + throw new DataSequenceException("需要获取的序列号长度不可为0或者空"); + } + IdAlloc idAlloc = idAllocConcurrentHashMap.get(businessCode); + if (ObjectUtil.isNull(idAlloc)) { + idAlloc = new IdAlloc(); + idAllocConcurrentHashMap.put(businessCode, idAlloc); + } + + if (idAlloc.getUsedNumber() == MagicNumConstant.ZERO) { + DataSequence dataSequence = getDataSequence(businessCode); + updateDataSequence(businessCode); + idAlloc.setStartNumber(dataSequence.getStart()); + idAlloc.setEndNumber(dataSequence.getStart() + dataSequence.getStep() - MagicNumConstant.ONE); + idAlloc.setUsedNumber(idAlloc.getEndNumber() - idAlloc.getStartNumber() + MagicNumConstant.ONE); + } + if (idAlloc.getUsedNumber() <= number) { + expansionUsedNumber(businessCode, number); + } + long returnStartNumber = idAlloc.getStartNumber(); + idAlloc.setStartNumber(idAlloc.getStartNumber() + number); + idAlloc.setUsedNumber(idAlloc.getUsedNumber() - number); + return returnStartNumber; + } + + /** + * 根据业务编码获取配置信息 + * @param businessCode 业务编码 + * @return DataSequence 数据索引 + */ + private DataSequence getDataSequence(String businessCode) { + DataSequence dataSequence = dataSequenceService.getSequence(businessCode); + if (dataSequence == null || dataSequence.getStart() == null || dataSequence.getStep() == null) { + throw new DataSequenceException("配置出错，请检查data_sequence表配置"); + } + return dataSequence; + } + + /** + * 根据业务编码更新起始值 + * @param businessCode 业务编码 + */ + private void updateDataSequence(String businessCode) { + dataSequenceService.updateSequenceStart(businessCode); + } + + /** + * 多次扩容 + * @param businessCode 业务编码 + * @param number 数量 + */ + private void expansionUsedNumber(String businessCode, int number) { + IdAlloc idAlloc = idAllocConcurrentHashMap.get(businessCode); + updateDataSequence(businessCode); + DataSequence dataSequenceNew = getDataSequence(businessCode); + idAlloc.setEndNumber(idAlloc.getEndNumber() + dataSequenceNew.getStep()); + idAlloc.setUsedNumber(idAlloc.getEndNumber() - idAlloc.getStartNumber() + MagicNumConstant.ONE); + if (idAlloc.getUsedNumber() <= number) { + expansionUsedNumber(businessCode, number); + } + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/HandleFileUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/HandleFileUtil.java new file mode 100644 index 0000000..baafdb9 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/HandleFileUtil.java @@ -0,0 +1,91 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.util; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.LineIterator; +import org.dubhe.datasetutil.common.constant.BusinessConstant; + +import java.io.File; +import java.io.IOException; + +/** + * @description 文件工具类 + * @date 2020-10-15 + */ +public class HandleFileUtil { + + /** + * 读取文件内容 + * + * @param file 文件对象 + * @return String 文件内容 + */ + public static String readFile(File file) throws IOException{ + StringBuilder stringBuffer = new StringBuilder(); + LineIterator fileContext = FileUtils.lineIterator(file,"UTF-8"); + while (fileContext.hasNext()) { + stringBuffer.append(fileContext.nextLine()); + } + return stringBuffer.toString(); + } + + + /** + * 获取文件名后缀名 + * + * @param fileName 文件名 + * @return String 文件后缀名 + */ + public static String readFileSuffixName(String fileName){ + return fileName.substring(fileName.lastIndexOf(".")); + } + + + /** + * 获取文件名(踢除后缀名) + * + * @param fileName 文件名 + * @return String 文件名(踢除后缀名) + */ + public static String readFileName(String fileName){ + return fileName.substring(0,fileName.lastIndexOf(".")); + } + + + /** + * 生成文件路径 + * + * @param businessCode 业务类型 + * @return String 文件路径 + */ + public static String generateFilePath(String businessCode){ + return BusinessConstant.FILE_SEPARATOR + businessCode; + } + + + /** + * 获取标签组名称 + * + * @param fileName 文件名称 + * @return String 标签组名 + */ + public static String getLabelGroupName(String fileName){ + return fileName.substring(fileName.indexOf(BusinessConstant.UNDERLINE) + 1,fileName.lastIndexOf(BusinessConstant.SPOT)); + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/LogUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/LogUtil.java new file mode 100644 index 0000000..1b429f9 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/LogUtil.java @@ -0,0 +1,300 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.util; + +import ch.qos.logback.classic.Level; +import com.alibaba.fastjson.JSON; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.dubhe.datasetutil.common.aspect.LogAspect; +import org.dubhe.datasetutil.common.enums.LogEnum; +import org.dubhe.datasetutil.domain.entity.LogInfo; +import org.slf4j.MDC; +import org.slf4j.MarkerFactory; +import org.slf4j.helpers.MessageFormatter; + +import java.util.Arrays; +import java.util.UUID; + +/** + * @description 日志工具类 + * @date 2020-06-29 + */ +@Slf4j +public class LogUtil { + + private static final String TRACE_TYPE = "TRACE_TYPE"; + + public static final String SCHEDULE_LEVEL = "SCHEDULE"; + + private static final String GLOBAL_REQUEST_LEVEL = "GLOBAL_REQUEST"; + + private static final String TRACE_LEVEL = "TRACE"; + + private static final String DEBUG_LEVEL = "DEBUG"; + + private static final String INFO_LEVEL = "INFO"; + + private static final String WARN_LEVEL = "WARN"; + + private static final String ERROR_LEVEL = "ERROR"; + + + public static void startScheduleTrace() { + MDC.put(TRACE_TYPE, SCHEDULE_LEVEL); + } + + public static void cleanTrace() { + MDC.clear(); + } + + /** + * info级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + + public static void info(LogEnum logType, Object... object) { + + logHandle(logType, Level.INFO, object); + } + + /** + * debug级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + public static void debug(LogEnum logType, Object... object) { + logHandle(logType, Level.DEBUG, object); + } + + /** + * error级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + public static void error(LogEnum logType, Object... object) { + errorObjectHandle(object); + logHandle(logType, Level.ERROR, object); + } + + /** + * warn级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + public static void warn(LogEnum logType, Object... object) { + logHandle(logType, Level.WARN, object); + } + + /** + * trace级别的日志 + * + * @param logType 日志类型 + * @param object 打印的日志参数 + */ + public static void trace(LogEnum logType, Object... object) { + logHandle(logType, Level.TRACE, object); + } + + /** + * 日志处理 + * + * @param logType 日志类型 + * @param level 日志级别 + * @param object 打印的日志参数 + */ + private static void logHandle(LogEnum logType, Level level, Object[] object) { + + LogInfo logInfo = generateLogInfo(logType, level, object); + + switch (logInfo.getLevel()) { + case TRACE_LEVEL: + log.trace(MarkerFactory.getMarker(TRACE_LEVEL), logJsonStringLengthLimit(logInfo)); + break; + case DEBUG_LEVEL: + log.debug(MarkerFactory.getMarker(DEBUG_LEVEL), logJsonStringLengthLimit(logInfo)); + break; + case GLOBAL_REQUEST_LEVEL: + logInfo.setLevel(null); + logInfo.setType(null); + logInfo.setLocation(null); + log.info(MarkerFactory.getMarker(GLOBAL_REQUEST_LEVEL), logJsonStringLengthLimit(logInfo)); + break; + case SCHEDULE_LEVEL: + log.info(MarkerFactory.getMarker(SCHEDULE_LEVEL), logJsonStringLengthLimit(logInfo)); + break; + case INFO_LEVEL: + log.info(MarkerFactory.getMarker(INFO_LEVEL), logJsonStringLengthLimit(logInfo)); + break; + case WARN_LEVEL: + log.warn(MarkerFactory.getMarker(WARN_LEVEL), logJsonStringLengthLimit(logInfo)); + break; + case ERROR_LEVEL: + log.error(MarkerFactory.getMarker(ERROR_LEVEL), logJsonStringLengthLimit(logInfo)); + break; + default: + } + + } + + + /** + * 日志信息组装的内部方法 + * + * @param logType 日志类型 + * @param level 日志级别 + * @param object 打印的日志参数 + * @return LogInfo 日志对象 + */ + private static LogInfo generateLogInfo(LogEnum logType, Level level, Object[] object) { + + + LogInfo logInfo = new LogInfo(); + // 日志类型检测 + if (!LogEnum.isLogType(logType)) { + level = Level.ERROR; + object = new Object[1]; + object[0] = "日志类型【".concat(String.valueOf(logType)).concat("】不正确！"); + logType = LogEnum.SYS_ERR; + } + + // 获取trace_id + if (StringUtils.isEmpty(MDC.get(LogAspect.TRACE_ID))) { + MDC.put(LogAspect.TRACE_ID, UUID.randomUUID().toString()); + } + // 设置logInfo的level,type,traceId属性 + logInfo.setLevel(level.levelStr) + .setType(logType.toString()) + .setTraceId(MDC.get(LogAspect.TRACE_ID)); + + + //自定义日志级别 + //LogEnum、 MDC中的 TRACE_TYPE 做日志分流标识 + if (Level.INFO.toInt() == level.toInt()) { + if (LogEnum.GLOBAL_REQ.equals(logType)) { + //info全局请求 + logInfo.setLevel(GLOBAL_REQUEST_LEVEL); + } else { + //schedule定时等 链路记录 + String traceType = MDC.get(TRACE_TYPE); + if (StringUtils.isNotBlank(traceType)) { + logInfo.setLevel(traceType); + } + } + } + + // 设置logInfo的堆栈信息 + setLogStackInfo(logInfo); + // 设置logInfo的info信息 + setLogInfo(logInfo, object); + // 截取logInfo的长度并转换成json字符串 + return logInfo; + } + + /** + * 设置loginfo的堆栈信息 + * + * @param logInfo 日志对象 + */ + private static void setLogStackInfo(LogInfo logInfo) { + StackTraceElement[] elements = Thread.currentThread().getStackTrace(); + if (elements.length >= 6) { + StackTraceElement element = elements[5]; + logInfo.setLocation(String.format("%s#%s:%s", element.getClassName(), element.getMethodName(), element.getLineNumber())); + } + } + + /** + * 限制log日志的长度并转换成json + * + * @param logInfo 日志对象 + * @return String 转换的json + */ + private static String logJsonStringLengthLimit(LogInfo logInfo) { + try { + + String jsonString = JSON.toJSONString(logInfo); + if (StringUtils.isBlank(jsonString)) { + return ""; + } + if (jsonString.length() > 10000) { + String trunk = logInfo.getInfo().toString().substring(0, 9000); + logInfo.setInfo(trunk); + jsonString = JSON.toJSONString(logInfo); + } + return jsonString; + + } catch (Exception e) { + logInfo.setLevel(Level.ERROR.levelStr).setType(LogEnum.SYS_ERR.toString()) + .setInfo("cannot serialize exception: " + ExceptionUtils.getStackTrace(e)); + return JSON.toJSONString(logInfo); + } + } + + /** + * 设置日志对象的info信息 + * + * @param logInfo 日志对象 + * @param object 打印的日志参数 + */ + private static void setLogInfo(LogInfo logInfo, Object[] object) { + + if (object.length > 1) { + logInfo.setInfo(MessageFormatter.arrayFormat(object[0].toString(), + Arrays.copyOfRange(object, 1, object.length)).getMessage()); + + } else if (object.length == 1 && object[0] instanceof Exception) { + logInfo.setInfo((ExceptionUtils.getStackTrace((Exception) object[0]))); + log.error((ExceptionUtils.getStackTrace((Exception) object[0]))); + } else if (object.length == 1) { + logInfo.setInfo(object[0] == null ? "" : object[0]); + } else { + logInfo.setInfo(""); + } + + } + + /** + * 处理Exception的情况 + * + * @param object 打印的日志参数 + */ + private static void errorObjectHandle(Object[] object) { + + if (object.length == 2 && object[1] instanceof Exception) { + log.error(String.valueOf(object[0]), (Exception) object[1]); + object[1] = ExceptionUtils.getStackTrace((Exception) object[1]); + + } else if (object.length >= 3) { + log.error(String.valueOf(object[0]), + Arrays.copyOfRange(object, 1, object.length)); + for (int i = 0; i < object.length; i++) { + if (object[i] instanceof Exception) { + object[i] = ExceptionUtils.getStackTrace((Exception) object[i]); + } + + } + } + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MinioUtil.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MinioUtil.java new file mode 100644 index 0000000..f5bb17e --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MinioUtil.java @@ -0,0 +1,64 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.util; + +import io.minio.MinioClient; +import io.minio.PutObjectOptions; +import org.dubhe.datasetutil.common.config.MinioConfig; +import org.dubhe.datasetutil.common.enums.LogEnum; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.InputStream; + +/** + * @description Minio工具类 + * @date 2020-09-17 + */ +@Component +public class MinioUtil { + + @Autowired + private MinioClient minioClient; + + @Autowired + private MinioConfig minioConfig; + + /** + * 上传文件 + * + * @param objectName 对象名称 + * @param inputStream 文件流 + * @throws Exception 上传异常 + */ + public void upLoadFile(String objectName, InputStream inputStream) throws Exception { + LogUtil.info(LogEnum.BIZ_DATASET,"文件上传名称为: 【" + objectName + "】"); + PutObjectOptions options = new PutObjectOptions(inputStream.available(), -1); + minioClient.putObject(minioConfig.getBucketName(), objectName, inputStream, options); + } + + /** + * 获取文件URL + * + * @param objectName 对象名称 + * @return String 文件路径 + */ + public String getUrl(String objectName) { + return minioConfig.getBucketName() + "/" + objectName; + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MyPreciseShardingAlgorithm.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MyPreciseShardingAlgorithm.java new file mode 100644 index 0000000..fb0c9e4 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/MyPreciseShardingAlgorithm.java @@ -0,0 +1,72 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.util; + +import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm; +import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue; +import org.dubhe.datasetutil.common.base.MagicNumConstant; +import org.dubhe.datasetutil.common.constant.BusinessConstant; +import org.dubhe.datasetutil.service.DataSequenceService; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Collection; + +/** + * @description 数据分片 + * @date 2020-09-21 + */ +public class MyPreciseShardingAlgorithm implements PreciseShardingAlgorithm{ + + @Autowired + private DataSequenceService dataSequenceService; + + /** + * 数据表分片 + * + * @param collection 集合 + * @param preciseShardingValue 分片值 + * @return 字符串 + */ + @Override + public String doSharding(Collection collection, PreciseShardingValue preciseShardingValue) { + long startIndex = MagicNumConstant.ONE; + long endIndex = MagicNumConstant.FIFTY; + dataSequenceService = SpringContextHolder.getBean(DataSequenceService.class); + String tableName = preciseShardingValue.getLogicTableName()+ BusinessConstant.UNDERLINE + preciseSharding(preciseShardingValue.getValue(),startIndex ,endIndex); + if(!dataSequenceService.checkTableExist(tableName)){ + dataSequenceService.createTable(tableName); + } + return tableName; + } + + /** + * 分片实现 + * + * @param indexId 起始位置 + * @param startIndex 起始值 + * @param endIndex 结束值 + * @return long 返回截止值 + */ + public long preciseSharding(long indexId,long startIndex , long endIndex){ + if(indexId > endIndex){ + startIndex = startIndex + BusinessConstant.INTERVAL_NUMBER; + endIndex = endIndex + BusinessConstant.INTERVAL_NUMBER; + return preciseSharding(indexId,startIndex,endIndex); + } + return endIndex / BusinessConstant.INTERVAL_NUMBER; + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/PrintUtils.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/PrintUtils.java new file mode 100644 index 0000000..938cce5 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/PrintUtils.java @@ -0,0 +1,156 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.util; + +/** + * @description 控制台字体颜色 + * @date 2020-10-22 + */ +public class PrintUtils { + /** + * 白色 + */ + public static final int WHITE = 30; + + /** + * 白色背景 + */ + public static final int WHITE_BACKGROUND = 40; + + /** + * 红色 + */ + public static final int RED = 31; + + /** + * 红色背景 + */ + public static final int RED_BACKGROUND = 41; + + /** + * 绿色 + */ + public static final int GREEN = 32; + + /** + * 绿色背景 + */ + public static final int GREEN_BACKGROUND = 42; + + /** + * 黄色 + */ + public static final int YELLOW = 33; + + /** + * 黄色背景 + */ + public static final int YELLOW_BACKGROUND = 43; + + /** + * 蓝色 + */ + public static final int BLUE = 34; + + /** + * 蓝色背景 + */ + public static final int BLUE_BACKGROUND = 44; + + /** + * 品红（洋红） + */ + public static final int MAGENTA = 35; + + /** + * 品红背景 + */ + public static final int MAGENTA_BACKGROUND = 45; + + /** + * 蓝绿 + */ + public static final int CYAN = 36; + + /** + * 蓝绿背景 + */ + public static final int CYAN_BACKGROUND = 46; + + /** + * 黑色 + */ + public static final int BLACK = 37; + + /** + * 黑色背景 + */ + public static final int BLACK_BACKGROUND = 47; + + /** + * 粗体 + */ + public static final int BOLD = 1; + + /** + * 斜体 + */ + public static final int ITATIC = 3; + + /** + * 下划线 + */ + public static final int UNDERLINE = 4; + + /** + * 反转 + */ + public static final int REVERSE = 7; + + /** + * 格式化 + * + * @param txt 文本 + * @param codes 信息 + * @return String 格式化后的内容 + */ + private static String FMT(String txt, int... codes) { + StringBuffer sb = new StringBuffer(); + for (int code : codes) { + sb.append(code + ";"); + } + String _code = sb.toString(); + if (_code.endsWith(";")) { + _code = _code.substring(0, _code.length() - 1); + } + return (char) 27 + "[" + _code + "m" + txt + (char) 27 + "[0m"; + } + + /** + * 打印并换行 + */ + public static void printLine(String txt, int... codes) { + System.out.println(FMT(txt, codes)); + } + + /** + * 默认打印红色文字 + */ + public static void PN(String txt) { + System.out.println(FMT(txt, new int[]{RED})); + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/SpringContextHolder.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/SpringContextHolder.java new file mode 100644 index 0000000..7f3e52b --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/SpringContextHolder.java @@ -0,0 +1,95 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.common.util; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +/** + * @description 上下文工具类 + * @date 2020-03-25 + */ +@Slf4j +public class SpringContextHolder implements ApplicationContextAware, DisposableBean { + + private static ApplicationContext applicationContext = null; + + /** + * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型. + * + * @param name bean名称 + * @return T 根据Bean名称转型的对象 + */ + @SuppressWarnings("unchecked") + public static T getBean(String name) { + assertContextInjected(); + return (T) applicationContext.getBean(name); + } + + /** + * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型. + * + * @param requiredType 要求类型 + * @return T 根据Bean名称转型的对象 + */ + public static T getBean(Class requiredType) { + assertContextInjected(); + return applicationContext.getBean(requiredType); + } + + /** + * 检查ApplicationContext不为空. + */ + private static void assertContextInjected() { + if (applicationContext == null) { + throw new IllegalStateException("applicaitonContext属性未注入, 请在applicationContext" + + ".xml中定义SpringContextHolder或在SpringBoot启动类中注册SpringContextHolder."); + } + } + + /** + * 清除SpringContextHolder中的ApplicationContext为Null. + */ + private static void clearHolder() { + log.debug("清除SpringContextHolder中的ApplicationContext:" + + applicationContext); + applicationContext = null; + } + + @Override + public void destroy() { SpringContextHolder.clearHolder(); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + if (SpringContextHolder.applicationContext != null) { + log.warn("SpringContextHolder中的ApplicationContext被覆盖, 原有ApplicationContext为:" + SpringContextHolder.applicationContext); + } + SpringContextHolder.applicationContext = applicationContext; + } + + /** + * 获取当前环境 + */ + public static String getActiveProfile(){ + return applicationContext.getEnvironment().getActiveProfiles()[0]; + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/ThreadUtils.java b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/ThreadUtils.java new file mode 100644 index 0000000..8e58cfc --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/common/util/ThreadUtils.java @@ -0,0 +1,72 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.common.util; + + +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.concurrent.*; + +/** + * @description 线程工具 + * @date 2020-10-19 + */ +@Slf4j +public class ThreadUtils { + + private ThreadUtils(){} + + /** + * 根据需要处理的数量创建线程数 + * + * @param listSize 集合数量 + * @return int 数量 + */ + public static int createThread(int listSize) { + return listSize / getNeedThreadNumber() == 0 ? 1 : listSize / getNeedThreadNumber(); + } + + + /** + * 获取需要创建的线程数 + * + * @return int 数量 + */ + public static int getNeedThreadNumber() { + final int numOfCores = Runtime.getRuntime().availableProcessors(); + final double blockingCoefficient = 0.8; + return (int) (numOfCores / (1 - blockingCoefficient)); + } + + /** + * 按要求分多线程执行 + * + * @param partitions 分线程集合 + * @throws Exception 线程执行异常 + */ + public static void runMultiThread(List> partitions) throws Exception { + final ExecutorService executorService = Executors.newFixedThreadPool(ThreadUtils.getNeedThreadNumber()); + final List> valueOfStocks = executorService.invokeAll(partitions); + Integer endCount = 0; + for (final Future value : valueOfStocks) { + endCount += value.get(); + } + log.warn("#-------------处理结束,成功处理文件 【" + endCount + "】个-------------#"); + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataFileMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataFileMapper.java new file mode 100644 index 0000000..0229e77 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataFileMapper.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.dubhe.datasetutil.domain.entity.DataFile; +import java.util.List; + +/** + * @description 数据集文件 Mapper接口 + * @date 2020-09-17 + */ +public interface DataFileMapper extends BaseMapper { + + /** + * 插入文件数据 + * + * @param listDataFile 文件数据集合 + */ + void saveBatchDataFile(@Param("listDataFile") List listDataFile); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataGroupLabelMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataGroupLabelMapper.java new file mode 100644 index 0000000..2b92942 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataGroupLabelMapper.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.dubhe.datasetutil.domain.entity.DataGroupLabel; + +import java.util.List; + +/** + * @description 标签组与标签的关系MAPPER + * @date 2020-10-21 + */ +public interface DataGroupLabelMapper extends BaseMapper { + /** + * 插入标签组与标签的关数据 + * + * @param listDataGroupLabel 标签组与标签数据集合 + */ + void saveDataGroupLabel(@Param("list") List listDataGroupLabel); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelGroupMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelGroupMapper.java new file mode 100644 index 0000000..f1ee840 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelGroupMapper.java @@ -0,0 +1,38 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.dao; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.dubhe.datasetutil.domain.entity.DataLabelGroup; + +/** + * @description 数据集标签组 + * @date 2020-10-14 + */ +public interface DataLabelGroupMapper extends BaseMapper { + /** + * 根据标签组名查询 + * + * @param labelGroupName 标签组名 + * @return int 标签组数量 + */ + @Select("select count(1) from data_label_group where name = #{labelGroupName} and deleted = 0") + int selectByLabelGroupName(@Param("labelGroupName") String labelGroupName); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelMapper.java new file mode 100644 index 0000000..1040584 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataLabelMapper.java @@ -0,0 +1,36 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.dubhe.datasetutil.domain.entity.DataLabel; + +import java.util.List; + +/** + * @description 数据集标签 + * @date 2020-10-14 + */ +public interface DataLabelMapper extends BaseMapper { + /** + * 批量保存数据集标签 + * + * @param listDataLabel 标签数据 + */ + void saveBatchDataLabel(@Param("listDataLabel") List listDataLabel); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataSequenceMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataSequenceMapper.java new file mode 100644 index 0000000..abf5d00 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataSequenceMapper.java @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; +import org.dubhe.datasetutil.domain.entity.DataSequence; + +/** + * @description 序列Mapper + * @date 2020-09-23 + */ +public interface DataSequenceMapper extends BaseMapper { + /** + * 根据业务编码查询序列 + * + * @param businessCode 业务信息 + * @return DataSequence 根据业务编码得到的序列 + */ + @Select("select id, business_code ,start, step from data_sequence where business_code = #{businessCode}") + DataSequence selectByBusiness(String businessCode); + + /** + * 根据业务编码更新序列起始值 + * + * @param businessCode 业务信息 + * @return DataSequence 根据业务编码更新序列起始值 + */ + @Update("update data_sequence set start = start + step where business_code = #{businessCode} ") + int updateStartByBusinessCode(String businessCode); + + /** + * 查询存在表的记录数 + * + * @param tableName 类型名称 + * @return int 根据类型查在表的记录数量 + */ + @Select("select count(1) from ${tableName}") + int checkTableExist(@Param("tableName") String tableName); + + /** + * 执行创建表 + * + * @param tableName 类型名称 + * @param oldTableName 旧类型名称 + */ + @Update({"CREATE TABLE ${tableName} AS select * from ${oldTableName} "}) + void createNewTable(@Param("tableName") String tableName, @Param("oldTableName") String oldTableName); + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataVersionFileMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataVersionFileMapper.java new file mode 100644 index 0000000..d8c107d --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DataVersionFileMapper.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.dubhe.datasetutil.domain.dto.DataVersionFile; + +import java.util.List; + +/** + * @description 数据集文件中间表 Mapper接口 + * @date 2020-9-17 + */ +public interface DataVersionFileMapper extends BaseMapper { + + /** + * 插入数据集文件中间表数据 + * + * @param listDataVersionFile 数据集文件中间表数据集合 + */ + void saveBatchDataFileVersion(@Param("listDataVersionFile") List listDataVersionFile); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetDataLabelMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetDataLabelMapper.java new file mode 100644 index 0000000..cff8aa6 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetDataLabelMapper.java @@ -0,0 +1,37 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.dubhe.datasetutil.domain.entity.DatasetDataLabel; + +import java.util.List; + +/** + * @description 数据集与标签关系 + * @date 2020-10-14 + */ +public interface DatasetDataLabelMapper extends BaseMapper { + + /** + * 批量保存数据集与标签关系 + * + * @param datasetDataLabelList 数据集与标签 + */ + void saveBatchDatasetDataLabel(@Param("datasetDataLabelList") List datasetDataLabelList); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetMapper.java b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetMapper.java new file mode 100644 index 0000000..f21cc5b --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/dao/DatasetMapper.java @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.dubhe.datasetutil.domain.entity.Dataset; + +/** + * @description 数据集 Mapper接口 + * @date 2020-9-17 + */ +public interface DatasetMapper extends BaseMapper { + + /** + * 根据数据集ID查询数据集 + * + * @param datasetId 数据集id + * @return Dataset 根据数据集ID得到数据集 + */ + @Select("select * from data_dataset where id = #{datasetId} and is_import = 1") + Dataset findDatasetById(@Param("datasetId") Long datasetId); + + /** + * 查询数据集标签数量 + * + * @param datasetId 数据集id + * @return int 数据集标签数量 + */ + @Select("select count(1) from data_dataset_label where dataset_id = #{datasetId}") + int findDataLabelById(@Param("datasetId") Long datasetId); + + /** + * 查询数据集文件数量 + * + * @param datasetId 数据集id + * @return int 数据集标签数量 + */ + @Select("select count(1) from data_file where dataset_id = #{datasetId}") + int findDataFileById(@Param("datasetId") Long datasetId); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/DataVersionFile.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/DataVersionFile.java new file mode 100644 index 0000000..a7c89ec --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/DataVersionFile.java @@ -0,0 +1,93 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.domain.dto; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; +import org.dubhe.datasetutil.common.base.BaseEntity; + +import java.io.Serializable; + + +/** + * @description 数据集文件关系类 + * @date 2020-9-17 + */ +@Data +public class DataVersionFile extends BaseEntity implements Serializable { + + /** + * id + */ + @TableId(type = IdType.AUTO) + private Long id; + + /** + * 数据集id + */ + private Long datasetId; + + /** + * 版本号 + */ + private String versionName; + + /** + * 文件id + */ + private Long fileId; + + /** + * 状态 + */ + private Integer status; + + /** + * 数据集状态 + */ + private Integer annotationStatus; + + /** + * 数据集状态备份 + */ + private Integer backupStatus; + + /** + * 发布是否转换 + */ + private Integer changed; + + public DataVersionFile() { + } + + /** + * 插入数据集版本文件关系 + * + * @param datasetId 数据集id + * @param fileId 文件id + * @param annotationStatus 数据集id + * @param status 状态 + * @return DataVersionFile 数据集版本文件表 + */ + public DataVersionFile(Long datasetId, Long fileId,Integer annotationStatus,Integer status) { + this.datasetId = datasetId; + this.fileId = fileId; + this.annotationStatus = annotationStatus; + this.status = status; + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/IdAlloc.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/IdAlloc.java new file mode 100644 index 0000000..87ae460 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/dto/IdAlloc.java @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.domain.dto; + +import lombok.Data; +import org.dubhe.datasetutil.common.base.MagicNumConstant; + +/** + * @description ID策略实体 + * @date 2020-10-16 + */ +@Data +public class IdAlloc { + + /** + * 起始位置 + */ + private long startNumber; + + /** + * 结束位置 + */ + private long endNumber; + + /** + * 可用数量 + */ + private long usedNumber; + + public IdAlloc() { + this.startNumber = MagicNumConstant.ZERO; + this.endNumber = MagicNumConstant.ZERO; + this.usedNumber = MagicNumConstant.ZERO; + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataFile.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataFile.java new file mode 100644 index 0000000..19a553b --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataFile.java @@ -0,0 +1,118 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; +import org.dubhe.datasetutil.common.base.BaseEntity; + +import java.awt.image.BufferedImage; +import java.io.Serializable; + +/** + * @description 文件类 + * @date 2020-09-17 + */ +@Data +public class DataFile extends BaseEntity implements Serializable { + + /** + * id + */ + @TableId(type = IdType.AUTO) + private Long id; + + /** + * 名称 + */ + private String name; + + /** + * 状态 + */ + private Integer status; + + /** + * 数据集id + */ + private Long datasetId; + + /** + * url + */ + private String url; + + /** + * 文件类型 + */ + private Integer fileType; + + /** + * 父id + */ + private Long pid; + + /** + * 帧间隔 + */ + private Integer frameInterval; + + /** + * 增强类型 + */ + private Integer enhanceType; + + /** + * 宽 + */ + private Integer width; + + /** + * 高 + */ + private Integer height; + + /** + * 拥有人id + */ + private Long originUserId; + + public DataFile() {} + + /** + * 插入文件表 + * + * @param name 文件名字 + * @param datasetId 数据集id + * @param url 文件路径 + * @param createUserId 创建人id + * @param read 文件宽高 + * @return DataFile file对象 + */ + public DataFile(String name, Long datasetId, String url, Long createUserId, BufferedImage read) { + this.name = name.substring(0, name.lastIndexOf(".")); + this.datasetId = datasetId; + this.url = url; + this.status = 101; + this.setDeleted(false); + this.originUserId = createUserId; + this.width = read.getWidth(); + this.height = read.getHeight(); + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataGroupLabel.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataGroupLabel.java new file mode 100644 index 0000000..139f6fc --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataGroupLabel.java @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.*; +import org.dubhe.datasetutil.common.base.BaseEntity; + +import java.io.Serializable; + +/** + * @description 标签组标签中间表 + * @date 2020-10-21 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(callSuper = false) +@Builder +@TableName("data_group_label") +public class DataGroupLabel extends BaseEntity implements Serializable { + + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 标签id + */ + @TableField(value = "label_id") + private Long labelId; + + /** + * 标签组id + */ + @TableField(value = "label_group_id") + private Long labelGroupId; + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabel.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabel.java new file mode 100644 index 0000000..4296edb --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabel.java @@ -0,0 +1,64 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.*; +import org.dubhe.datasetutil.common.base.BaseEntity; + +import java.io.Serializable; + +/** + * @description 数据集标签实体 + * @date 2020-10-14 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(callSuper = false) +@Builder +@TableName("data_label") +public class DataLabel extends BaseEntity implements Serializable { + + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 名称 + */ + @TableField(value = "name") + private String name; + + /** + * 颜色 + */ + @TableField(value = "color") + private String color; + + /** + * 类型 + */ + @TableField(value = "type") + private Long type; + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabelGroup.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabelGroup.java new file mode 100644 index 0000000..4bc6120 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataLabelGroup.java @@ -0,0 +1,70 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.*; +import org.dubhe.datasetutil.common.base.BaseEntity; + +import java.io.Serializable; + +/** + * @description 标签组实体 + * @date 2020-10-14 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(callSuper = false) +@Builder +@TableName("data_label_group") +public class DataLabelGroup extends BaseEntity implements Serializable { + + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 名称 + */ + @TableField(value = "name") + private String name; + + /** + * 描述 + */ + @TableField(value = "remark") + private String remark; + + /** + * 类型 + */ + @TableField(value = "type") + private Long type; + + /** + * 拥有人id + */ + @TableField(value = "origin_user_id") + private Long originUserId; + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataSequence.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataSequence.java new file mode 100644 index 0000000..52fa0e0 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DataSequence.java @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +/** + * @description 序列表 + * @date 2020-09-23 + */ +@Data +@TableName("data_sequence") +public class DataSequence { + + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 业务code + */ + @TableField(value = "business_code") + private String businessCode; + + /** + * 启动 + */ + @TableField(value = "start") + private Long start; + + /** + * 步数 + */ + @TableField(value = "step") + private Long step; + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/Dataset.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/Dataset.java new file mode 100644 index 0000000..5abc7bc --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/Dataset.java @@ -0,0 +1,145 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.domain.entity; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; +import org.dubhe.datasetutil.common.base.BaseEntity; + +import java.io.Serializable; +import java.sql.Timestamp; + +/** + * @description 数据集类 + * @date 2020-9-17 + */ +@Data +@TableName("data_dataset") +public class Dataset extends BaseEntity implements Serializable { + + /** + * 删除标识 + */ + @TableField("deleted") + private Boolean deleted = false; + + /** + * 数据集名称 + */ + private String name; + + /** + * 数据集备注 + */ + private String remark; + + /** + * 类型 + */ + private Integer type; + + /** + * 数据集类型 + */ + private Integer dataType; + + /** + * 数据集状态 + */ + private Integer annotateType; + + /** + * 任务id + */ + private Long teamId; + + /** + * id + */ + @TableId(type = IdType.AUTO) + private Long id; + + /** + * url + */ + private String uri; + + /** + * 状态 + */ + private Integer status; + + /** + * 当前版本号 + */ + private String currentVersionName; + + /** + * 是否为导入 + */ + @TableField(value = "is_import") + private boolean isImport; + + /** + * 用户导入数据集压缩包地址 + */ + private String archiveUrl; + + /** + * 解压状态 + */ + private Integer decompressState; + + /** + * 解压失败原因 + */ + private String decompressFailReason; + + /** + * 是否置顶 + */ + @TableField(value = "is_top") + private boolean isTop; + + /** + * 创建时间 + */ + private Timestamp createTime; + + /** + * 修改时间 + */ + private Timestamp updateTime; + + /** + * 创建人id + */ + private Long createUserId; + + /** + * 修改人id + */ + private Long updateUserId; + + /** + * 拥有人id + */ + private Long originUserId; + + public Dataset() {} + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DatasetDataLabel.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DatasetDataLabel.java new file mode 100644 index 0000000..da6b84d --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/DatasetDataLabel.java @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.domain.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.*; +import org.dubhe.datasetutil.common.base.BaseEntity; + +import java.io.Serializable; + +/** + * @description 数据集与标签关系实体 + * @date 2020-10-14 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(callSuper = false) +@Builder +@TableName("data_label") +public class DatasetDataLabel extends BaseEntity implements Serializable { + + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + /** + * 数据集id + */ + @TableField(value = "dataset_id") + private Long datasetId; + + /** + * 标签id + */ + @TableField(value = "label_id") + private Long labelId; + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/LogInfo.java b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/LogInfo.java new file mode 100644 index 0000000..69924c5 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/domain/entity/LogInfo.java @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ + +package org.dubhe.datasetutil.domain.entity; + +import com.alibaba.fastjson.annotation.JSONField; +import com.xiaoleilu.hutool.date.DateUtil; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; + +/** + * @description 日志对象封装类 + * @date 2020-06-29 + */ +@Data +@Accessors(chain = true) +public class LogInfo implements Serializable { + + /** + * id + */ + @JSONField(ordinal = 1) + private String traceId; + + /** + * 类型 + */ + @JSONField(ordinal = 2) + private String type; + + /** + * 等级 + */ + @JSONField(ordinal = 3) + private String level; + + /** + * 位置 + */ + @JSONField(ordinal = 4) + private String location; + + /** + * 时间 + */ + @JSONField(ordinal = 5) + private String time = DateUtil.now(); + + /** + * 描述 + */ + @JSONField(ordinal = 6) + private Object info; + + public void setInfo(Object info) { + this.info = info; + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImageUploadHandle.java b/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImageUploadHandle.java new file mode 100644 index 0000000..312d5e5 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImageUploadHandle.java @@ -0,0 +1,140 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.handle; + +import com.xiaoleilu.hutool.io.FileUtil; +import lombok.extern.slf4j.Slf4j; +import org.dubhe.datasetutil.common.constant.BusinessConstant; +import org.dubhe.datasetutil.common.util.DateUtil; +import org.dubhe.datasetutil.common.util.GeneratorKeyUtil; +import org.dubhe.datasetutil.common.util.MinioUtil; +import org.dubhe.datasetutil.common.util.ThreadUtils; +import org.dubhe.datasetutil.domain.entity.DataFile; +import org.dubhe.datasetutil.domain.dto.DataVersionFile; +import org.dubhe.datasetutil.domain.entity.Dataset; +import org.dubhe.datasetutil.service.DataFileService; +import org.dubhe.datasetutil.service.DataVersionFileService; +import org.dubhe.datasetutil.service.DatasetService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @description 上传图片工具类 + * @date 2020-09-17 + */ +@Slf4j +@Component +public class DatasetImageUploadHandle { + + @Autowired + private MinioUtil minioUtil; + + @Autowired + private DataFileService dataFileService; + + @Autowired + private DataVersionFileService dataVersionFileService; + + @Autowired + private DatasetService datasetService; + + @Autowired + private GeneratorKeyUtil generatorKeyUtil; + + /** + * 启动线程 + * + * @param imagePath 本地文件地址 + * @param datasetId 数据集Id + */ + public void execute(String imagePath, Long datasetId) throws Exception { + log.info("#-------------开始处理,时间[" + DateUtil.getNowStr() + "]-------------#"); + List fileNames = FileUtil.listFileNames(imagePath); + log.info("#-------------文件数量[" + fileNames.size() + "]------------------------"); + String fileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + datasetId + + BusinessConstant.FILE_SEPARATOR + BusinessConstant.IMAGE_ORIGIN + BusinessConstant.FILE_SEPARATOR; + List> partitions = new ArrayList<>(); + int oneSize = ThreadUtils.createThread(fileNames.size()); + List need = new ArrayList<>(); + AtomicInteger atomicInteger = new AtomicInteger(0); + for (String fileName : fileNames) { + need.add(fileName); + if (need.size() == oneSize || atomicInteger.intValue() == fileNames.size() - 1) { + List now = new ArrayList<>(need); + need.clear(); + partitions.add(() -> run(datasetId, now, fileBaseDir, imagePath)); + } + atomicInteger.getAndIncrement(); + } + ThreadUtils.runMultiThread(partitions); + } + + /** + * 插入数据库数据 + * + * @param datasetId 数据集Id + * @param fileNames 文件Name + * @param fileBaseDir 文件路径 + * @param imagePath 文件地址 + * @return java.lang.Integer 成功数量 + */ + public Integer run(Long datasetId, List fileNames, String fileBaseDir, String imagePath) { + Integer success = 0; + Dataset dataset = datasetService.findCreateUserIdById(datasetId); + List dataFiles = new ArrayList<>(); + List dataVersionFiles = new ArrayList<>(); + for (int i = 0; i < fileNames.size(); i++) { + try { + minioUtil.upLoadFile(fileBaseDir + fileNames.get(i), FileUtil.getInputStream(imagePath + BusinessConstant.FILE_SEPARATOR + fileNames.get(i))); + BufferedImage read = ImageIO.read(new File(imagePath + BusinessConstant.FILE_SEPARATOR + fileNames.get(i))); + success++; + dataFiles.add(new DataFile(fileNames.get(i), datasetId, minioUtil.getUrl(fileBaseDir + fileNames.get(i)), dataset.getCreateUserId(), read)); + + if (dataFiles.size() % 500 == 0 || i == fileNames.size() - 1) { + long startDataFileIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_FILE, dataFiles.size()); + for (DataFile dataFileEntity : dataFiles) { + dataFileEntity.setId(startDataFileIndex++); + } + + dataFileService.saveBatchDataFile(dataFiles); + for (DataFile file : dataFiles) { + dataVersionFiles.add(new DataVersionFile(datasetId, file.getId(), 101, 0)); + } + long startDataFileVersionIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_VERSION_FILE, dataVersionFiles.size()); + for (DataVersionFile dataVersionFile : dataVersionFiles) { + dataVersionFile.setId(startDataFileVersionIndex++); + } + dataVersionFileService.saveBatchDataFileVersion(dataVersionFiles); + dataVersionFiles.clear(); + dataFiles.clear(); + } + } catch (Exception e) { + log.error("{}", e); + } + } + return success; + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImportHandle.java b/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImportHandle.java new file mode 100644 index 0000000..f456043 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/handle/DatasetImportHandle.java @@ -0,0 +1,522 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.handle; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.TypeReference; +import com.xiaoleilu.hutool.io.FileUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.dubhe.datasetutil.common.base.MagicNumConstant; +import org.dubhe.datasetutil.common.config.MinioConfig; +import org.dubhe.datasetutil.common.constant.BusinessConstant; +import org.dubhe.datasetutil.common.constant.FileStateCodeConstant; +import org.dubhe.datasetutil.common.exception.ImportDatasetException; +import org.dubhe.datasetutil.common.util.*; +import org.dubhe.datasetutil.domain.entity.*; +import org.dubhe.datasetutil.domain.dto.DataVersionFile; +import org.dubhe.datasetutil.service.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.sql.Timestamp; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +/** + * @description 导入数据集工具类 + * @date 2020-10-12 + */ +@Slf4j +@Component +public class DatasetImportHandle { + + @Autowired + private DatasetService datasetService; + + @Autowired + private DataLabelGroupService dataLabelGroupService; + + @Autowired + private DataGroupLabelService dataGroupLabelService; + + @Autowired + private DataLabelService dataLabelService; + + @Autowired + private DataFileService dataFileService; + + @Autowired + private DatasetDataLabelService datasetDataLabelService; + + @Autowired + private DataVersionFileService dataVersionFileService; + + @Autowired + private MinioUtil minioUtil; + + @Autowired + private MinioConfig minioConfig; + + @Autowired + private GeneratorKeyUtil generatorKeyUtil; + + /** + * 可支持的图片格式集合 + */ + private static final List SUFFIX_LIST = new ArrayList<>(); + + /** + * 标注文件中JSON的key + */ + static List annotationFileContextKey = new ArrayList<>(); + + /** + * 加载静态集合数据 + */ + static { + SUFFIX_LIST.add(".jpg"); + SUFFIX_LIST.add(".png"); + SUFFIX_LIST.add(".bmp"); + SUFFIX_LIST.add(".jpeg"); + + annotationFileContextKey.add("score"); + annotationFileContextKey.add("area"); + annotationFileContextKey.add("name"); + annotationFileContextKey.add("bbox"); + annotationFileContextKey.add("segmentation"); + annotationFileContextKey.add("iscrowd"); + } + + /** + * 导入数据集 + * + * @param scanner 输入 + */ + public void importDataset(Scanner scanner) throws Exception { + Dataset dataset = verificationDatasetId(scanner); + String filePath = verificationFilePath(scanner); + File labelJsonFile = verificationFile(filePath); + DataLabelGroup dataLabelGroup = saveDataLabelGroup(HandleFileUtil.getLabelGroupName(labelJsonFile.getName()), dataset); + List dataLabelList = readLabelContext(labelJsonFile); + saveDataLabel(dataset, dataLabelList, dataLabelGroup.getId()); + executeUploadAndSave(dataLabelList, filePath, dataset); + datasetService.updateDatasetStatus(dataset); + log.warn(""); + PrintUtils.printLine(" Success: 执行成功 ", PrintUtils.GREEN); + log.warn(""); + log.warn("# 是否结束? Y / N #"); + Scanner scannerExit = new Scanner(System.in); + if (BusinessConstant.Y.toLowerCase().equals(scannerExit.nextLine().toLowerCase())) { + System.exit(0); + } + } + + /** + * 检查文件结构 、类型 + * + * @param globalFilePath 文件路径 + * @return file 标签文件 + */ + public File verificationFile(String globalFilePath) throws IOException { + File labelRootFiles = new File(globalFilePath); + File imageRootFiles = new File(globalFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN)); + File annotationRootFiles = new File(globalFilePath + HandleFileUtil.generateFilePath(BusinessConstant.ANNOTATION)); + if (imageRootFiles.list() == null || annotationRootFiles.listFiles() == null) { + throw new ImportDatasetException(" 【" + globalFilePath + "】目录中的图片目录(origin)或者标注文件目录(annotation)的文件夹为空 "); + } + File labelJsonFile = null; + for (File file : Objects.requireNonNull(labelRootFiles.listFiles())) { + if (file.isFile() && file.getName().toLowerCase().startsWith(BusinessConstant.LABEL.toLowerCase()) + && file.getName().contains(BusinessConstant.UNDERLINE) + && file.getName().toLowerCase().endsWith(BusinessConstant.SUFFIX_JSON.toLowerCase())) { + labelJsonFile = file; + } + } + if (labelJsonFile == null) { + throw new ImportDatasetException(" 【" + globalFilePath + "】目录中未找到标签组文件"); + } + dealLabelGroup(labelJsonFile.getName()); + List dataLabelList = readLabelContext(labelJsonFile); + Map> dataLabelMap = dataLabelList.stream().collect(Collectors.groupingBy(DataLabel::getName)); + for (Map.Entry> entry : dataLabelMap.entrySet()) { + if (entry.getValue().size() > 1) { + throw new ImportDatasetException(" 标签组中标签存在重复标签：【" + entry.getKey() + "】"); + } + } + File[] imageFiles = imageRootFiles.listFiles(); + if (imageFiles == null || imageFiles.length == 0) { + throw new ImportDatasetException(" 图片文件下不存在图片文件 "); + } + for (File imageFile : imageFiles) { + String suffixFileName = imageFile.getName().substring(imageFile.getName().lastIndexOf(BusinessConstant.SPOT)); + if (!SUFFIX_LIST.contains(suffixFileName.toLowerCase())) { + throw new ImportDatasetException(" 图片文件文件夹中存在非法格式 "); + } + } + File[] annotationFiles = annotationRootFiles.listFiles(); + if (annotationFiles == null || annotationFiles.length == 0) { + throw new ImportDatasetException(" 图片文件下不存在标注文件 "); + } + for (File annotationFile : annotationFiles) { + if (!annotationFile.getName().toLowerCase().endsWith(BusinessConstant.SUFFIX_JSON.toLowerCase())) { + throw new ImportDatasetException(" 标注文件文件夹中存在非法格式 "); + } + } + return labelJsonFile; + } + + /** + * 启动线程 + * + * @param filePath 本地根目录 + * @param dataset 数据集 + */ + public void executeUploadAndSave(List dataLabelList, String filePath, Dataset dataset) throws Exception { + String localImageFilePath = filePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN); + List imageFileNameList = FileUtil.listFileNames(localImageFilePath); + log.info("需要处理： 【" + imageFileNameList.size() + "】张图片"); + int oneSize = ThreadUtils.createThread(imageFileNameList.size()); + log.info("需要创建线程数： 【" + oneSize + "】 条"); + List> partitions = new ArrayList<>(); + List need = new ArrayList<>(); + AtomicInteger atomicInteger = new AtomicInteger(MagicNumConstant.ZERO); + for (String fileName : imageFileNameList) { + need.add(fileName); + if (need.size() == oneSize || atomicInteger.intValue() == imageFileNameList.size() - MagicNumConstant.ONE) { + List fileNameList = new ArrayList<>(need); + need.clear(); + partitions.add(() -> runTask(dataLabelList, dataset, fileNameList, filePath)); + } + atomicInteger.getAndIncrement(); + } + ThreadUtils.runMultiThread(partitions); + } + + /** + * 实际执行任务 + * + * @param dataset 数据集 + * @param fileNameList 文件名字集合 + * @param dataSetRootFilePath 文件路径 + * @return Integer 执行次数 + */ + private Integer runTask(List dataLabelList, Dataset dataset, List fileNameList, String dataSetRootFilePath) throws Exception { + List dataFilesList = new ArrayList<>(); + AtomicInteger atomicInteger = new AtomicInteger(MagicNumConstant.ZERO); + String imageFileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + dataset.getId() + + BusinessConstant.FILE_SEPARATOR + BusinessConstant.IMAGE_ORIGIN + BusinessConstant.FILE_SEPARATOR; + String annotationFileBaseDir = BusinessConstant.MINIO_ROOT_PATH + BusinessConstant.FILE_SEPARATOR + dataset.getId() + + BusinessConstant.FILE_SEPARATOR + BusinessConstant.ANNOTATION + BusinessConstant.FILE_SEPARATOR; + for (String fileName : fileNameList) { + String imageUploadFile = imageFileBaseDir + fileName; + String annotationFileName = HandleFileUtil.readFileName(fileName); + File annotationFile = new File(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.ANNOTATION) + BusinessConstant.FILE_SEPARATOR + annotationFileName + BusinessConstant.SUFFIX_JSON.toLowerCase()); + JSONArray jsonArray = replaceJsonNode(annotationFile, dataLabelList); + minioUtil.upLoadFile(imageUploadFile, FileUtil.getInputStream(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN) + BusinessConstant.FILE_SEPARATOR + fileName)); + minioUtil.upLoadFile(annotationFileBaseDir + annotationFileName, IOUtils.toInputStream(jsonArray.toString(), StandardCharsets.UTF_8.name())); + DataFile dataFile = new DataFile(); + dataFile.setName(annotationFileName); + dataFile.setUrl(minioConfig.getBucketName() + BusinessConstant.FILE_SEPARATOR + imageUploadFile); + dataFile.setStatus(FileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE); + dataFile.setDatasetId(dataset.getId()); + dataFile.setFileType(MagicNumConstant.ZERO); + dataFile.setPid(MagicNumConstant.ZERO_LONG); + dataFile.setCreateUserId(dataset.getCreateUserId()); + try { + BufferedImage image = ImageIO.read(new File(dataSetRootFilePath + HandleFileUtil.generateFilePath(BusinessConstant.IMAGE_ORIGIN) + BusinessConstant.FILE_SEPARATOR + fileName)); + dataFile.setWidth(image.getWidth()); + dataFile.setHeight(image.getHeight()); + } catch (IOException e) { + throw new ImportDatasetException(" 读取图片高和宽失败 "); + } + dataFile.setOriginUserId(dataset.getCreateUserId()); + dataFilesList.add(dataFile); + if (dataFilesList.size() % MagicNumConstant.FIVE_HUNDRED == MagicNumConstant.ZERO || atomicInteger.intValue() == fileNameList.size() - MagicNumConstant.ONE) { + long startDataFileIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_FILE, dataFilesList.size()); + for (DataFile dataFileEntity : dataFilesList) { + dataFileEntity.setId(startDataFileIndex++); + } + saveDataFile(dataFilesList); + List dataVersionFileList = new ArrayList<>(); + for (DataFile file : dataFilesList) { + DataVersionFile dataVersionFile = new DataVersionFile(); + dataVersionFile.setDatasetId(dataset.getId()); + dataVersionFile.setFileId(file.getId()); + dataVersionFile.setStatus(MagicNumConstant.ZERO); + dataVersionFile.setAnnotationStatus(FileStateCodeConstant.ANNOTATION_COMPLETE_FILE_STATE); + dataVersionFileList.add(dataVersionFile); + } + long startDataFileVersionIndex = generatorKeyUtil.getSequenceByBusinessCode(BusinessConstant.DATA_VERSION_FILE, dataVersionFileList.size()); + for (DataVersionFile dataVersionFile : dataVersionFileList) { + dataVersionFile.setId(startDataFileVersionIndex++); + } + saveDataVersionFile(dataVersionFileList); + dataVersionFileList.clear(); + dataFilesList.clear(); + } + atomicInteger.getAndIncrement(); + } + return atomicInteger.getAndIncrement(); + + } + + /** + * 检查并且替换JSON中的节点 + * + * @param annotationFile 标注文件 + * @param dataLabelList 数据集集合 + */ + public JSONArray replaceJsonNode(File annotationFile, List dataLabelList) throws IOException { + JSONArray jsonArray = new JSONArray(); + if (annotationFile.exists()) { + String annotationFileContext = HandleFileUtil.readFile(annotationFile); + jsonArray = JSONArray.parseArray(annotationFileContext); + if (!jsonArray.isEmpty()) { + replaceAllNode(jsonArray, dataLabelList); + } + } + return jsonArray; + } + + /** + * 替换节点值 + * + * @param jsonArray 标注文件集合 + * @param dataLabelList 标签集合 + */ + public void replaceAllNode(JSONArray jsonArray, List dataLabelList) { + for (int i = MagicNumConstant.ZERO; i < jsonArray.size(); i++) { + JSONObject jsonObject = jsonArray.getJSONObject(i); + jsonObject.put("category_id", findDataLabelId(dataLabelList, jsonObject.get("name").toString())); + jsonObject.remove("name"); + } + } + + /** + * 查询需要替换的节点 + * + * @param dataLabelList 标签集合 + * @param objectValue 替换的节点值 + * @return long 替换标签的Id + */ + public long findDataLabelId(List dataLabelList, String objectValue) { + Optional matchedDataLabel = dataLabelList.stream().filter(dataLabel -> objectValue.equals(dataLabel.getName())).findAny(); + if (!matchedDataLabel.isPresent()) { + throw new ImportDatasetException(" 标注文件中name的值不存在于标签中!"); + } + return matchedDataLabel.get().getId(); + } + + /** + * 获取JSON中所有的key + * + * @param jsonArray json集合 + * @return JSON中所有的key + */ + public List getJsonKeyList(JSONArray jsonArray) { + List listKey = new ArrayList<>(); + for (Object object : jsonArray) { + LinkedHashMap jsonMap = JSON.parseObject(object.toString(), new TypeReference>() { + }); + for (Map.Entry entry : jsonMap.entrySet()) { + listKey.add(entry.getKey()); + } + } + return listKey.stream().distinct().collect(Collectors.toList()); + } + + /** + * 读取标签文件中标签数据 + * + * @param file 标签文件 + * @return List 标签数据集合 + */ + public List readLabelContext(File file) throws IOException { + String fileContext = HandleFileUtil.readFile(file); + List dataLabelList = JSONArray.parseArray(fileContext, DataLabel.class); + for (DataLabel dataLabel : dataLabelList) { + if (StringUtils.isEmpty(dataLabel.getName()) || StringUtils.isEmpty(dataLabel.getColor())) { + throw new ImportDatasetException(" 标签文件不规范,未能读到 'name' 或者 'color' "); + } + } + return dataLabelList; + } + + /** + * 检查标签组是否有重名 + * + * @param labelGroupName 标签组名称 + */ + public void dealLabelGroup(String labelGroupName) { + String groupName = HandleFileUtil.getLabelGroupName(labelGroupName); + int count = dataLabelGroupService.selectByLabelGroupName(groupName); + if (count > MagicNumConstant.ZERO) { + throw new ImportDatasetException(" 标签组名称【" + groupName + "】已存在，请修改label_xxx.json文件名 "); + } + } + + /** + * 保存标签组名称 + * + * @param labelGroupName 标签组名称 + * @param dataset 数据集 + * @return DataLabelGroup 数据集标签组 + */ + public DataLabelGroup saveDataLabelGroup(String labelGroupName, Dataset dataset) { + long timeStamp = System.currentTimeMillis(); + DataLabelGroup dataLabelGroup = new DataLabelGroup(); + dataLabelGroup.setName(labelGroupName); + dataLabelGroup.setOriginUserId(dataset.getCreateUserId()); + dataLabelGroup.setType(MagicNumConstant.ZERO_LONG); + dataLabelGroup.setCreateUserId(dataset.getCreateUserId()); + dataLabelGroup.setCreateTime(new Timestamp(timeStamp)); + dataLabelGroup.setUpdateTime(new Timestamp(timeStamp)); + dataLabelGroupService.saveDataGroupLabel(dataLabelGroup); + return dataLabelGroup; + } + + /** + * 批量保存标签 + * + * @param dataset 数据集 + * @param listDataLabel 标签集合 + * @param dataLabelGroupId 标签组Id + */ + public void saveDataLabel(Dataset dataset, List listDataLabel, Long dataLabelGroupId) { + listDataLabel.forEach(dataLabel -> dataLabel.setCreateUserId(dataset.getCreateUserId())); + dataLabelService.saveBatchDataLabel(listDataLabel); + List listDatasetDataLabel = new ArrayList<>(); + for (DataLabel dataLabel : listDataLabel) { + DatasetDataLabel datasetDataLabel = new DatasetDataLabel(); + datasetDataLabel.setLabelId(dataLabel.getId()); + datasetDataLabel.setDatasetId(dataset.getId()); + listDatasetDataLabel.add(datasetDataLabel); + } + saveDatasetDataLabel(listDatasetDataLabel); + + List listDataGroupLabel = new ArrayList<>(); + for (DatasetDataLabel datasetDataLabel : listDatasetDataLabel) { + DataGroupLabel dataGroupLabel = new DataGroupLabel(); + dataGroupLabel.setLabelId(datasetDataLabel.getId()); + dataGroupLabel.setLabelGroupId(dataLabelGroupId); + listDataGroupLabel.add(dataGroupLabel); + } + saveDatasetDataGroupLabel(listDataGroupLabel); + + } + + /** + * 批量保存文件 + * + * @param listDataFile file集合 + */ + public void saveDataFile(List listDataFile) { + dataFileService.saveBatchDataFile(listDataFile); + } + + /** + * 批量保存文件版本数据 + * + * @param listDataVersionFile 文件版本数据 + */ + public void saveDataVersionFile(List listDataVersionFile) { + dataVersionFileService.saveBatchDataFileVersion(listDataVersionFile); + } + + + /** + * 批量保存标签与数据集关系表 + * + * @param listDatasetDataLabel 标签与数据集关系表 + */ + public void saveDatasetDataLabel(List listDatasetDataLabel) { + datasetDataLabelService.saveBatchDatasetDataLabel(listDatasetDataLabel); + } + + /** + * 批量保存标签与标签组的关系 + * + * @param listDataGroupLabel 标签与标签组集合 + */ + public void saveDatasetDataGroupLabel(List listDataGroupLabel) { + dataGroupLabelService.saveDataGroupLabel(listDataGroupLabel); + } + + /** + * 查询数据集 + * + * @param datasetId 数据集Id + * @return Dataset 根据数据集ID查询返回的数据集 + */ + private Dataset findDataset(Long datasetId) { + return datasetService.findDatasetById(datasetId); + } + + /** + * 校验数据集ID + * + * @param scanner 控制台输入参数 + * @return Dataset 数据集 + */ + public Dataset verificationDatasetId(Scanner scanner) { + log.warn("# 请输入数据集ID #"); + String datasetIdStr = scanner.nextLine(); + long datasetId; + try { + datasetId = Long.parseLong(datasetIdStr.trim()); + } catch (Exception e) { + throw new ImportDatasetException(" 数据集ID非法,请重新输入 "); + } + Dataset dataset = findDataset(datasetId); + if (dataset == null) { + throw new ImportDatasetException(" 数据集ID不存在，请重新输入 "); + } + int countDataLabel = datasetService.findDataLabelById(dataset.getId()); + int countDataFile = datasetService.findDataFileById(dataset.getId()); + if (countDataLabel > MagicNumConstant.ZERO || countDataFile > MagicNumConstant.ZERO) { + throw new ImportDatasetException(" 当前数据集文件已存在，请勿重新导入 "); + } + return dataset; + } + + /** + * 校验文件路径 + * + * @param scanner 输入控制台 + * @return String 字符串 + */ + public String verificationFilePath(Scanner scanner) { + log.warn("# 请输入待导入数据集绝对路径地址 #"); + String filePath = scanner.nextLine(); + File file = new File(filePath.trim()); + if (!file.exists()) { + throw new ImportDatasetException(" 【" + filePath + "】 文件路径不存在,请重新输入"); + } + return filePath; + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataFileService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataFileService.java new file mode 100644 index 0000000..c100a35 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataFileService.java @@ -0,0 +1,35 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service; + +import org.dubhe.datasetutil.domain.entity.DataFile; +import java.util.List; + +/** + * @description 文件服务 + * @date 2020-09-17 + */ +public interface DataFileService { + + /** + * 批量写入文件内容 + * + * @param dataFiles 文件数据集合 + */ + void saveBatchDataFile(List dataFiles); + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataGroupLabelService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataGroupLabelService.java new file mode 100644 index 0000000..5c5c0ce --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataGroupLabelService.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service; + +import org.dubhe.datasetutil.domain.entity.DataGroupLabel; + +import java.util.List; + +/** + * @description 标签与标签组的服务接口 + * @date 2020-10-21 + */ +public interface DataGroupLabelService { + /** + * 保存标签与标签组的关系 + * + * @param listDataGroupLabel 标签与标签组数据 + */ + void saveDataGroupLabel(List listDataGroupLabel); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelGroupService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelGroupService.java new file mode 100644 index 0000000..9f2f1c4 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelGroupService.java @@ -0,0 +1,40 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service; + +import org.dubhe.datasetutil.domain.entity.DataLabelGroup; + +/** + * @description 数据集标签组服务接口 + * @date 2020-10-14 + */ +public interface DataLabelGroupService { + /** + * 根据标签组名查询 + * + * @param labelGroupName 标签组名称 + * @return int 数量 + */ + int selectByLabelGroupName(String labelGroupName); + + /** + * 保存数据集标签组标签 + * + * @param dataLabelGroup 标签组名称 + */ + void saveDataGroupLabel(DataLabelGroup dataLabelGroup); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelService.java new file mode 100644 index 0000000..a4a98fa --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataLabelService.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service; + +import org.dubhe.datasetutil.domain.entity.DataLabel; + +import java.util.List; + +/** + * @description 数据集标签服务接口 + * @date 2020-10-14 + */ +public interface DataLabelService { + /** + * 批量保存数据集标签 + * + * @param listDataLabel 数据集标签集合 + */ + void saveBatchDataLabel(List listDataLabel); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataSequenceService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataSequenceService.java new file mode 100644 index 0000000..5199174 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataSequenceService.java @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service; + + +import org.dubhe.datasetutil.domain.entity.DataSequence; + +/** + * @description 获取序列服务接口 + * @date 2020-09-23 + */ +public interface DataSequenceService { + /** + * 根据业务编码获取序列号 + * + * @param businessCode 业务编码 + * @return DataSequence 数据索引 + */ + DataSequence getSequence(String businessCode); + /** + * 根据业务编码更新起点 + * + * @param businessCode 业务编码 + * @return int 数量 + */ + int updateSequenceStart(String businessCode); + + /** + * 检查表是否存在 + * + * @param tableName 表名 + * @return boolean 存在结果 + */ + boolean checkTableExist(String tableName); + + /** + * 执行存储过程创建表 + * + * @param tableId 表ID + */ + void createTable(String tableId); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataVersionFileService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataVersionFileService.java new file mode 100644 index 0000000..5b3d5a9 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DataVersionFileService.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service; + +import org.dubhe.datasetutil.domain.dto.DataVersionFile; + +import java.util.List; + +/** + * @description 数据集文件关系服务 + * @date 2020-9-17 + */ +public interface DataVersionFileService { + /** + * 插入数据集文件数据 + * + * @param dataVersionFiles 数据集文件数据集合 + */ + void saveBatchDataFileVersion(List dataVersionFiles); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetDataLabelService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetDataLabelService.java new file mode 100644 index 0000000..6752391 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetDataLabelService.java @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service; + +import org.dubhe.datasetutil.domain.entity.DatasetDataLabel; + +import java.util.List; + +/** + * @description 数据集与标签关系服务接口 + * @date 2020-10-14 + */ +public interface DatasetDataLabelService { + /** + * 批量保存数据集与标签关系 + * + * @param listDatasetDataLabel 数据集标签集合 + */ + void saveBatchDatasetDataLabel(List listDatasetDataLabel); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetService.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetService.java new file mode 100644 index 0000000..06d2b8b --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/DatasetService.java @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service; + +import org.dubhe.datasetutil.domain.entity.Dataset; + +/** + * @description 数据集服务 + * @date 2020-9-17 + */ +public interface DatasetService { + /** + * 根据数据集Id查询创建人Id + * + * @param datasetId 数据集Id + * @return Dataset 数据集 + */ + Dataset findCreateUserIdById(Long datasetId); + + /** + * 根据ID查询数据集 + * + * @param datasetId 数据集Id + * @return Dataset 数据集 + */ + Dataset findDatasetById(Long datasetId); + + /** + * 更新数据集状态 + * + * @param dataset 数据集 + * @return int 数量 + */ + int updateDatasetStatus(Dataset dataset); + + /** + * 查询数据集标签数量 + * + * @param datasetId 数据集ID + * @return int 数量 + */ + int findDataLabelById(Long datasetId); + + /** + * 查询数据集文件数量 + * + * @param datasetId 数据集ID + * @return int 数量 + */ + int findDataFileById(Long datasetId); +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataFileServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataFileServiceImpl.java new file mode 100644 index 0000000..1186efe --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataFileServiceImpl.java @@ -0,0 +1,60 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.dubhe.datasetutil.common.constant.BusinessConstant; +import org.dubhe.datasetutil.dao.DataFileMapper; +import org.dubhe.datasetutil.domain.entity.DataFile; +import org.dubhe.datasetutil.service.DataFileService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @description 数据集文件 服务实现类 + * @date 2020-09-17 + */ +@Slf4j +@Service +public class DataFileServiceImpl extends ServiceImpl implements DataFileService { + + @Autowired + private DataFileMapper dataFileMapper; + + /** + * 批量写入文件内容 + * + * @param listDataFile 文件数据集合 + */ + @Override + public void saveBatchDataFile(List listDataFile) { + int listSize = listDataFile.size(); + if (listSize > BusinessConstant.SUB_LENGTH) { + List> partitionList = Lists.partition(listDataFile, BusinessConstant.SUB_LENGTH); + for (List subPartitionList : partitionList) { + dataFileMapper.saveBatchDataFile(subPartitionList); + } + } else { + dataFileMapper.saveBatchDataFile(listDataFile); + } + } + +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataGroupLabelServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataGroupLabelServiceImpl.java new file mode 100644 index 0000000..dd9dde0 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataGroupLabelServiceImpl.java @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.google.common.collect.Lists; +import org.dubhe.datasetutil.common.constant.BusinessConstant; +import org.dubhe.datasetutil.dao.DataGroupLabelMapper; +import org.dubhe.datasetutil.domain.entity.DataGroupLabel; +import org.dubhe.datasetutil.service.DataGroupLabelService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @description 标签与标签组的服务接口实现 + * @date 2020-10-21 + */ +@Service +public class DataGroupLabelServiceImpl extends ServiceImpl implements DataGroupLabelService { + + @Autowired + private DataGroupLabelMapper dataGroupLabelMapper; + + /** + * 插入标签组 + * + * @param listDataGroupLabel 标签组数据 + */ + @Override + public void saveDataGroupLabel(List listDataGroupLabel) { + int listSize = listDataGroupLabel.size(); + if (listSize > BusinessConstant.SUB_LENGTH) { + List> partitionList = Lists.partition(listDataGroupLabel, BusinessConstant.SUB_LENGTH); + for (List subPartitionList : partitionList) { + dataGroupLabelMapper.saveDataGroupLabel(subPartitionList); + } + } else { + dataGroupLabelMapper.saveDataGroupLabel(listDataGroupLabel); + } + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelGroupServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelGroupServiceImpl.java new file mode 100644 index 0000000..47b936b --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelGroupServiceImpl.java @@ -0,0 +1,60 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.dubhe.datasetutil.dao.DataGroupLabelMapper; +import org.dubhe.datasetutil.dao.DataLabelGroupMapper; +import org.dubhe.datasetutil.domain.entity.DataGroupLabel; +import org.dubhe.datasetutil.domain.entity.DataLabelGroup; +import org.dubhe.datasetutil.service.DataLabelGroupService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * @description 数据集标签组服务实现接口 + * @date 2020-10-14 + */ +@Service +public class DataLabelGroupServiceImpl extends ServiceImpl implements DataLabelGroupService { + + @Autowired + private DataLabelGroupMapper dataLabelGroupMapper; + + /** + * 根据标签组名查询 + * + * @param labelGroupName 标签组名称 + * @return int 数量 + */ + @Override + public int selectByLabelGroupName(String labelGroupName) { + return dataLabelGroupMapper.selectByLabelGroupName(labelGroupName); + } + + /** + * 保存标签组名称 + * + * @param dataLabelGroup 标签组名称 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public void saveDataGroupLabel(DataLabelGroup dataLabelGroup) { + dataLabelGroupMapper.insert(dataLabelGroup); + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelServiceImpl.java new file mode 100644 index 0000000..0d2fa0b --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataLabelServiceImpl.java @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service.impl; + +import com.google.common.collect.Lists; +import org.dubhe.datasetutil.common.constant.BusinessConstant; +import org.dubhe.datasetutil.dao.DataLabelMapper; +import org.dubhe.datasetutil.domain.entity.DataLabel; +import org.dubhe.datasetutil.service.DataLabelService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @description 数据集标签服务接口实现 + * @date 2020-10-14 + */ +@Service +public class DataLabelServiceImpl implements DataLabelService { + + @Autowired + private DataLabelMapper dataLabelMapper; + + /** + * 批量保存数据集标签 (分批) + * + * @param listDataLabel 数据集标签 + */ + @Override + public void saveBatchDataLabel(List listDataLabel) { + int listSize = listDataLabel.size(); + if(listSize > BusinessConstant.SUB_LENGTH){ + List> partitionList = Lists.partition(listDataLabel,BusinessConstant.SUB_LENGTH); + for (List subPartitionList: partitionList) { + dataLabelMapper.saveBatchDataLabel(subPartitionList); + } + }else{ + dataLabelMapper.saveBatchDataLabel(listDataLabel); + } + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataSequenceServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataSequenceServiceImpl.java new file mode 100644 index 0000000..22f1dbb --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataSequenceServiceImpl.java @@ -0,0 +1,78 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service.impl; + +import org.dubhe.datasetutil.dao.DataSequenceMapper; +import org.dubhe.datasetutil.domain.entity.DataSequence; +import org.dubhe.datasetutil.service.DataSequenceService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + + +/** + * @description 获取序列服务接口实现 + * @date 2020-09-23 + */ +@Service +public class DataSequenceServiceImpl implements DataSequenceService { + + @Autowired + private DataSequenceMapper dataSequenceMapper; + + @Override + public DataSequence getSequence(String businessCode) { + return dataSequenceMapper.selectByBusiness(businessCode); + } + + /** + * 更新数据索引 + * + * @param businessCode 业务编码 + * @return int 数量 + */ + @Override + public int updateSequenceStart(String businessCode) { + return dataSequenceMapper.updateStartByBusinessCode(businessCode); + } + + /** + * 检查表名 + * + * @param tableName 表名 + * @return true/false + */ + @Override + public boolean checkTableExist(String tableName) { + try { + dataSequenceMapper.checkTableExist(tableName); + return true; + }catch (Exception e){ + return false; + } + } + + /** + * 创建表 + * + * @param tableName 类型名称 + */ + @Override + public void createTable(String tableName) { + String oldTableName = tableName.substring(0,tableName.lastIndexOf("_")); + dataSequenceMapper.createNewTable(tableName,oldTableName); + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataVersionFileServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataVersionFileServiceImpl.java new file mode 100644 index 0000000..a032328 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DataVersionFileServiceImpl.java @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.dubhe.datasetutil.dao.DataVersionFileMapper; +import org.dubhe.datasetutil.domain.dto.DataVersionFile; +import org.dubhe.datasetutil.service.DataVersionFileService; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @description 数据集文件 服务实现类 + * @date 2020-09-17 + */ +@Service +public class DataVersionFileServiceImpl extends ServiceImpl implements DataVersionFileService { + + + /** + * 插入数据集文件数据 + * + * @param listDataVersionFile 数据集文件数据集合 + */ + @Override + public void saveBatchDataFileVersion(List listDataVersionFile) { + baseMapper.saveBatchDataFileVersion(listDataVersionFile); + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetDataLabelServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetDataLabelServiceImpl.java new file mode 100644 index 0000000..6f0cf52 --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetDataLabelServiceImpl.java @@ -0,0 +1,57 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service.impl; + +import com.google.common.collect.Lists; +import org.dubhe.datasetutil.common.base.MagicNumConstant; +import org.dubhe.datasetutil.dao.DatasetDataLabelMapper; +import org.dubhe.datasetutil.domain.entity.DatasetDataLabel; +import org.dubhe.datasetutil.service.DatasetDataLabelService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +/** + * @description 数据集和标签服务实现类 + * @date 2020-10-14 + */ +@Service +public class DatasetDataLabelServiceImpl implements DatasetDataLabelService { + + @Autowired + private DatasetDataLabelMapper datasetDataLabelMapper; + + private static final int SUB_LENGTH = MagicNumConstant.FIVE_THOUSAND; + + /** + * 批量保存数据集标签 + * + * @param listDatasetDataLabel 数据集标签集合 + */ + @Override + public void saveBatchDatasetDataLabel(List listDatasetDataLabel) { + if(listDatasetDataLabel.size() > SUB_LENGTH){ + List> partitionList = Lists.partition(listDatasetDataLabel,SUB_LENGTH); + for (List subPartitionList: partitionList) { + datasetDataLabelMapper.saveBatchDatasetDataLabel(subPartitionList); + } + }else{ + datasetDataLabelMapper.saveBatchDatasetDataLabel(listDatasetDataLabel); + } + } +} diff --git a/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetServiceImpl.java b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetServiceImpl.java new file mode 100644 index 0000000..10ab8cc --- /dev/null +++ b/dataset-util/src/main/java/org/dubhe/datasetutil/service/impl/DatasetServiceImpl.java @@ -0,0 +1,92 @@ +/** + * Copyright 2020 Zhejiang Lab. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================= + */ +package org.dubhe.datasetutil.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.dubhe.datasetutil.common.constant.DataStateCodeConstant; +import org.dubhe.datasetutil.dao.DatasetMapper; +import org.dubhe.datasetutil.domain.entity.Dataset; +import org.dubhe.datasetutil.service.DatasetService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * @description 数据集 服务实现类 + * @date 2020-09-17 + */ +@Service +public class DatasetServiceImpl extends ServiceImpl implements DatasetService { + + @Autowired + private DatasetMapper datasetMapper; + + /** + * 根据数据集Id查询创建人Id + * + * @param datasetId 数据集Id + * @return dataset 数据集合 + */ + @Override + public Dataset findCreateUserIdById(Long datasetId) { + return baseMapper.selectById(datasetId); + } + + /** + * 根据ID 查询数据集 + * + * @param datasetId 数据集ID + * @return Dataset 数据集 + */ + @Override + public Dataset findDatasetById(Long datasetId) { + return datasetMapper.findDatasetById(datasetId); + } + + /** + * 更新数据集状态 + * + * @param dataset 数据集 + * @return int 数量 + */ + @Override + public int updateDatasetStatus(Dataset dataset) { + dataset.setStatus(DataStateCodeConstant.ANNOTATION_COMPLETE_STATE); + return datasetMapper.updateById(dataset); + } + + /** + * 查询数据集数据标签数量 + * + * @param datasetId 数据集ID + * @return int 数量 + */ + @Override + public int findDataLabelById(Long datasetId) { + return datasetMapper.findDataLabelById(datasetId); + } + + /** + * 查询数据集文件数量 + * + * @param datasetId 数据集ID + * @return int 数量 + */ + @Override + public int findDataFileById(Long datasetId) { + return datasetMapper.findDataFileById(datasetId); + } +} diff --git a/dataset-util/src/main/resources/banner.txt b/dataset-util/src/main/resources/banner.txt new file mode 100644 index 0000000..e69de29 diff --git a/dataset-util/src/main/resources/logback-spring-dev.xml b/dataset-util/src/main/resources/logback-spring-dev.xml new file mode 100644 index 0000000..90e2344 --- /dev/null +++ b/dataset-util/src/main/resources/logback-spring-dev.xml @@ -0,0 +1,254 @@ + + + dubhe + + + + + + + + ${log.pattern} + + + WARN + INFO + ACCEPT + DENY + + + + + + logs/${log.path}/info/dubhe-info.log + + logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + INFO + INFO,K8S_CALLBACK + ACCEPT + DENY + + + + + + logs/${log.path}/debug/dubhe-debug.log + + logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + DEBUG + DEBUG + ACCEPT + DENY + + + + + + logs/${log.path}/error/dubhe-error.log + + logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + ERROR + ERROR + ACCEPT + DENY + + + + + + logs/${log.path}/warn/dubhe-warn.log + + logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + + WARN + WARN + ACCEPT + DENY + + + + + + logs/${log.path}/trace/dubhe-trace.log + + logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + TRACE + TRACE + ACCEPT + DENY + + + + + + + logs/${log.path}/info/dubhe-schedule.log + + logs/${log.path}/info/dubhe-${app.active}-schedule-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + INFO + SCHEDULE + ACCEPT + DENY + + + + + + logs/${log.path}/info/dubhe-request.log + + logs/${log.path}/info/dubhe-${app.active}-request-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + INFO + + GLOBAL_REQUEST + ACCEPT + DENY + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dataset-util/src/main/resources/logback-spring-test.xml b/dataset-util/src/main/resources/logback-spring-test.xml new file mode 100644 index 0000000..038359c --- /dev/null +++ b/dataset-util/src/main/resources/logback-spring-test.xml @@ -0,0 +1,248 @@ + + + dubhe + + + + + + + + + ${log.pattern} + + + + + + logs/${log.path}/info/dubhe-info.log + + logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + INFO + INFO,K8S_CALLBACK + ACCEPT + DENY + + + + + + logs/${log.path}/debug/dubhe-debug.log + + logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + DEBUG + DEBUG + ACCEPT + DENY + + + + + + logs/${log.path}/error/dubhe-error.log + + logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + ERROR + ERROR + ACCEPT + DENY + + + + + + logs/${log.path}/warn/dubhe-warn.log + + logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + + WARN + WARN + ACCEPT + DENY + + + + + + logs/${log.path}/trace/dubhe-trace.log + + logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + TRACE + TRACE + ACCEPT + DENY + + + + + + + logs/${log.path}/info/dubhe-schedule.log + + logs/${log.path}/info/dubhe-${app.active}-schedule-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + INFO + SCHEDULE + ACCEPT + DENY + + + + + + logs/${log.path}/info/dubhe-request.log + + logs/${log.path}/info/dubhe-${app.active}-request-%d{yyyy-MM-dd}.%i.log + + + 50MB + 7 + 250MB + + + %m%n + + + true + + INFO + + GLOBAL_REQUEST + ACCEPT + DENY + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dataset-util/src/main/resources/mapper/DataFileMapper.xml b/dataset-util/src/main/resources/mapper/DataFileMapper.xml new file mode 100644 index 0000000..1f7b10f --- /dev/null +++ b/dataset-util/src/main/resources/mapper/DataFileMapper.xml @@ -0,0 +1,22 @@ + + + + + + insert into data_file (id,`name`,dataset_id,status,url,enhance_type,width,height,origin_user_id,create_user_id,pid) + values + + (#{file.id},#{file.name},#{file.datasetId},#{file.status}, + #{file.url},#{file.enhanceType},#{file.width},#{file.height},#{file.originUserId},#{file.createUserId} + + + ,#{file.pid} + + + ,0 + + + ) + + + \ No newline at end of file diff --git a/dataset-util/src/main/resources/mapper/DataGroupLabelMapper.xml b/dataset-util/src/main/resources/mapper/DataGroupLabelMapper.xml new file mode 100644 index 0000000..91059a9 --- /dev/null +++ b/dataset-util/src/main/resources/mapper/DataGroupLabelMapper.xml @@ -0,0 +1,12 @@ + + + + + + insert into data_group_label (label_id,label_group_id) + values + + (#{dataGroupLabel.labelId},#{dataGroupLabel.labelGroupId}) + + + \ No newline at end of file diff --git a/dataset-util/src/main/resources/mapper/DataLabelMapper.xml b/dataset-util/src/main/resources/mapper/DataLabelMapper.xml new file mode 100644 index 0000000..20d80cb --- /dev/null +++ b/dataset-util/src/main/resources/mapper/DataLabelMapper.xml @@ -0,0 +1,12 @@ + + + + + + insert into data_label (`name`,color,create_user_id) + values + + ( #{dataLabel.name},#{dataLabel.color},#{dataLabel.createUserId}) + + + \ No newline at end of file diff --git a/dataset-util/src/main/resources/mapper/DataVersionFileMapper.xml b/dataset-util/src/main/resources/mapper/DataVersionFileMapper.xml new file mode 100644 index 0000000..5eb4245 --- /dev/null +++ b/dataset-util/src/main/resources/mapper/DataVersionFileMapper.xml @@ -0,0 +1,12 @@ + + + + + + insert into data_dataset_version_file (id,dataset_id,file_id,annotation_status,status) + values + + (#{temp.id},#{temp.datasetId},#{temp.fileId},#{temp.annotationStatus},#{temp.status}) + + + \ No newline at end of file diff --git a/dataset-util/src/main/resources/mapper/DatasetDataLabelMapper.xml b/dataset-util/src/main/resources/mapper/DatasetDataLabelMapper.xml new file mode 100644 index 0000000..59c4e56 --- /dev/null +++ b/dataset-util/src/main/resources/mapper/DatasetDataLabelMapper.xml @@ -0,0 +1,12 @@ + + + + + + insert into data_dataset_label(dataset_id,label_id ) + values + + (#{datasetDataLabel.datasetId},#{datasetDataLabel.labelId}) + + + \ No newline at end of file diff --git a/dataset-util/src/main/resources/mapper/DatasetMapper.xml b/dataset-util/src/main/resources/mapper/DatasetMapper.xml new file mode 100644 index 0000000..7e7617f --- /dev/null +++ b/dataset-util/src/main/resources/mapper/DatasetMapper.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file