diff --git a/LICENSE b/LICENSE index ef4ea8b..d645695 100644 --- a/LICENSE +++ b/LICENSE @@ -200,12 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -Other dependencies and licenses: ----------------------------------------------------------------------------------------- - -Open Source Software Licensed Under the Apache License, Version 2.0: -The below software in this distribution may have been modified. ----------------------------------------------------------------------------------------- -1. EL-ADMIN -Copyright 2019-2020 Zheng Jie \ No newline at end of file diff --git a/README.md b/README.md index 7652fa9..4d68fbb 100644 --- a/README.md +++ b/README.md @@ -35,14 +35,6 @@ ## 技术架构 ![技术架构](http://cdn.qjycloud.com/tech-arc.jpg "技术架构") -## 技术栈 -- 后端: [Spring Boot](https://spring.io/projects/spring-boot) -- 前端: [Vue.js](https://vuejs.org/), [Element](https://element.eleme.cn/) -- 数据处理 [Yolo](https://pjreddie.com/darknet/yolo/) ... -- 可视化: [Django](https://www.djangoproject.com/) ... -- 中间件: [MySQL](https://www.mysql.com/), [MyBatis-Plus](https://mp.baomidou.com/), [Redis](https://redis.io/) -- 基础设施: [Docker](https://www.docker.com/), [Kubernetes](https://kubernetes.io/) - ## 反馈问题 - [在线社区](http://www.aiiaos.cn/index.php?s=/forum/index/forum/id/45.html) diff --git a/dubhe-server/README.md b/dubhe-server/README.md index 609205f..b6fa52c 100644 --- a/dubhe-server/README.md +++ b/dubhe-server/README.md @@ -1,8 +1,6 @@ -# 之江天枢-服务端 +# 一站式开发平台-服务端 -**之江天枢一站式人工智能开源平台**（简称：**之江天枢**），包括海量数据处理、交互式模型构建（包含Notebook和模型可视化）、AI模型高效训练。多维度产品形态满足从开发者到大型企业的不同需求，将提升人工智能技术的研发效率、扩大算法模型的应用范围，进一步构建人工智能生态“朋友圈”。 - -## 源码部署 +## 本地开发 ### 准备环境 安装如下软件环境。 @@ -11,13 +9,6 @@ - Maven: 3.0+ - MYSQL: 5.5.0+ -### 下载源码 -``` bash -git clone https://codeup.teambition.com/zhejianglab/dubhe-server.git -# 进入项目根目录 -cd dubhe-server -``` - ### 创建DB 在MySQL中依次执行如下sql文件 ``` @@ -29,31 +20,14 @@ sql/v1/02-Dubhe-DML.sql ### 配置 根据实际情况修改如下配置文件。 ``` -dubhe-admin/src/main/resources/config/application-prod.yml +dubhe-admin/src/main/resources/config/application-dev.yml ``` -### 构建 -``` bash -# 构建，生成的 jar 包位于 ./dubhe-admin/target/dubhe-admin-1.0.jar -mvn clean compile package +### 启动： ``` - -### 启动 -``` bash -# 指定启动环境为 prod -java -jar ./dubhe-admin/target/dubhe-admin-1.0.jar --spring.profiles.active=prod +mvn spring-boot:run ``` -## 本地开发 - -### 必要条件： - 导入maven项目，下载所需的依赖包 - mysql下创建数据库dubhe，初始化数据脚本 - 安装redis - -### 启动： - mvn spring-boot:run - ## 代码结构： ``` ├── common 公共模块 diff --git a/webapp/README.md b/webapp/README.md index 9a37041..b31ee0e 100644 --- a/webapp/README.md +++ b/webapp/README.md @@ -1,78 +1,10 @@ # 一站式开发平台-前端 -**天枢人工智能开源开放平台**（简称：**天枢平台**）是天枢平台由之江实验室牵头，联合北京一流科技、中国信通院和浙江大学共同自研的人工智能开源平台。整个平台由一站式AI模型开发平台、高性能深度学习框架和模型炼知框架三大子系统组成。 - -其中， **一站式AI模型开发平台面**（简称：**一站式开发平台**）面向AI模型生产的生命周期，提供了包括数据处理、模型开发、模型训练和模型管理等功能，方便用户一站式构建AI算法。 - -## 特性 -* 一站式开发 -* 集成先进算法 -* 灵活易用 -* 性能优越 - -## 预览 -![概览](/public/dubhe_dashboard.png "概览") - -## 源码部署 - -### 1. 下载源码 - -``` bash -git clone https://codeup.teambition.com/zhejianglab/dubhe-web.git - -# 进入根目录 -cd dubhe-web - -``` -### 2. 配置 - -根据需要修改如下配置文件 -``` -config/index.js -settings.js -.env.production -``` - -### 3. 构建 - -``` bash -# 安装项目依赖 -npm install - -# 构建生产环境 -npm run build:prod -``` - -### 4. 部署 - -- 构建完成后会在根目录生成 dist 文件夹，并将该文件夹上传至服务器； -- 在服务器 nginx.conf 文件中添加如下配置； - -``` nginx -server { - listen 80; # 端口 - server_name localhost; # 域名/外网IP - - location / { - root /home/wwwroot/dubhe-web/dist; # dist 文件夹根目录 - index index.html; - try_files $uri $uri/ /index.html; - } -} - -``` - -- 保存 `nginx.conf` 并重启 Nginx 使之生效。 - - ## 本地开发 ``` bash -# 下载源码 -git clone https://codeup.teambition.com/zhejianglab/dubhe-web.git - -# 进入项目根目录 -cd dubhe-web +# 进入前端项目根目录 +cd webapp # 安装依赖 npm install diff --git a/webapp/src/components/UploadForm/index.vue b/webapp/src/components/UploadForm/index.vue index d4a5784..6c54850 100644 --- a/webapp/src/components/UploadForm/index.vue +++ b/webapp/src/components/UploadForm/index.vue @@ -55,6 +55,7 @@ export default { type: Boolean, default: false, }, + transformFile: Function, hash: { type: Boolean, default: true, @@ -67,7 +68,7 @@ export default { }, }, setup(props, ctx) { - const { toggleVisible, request } = props; + const { toggleVisible, request, transformFile } = props; const formRef = ref(null); const state = reactive({ visible: props.visible, @@ -98,7 +99,7 @@ export default { state.uploading = true; // 开始调用上传接口 - uploadReqeust && uploadReqeust({ ...props.params, fileList: renameFileList }, handleUploadProgress) + uploadReqeust && uploadReqeust({ ...props.params, fileList: renameFileList, transformFile }, handleUploadProgress) .then(res => { const outputPath = getFileOutputPath(renameFileList, props.params); state.uploading = false; diff --git a/webapp/src/components/UploadForm/inline.vue b/webapp/src/components/UploadForm/inline.vue index 6c24639..6282309 100644 --- a/webapp/src/components/UploadForm/inline.vue +++ b/webapp/src/components/UploadForm/inline.vue @@ -38,6 +38,7 @@ export default { type: Boolean, default: false, }, + transformFile: Function, hash: { type: Boolean, default: true, @@ -48,7 +49,7 @@ export default { }, }, setup(props, ctx) { - const { request } = props; + const { request, transformFile } = props; const formRef = ref(null); const state = reactive({ uploading: false, @@ -71,7 +72,7 @@ export default { ctx.emit('uploadStart'); const uploadReqeust = request || minIOUpload; // 开始调用上传接口 - return uploadReqeust({ ...props.params, fileList: renameFileList }, callback) + return uploadReqeust({ ...props.params, fileList: renameFileList, transformFile }, callback) .then(res => { const outputPath = getFileOutputPath(renameFileList, props.params); state.uploading = false; diff --git a/webapp/src/components/UploadForm/util.js b/webapp/src/components/UploadForm/util.js index 4c6151e..1268f86 100644 --- a/webapp/src/components/UploadForm/util.js +++ b/webapp/src/components/UploadForm/util.js @@ -87,7 +87,7 @@ export const putObject = (uploadUrl, file, options = {}) => { }; // 默认通过 minIO 上传 -export const minIOUpload = async({ objectPath, fileList }, callback) => { +export const minIOUpload = async({ objectPath, fileList, transformFile }, callback) => { // add 进度条 let resolved = 0; @@ -97,7 +97,7 @@ export const minIOUpload = async({ objectPath, fileList }, callback) => { const uploadPrefix = `${minIOPrefix}/${bucketName}`; const objectName = `${objectPath}/${d.name}`; - const result = await putObject(`${uploadPrefix}/${objectName}`, d.raw, { + const fileRes = await putObject(`${uploadPrefix}/${objectName}`, d.raw, { objectName, callback, }); @@ -110,11 +110,18 @@ export const minIOUpload = async({ objectPath, fileList }, callback) => { if (typeof callback === 'function' && fileList.length > 1) { callback(resolved, fileList.length); } - return result; + + // 视频不做转换 + if (isValidVideoFile(d)) return fileRes; + + if (typeof transformFile === 'function') { + const transformed = await transformFile(fileRes, d); + return transformed; + } + return fileRes; }; const result = await pMap(fileList, mapper, {concurrency: 10}); - return result; }; @@ -125,3 +132,26 @@ export const hashify = (name, hash) => { export const getFileOutputPath = (rawFiles, { objectPath }) => { return rawFiles.map(d => `${bucketHost}/${bucketName}/${objectPath}/${d.name}`); }; + +// 对文件进行自定义转换 +export const transformFile = (result, file) => { + return new Promise((resolve) => { + const reader = new FileReader(); + reader.addEventListener("load", () => { + const img = new Image(); + img.onload = () => resolve({ + ...result, + data: { + ...result.data, + meta: { + width: img.width, + height: img.height, + }, + }, + }); + img.src = reader.result; + }, false); + + reader.readAsDataURL(file.raw); + }); +}; diff --git a/webapp/src/views/dataset/annotate/thumbContainer/index.vue b/webapp/src/views/dataset/annotate/thumbContainer/index.vue index 3816376..0d0b5bf 100644 --- a/webapp/src/views/dataset/annotate/thumbContainer/index.vue +++ b/webapp/src/views/dataset/annotate/thumbContainer/index.vue @@ -78,6 +78,7 @@ ref="uploaderRef" action="fakeApi" :visible="thumbState.showDialog" + :transformFile="withDimensionFile" :toggleVisible="handleClose" :params="uploadParams" @uploadSuccess="uploadSuccess" @@ -92,7 +93,7 @@ import { Message } from 'element-ui'; import { pick } from 'lodash'; import UploadForm from '@/components/UploadForm'; -import { fileTypeEnum, getImgFromMinIO, withDimensionFiles } from '@/views/dataset/util'; +import { fileTypeEnum, getImgFromMinIO, withDimensionFile } from '@/views/dataset/util'; import { submit } from '@/api/preparation/datafile'; import { detectFileList, queryFileOffset } from '@/api/preparation/dataset'; import List from './list'; @@ -160,9 +161,8 @@ export default { const uploadSuccess = async(res) => { const files = getImgFromMinIO(res); - const _files = await withDimensionFiles(files); // 提交业务上传 - submit(datasetId.value, _files).then(() => { + submit(datasetId.value, files).then(() => { Message.success('上传成功'); updateList({ type: thumbState.type }); }); @@ -224,6 +224,7 @@ export default { return { thumbState, + withDimensionFile, uploadParams, dropdownList, handleUpload, diff --git a/webapp/src/views/dataset/classify.vue b/webapp/src/views/dataset/classify.vue index e896acd..d645b98 100644 --- a/webapp/src/views/dataset/classify.vue +++ b/webapp/src/views/dataset/classify.vue @@ -19,6 +19,7 @@ 0) { - submit(this.datasetId, _files).then(() => { + if (files.length > 0) { + submit(this.datasetId, files).then(() => { this.$message({ message: '上传文件成功', type: 'success', diff --git a/webapp/src/views/dataset/list/index.vue b/webapp/src/views/dataset/list/index.vue index d98f167..517dea4 100644 --- a/webapp/src/views/dataset/list/index.vue +++ b/webapp/src/views/dataset/list/index.vue @@ -125,6 +125,7 @@ ref="initFileUploadForm" action="fakeApi" :params="uploadParams" + :transformFile="withDimensionFile" v-bind="optionCreateProps" @uploadSuccess="uploadSuccess" @uploadError="uploadError" @@ -203,6 +204,7 @@ :visible="uploadDialogVisible" :toggleVisible="toggleUploadFormClose" :params="uploadParams" + :transformFile="withDimensionFile" v-bind="optionImportProps" @uploadSuccess="uploadSuccess" @uploadError="uploadError" @@ -446,7 +448,7 @@ import InfoSelect from '@/components/InfoSelect'; import { getAutoLabels } from '@/api/preparation/datalabel'; import { submit, submitVideo } from '@/api/preparation/datafile'; -import { getImgFromMinIO, getFullFileUrl, annotationMap, dataTypeMap, annotationProgressMap, decompressProgressMap, datasetStatusMap, withDimensionFiles } from '@/views/dataset/util'; +import { getImgFromMinIO, annotationMap, dataTypeMap, annotationProgressMap, decompressProgressMap, datasetStatusMap, withDimensionFile } from '@/views/dataset/util'; import Edit from '@/components/InlineTableEdit'; import BaseModal from '@/components/BaseModal'; import { toFixed, isEqualByProp, formatDateTime, downloadZipFromObjectPath } from '@/utils'; @@ -586,6 +588,10 @@ export default { return String(state.dataset.activePanel); }, }), + // 文件上传前携带尺寸信息 + withDimensionFile() { + return withDimensionFile; + }, // 自定义上传数据集 isImport() { return isImport; @@ -1132,41 +1138,14 @@ export default { } }, - // 获取文件信息 - async checkImg (file){ - const fileUrl = getFullFileUrl(file); - return new Promise((resolve, reject) => { - const img = new Image(); - img.onload = () => resolve({ - width: img.width, - height: img.height, - ...file, - }); - img.onerror = (err) => reject(err); - - img.src = fileUrl; - }); - }, - - // 上传文件之前加一层转换 - async getTransformFiles(files) { - return Promise.all(files.map(file => this.checkImg(file))); - }, - // 将文件上传和视频上传统一 - async uploader(datasetId, files, options = {}) { + async uploader(datasetId, files) { const datasetInfo = await this.queryDatasetDetail(datasetId); - const { transformFile } = options; // 点击导入操作 const { dataType } = datasetInfo || {}; // 文件上传 if (dataType === 0) { - let _files = files.slice(); - // 对文件进行转换，生成宽、高信息 - if (typeof transformFile === 'function') { - _files = await transformFile(files); - } - return submit(datasetId, _files); + return submit(datasetId, files); } if (dataType === 1) { // 根据是否通过点击导入按钮来区分 frameInterval 来源 const frameInterval = this.importRow @@ -1194,9 +1173,7 @@ export default { const successMessage = [0, 1].includes(this.chosenDatasetStatus) ? '上传文件成功' : '上传文件成功，若数据集状态未及时更新，请手动刷新页面'; if (files.length > 0) { - this.uploader(this.chosenDatasetId, files, { - transformFile: withDimensionFiles, - }).then(() => { + this.uploader(this.chosenDatasetId, files).then(() => { this.$message({ message: successMessage, duration: 5000, @@ -1244,9 +1221,11 @@ export default { return toFixed(allFinished / (allFinished + progress.unfinished), 2, 0); }, parseDataType(row, column, cellValue = 0) { + if(row.import) return "自定义"; return dataTypeMap[cellValue]; }, parseAnnotateType(row, column, cellValue) { + if(row.import) return "自定义"; return (annotationMap[cellValue] || {}).name || ''; }, parseStatus(row, column, cellValue = 0) { diff --git a/webapp/src/views/dataset/util.js b/webapp/src/views/dataset/util.js index f43a7dd..ca6159c 100644 --- a/webapp/src/views/dataset/util.js +++ b/webapp/src/views/dataset/util.js @@ -74,9 +74,33 @@ export const stringifyAnnotations = (annotations) => { const buildImgUrl = (list = []) => { return list.map(d => ({ url: `${bucketName}/${d.data.objectName}`, + ...(d.data.meta || {}), // 附加的信息，目前只包括 width, height })); }; +// 对文件进行自定义转换 +export const withDimensionFile = (result, file) => { + return new Promise((resolve) => { + const reader = new FileReader(); + reader.addEventListener("load", () => { + const img = new Image(); + img.onload = () => resolve({ + ...result, + data: { + ...result.data, + meta: { + width: img.width, + height: img.height, + }, + }, + }); + img.src = reader.result; + }, false); + + reader.readAsDataURL(file.raw); + }); +}; + export const getImgFromMinIO = (res) => { return buildImgUrl(res); }; @@ -110,6 +134,7 @@ export const transformFile = (rawFile, callback) => { return res; }; +// deprecated // 获取文件信息 async function checkImg (file){ const fileUrl = getFullFileUrl(file); @@ -126,6 +151,7 @@ async function checkImg (file){ }); } +// deprecated // 上传文件之前加一层转换 export const withDimensionFiles = async(files) => { return Promise.all(files.map(file => checkImg(file)));