@@ -200,12 +200,3 @@ | |||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
See the License for the specific language governing permissions and | |||
limitations under the License. | |||
Other dependencies and licenses: | |||
---------------------------------------------------------------------------------------- | |||
Open Source Software Licensed Under the Apache License, Version 2.0: | |||
The below software in this distribution may have been modified. | |||
---------------------------------------------------------------------------------------- | |||
1. EL-ADMIN | |||
Copyright 2019-2020 Zheng Jie |
@@ -35,14 +35,6 @@ | |||
## 技术架构 | |||
![技术架构](http://cdn.qjycloud.com/tech-arc.jpg "技术架构") | |||
## 技术栈 | |||
- 后端: [Spring Boot](https://spring.io/projects/spring-boot) | |||
- 前端: [Vue.js](https://vuejs.org/), [Element](https://element.eleme.cn/) | |||
- 数据处理 [Yolo](https://pjreddie.com/darknet/yolo/) ... | |||
- 可视化: [Django](https://www.djangoproject.com/) ... | |||
- 中间件: [MySQL](https://www.mysql.com/), [MyBatis-Plus](https://mp.baomidou.com/), [Redis](https://redis.io/) | |||
- 基础设施: [Docker](https://www.docker.com/), [Kubernetes](https://kubernetes.io/) | |||
## 反馈问题 | |||
- [在线社区](http://www.aiiaos.cn/index.php?s=/forum/index/forum/id/45.html) | |||
@@ -1,8 +1,6 @@ | |||
# 之江天枢-服务端 | |||
# 一站式开发平台-服务端 | |||
**之江天枢一站式人工智能开源平台**(简称:**之江天枢**),包括海量数据处理、交互式模型构建(包含Notebook和模型可视化)、AI模型高效训练。多维度产品形态满足从开发者到大型企业的不同需求,将提升人工智能技术的研发效率、扩大算法模型的应用范围,进一步构建人工智能生态“朋友圈”。 | |||
## 源码部署 | |||
## 本地开发 | |||
### 准备环境 | |||
安装如下软件环境。 | |||
@@ -11,13 +9,6 @@ | |||
- Maven: 3.0+ | |||
- MYSQL: 5.5.0+ | |||
### 下载源码 | |||
``` bash | |||
git clone https://codeup.teambition.com/zhejianglab/dubhe-server.git | |||
# 进入项目根目录 | |||
cd dubhe-server | |||
``` | |||
### 创建DB | |||
在MySQL中依次执行如下sql文件 | |||
``` | |||
@@ -29,31 +20,14 @@ sql/v1/02-Dubhe-DML.sql | |||
### 配置 | |||
根据实际情况修改如下配置文件。 | |||
``` | |||
dubhe-admin/src/main/resources/config/application-prod.yml | |||
dubhe-admin/src/main/resources/config/application-dev.yml | |||
``` | |||
### 构建 | |||
``` bash | |||
# 构建,生成的 jar 包位于 ./dubhe-admin/target/dubhe-admin-1.0.jar | |||
mvn clean compile package | |||
### 启动: | |||
``` | |||
### 启动 | |||
``` bash | |||
# 指定启动环境为 prod | |||
java -jar ./dubhe-admin/target/dubhe-admin-1.0.jar --spring.profiles.active=prod | |||
mvn spring-boot:run | |||
``` | |||
## 本地开发 | |||
### 必要条件: | |||
导入maven项目,下载所需的依赖包 | |||
mysql下创建数据库dubhe,初始化数据脚本 | |||
安装redis | |||
### 启动: | |||
mvn spring-boot:run | |||
## 代码结构: | |||
``` | |||
├── common 公共模块 | |||
@@ -1,78 +1,10 @@ | |||
# 一站式开发平台-前端 | |||
**天枢人工智能开源开放平台**(简称:**天枢平台**)是天枢平台由之江实验室牵头,联合北京一流科技、中国信通院和浙江大学共同自研的人工智能开源平台。整个平台由一站式AI模型开发平台、高性能深度学习框架和模型炼知框架三大子系统组成。 | |||
其中, **一站式AI模型开发平台面**(简称:**一站式开发平台**)面向AI模型生产的生命周期,提供了包括数据处理、模型开发、模型训练和模型管理等功能,方便用户一站式构建AI算法。 | |||
## 特性 | |||
* 一站式开发 | |||
* 集成先进算法 | |||
* 灵活易用 | |||
* 性能优越 | |||
## 预览 | |||
![概览](/public/dubhe_dashboard.png "概览") | |||
## 源码部署 | |||
### 1. 下载源码 | |||
``` bash | |||
git clone https://codeup.teambition.com/zhejianglab/dubhe-web.git | |||
# 进入根目录 | |||
cd dubhe-web | |||
``` | |||
### 2. 配置 | |||
根据需要修改如下配置文件 | |||
``` | |||
config/index.js | |||
settings.js | |||
.env.production | |||
``` | |||
### 3. 构建 | |||
``` bash | |||
# 安装项目依赖 | |||
npm install | |||
# 构建生产环境 | |||
npm run build:prod | |||
``` | |||
### 4. 部署 | |||
- 构建完成后会在根目录生成 dist 文件夹,并将该文件夹上传至服务器; | |||
- 在服务器 nginx.conf 文件中添加如下配置; | |||
``` nginx | |||
server { | |||
listen 80; # 端口 | |||
server_name localhost; # 域名/外网IP | |||
location / { | |||
root /home/wwwroot/dubhe-web/dist; # dist 文件夹根目录 | |||
index index.html; | |||
try_files $uri $uri/ /index.html; | |||
} | |||
} | |||
``` | |||
- 保存 `nginx.conf` 并重启 Nginx 使之生效。 | |||
## 本地开发 | |||
``` bash | |||
# 下载源码 | |||
git clone https://codeup.teambition.com/zhejianglab/dubhe-web.git | |||
# 进入项目根目录 | |||
cd dubhe-web | |||
# 进入前端项目根目录 | |||
cd webapp | |||
# 安装依赖 | |||
npm install | |||
@@ -55,6 +55,7 @@ export default { | |||
type: Boolean, | |||
default: false, | |||
}, | |||
transformFile: Function, | |||
hash: { | |||
type: Boolean, | |||
default: true, | |||
@@ -67,7 +68,7 @@ export default { | |||
}, | |||
}, | |||
setup(props, ctx) { | |||
const { toggleVisible, request } = props; | |||
const { toggleVisible, request, transformFile } = props; | |||
const formRef = ref(null); | |||
const state = reactive({ | |||
visible: props.visible, | |||
@@ -98,7 +99,7 @@ export default { | |||
state.uploading = true; | |||
// 开始调用上传接口 | |||
uploadReqeust && uploadReqeust({ ...props.params, fileList: renameFileList }, handleUploadProgress) | |||
uploadReqeust && uploadReqeust({ ...props.params, fileList: renameFileList, transformFile }, handleUploadProgress) | |||
.then(res => { | |||
const outputPath = getFileOutputPath(renameFileList, props.params); | |||
state.uploading = false; | |||
@@ -38,6 +38,7 @@ export default { | |||
type: Boolean, | |||
default: false, | |||
}, | |||
transformFile: Function, | |||
hash: { | |||
type: Boolean, | |||
default: true, | |||
@@ -48,7 +49,7 @@ export default { | |||
}, | |||
}, | |||
setup(props, ctx) { | |||
const { request } = props; | |||
const { request, transformFile } = props; | |||
const formRef = ref(null); | |||
const state = reactive({ | |||
uploading: false, | |||
@@ -71,7 +72,7 @@ export default { | |||
ctx.emit('uploadStart'); | |||
const uploadReqeust = request || minIOUpload; | |||
// 开始调用上传接口 | |||
return uploadReqeust({ ...props.params, fileList: renameFileList }, callback) | |||
return uploadReqeust({ ...props.params, fileList: renameFileList, transformFile }, callback) | |||
.then(res => { | |||
const outputPath = getFileOutputPath(renameFileList, props.params); | |||
state.uploading = false; | |||
@@ -87,7 +87,7 @@ export const putObject = (uploadUrl, file, options = {}) => { | |||
}; | |||
// 默认通过 minIO 上传 | |||
export const minIOUpload = async({ objectPath, fileList }, callback) => { | |||
export const minIOUpload = async({ objectPath, fileList, transformFile }, callback) => { | |||
// add 进度条 | |||
let resolved = 0; | |||
@@ -97,7 +97,7 @@ export const minIOUpload = async({ objectPath, fileList }, callback) => { | |||
const uploadPrefix = `${minIOPrefix}/${bucketName}`; | |||
const objectName = `${objectPath}/${d.name}`; | |||
const result = await putObject(`${uploadPrefix}/${objectName}`, d.raw, { | |||
const fileRes = await putObject(`${uploadPrefix}/${objectName}`, d.raw, { | |||
objectName, | |||
callback, | |||
}); | |||
@@ -110,11 +110,18 @@ export const minIOUpload = async({ objectPath, fileList }, callback) => { | |||
if (typeof callback === 'function' && fileList.length > 1) { | |||
callback(resolved, fileList.length); | |||
} | |||
return result; | |||
// 视频不做转换 | |||
if (isValidVideoFile(d)) return fileRes; | |||
if (typeof transformFile === 'function') { | |||
const transformed = await transformFile(fileRes, d); | |||
return transformed; | |||
} | |||
return fileRes; | |||
}; | |||
const result = await pMap(fileList, mapper, {concurrency: 10}); | |||
return result; | |||
}; | |||
@@ -125,3 +132,26 @@ export const hashify = (name, hash) => { | |||
export const getFileOutputPath = (rawFiles, { objectPath }) => { | |||
return rawFiles.map(d => `${bucketHost}/${bucketName}/${objectPath}/${d.name}`); | |||
}; | |||
// 对文件进行自定义转换 | |||
export const transformFile = (result, file) => { | |||
return new Promise((resolve) => { | |||
const reader = new FileReader(); | |||
reader.addEventListener("load", () => { | |||
const img = new Image(); | |||
img.onload = () => resolve({ | |||
...result, | |||
data: { | |||
...result.data, | |||
meta: { | |||
width: img.width, | |||
height: img.height, | |||
}, | |||
}, | |||
}); | |||
img.src = reader.result; | |||
}, false); | |||
reader.readAsDataURL(file.raw); | |||
}); | |||
}; |
@@ -78,6 +78,7 @@ | |||
ref="uploaderRef" | |||
action="fakeApi" | |||
:visible="thumbState.showDialog" | |||
:transformFile="withDimensionFile" | |||
:toggleVisible="handleClose" | |||
:params="uploadParams" | |||
@uploadSuccess="uploadSuccess" | |||
@@ -92,7 +93,7 @@ import { Message } from 'element-ui'; | |||
import { pick } from 'lodash'; | |||
import UploadForm from '@/components/UploadForm'; | |||
import { fileTypeEnum, getImgFromMinIO, withDimensionFiles } from '@/views/dataset/util'; | |||
import { fileTypeEnum, getImgFromMinIO, withDimensionFile } from '@/views/dataset/util'; | |||
import { submit } from '@/api/preparation/datafile'; | |||
import { detectFileList, queryFileOffset } from '@/api/preparation/dataset'; | |||
import List from './list'; | |||
@@ -160,9 +161,8 @@ export default { | |||
const uploadSuccess = async(res) => { | |||
const files = getImgFromMinIO(res); | |||
const _files = await withDimensionFiles(files); | |||
// 提交业务上传 | |||
submit(datasetId.value, _files).then(() => { | |||
submit(datasetId.value, files).then(() => { | |||
Message.success('上传成功'); | |||
updateList({ type: thumbState.type }); | |||
}); | |||
@@ -224,6 +224,7 @@ export default { | |||
return { | |||
thumbState, | |||
withDimensionFile, | |||
uploadParams, | |||
dropdownList, | |||
handleUpload, | |||
@@ -19,6 +19,7 @@ | |||
<UploadForm | |||
action="fakeApi" | |||
:visible="uploadDialogVisible" | |||
:transformFile="withDimensionFile" | |||
:toggleVisible="handleClose" | |||
:params="uploadParams" | |||
@uploadSuccess="uploadSuccess" | |||
@@ -164,7 +165,7 @@ import { without, isNil } from 'lodash'; | |||
import { Message } from 'element-ui'; | |||
import { queryDataEnhanceList } from '@/api/preparation/dataset'; | |||
import { transformFile, transformFiles , getImgFromMinIO, dataEnhanceMap, withDimensionFiles } from '@/views/dataset/util'; | |||
import { transformFile, transformFiles, getImgFromMinIO, dataEnhanceMap, withDimensionFile } from '@/views/dataset/util'; | |||
import crudDataFile, { list, del , submit } from '@/api/preparation/datafile'; | |||
import { getAutoLabels, getLabels, createLabel } from '@/api/preparation/datalabel'; | |||
import { batchFinishAnnotation } from '@/api/preparation/annotation'; | |||
@@ -232,6 +233,10 @@ export default { | |||
}; | |||
}, | |||
computed: { | |||
// 文件上传前携带尺寸信息 | |||
withDimensionFile() { | |||
return withDimensionFile; | |||
}, | |||
uploadParams() { | |||
return { | |||
datasetId: this.datasetId, | |||
@@ -412,10 +417,9 @@ export default { | |||
}, | |||
async uploadSuccess(res) { | |||
const files = getImgFromMinIO(res); | |||
const _files = await withDimensionFiles(files); | |||
// 提交业务上传 | |||
if (_files.length > 0) { | |||
submit(this.datasetId, _files).then(() => { | |||
if (files.length > 0) { | |||
submit(this.datasetId, files).then(() => { | |||
this.$message({ | |||
message: '上传文件成功', | |||
type: 'success', | |||
@@ -125,6 +125,7 @@ | |||
ref="initFileUploadForm" | |||
action="fakeApi" | |||
:params="uploadParams" | |||
:transformFile="withDimensionFile" | |||
v-bind="optionCreateProps" | |||
@uploadSuccess="uploadSuccess" | |||
@uploadError="uploadError" | |||
@@ -203,6 +204,7 @@ | |||
:visible="uploadDialogVisible" | |||
:toggleVisible="toggleUploadFormClose" | |||
:params="uploadParams" | |||
:transformFile="withDimensionFile" | |||
v-bind="optionImportProps" | |||
@uploadSuccess="uploadSuccess" | |||
@uploadError="uploadError" | |||
@@ -446,7 +448,7 @@ import InfoSelect from '@/components/InfoSelect'; | |||
import { getAutoLabels } from '@/api/preparation/datalabel'; | |||
import { submit, submitVideo } from '@/api/preparation/datafile'; | |||
import { getImgFromMinIO, getFullFileUrl, annotationMap, dataTypeMap, annotationProgressMap, decompressProgressMap, datasetStatusMap, withDimensionFiles } from '@/views/dataset/util'; | |||
import { getImgFromMinIO, annotationMap, dataTypeMap, annotationProgressMap, decompressProgressMap, datasetStatusMap, withDimensionFile } from '@/views/dataset/util'; | |||
import Edit from '@/components/InlineTableEdit'; | |||
import BaseModal from '@/components/BaseModal'; | |||
import { toFixed, isEqualByProp, formatDateTime, downloadZipFromObjectPath } from '@/utils'; | |||
@@ -586,6 +588,10 @@ export default { | |||
return String(state.dataset.activePanel); | |||
}, | |||
}), | |||
// 文件上传前携带尺寸信息 | |||
withDimensionFile() { | |||
return withDimensionFile; | |||
}, | |||
// 自定义上传数据集 | |||
isImport() { | |||
return isImport; | |||
@@ -1132,41 +1138,14 @@ export default { | |||
} | |||
}, | |||
// 获取文件信息 | |||
async checkImg (file){ | |||
const fileUrl = getFullFileUrl(file); | |||
return new Promise((resolve, reject) => { | |||
const img = new Image(); | |||
img.onload = () => resolve({ | |||
width: img.width, | |||
height: img.height, | |||
...file, | |||
}); | |||
img.onerror = (err) => reject(err); | |||
img.src = fileUrl; | |||
}); | |||
}, | |||
// 上传文件之前加一层转换 | |||
async getTransformFiles(files) { | |||
return Promise.all(files.map(file => this.checkImg(file))); | |||
}, | |||
// 将文件上传和视频上传统一 | |||
async uploader(datasetId, files, options = {}) { | |||
async uploader(datasetId, files) { | |||
const datasetInfo = await this.queryDatasetDetail(datasetId); | |||
const { transformFile } = options; | |||
// 点击导入操作 | |||
const { dataType } = datasetInfo || {}; | |||
// 文件上传 | |||
if (dataType === 0) { | |||
let _files = files.slice(); | |||
// 对文件进行转换,生成宽、高信息 | |||
if (typeof transformFile === 'function') { | |||
_files = await transformFile(files); | |||
} | |||
return submit(datasetId, _files); | |||
return submit(datasetId, files); | |||
} if (dataType === 1) { | |||
// 根据是否通过点击导入按钮来区分 frameInterval 来源 | |||
const frameInterval = this.importRow | |||
@@ -1194,9 +1173,7 @@ export default { | |||
const successMessage = [0, 1].includes(this.chosenDatasetStatus) | |||
? '上传文件成功' : '上传文件成功,若数据集状态未及时更新,请手动刷新页面'; | |||
if (files.length > 0) { | |||
this.uploader(this.chosenDatasetId, files, { | |||
transformFile: withDimensionFiles, | |||
}).then(() => { | |||
this.uploader(this.chosenDatasetId, files).then(() => { | |||
this.$message({ | |||
message: successMessage, | |||
duration: 5000, | |||
@@ -1244,9 +1221,11 @@ export default { | |||
return toFixed(allFinished / (allFinished + progress.unfinished), 2, 0); | |||
}, | |||
parseDataType(row, column, cellValue = 0) { | |||
if(row.import) return "自定义"; | |||
return dataTypeMap[cellValue]; | |||
}, | |||
parseAnnotateType(row, column, cellValue) { | |||
if(row.import) return "自定义"; | |||
return (annotationMap[cellValue] || {}).name || ''; | |||
}, | |||
parseStatus(row, column, cellValue = 0) { | |||
@@ -74,9 +74,33 @@ export const stringifyAnnotations = (annotations) => { | |||
const buildImgUrl = (list = []) => { | |||
return list.map(d => ({ | |||
url: `${bucketName}/${d.data.objectName}`, | |||
...(d.data.meta || {}), // 附加的信息,目前只包括 width, height | |||
})); | |||
}; | |||
// 对文件进行自定义转换 | |||
export const withDimensionFile = (result, file) => { | |||
return new Promise((resolve) => { | |||
const reader = new FileReader(); | |||
reader.addEventListener("load", () => { | |||
const img = new Image(); | |||
img.onload = () => resolve({ | |||
...result, | |||
data: { | |||
...result.data, | |||
meta: { | |||
width: img.width, | |||
height: img.height, | |||
}, | |||
}, | |||
}); | |||
img.src = reader.result; | |||
}, false); | |||
reader.readAsDataURL(file.raw); | |||
}); | |||
}; | |||
export const getImgFromMinIO = (res) => { | |||
return buildImgUrl(res); | |||
}; | |||
@@ -110,6 +134,7 @@ export const transformFile = (rawFile, callback) => { | |||
return res; | |||
}; | |||
// deprecated | |||
// 获取文件信息 | |||
async function checkImg (file){ | |||
const fileUrl = getFullFileUrl(file); | |||
@@ -126,6 +151,7 @@ async function checkImg (file){ | |||
}); | |||
} | |||
// deprecated | |||
// 上传文件之前加一层转换 | |||
export const withDimensionFiles = async(files) => { | |||
return Promise.all(files.map(file => checkImg(file))); | |||