update data process

dubhe_data_process/LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

dubhe_data_process/README.md

@@ -4,30 +4,96 @@
 
 ## 算法部署
 
-部署请参考 http://tianshu.org.cn/?/course 中文档**部署数据处理算法**
+源码部署
+准备环境
 
-## 代码结构：
+ubuntu系统 版本18.04及以上
+python 3.7+
+redis  5.0+
+oneflow 框架
+
+## 下载源码
+http://repo.codelab.org.cn/codeup/codelab/Dubhe.git
+
+## 进入项目根目录
+cd dubhe_data_process
+
+## 启动算法 (参数指定需要启动的算法)
+python main.py track 
+
+具体部署流程请参考 http://tianshu.org.cn/?/course 中文档**部署数据处理算法**
+
+## 快速上手：
+
+### 代码结构：
 
 ```
+├── 
+├── common                              基础工具
+|   ├── config
+|   ├── constant
+|   ├── util
+├── log
+├── of_model                            oneflow模型文件
+├── program
+|   ├── abstract
+|       ├── actuator.py                 执行器抽象类
+|       ├── algorithm.py                算法抽象类 
+|       ├── storage.py                  存储抽象类
+|   ├── exec
+|       ├── annotation                  目标检测
+|       ├── imagenet                    图像分类
+|       ├── imgprocess                  数据增强
+|       ├── lung_segmentation           肺部分割
+|       ├── ofrecord                    ofrecord转换
+|       ├── text_classification         文本分类
+|       ├── track                       目标跟踪
+|       ├── videosample                 视频采样
+|   ├── impl
+|       ├── config_actuator.py          执行器配置实现
+|       ├── redis_storage.py            redis存储
+|   ├── thread
+├── script                              脚本
 ├── LICENSE
-├── README.md
-├── algorithm-annotation.py  #目标检测和图像分类算法
-├── algorithm-imagenet.py    #图像分类中imagenet标签处理算法
-├── algorithm-imgprocess.py  #数据增强算法
-├── algorithm-ofrecord.py    #ofrecord数据转换算法
-├── algorithm-track.py       #跟踪算法
-├── algorithm-videosample.py #视频采样算法
-├── annotation.py
-├── common                   #基础工具
-├── data
-├── imagenet.py
-├── imgprocess.py
-├── luascript
-├── of_model                 #oneflow模型文件
-├── ofrecord.py
-├── predict_with_print_box.py
-├── taskexecutor.py
-├── track.py
-├── track_only
-└── videosample.py
-```
+├── main.py
+└── README.md
+```
+
+### 算法接入：
+
+#### 算法文件
+[algorithm.py](./program/abstract/algorithm.py) 需要实现此算法抽象类
+
+算法文件目录放在 program/exec 下，实现 program/abstract 目录下的 algoriyhm.py 文件中的 Algorithm 类，
+其中 __init__ 方法和 execut 方法需要实现，__init__ 方法为算法的初始化操作，execute 为算法执行入口，入参
+为 jsonObject，返回值为 finish_data（算法执行完成放入 redis 中的信息）以及布尔类型（算法执行成功或者失败）
+
+#### config.json文件
+在 program/exec 的每个算法目录下，需要有 config.json 文件，用户启动 main.py 时通过参数来指定需要执行的算
+法(参数与算法目录名称相同)
+
+### config.json模板
+
+#### 算法不需要使用GPU时的config.json
+[config.json](./common/template/config.json)
+
+用户需要提供的参数:
+- step1："paramLocal"算法处理中队列名称
+- step2:"module","class"替换为需要接入的算法
+- step4:"paramLocal" 中"algorithm_task_queue","algorithm_processing_queue"替换为需要接入算法的待处理任务队列和处理中任务队列
+- step:5:"module","class"替换为需要接入的算法
+- step6:"paramLocal" 中"algorithm_task_queue","algorithm_processing_queue"替换为需要接入算法的处理成功和处理失败队列
+
+#### 算法需要使用GPU时的config.json
+[config_GPU.json](./common/template/config_GPU.json)
+
+用户需要提供的参数:
+- step1："paramLocal"算法处理中队列名称
+- step3:"module","class"替换为需要接入的算法
+- step5:"paramLocal" 中"algorithm_task_queue","algorithm_processing_queue"替换为需要接入算法的待处理任务队列和处理中任务队列
+- step:6:"module","class"替换为需要接入的算法
+- step7:"paramLocal" 中"algorithm_task_queue","algorithm_processing_queue"替换为需要接入算法的处理成功和处理失败队列
+
+## 开发者指南
+若用户需了解算法接入实现细节，请参考官方文档：开发人员自定义算法接入规范
+

dubhe_data_process/common/RedisUtil.py

@@ -1,42 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-import redis
-import sys
-
-
-def getRedisConnection(host, port, db, password):
-    return redis.Redis(host=host, port=port, db=db, password=password)
-
-
-def getOneMinScoreElement(f, queue):
-    return f.zrangebyscore(queue, 0, sys.maxsize, 0, 1)
-
-
-def deleteElement(f, queue, element):
-    f.zrem(queue, element)
-
-# get bu key
-def getByKey(f, key):
-   print(key)
-   return f.get(key);
-
-
-def pushToQueue(f, key, value):
-    f.rpush(key, value)

dubhe_data_process/common/config.py → dubhe_data_process/common/config/config.py

@@ -1,22 +1,22 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
-# coding:utf-8
 
 import json
 
@@ -70,8 +70,8 @@ imgProcessFailedQueue = "imgProcess_failed"
 
 threadCount = 5
 
-configPath = "/root/algorithm/config.json"
-sign = "/root/algorithm/sign"
+configPath = "/Users/wangwei/Downloads/algorithm/config.json"
+sign = "/Users/wangwei/Downloads/algorithm/sign"
 
 
 def loadJsonData(path):

dubhe_data_process/common/log_config.py → dubhe_data_process/common/config/log_config.py

@@ -1,24 +1,22 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
-# !/usr/bin/env python3
-# -*- coding: utf-8 -*-
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
 
 import os
 import logging

dubhe_data_process/common/config/url.json

@@ -1 +0,0 @@
-{"test": "", "dev": "", "temp": ""}

dubhe_data_process/common/data/coco.names

@@ -1,80 +0,0 @@
-person
-bicycle
-car
-motorbike
-aeroplane
-bus
-train
-truck
-boat
-traffic light
-fire hydrant
-stop sign
-parking meter
-bench
-bird
-cat
-dog
-horse
-sheep
-cow
-elephant
-bear
-zebra
-giraffe
-backpack
-umbrella
-handbag
-tie
-suitcase
-frisbee
-skis
-snowboard
-sports ball
-kite
-baseball bat
-baseball glove
-skateboard
-surfboard
-tennis racket
-bottle
-wine glass
-cup
-fork
-knife
-spoon
-bowl
-banana
-apple
-sandwich
-orange
-broccoli
-carrot
-hot dog
-pizza
-donut
-cake
-chair
-sofa
-pottedplant
-bed
-diningtable
-toilet
-tvmonitor
-laptop
-mouse
-remote
-keyboard
-cell phone
-microwave
-oven
-toaster
-sink
-refrigerator
-book
-clock
-vase
-scissors
-teddy bear
-hair drier
-toothbrush

dubhe_data_process/common/predict_with_print_box.py

@@ -1,277 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-import json
-import time
-
-import cv2
-import numpy as np
-import oneflow_yolov3
-from yolo_net import YoloPredictNet
-
-import oneflow as flow
-
-
-'''Init oneflow config'''
-model_load_dir = "of_model/yolov3_model_python/"
-label_to_name_file = "coco.names"
-use_tensorrt = 0
-gpu_num_per_node = 1
-batch_size = 16
-image_height = 608
-image_width = 608
-flow.config.load_library(oneflow_yolov3.lib_path())
-func_config = flow.FunctionConfig()
-func_config.default_distribute_strategy(flow.distribute.consistent_strategy())
-func_config.default_data_type(flow.float)
-if use_tensorrt != 0:
-    func_config.use_tensorrt(True)
-label_2_name = []
-with open(label_to_name_file, 'r') as f:
-    label_2_name = f.readlines()
-nms = True
-print("nms:", nms)
-input_blob_def_dict = {
-    "images": flow.FixedTensorDef((batch_size, 3, image_height, image_width), dtype=flow.float),
-    "origin_image_info": flow.FixedTensorDef((batch_size, 2), dtype=flow.int32),
-}
-
-
-def xywh_2_x1y1x2y2(x, y, w, h, origin_image):
-    """The format of box transform"""
-    x1 = (x - w / 2.) * origin_image[1]
-    x2 = (x + w / 2.) * origin_image[1]
-    y1 = (y - h / 2.) * origin_image[0]
-    y2 = (y + h / 2.) * origin_image[0]
-    return x1, y1, x2, y2
-
-
-def batch_boxes(positions, probs, origin_image_info):
-    """The images postprocessing"""
-    batch_size = positions.shape[0]
-    batch_list = []
-    if nms == True:
-        for k in range(batch_size):
-            box_list = []
-            for i in range(1, 81):
-                for j in range(positions.shape[2]):
-                    if positions[k][i][j][2] != 0 and positions[k][i][j][3] != 0 and probs[k][i][j] != 0:
-                        x1, y1, x2, y2 = xywh_2_x1y1x2y2(positions[k][i][j][0], positions[k][i][j][1],
-                                                         positions[k][i][j][2], positions[k][i][j][3],
-                                                         origin_image_info[k])
-                        bbox = [i - 1, x1, y1, x2, y2, probs[k][i][j]]
-                        box_list.append(bbox)
-            batch_list.append(np.asarray(box_list))
-    else:
-        for k in range(batch_size):
-            box_list = []
-            for j in range(positions.shape[1]):
-                for i in range(1, 81):
-                    if positions[k][j][2] != 0 and positions[k][j][3] != 0 and probs[k][j][i] != 0:
-                        x1, y1, x2, y2 = xywh_2_x1y1x2y2(positions[k][j][0], positions[k][j][1], positions[k][j][2],
-                                                         positions[k][j][3], origin_image_info[k])
-                        bbox = [i - 1, x1, y1, x2, y2, probs[k][j][i]]
-                        box_list.append(bbox)
-            batch_list.append(np.asarray(box_list))
-    return batch_list
-
-
-@flow.function(func_config)
-def yolo_user_op_eval_job(images=input_blob_def_dict["images"],
-                          origin_image_info=input_blob_def_dict["origin_image_info"]):
-    """The model inference"""
-    yolo_pos_result, yolo_prob_result = YoloPredictNet(images, origin_image_info, trainable=False)
-    yolo_pos_result = flow.identity(yolo_pos_result, name="yolo_pos_result_end")
-    yolo_prob_result = flow.identity(yolo_prob_result, name="yolo_prob_result_end")
-    return yolo_pos_result, yolo_prob_result, origin_image_info
-
-
-def yolo_show(image_path_list, batch_list):
-    """Debug the result of Yolov3"""
-    font = cv2.FONT_HERSHEY_SIMPLEX
-    for img_path, batch in zip(image_path_list, batch_list):
-        result_list = batch.tolist()
-        img = cv2.imread(img_path)
-        for result in result_list:
-            cls = int(result[0])
-            bbox = result[1:-1]
-            score = result[-1]
-            print('img_file:', img_path)
-            print('cls:', cls)
-            print('bbox:', bbox)
-            c = ((int(bbox[0]) + int(bbox[2])) / 2, (int(bbox[1] + int(bbox[3])) / 2))
-            cv2.rectangle(img, (int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3])), (0, 255, 255), 1)
-            cv2.putText(img, str(cls), (int(c[0]), int(c[1])), font, 1, (0, 0, 255), 1)
-        result_name = img_path.split('/')[-1]
-        cv2.imwrite("data/results/" + result_name, img)
-
-
-def resize_image(img, origin_h, origin_w, image_height, image_width):
-    """The resize of image preprocessing"""
-    w = image_width
-    h = image_height
-    resized = np.zeros((3, image_height, image_width), dtype=np.float32)
-    part = np.zeros((3, origin_h, image_width), dtype=np.float32)
-    w_scale = (float)(origin_w - 1) / (w - 1)
-    h_scale = (float)(origin_h - 1) / (h - 1)
-
-    for c in range(w):
-        if c == w - 1 or origin_w == 1:
-            val = img[:, :, origin_w - 1]
-        else:
-            sx = c * w_scale
-            ix = int(sx)
-            dx = sx - ix
-            val = (1 - dx) * img[:, :, ix] + dx * img[:, :, ix + 1]
-        part[:, :, c] = val
-    for r in range(h):
-        sy = r * h_scale
-        iy = int(sy)
-        dy = sy - iy
-        val = (1 - dy) * part[:, iy, :]
-        resized[:, r, :] = val
-        if r == h - 1 or origin_h == 1:
-            continue
-        resized[:, r, :] = resized[:, r, :] + dy * part[:, iy + 1, :]
-    return resized
-
-
-def batch_image_preprocess_v2(img_path_list, image_height, image_width):
-    """The images preprocessing"""
-    result_list = []
-    origin_info_list = []
-    for img_path in img_path_list:
-        img = cv2.imread(img_path, cv2.IMREAD_COLOR)
-        img = img.transpose(2, 0, 1).astype(np.float32)  # hwc->chw
-        img = img / 255  # /255
-        img[[0, 1, 2], :, :] = img[[2, 1, 0], :, :]  # bgr2rgb
-
-        w = image_width
-        h = image_height
-        origin_h = img.shape[1]
-        origin_w = img.shape[2]
-        new_w = origin_w
-        new_h = origin_h
-        if w / origin_w < h / origin_h:
-            new_w = w
-            new_h = origin_h * w // origin_w
-        else:
-            new_h = h
-            new_w = origin_w * h // origin_h
-        resize_img = resize_image(img, origin_h, origin_w, new_h, new_w)
-
-        dw = (w - new_w) // 2
-        dh = (h - new_h) // 2
-
-        padh_before = int(dh)
-        padh_after = int(h - new_h - padh_before)
-        padw_before = int(dw)
-        padw_after = int(w - new_w - padw_before)
-        result = np.pad(resize_img, pad_width=((0, 0), (padh_before, padh_after), (padw_before, padw_after)),
-                        mode='constant', constant_values=0.5)
-        origin_image_info = [origin_h, origin_w]
-        result_list.append(result)
-        origin_info_list.append(origin_image_info)
-    results = np.asarray(result_list).astype(np.float32)
-    origin_image_infos = np.asarray(origin_info_list).astype(np.int32)
-    return results, origin_image_infos
-
-
-def coco_format(type_, id_list, file_list, result_list, label_list, coco_flag=0):
-    """Transform the annotations to coco format"""
-    annotations = []
-    for i, result in enumerate(result_list):
-        temp = {}
-        id_name = id_list[i]
-        file_path = file_list[i]
-        temp['id'] = id_name
-        temp['annotation'] = []
-        im = cv2.imread(file_path)
-        height, width, _ = im.shape
-        if result.shape[0] == 0:
-            temp['annotation'] = json.dumps(temp['annotation'])
-            annotations.append(temp)
-            continue
-        else:
-            for j in range(result.shape[0]):
-                cls_id = int(result[j][0]) + 1 + coco_flag
-                x1 = result[j][1]
-                x2 = result[j][3]
-                y1 = result[j][2]
-                y2 = result[j][4]
-                score = result[j][5]
-                width = max(0, x2 - x1)
-                height = max(0, y2 - y1)
-                if cls_id in label_list:
-                    temp['annotation'].append({
-                        'area': width * height,
-                        'bbox': [x1, y1, width, height],
-                        'category_id': cls_id,
-                        'iscrowd': 0,
-                        'segmentation': [[x1, y1, x2, y1, x2, y2, x1, y2]],
-                        'score': score
-                    })
-        if type_ == 2 and len(temp['annotation']) > 0:
-            temp['annotation'] = [temp['annotation'][0]]
-            temp['annotation'][0].pop('area')
-            temp['annotation'][0].pop('bbox')
-            temp['annotation'][0].pop('iscrowd')
-            temp['annotation'][0].pop('segmentation')
-        temp['annotation'] = json.dumps(temp['annotation'])
-        annotations.append(temp)
-    return annotations
-
-
-class YoloInference(object):
-    """Yolov3 detection inference"""
-
-    def __init__(self, label_log):
-        self.label_log = label_log
-        flow.config.gpu_device_num(gpu_num_per_node)
-        flow.env.ctrl_port(9789)
-
-        check_point = flow.train.CheckPoint()
-        if not model_load_dir:
-            check_point.init()
-        else:
-            check_point.load(model_load_dir)
-        print("Load check_point success")
-        self.label_log.info("Load check_point success")
-
-    def yolo_inference(self, type_, id_list, image_path_list, label_list, coco_flag=0):
-        annotations = []
-        try:
-            if len(image_path_list) == 16:
-                t0 = time.time()
-                images, origin_image_info = batch_image_preprocess_v2(image_path_list, image_height, image_width)
-                yolo_pos, yolo_prob, origin_image_info = yolo_user_op_eval_job(images, origin_image_info).get()
-                batch_list = batch_boxes(yolo_pos, yolo_prob, origin_image_info)
-                annotations = coco_format(type_, id_list, image_path_list, batch_list, label_list, coco_flag)
-                t1 = time.time()
-                print('t1-t0:', t1 - t0)
-        except:
-            print("Forward Error")
-            self.label_log.error("Forward Error")
-            for i, image_path in enumerate(image_path_list):
-                temp = {}
-                id_name = id_list[i]
-                temp['id'] = id_name
-                temp['annotation'] = []
-                temp['annotation'] = json.dumps(temp['annotation'])
-                annotations.append(temp)
-        return annotations

dubhe_data_process/common/select_gpu.py

@@ -1,47 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-import os
-import random
-import sys
-import pynvml
-import logging
-
-pid = os.getpid()
-pynvml.nvmlInit()
-
-
-def select_gpu():
-    deviceCount = pynvml.nvmlDeviceGetCount()
-    gpu_usable = []
-    for i in range(deviceCount):
-        logging.info('-------------get GPU information--------------')
-        handle = pynvml.nvmlDeviceGetHandleByIndex(i)
-        logging.info("Device:%s %s", i, pynvml.nvmlDeviceGetName(handle))
-        gpu_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
-        logging.info('free:%s MB', gpu_info.free / (1000 * 1000))
-        if gpu_info.free / (1000 * 1000) > 3072:
-            gpu_usable.append(i)
-    gpu_usable_num = len(gpu_usable)
-    if gpu_usable_num == 0:
-        logging.info('No GPU is currently available')
-        sys.exit()
-    random_gpu = random.randint(0, gpu_usable_num - 1)
-    os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_usable[random_gpu])
-    logging.info('use GPU:%s %s', gpu_usable[random_gpu], pynvml.nvmlDeviceGetName(handle))
-

dubhe_data_process/common/template/config.json

@@ -0,0 +1,82 @@
+{
+  "annotation": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "algorithm_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "初始化",
+      "module": "program.exec.algorithm.algorithm",
+      "class": "Algorithm",
+      "method": "__init__",
+      "paramType": 0
+    },
+    {
+      "step": 3,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 3
+    },
+    {
+      "step": 4,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "algorithm_task_queue",
+        "algorithm_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 3
+    },
+    {
+      "step": 5,
+      "desc": "执行任务",
+      "module": "program.exec.algorithm.algorithm",
+      "class": "Algorithm",
+      "method": "execute",
+      "paramType": 1,
+      "param": [
+        4.2
+      ]
+    },
+    {
+      "step": 6,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "algorithm_finished_queue",
+        "algorithm_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        5.1,
+        5.2
+      ],
+      "jump": 3
+    }
+  ]
+}

dubhe_data_process/common/template/config_GPU.json

@@ -0,0 +1,90 @@
+{
+  "annotation": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "algorithm_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "选择GPU",
+      "module": "common.util.public.select_gpu",
+      "class": "Select_gpu",
+      "method": "select_gpu",
+      "paramType": 0
+    },
+    {
+      "step": 3,
+      "desc": "初始化",
+      "module": "program.exec.algorithm.algorithm",
+      "class": "Algorithm",
+      "method": "__init__",
+      "paramType": 0
+    },
+    {
+      "step": 4,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 4
+    },
+    {
+      "step": 5,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "algorithm_task_queue",
+        "algorithm_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 4
+    },
+    {
+      "step": 6,
+      "desc": "执行任务",
+      "module": "program.exec.algorithm.algorithm",
+      "class": "Algorithm",
+      "method": "execute",
+      "paramType": 1,
+      "param": [
+        5.2
+      ]
+    },
+    {
+      "step": 7,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "algorithm_finished_queue",
+        "algorithm_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        6.1,
+        6.2
+      ],
+      "jump": 4
+    }
+  ]
+}

dubhe_data_process/common/augment_utils/ACE.py → dubhe_data_process/common/util/algorithm/ACE.py

@@ -1,27 +1,29 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
 import cv2
 import numpy as np
 import math
+
 para = {}
 
+
 def ACE(img, ratio=4, radius=300):
     """The implementation of ACE"""
     global para
@@ -36,12 +38,12 @@ def ACE(img, ratio=4, radius=300):
                 if not h and not w:
                     continue
                 para_mat[radius + h, radius + w] = 1.0 / \
-                    math.sqrt(h ** 2 + w ** 2)
+                                                   math.sqrt(h ** 2 + w ** 2)
         para_mat /= para_mat.sum()
         para[radius] = para_mat
     h, w = img.shape[:2]
-    p_h, p_w = [0] * radius + list(range(h)) + [h - 1] * radius,\
-             [0] * radius + list(range(w)) + [w - 1] * radius
+    p_h, p_w = [0] * radius + list(range(h)) + [h - 1] * radius, \
+               [0] * radius + list(range(w)) + [w - 1] * radius
     temp = img[np.ix_(p_h, p_w)]
     res = np.zeros(img.shape)
     for i in range(radius * 2 + 1):
@@ -52,6 +54,7 @@ def ACE(img, ratio=4, radius=300):
                     np.clip((img - temp[i:i + h, j:j + w]) * ratio, -1, 1))
     return res
 
+
 def ACE_channel(img, ratio, radius):
     """The implementation of ACE through individual channel"""
     h, w = img.shape[:2]
@@ -64,6 +67,7 @@ def ACE_channel(img, ratio, radius):
     re = up_temp + ACE(img, ratio, radius) - ACE(up_ori, ratio, radius)
     return re
 
+
 def ACE_color(img, ratio=4, radius=3):
     """Enhance the image through RGB channels"""
     re = np.zeros(img.shape)
@@ -71,6 +75,7 @@ def ACE_color(img, ratio=4, radius=3):
         re[:, :, c] = reprocessImage(ACE_channel(img[:, :, c], ratio, radius))
     return re
 
+
 def reprocessImage(img):
     """Reprocess and map the image to [0,1]"""
     ht = np.histogram(img, 2000)
@@ -80,8 +85,7 @@ def reprocessImage(img):
     except:
         left = 1999
     try:
-        right = next(y for y in range(len(d)-1,0,-1) if d[y] <= 0.995)
+        right = next(y for y in range(len(d) - 1, 0, -1) if d[y] <= 0.995)
     except:
         right = 1
     return np.clip((img - ht[1][left]) / (ht[1][right] - ht[1][left]), 0, 1)
-

dubhe_data_process/common/augment_utils/dehaze.py → dubhe_data_process/common/util/algorithm/dehaze.py

@@ -1,26 +1,22 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-  Reference:
-    - [Single Image Haze Removal Using Dark Channel Prior]
-      (http://kaiminghe.com/publications/cvpr09.pdf) (CVPR 2009)
-"""
 # !/usr/bin/env python
 # -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
 import cv2
 import numpy as np
 

dubhe_data_process/common/augment_utils/hist_equalize.py → dubhe_data_process/common/util/algorithm/hist_equalize.py

@@ -1,22 +1,22 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
 import cv2
 import numpy as np
 

dubhe_data_process/common/of_cnn_resnet.py → dubhe_data_process/common/util/algorithm/of_cnn_resnet.py

@@ -1,22 +1,23 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
-# -*- coding:utf-8 -*-
+
 
 from __future__ import absolute_import
 from __future__ import division
@@ -34,7 +35,7 @@ sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
 
 def init_resnet():
     """Initialize ResNet with pretrained weights"""
-    model_load_dir = '../of_model/resnet_v15_of_best_model_val_top1_773/'
+    model_load_dir = 'of_model/resnet_v15_of_best_model_val_top1_773/'
     assert os.path.isdir(model_load_dir)
     check_point = flow.train.CheckPoint()
     check_point.load(model_load_dir)

dubhe_data_process/common/yolo_net.py → dubhe_data_process/common/util/algorithm/yolo_net.py

@@ -1,23 +1,21 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-  Reference:
-    - [YOLOv3: An Incremental Improvement]
-      (https://arxiv.org/abs/1804.02767)
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
 
 import oneflow as flow

dubhe_data_process/common/util/public/RedisUtil.py

@@ -0,0 +1,42 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+import redis
+import sys
+
+
+def getRedisConnection(host, port, db, password):
+    return redis.Redis(host=host, port=port, db=db, password=password)
+
+
+def getOneMinScoreElement(f, queue):
+    return f.zrangebyscore(queue, 0, sys.maxsize, 0, 1)
+
+
+def deleteElement(f, queue, element):
+    f.zrem(queue, element)
+
+# get bu key
+def getByKey(f, key):
+   print(key)
+   return f.get(key);
+
+
+def pushToQueue(f, key, value):
+    f.rpush(key, value)

dubhe_data_process/common/util/public/json_util.py

@@ -0,0 +1,17 @@
+# _*_ coding:utf-8 _*_
+import json
+
+
+class JsonUtil:
+
+    def __init__(self):
+        pass
+
+    # noinspection PyMethodMayBeStatic
+    def load_json(self):
+        """
+        read json file
+        """
+        with open(self, encoding="utf-8") as f:
+            json_object = json.loads(f.read())
+        return json_object

dubhe_data_process/common/util/public/select_gpu.py

@@ -0,0 +1,50 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+import os
+import random
+import sys
+import pynvml
+import logging
+
+pid = os.getpid()
+pynvml.nvmlInit()
+
+
+class Select_gpu:
+
+    @staticmethod
+    def select_gpu():
+        deviceCount = pynvml.nvmlDeviceGetCount()
+        gpu_usable = []
+        for i in range(deviceCount):
+            logging.info('-------------get GPU information--------------')
+            handle = pynvml.nvmlDeviceGetHandleByIndex(i)
+            logging.info("Device:%s %s", i, pynvml.nvmlDeviceGetName(handle))
+            gpu_info = pynvml.nvmlDeviceGetMemoryInfo(handle)
+            logging.info('free:%s MB', gpu_info.free / (1000 * 1000))
+            if gpu_info.free / (1000 * 1000) > 3072:
+                gpu_usable.append(i)
+        gpu_usable_num = len(gpu_usable)
+        if gpu_usable_num == 0:
+            logging.info('No GPU is currently available')
+            sys.exit()
+        random_gpu = random.randint(0, gpu_usable_num - 1)
+        os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_usable[random_gpu])
+        logging.info('use GPU:%s %s', gpu_usable[random_gpu], pynvml.nvmlDeviceGetName(handle))

dubhe_data_process/entrance/algorithm-annotation.py

@@ -1,61 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-import threading
-
-import time
-import sys
-sys.path.append("../")
-import common.RedisUtil as f
-from common import config as config
-from entrance.executor import annotation as annotation, taskexecutor
-import luascript.starttaskscript as start_script
-import logging
-from common import select_gpu as gpu
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
-                    level=logging.DEBUG)
-
-if __name__ == '__main__':
-    """Automatic annotation algorithm entry."""
-    gpu.select_gpu()
-    jsonData = config.loadJsonData(config.configPath)
-    redisClient = f.getRedisConnection(jsonData["ip"], jsonData["port"], jsonData["database"], jsonData["password"])
-    logging.info('init redis client %s', redisClient)
-    t = threading.Thread(target=taskexecutor.delayKeyThread, args=(redisClient,))
-    t.setDaemon(True)
-    t.start()
-    annotation._init()
-    while 1:
-        try:
-            if config.loadJsonData(config.sign) == 0:
-                logging.info('not to execute new task')
-                time.sleep(1)
-            else:
-                logging.info('get one task')
-                element = redisClient.eval(start_script.startTaskLua, 1, config.queue,
-                                           config.annotationStartQueue, int(time.time()))
-                if len(element) > 0:
-                    taskexecutor.annotationExecutor(redisClient, element[0]);
-                else:
-                    logging.info('task queue is empty.')
-                    time.sleep(1)
-        except Exception as e:
-            logging.error('except:', e)
-            time.sleep(1)

dubhe_data_process/entrance/algorithm-imagenet.py

@@ -1,69 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-
-import json
-import threading
-import time
-import sys
-sys.path.append("../")
-from entrance.executor import imagenet as imagenet
-import common.RedisUtil as f
-import common.config as config
-import luascript.starttaskscript as start_script
-import logging
-import common.select_gpu as gpu
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
-                    level=logging.DEBUG)
-
-if __name__ == '__main__':
-    """Imagenet algorithm entry."""
-    gpu.select_gpu()
-    jsonData = config.loadJsonData(config.configPath)
-    redisClient = f.getRedisConnection(jsonData["ip"], jsonData["port"], jsonData["database"], jsonData["password"])
-    logging.info('init redis client %s', redisClient)
-    t = threading.Thread(target=imagenet.delayKeyThread, args=(redisClient,))
-    t.setDaemon(True)
-    t.start()
-    imagenet._init()
-    while 1:
-        try:
-            if config.loadJsonData(config.sign) == 0:
-                logging.info('not to execute new task')
-                time.sleep(1)
-            else:
-                logging.info('get one task')
-                element = redisClient.eval(start_script.startTaskLua, 1, config.imagenetTaskQueue,
-                                           config.imagenetStartQueue, int(time.time()))
-                if len(element) > 0:
-                    key = element[0].decode()
-                    jsonStr = f.getByKey(redisClient, key.replace('"', ''));
-                    result = imagenet.process(jsonStr, element[0])
-                    logging.info("result:", result)
-
-                    logging.info('save result to redis')
-                    f.pushToQueue(redisClient, config.imagenetFinishQueue, json.dumps(result))
-                    redisClient.zrem(config.imagenetStartQueue, element[0])
-                else:
-                    logging.info('task queue is empty.')
-                    time.sleep(2)
-        except Exception as e:
-            logging.error('except:', e)
-            time.sleep(1)

dubhe_data_process/entrance/algorithm-imgprocess.py

@@ -1,55 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-import threading
-import time
-import sys
-sys.path.append("../")
-import common.RedisUtil as f
-import luascript.starttaskscript as start_script
-import common.config as config
-import logging
-from entrance.executor import imgprocess
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
-                    level=logging.DEBUG)
-
-if __name__ == '__main__':
-    """Enhancement algorithm entry."""
-    jsonData = config.loadJsonData(config.configPath)
-    redisClient = f.getRedisConnection(jsonData["ip"], jsonData["port"], jsonData["database"], jsonData["password"])
-    logging.info('init redis client %s', redisClient)
-    t = threading.Thread(target=imgprocess.delayKeyThread, args=(redisClient,))
-    t.setDaemon(True)
-    t.start()
-    while 1:
-        try:
-            if config.loadJsonData(config.sign) == 0:
-                logging.info('not to execute new task')
-                time.sleep(5)
-            else:
-                enhanceTaskId = redisClient.eval(start_script.startTaskLua, 1, config.imgProcessTaskQueue,
-                                                 config.imgProcessStartQueue, int(time.time()))
-                if len(enhanceTaskId) > 0:
-                    imgprocess.start_enhance_task(enhanceTaskId, redisClient)
-                else:
-                    logging.info('task queue is empty.')
-                    time.sleep(5)
-        except Exception as e:
-            logging.error('except:', e)
-            time.sleep(1)

dubhe_data_process/entrance/algorithm-lungsegmenatation.py

@@ -1,63 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-import json
-import threading
-from datetime import datetime
-import time
-import sys
-sys.path.append("../")
-import common.RedisUtil as f
-import luascript.starttaskscript as start_script
-import common.config as config
-import logging
-from entrance.executor import lungsegmentation as lungseg
-import redis
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s', level=logging.DEBUG)
-
-if __name__ == '__main__':
-    """Lung segmentation algorithm based on CT image dcm entry."""
-    jsonData = config.loadJsonData(config.configPath)
-    redisClient = f.getRedisConnection(jsonData["ip"], jsonData["port"], jsonData["database"], jsonData["password"])
-    logging.info('init redis client %s', redisClient)
-    t = threading.Thread(target=lungseg.delayKeyThread, args=(redisClient,))
-    t.setDaemon(True)
-    t.start()
-    while 1:
-        try:
-            # if config.loadJsonData(config.sign) == 0:
-            #     logging.info('not to execute new task')
-            #     time.sleep(5)
-            # else:
-            logging.info("read redis:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S"))
-            lungTask = redisClient.eval(start_script.startTaskLua, 1, config.dcmTaskQueue, config.dcmStartQueue, int(time.time()))
-            if len(lungTask) > 0:
-                logging.info("start process.")
-                key = lungTask[0].decode()
-                jsonStr = f.getByKey(redisClient, key.replace('"', ''))
-                if lungseg.process(jsonStr, lungTask[0]):
-                    f.pushToQueue(redisClient, config.dcmFinishQueue, key)
-                    redisClient.zrem(config.dcmStartQueue, lungTask[0])
-                logging.info('success.')
-            else:
-                logging.info('task queue is empty.')
-                time.sleep(1)
-        except Exception as e:
-            logging.error('except:', e)
-            time.sleep(1)

dubhe_data_process/entrance/algorithm-ofrecord.py

@@ -1,80 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-
-import os
-import json
-import threading
-import time
-import sys
-sys.path.append("../")
-import common.RedisUtil as f
-import common.config as config
-import luascript.starttaskscript as start_script
-import logging
-import traceback
-from entrance.executor import ofrecord
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',level=logging.DEBUG)
-
-basePath = '/nfs/'
-descPath = 'ofrecord/train'
-
-if __name__ == '__main__':
-    """Ofrecord algorithm entry."""
-    jsonData = config.loadJsonData(config.configPath)
-    redisClient = f.getRedisConnection(jsonData["ip"], jsonData["port"], jsonData["database"], jsonData["password"])
-    logging.info('init redis client %s', redisClient)
-    t = threading.Thread(target=ofrecord.delayKeyThread, args=(redisClient,))
-    t.setDaemon(True)
-    t.start()
-    while 1:
-        try:
-            if config.loadJsonData(config.sign) == 0:
-                logging.info('not to execute new task')
-                time.sleep(1)
-            else:
-                element = redisClient.eval(start_script.startTaskLua, 1, config.ofrecordTaskQueue,
-                                           config.ofrecordStartQueue, int(time.time()))
-                if len(element) > 0:
-                    key = element[0].decode()
-                    detail = f.getByKey(redisClient, key.replace('"', ''))
-                    jsonStr = json.loads(detail.decode())
-                    label_map = {}
-                    index = 0
-                    for item in jsonStr["datasetLabels"].keys():
-                        if index >= 0 and item != '@type':
-                            label_map[item] = jsonStr["datasetLabels"][item]
-                        index += 1
-                    ofrecord.execute(os.path.join(basePath, jsonStr["datasetPath"]),
-                                     os.path.join(basePath, jsonStr["datasetPath"], descPath),
-                                     label_map,
-                                     jsonStr["files"],
-                                     jsonStr["partNum"],
-                                     element[0])
-                    logging.info('save result to redis')
-                    f.pushToQueue(redisClient, config.ofrecordFinishQueue, key)
-                    redisClient.zrem(config.ofrecordStartQueue, element[0])
-                else:
-                    logging.info('task queue is empty.')
-                    time.sleep(2)
-        except Exception as e:
-            logging.error('except:', e)
-            redisClient.zrem(config.ofrecordStartQueue, element[0])
-            time.sleep(1)

dubhe_data_process/entrance/algorithm-text-classification.py

@@ -1,58 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-import sys
-sys.path.append("../")
-import threading
-import time
-import common.RedisUtil as f
-import common.config as config
-from entrance.executor import text_classification as classify, text_taskexecutor
-import luascript.starttaskscript as start_script
-import logging
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
-                    level=logging.DEBUG)
-
-if __name__ == '__main__':
-    """Automatic text classification algorithm entry."""
-    jsonData = config.loadJsonData(config.configPath)
-    redisClient = f.getRedisConnection(jsonData["ip"], jsonData["port"], jsonData["database"], jsonData["password"])
-    logging.info('init redis client %s', redisClient)
-    t = threading.Thread(target=text_taskexecutor.delayKeyThread, args=(redisClient,))
-    t.setDaemon(True)
-    t.start()
-    classify._init() #
-    while 1:
-        try:
-            if config.loadJsonData(config.sign) == 0:
-                logging.info('not to execute new task')
-                time.sleep(1)
-            else:
-                logging.info('get one task')
-                element = redisClient.eval(start_script.startTaskLua, 1, config.textClassificationQueue,
-                                           config.textClassificationStartQueue, int(time.time()))
-                if len(element) > 0:
-                    text_taskexecutor.textClassificationExecutor(redisClient, element[0])
-                else:
-                    logging.info('task queue is empty.')
-                    time.sleep(1)
-        except Exception as e:
-            logging.error('except:', e)
-            time.sleep(1)

dubhe_data_process/entrance/algorithm-track.py

@@ -1,66 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-import threading
-import time
-import sys
-sys.path.append("../")
-import common.RedisUtil as f
-import common.config as config
-import luascript.starttaskscript as start_script
-import logging
-from entrance.executor import track
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
-                    level=logging.DEBUG)
-
-if __name__ == '__main__':
-    """Track algorithm entry."""
-    jsonData = config.loadJsonData(config.configPath)
-    redisClient = f.getRedisConnection(jsonData["ip"], jsonData["port"], jsonData["database"], jsonData["password"])
-    logging.info('init redis client %s', redisClient)
-    t = threading.Thread(target=track.delayKeyThread, args=(redisClient,))
-    t.setDaemon(True)
-    t.start()
-    while 1:
-        try:
-            if config.loadJsonData(config.sign) == 0:
-                logging.info('not to execute new task')
-                time.sleep(1)
-            else:
-                logging.info('get one task')
-                element = redisClient.eval(start_script.startTaskLua, 1, config.trackTaskQueue,
-                                           config.trackStartQueue, int(time.time()))
-                if len(element) > 0:
-                    key = element[0].decode()
-                    jsonStr = f.getByKey(redisClient, key.replace('"', ''));
-                    if track.trackProcess(jsonStr, element[0]):
-                        f.pushToQueue(redisClient, config.trackFinishQueue, key)
-                        redisClient.zrem(config.trackStartQueue, element[0])
-                        logging.info('success')
-                    else:
-                        f.pushToQueue(redisClient, config.trackFailedQueue, key)
-                        redisClient.zrem(config.trackStartQueue, element[0])
-                        logging.info('failed')
-                else:
-                    logging.info('task queue is empty.')
-                    time.sleep(1)
-        except Exception as e:
-            logging.error('except:', e)
-            time.sleep(1)

dubhe_data_process/entrance/algorithm-videosample.py

@@ -1,63 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-import json
-import threading
-from datetime import datetime
-import time
-import sys
-sys.path.append("../")
-import common.RedisUtil as f
-import luascript.starttaskscript as start_script
-import common.config as config
-import logging
-from entrance.executor import videosample
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
-                    level=logging.DEBUG)
-
-if __name__ == '__main__':
-    """VideoSample algorithm entry."""
-    jsonData = config.loadJsonData(config.configPath)
-    redisClient = f.getRedisConnection(jsonData["ip"], jsonData["port"], jsonData["database"], jsonData["password"])
-    logging.info('init redis client %s', redisClient)
-    t = threading.Thread(target=videosample.delayKeyThread, args=(redisClient,))
-    t.setDaemon(True)
-    t.start()
-    while 1:
-        try:
-            if config.loadJsonData(config.sign) == 0:
-                logging.info('not to execute new task')
-                time.sleep(5)
-            else:
-                logging.info("read redis:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S"))
-                sampleTask = redisClient.eval(start_script.startTaskLua, 1, config.videoPendingQueue,
-                                              config.videoStartQueue, int(time.time()))
-                logging.info(int(time.time()))
-                if len(sampleTask) > 0:
-                    datasetId = json.loads(sampleTask[0])['datasetIdKey']
-                    taskParameters = json.loads(redisClient.get("videoSample:" + str(datasetId)))
-                    path = taskParameters['path']
-                    frameList = taskParameters['frames']
-                    videosample.sampleProcess(datasetId, path, frameList, redisClient)
-                else:
-                    logging.info('task queue is empty.')
-                    time.sleep(5)
-        except Exception as e:
-            logging.error('except:', e)
-            time.sleep(1)

dubhe_data_process/entrance/executor/annotation.py

@@ -1,46 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-import time
-import sys
-sys.path.append("../../")
-from entrance.executor import predict_with_print_box as yolo_demo
-from common.log_config import setup_log
-
-
-label_log = setup_log('dev', 'label.log')
-
-
-def _init():
-    print('init yolo_obj')
-    global yolo_obj
-    yolo_obj = yolo_demo.YoloInference(label_log)
-
-
-def _annotation(type_, image_path_list, id_list, annotation_url_list, label_list, coco_flag=0):
-    """Perform automatic annotation task."""
-    image_num = len(image_path_list)
-    if image_num < 16:
-       for i in range(16 - image_num):
-           image_path_list.append(image_path_list[0])
-           id_list.append(id_list[0])
-           annotation_url_list.append(annotation_url_list[0])
-    image_num = len(image_path_list)
-    annotations = yolo_obj.yolo_inference(type_, id_list, annotation_url_list, image_path_list, label_list, coco_flag)
-    return annotations[0:image_num] 

dubhe_data_process/entrance/executor/imagenet.py

@@ -1,103 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-import os
-import sched
-import sys
-
-sys.path.append("../../common")
-import logging
-import time
-import json
-import common.of_cnn_resnet as of_cnn_resnet
-import numpy as np
-import luascript.delaytaskscript as delay_script
-import common.config as config
-from datetime import datetime
-
-schedule = sched.scheduler(time.time, time.sleep)
-
-base_path = "/nfs/"
-delayId = ""
-
-
-def _init():
-    of_cnn_resnet.init_resnet()
-    logging.info('env init finished')
-
-
-def process(task_dict, key):
-    """Imagenet task method.
-        Args:
-            task_dict: imagenet task details.
-            key: imagenet task key.
-    """
-    global delayId
-    delayId = "\"" + eval(str(key, encoding="utf-8")) + "\""
-    task_dict = json.loads(task_dict)
-    id_list = []
-    image_path_list = []
-    annotation_path_list = []
-    for file in task_dict["files"]:
-        id_list.append(file["id"])
-        image_path = base_path + file["url"]
-        image_path_list.append(image_path)
-        annotation_url = image_path.replace("origin/", "annotation/")
-        annotation_path_list.append(os.path.splitext(annotation_url)[0])
-        isExists = os.path.exists(os.path.dirname(annotation_url))
-        if not isExists:
-            os.makedirs(os.path.dirname(annotation_url))
-    label_list = task_dict["labels"]
-    image_num = len(image_path_list)
-    annotations = []
-    for inds in range(len(image_path_list)):
-        temp = {}
-        temp['id'] = id_list[inds]
-        score, ca_id = of_cnn_resnet.resnet_inf(image_path_list[inds])
-        temp['annotation'] = [{'category_id': int(ca_id), 'score': np.float(score)}]
-        temp['annotation'] = json.dumps(temp['annotation'])
-        annotations.append(temp)
-        with open(annotation_path_list[inds], 'w') as w:
-            w.write(temp['annotation'])
-    result = {"annotations": annotations, "task": key.decode()}
-    return result
-
-
-def delaySchduled(inc, redisClient):
-    """Delay task method.
-        Args:
-            inc: scheduled task time.
-            redisClient: redis client.
-    """
-    try:
-        print("delay:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S"))
-        redisClient.eval(delay_script.delayTaskLua, 1, config.imagenetStartQueue, delayId, int(time.time()))
-        schedule.enter(inc, 0, delaySchduled, (inc, redisClient))
-    except Exception as e:
-        print("delay error" + e)
-
-
-def delayKeyThread(redisClient):
-    """Delay task thread.
-        Args:
-            redisClient: redis client.
-    """
-    schedule.enter(0, 0, delaySchduled, (5, redisClient))
-    schedule.run()

dubhe_data_process/entrance/executor/imgprocess.py

@@ -1,189 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# !/usr/bin/env python3
-# -*- coding: utf-8 -*-
-from datetime import datetime
-import sched
-import os
-import cv2
-import numpy as np
-import logging
-import time
-import json
-import argparse
-import sys
-import codecs
-import shutil
-
-import luascript.delaytaskscript as delay_script
-import common.config as config
-
-from common.augment_utils.ACE import ACE_color
-from common.augment_utils.dehaze import deHaze, addHaze
-from common.augment_utils.hist_equalize import adaptive_hist_equalize
-from common.log_config import setup_log
-
-schedule = sched.scheduler(time.time, time.sleep)
-
-delayId = ""
-finish_key = {}
-re_task_id = {}
-sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
-
-# task url suffix
-img_pro_url = 'api/data/datasets/'
-
-# arguments
-parser = argparse.ArgumentParser(description="config for image augmentation server")
-parser.add_argument("-m", "--mode", type=str, default="test", required=False)
-args = parser.parse_args()
-
-# url concat(ip + port + suffix)
-url_json = '../common/config/url.json'
-with open(url_json) as f:
-    url_dict = json.loads(f.read())
-img_pro_url = url_dict[args.mode] + img_pro_url
-
-# creat task quene
-base_path = "/nfs/"
-
-# create log path and file
-des_folder = os.path.join('./log', args.mode)
-if not os.path.exists(des_folder):
-    os.makedirs(des_folder)
-
-logging = setup_log(args.mode, 'enhance-' + args.mode + '.log')
-enhanceTaskId = ""
-
-
-def start_enhance_task(enhanceTaskId, redisClient):
-    """Enhance task method.
-        Args:
-            enhanceTaskId: enhance task id.
-            redisClient: redis client.
-    """
-    global delayId
-    detailKey = 'imgProcess:' + eval(str(enhanceTaskId[0], encoding="utf-8"))
-    delayId = "\"" + eval(str(enhanceTaskId[0], encoding="utf-8")) + "\""
-    print(detailKey)
-    taskParameters = json.loads(redisClient.get(detailKey).decode())
-    dataset_id = taskParameters['id']
-    img_save_path = taskParameters['enhanceFilePath']
-    ann_save_path = taskParameters["enhanceAnnotationPath"]
-    file_list = taskParameters['fileDtos']
-    nums_, img_path_list, ann_path_list = img_ann_list_gen(file_list)
-    process_type = taskParameters['type']
-    re_task_id = eval(str(enhanceTaskId[0], encoding="utf-8"))
-    img_process_config = [dataset_id, img_save_path,
-                          ann_save_path, img_path_list,
-                          ann_path_list, process_type, re_task_id]
-    image_enhance_process(img_process_config, redisClient)
-    logging.info(str(nums_) + ' images for augment')
-
-
-def img_ann_list_gen(file_list):
-    """Analyze the json request and convert to list"""
-    nums_ = len(file_list)
-    img_list = []
-    ann_list = []
-    for i in range(nums_):
-        img_list.append(file_list[i]['filePath'])
-        ann_list.append(file_list[i]['annotationPath'])
-    return nums_, img_list, ann_list
-
-
-def image_enhance_process(img_task, redisClient):
-    """The implementation of image augmentation thread"""
-    global img_pro_url
-    global finish_key
-    global re_task_id
-    logging.info('img_process server start'.center(66, '-'))
-    logging.info(img_pro_url)
-    try:
-        dataset_id = img_task[0]
-        img_save_path = img_task[1]
-        ann_save_path = img_task[2]
-        img_list = img_task[3]
-        ann_list = img_task[4]
-        method = img_task[5]
-        re_task_id = img_task[6]
-        suffix = '_enchanced_' + re_task_id
-        logging.info("dataset_id " + str(dataset_id))
-
-        finish_key = {"processKey": re_task_id}
-        finish_data = {"id": re_task_id,
-                       "suffix": suffix}
-        for j in range(len(ann_list)):
-            img_path = img_list[j]
-            ann_path = ann_list[j]
-            img_process(suffix, img_path, ann_path,
-                        img_save_path, ann_save_path, method)
-
-        redisClient.set("imgProcess:finished:" + re_task_id, json.dumps(finish_data))
-        redisClient.zrem(config.imgProcessStartQueue, "\"" + re_task_id + "\"")
-        redisClient.lpush(config.imgProcessFinishQueue, json.dumps(finish_key, separators=(',', ':')))
-        logging.info('suffix:' + suffix)
-        logging.info("End img_process of dataset:" + str(dataset_id))
-
-    except Exception as e:
-        redisClient.lpush(config.imgProcessFailedQueue, json.dumps(finish_key, separators=(',', ':')))
-        redisClient.zrem(config.imgProcessStartQueue, "\"" + re_task_id + "\"")
-        logging.info(img_pro_url)
-        logging.error("Error imgProcess")
-        logging.error(e)
-    time.sleep(0.01)
-
-
-def img_process(suffix, img_path, ann_path, img_save_path, ann_save_path, method_ind):
-    """Process images and save in specified path"""
-    inds2method = {1: deHaze, 2: addHaze, 3: ACE_color, 4: adaptive_hist_equalize}
-    method = inds2method[method_ind]
-    img_raw = cv2.imdecode(np.fromfile(img_path.encode('utf-8'), dtype=np.uint8), 1)
-    img_suffix = os.path.splitext(img_path)[-1]
-    ann_name = os.path.basename(ann_path)
-    if method_ind <= 3:
-        processed_img = method(img_raw / 255.0) * 255
-    else:
-        processed_img = method(img_raw)
-    cv2.imwrite(img_save_path + "/" + ann_name + suffix + img_suffix,
-                processed_img.astype(np.uint8))
-    shutil.copyfile(ann_path.encode('utf-8'), (ann_path + suffix).encode('utf-8'))
-
-
-def delaySchduled(inc, redisClient):
-    """Delay task method.
-         Args:
-            inc: scheduled task time.
-            redisClient: redis client.
-    """
-    try:
-        logging.info("delay:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S") + ":" + delayId)
-        redisClient.eval(delay_script.delayTaskLua, 1, config.imgProcessStartQueue, delayId, int(time.time()))
-        schedule.enter(inc, 0, delaySchduled, (inc, redisClient))
-    except Exception as e:
-        print("delay error" + e)
-
-
-def delayKeyThread(redisClient):
-    """Delay task thread.
-        Args:
-            redisClient: redis client.
-    """
-    schedule.enter(0, 0, delaySchduled, (5, redisClient))
-    schedule.run()

dubhe_data_process/entrance/executor/lungsegmentation.py

@@ -1,175 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-import sched
-import sys
-
-sys.path.append("../../")
-import logging
-import time
-import json
-import numpy as np
-import luascript.delaytaskscript as delay_script
-import common.config as config
-from datetime import datetime
-from skimage.morphology import disk, binary_erosion, binary_closing
-from skimage.measure import label,regionprops, find_contours
-from skimage.filters import roberts
-from scipy import ndimage as ndi
-from skimage.segmentation import clear_border
-import pydicom as dicom
-import os
-import logging
-
-schedule = sched.scheduler(time.time, time.sleep)
-
-base_path = "/nfs/"
-delayId = ""
-
-
-def process(task_dict, key):
-    """Lung segmentation based on dcm task method.
-        Args:
-            task_dict: imagenet task details.
-            key: imagenet task key.
-    """
-    global delayId
-    delayId = "\"" + eval(str(key, encoding="utf-8")) + "\""
-    task_dict = json.loads(task_dict)
-    base_path = task_dict["annotationPath"]
-    if not os.path.exists(base_path):
-        logging.info("make annotation path.")
-        os.makedirs(base_path)
-    
-    for dcm in task_dict["dcms"]:
-        image, image_path = preprocesss_dcm_image(dcm)
-        # segmentation and wirte coutours to result_path
-        result_path = os.path.join(base_path, image_path)
-        contour(segmentation(image), result_path)
-    
-    logging.info("all dcms in one task are processed.")
-    return True
-
-def preprocesss_dcm_image(path):
-    """Load and preprocesss dcm image.
-        Args:
-            path: dcm file path.
-    """
-    # result_path = os.path.basename(path).split(".", 1)[0] + ".json"
-    result_path = ".".join(os.path.basename(path).split(".")[0:-1]) + ".json"
-    dcm = dicom.dcmread(path)
-    image = dcm.pixel_array.astype(np.int16)
-    
-    # Set outside-of-scan pixels to 0.
-    image[image == -2000] = 0
-
-    # Convert to Hounsfield units (HU)
-    intercept = dcm.RescaleIntercept
-    slope = dcm.RescaleSlope
-
-    if slope != 1:
-        image = slope * image.astype(np.float64)
-        image = image.astype(np.int16)
-
-    image += np.int16(intercept)
-    logging.info("preprocesss_dcm_image done.")
-    return np.array(image, dtype=np.int16), result_path
-
-def segmentation(image):
-    """Segments the lung from the given 2D slice.
-        Args:
-            image: single image in one dcm.
-    """
-    # Step 1: Convert into a binary image.
-    binary = image < -350
-
-    # Step 2: Remove the blobs connected to the border of the image. 
-    cleared = clear_border(binary)
-
-    # Step 3: Label the image. 
-    label_image = label(cleared)
-
-    # Step 4: Keep the labels with 2 largest areas. 
-    areas = [r.area for r in regionprops(label_image)]
-    areas.sort()
-    if len(areas) > 2:
-        for region in regionprops(label_image):
-            if region.area < areas[-2]:
-                for coordinates in region.coords:
-                    label_image[coordinates[0], coordinates[1]] = 0
-    binary = label_image > 0
-
-    # Step 5: Erosion operation with a disk of radius 2. This operation is seperate the lung nodules attached to the blood vessels.
-    selem = disk(1)
-    binary = binary_erosion(binary, selem)
-    
-    # Step 6: Closure operation with a disk of radius 10. This operation is to keep nodules attached to the lung wall. 
-    selem = disk(16)
-    binary = binary_closing(binary, selem)
-
-    # Step 7: Fill in the small holes inside the binary mask of lungs.
-    for _ in range(3):
-        edges = roberts(binary)
-        binary = ndi.binary_fill_holes(edges)
-    logging.info("lung segmentation done.")
-    return binary
-
-def contour(image, path):
-    """Get contours of segmentation.
-        Args:
-            seg: segmentation of lung.
-    """
-    result = []
-    contours = find_contours(image, 0.5)
-    if len(contours) > 2:
-        contours.sort(key = lambda x: int(x.shape[0]))
-        contours = contours[-2:]
-
-    for n, contour in enumerate(contours):
-        # result.append({"type":n, "annotation":contour.tolist()})
-        result.append({"type":n, "annotation":np.flip(contour, 1).tolist()})
-    
-    # write json
-    with open(path, 'w') as f:
-        json.dump(result, f)
-    logging.info("write {} done.".format(path))
-
-
-def delaySchduled(inc, redisClient):
-    """Delay task method.
-        Args:
-            inc: scheduled task time.
-            redisClient: redis client.
-    """
-    try:
-        print("delay:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S"))
-        redisClient.eval(delay_script.delayTaskLua, 1, config.dcmStartQueue, delayId, int(time.time()))
-        schedule.enter(inc, 0, delaySchduled, (inc, redisClient))
-    except Exception as e:
-        print("delay error" + e)
-
-
-def delayKeyThread(redisClient):
-    """Delay task thread.
-        Args:
-            redisClient: redis client.
-    """
-    schedule.enter(0, 0, delaySchduled, (5, redisClient))
-    schedule.run()

dubhe_data_process/entrance/executor/ofrecord.py

@@ -1,181 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# -*- coding: utf-8 -*-
-
-import logging
-import json
-import os
-import struct
-import cv2
-import sched
-import numpy as np
-import oneflow.core.record.record_pb2 as of_record
-import luascript.delaytaskscript as delay_script
-import time
-import common.config as config
-from datetime import datetime
-
-schedule = sched.scheduler(time.time, time.sleep)
-
-delayId = ""
-
-class ImageCoder(object):
-    """Helper class that provides image coding utilities."""
-
-    def __init__(self, size=None):
-        self.size = size
-
-    def _resize(self, image_data):
-        if self.size is not None and image_data.shape[:2] != self.size:
-            return cv2.resize(image_data, self.size)
-        return image_data
-
-    def image_to_jpeg(self, image_data):
-        image_data = cv2.imdecode(np.frombuffer(image_data, np.uint8), 1)
-        image_data = self._resize(image_data)
-        return cv2.imencode(".jpg", image_data)[1].tobytes(
-        ), image_data.shape[0], image_data.shape[1]
-
-
-def _process_image(filename, coder):
-    """Process a single image file.
-        Args:
-            filename: string, path to an image file e.g., '/path/to/example.JPG'.
-            coder: instance of ImageCoder to provide image coding utils.
-        Returns:
-            image_buffer: string, JPEG encoding of RGB image.
-            height: integer, image height in pixels.
-            width: integer, image width in pixels.
-    """
-    # Read the image file.
-    with open(filename, 'rb') as f:
-        image_data = f.read()
-        image_data, height, width = coder.image_to_jpeg(image_data)
-
-        return image_data, height, width
-
-
-def _bytes_feature(value):
-    """Wrapper for inserting bytes features into Example proto."""
-    return of_record.Feature(bytes_list=of_record.BytesList(value=[value]))
-
-
-def dense_to_one_hot(labels_dense, num_classes):
-    """Convert class labels from scalars to one-hot vectors."""
-    num_labels = labels_dense.shape[0]
-    index_offset = np.arange(num_labels) * num_classes
-    labels_one_hot = np.zeros((num_labels, num_classes))
-    labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1
-    return labels_one_hot
-
-
-def extract_img_label(names, path):
-    """Extract the images and labels into np array [index].
-    Args:
-      f: A file object that contain images and annotations.
-    Returns:
-      data: A 4D uint8 np array [index, h, w, depth].
-      labels: a 1D uint8 np array.
-      num_img: the number of images.
-    """
-    train_img = os.path.join(path, 'origin/')
-    train_label = os.path.join(path, 'annotation/')
-    num_imgs = len(names)
-    data = []
-    labels = []
-    print('^^^^^^^^^^ start img_set for sycle')
-    for i in names:
-        name = os.path.splitext(i)[0]
-        print(name)
-        coder = ImageCoder((224, 224))
-        image_buffer, height, width = _process_image(
-            os.path.join(train_img, i), coder)
-
-        data += [image_buffer]
-
-        if os.path.exists(os.path.join(train_label, name)):
-
-            with open(os.path.join(train_label, name), "r", encoding='utf-8') as jsonFile:
-                la = json.load(jsonFile)
-                if la:
-                    labels += [la[0]['category_id']]
-                else:
-                    data.pop()
-                    num_imgs -= 1
-        else:
-            print('File is not found')
-    print('^^^^^^^^^ img_set for end')
-    data = np.array(data)
-    labels = np.array(labels)
-    print(data.shape, labels.shape)
-    return num_imgs, data, labels
-
-
-def execute(src_path, desc, label_map, files, part_id, key):
-    """Execute ofrecord task method."""
-    global delayId
-    delayId = delayId = "\"" + eval(str(key, encoding="utf-8")) + "\""
-    logging.info(part_id)
-    num_imgs, images, labels = extract_img_label(files, src_path)
-    keys = sorted(list(map(int, label_map.keys())))
-    for i in range(len(keys)):
-        label_map[str(keys[i])] = i
-    if not num_imgs:
-        return False, 0, 0
-    try:
-        os.makedirs(desc)
-    except Exception as e:
-        print('{} exists.'.format(desc))
-    filename = 'part-{}'.format(part_id)
-    filename = os.path.join(desc, filename)
-    f = open(filename, 'wb')
-    print(filename)
-    for i in range(num_imgs):
-        img = images[i]
-        label = label_map[str(labels[i])]
-        sample = of_record.OFRecord(feature={
-            'class/label': of_record.Feature(int32_list=of_record.Int32List(value=[label])),
-            'encoded': _bytes_feature(img)
-        })
-        size = sample.ByteSize()
-        f.write(struct.pack("q", size))
-        f.write(sample.SerializeToString())
-    if f:
-        f.close()
-
-def delaySchduled(inc, redisClient):
-    """Delay task method.
-        Args:
-            inc: scheduled task time.
-            redisClient: redis client.
-    """
-    try:
-        print("delay:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S"))
-        redisClient.eval(delay_script.delayTaskLua, 1, config.ofrecordStartQueue, delayId, int(time.time()))
-        schedule.enter(inc, 0, delaySchduled, (inc, redisClient))
-    except Exception as e:
-        print("delay error" + e)
-
-def delayKeyThread(redisClient):
-    """Delay task thread.
-        Args:
-            redisClient: redis client.
-    """
-    schedule.enter(0, 0, delaySchduled, (5, redisClient))
-    schedule.run()

dubhe_data_process/entrance/executor/taskexecutor.py

@@ -1,108 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-
-import codecs
-import os
-import sched
-import sys
-import json
-import logging
-import time
-import common.RedisUtil as f
-import common.config as config
-from entrance.executor import annotation as annotation
-from datetime import datetime
-import luascript.delaytaskscript as delay_script
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
-                    level=logging.DEBUG)
-
-schedule = sched.scheduler(time.time, time.sleep)
-sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
-
-delayId = ""
-
-
-def annotationExecutor(redisClient, key):
-    """Annotation task method.
-                Args:
-                  redisClient: redis client.
-                  key: annotation task key.
-    """
-    global delayId
-    print('-------------process one-----------------')
-    try:
-        delayId = "\"" + eval(str(key, encoding="utf-8")) + "\""
-        logging.info('get element is  {0}'.format(key))
-        key = key.decode()
-        jsonStr = f.getByKey(redisClient, key.replace('"', ''));
-        print(jsonStr)
-        jsonObject = json.loads(jsonStr.decode('utf-8'));
-        image_path_list = []
-        id_list = []
-        annotation_url_list = []
-        label_list = []
-        label_list = jsonObject['labels']
-        for fileObject in jsonObject['files']:
-            pic_url = '/nfs/' + fileObject['url']
-            image_path_list.append(pic_url)
-            annotation_url = pic_url.replace("origin/", "annotation/")
-            annotation_url_list.append(os.path.splitext(annotation_url)[0])
-            isExists = os.path.exists(os.path.dirname(annotation_url))
-            if not isExists:
-                os.makedirs(os.path.dirname(annotation_url))
-            id_list.append(fileObject['id'])
-        print(image_path_list)
-        print(annotation_url_list)
-        print(label_list)
-        coco_flag = 0
-        if "labelType" in jsonObject:
-            label_type = jsonObject['labelType']
-            if label_type == 3:
-                coco_flag = 80
-        annotations = annotation._annotation(0, image_path_list, id_list, annotation_url_list, label_list, coco_flag);
-        result = {"task": key, "annotations": annotations}
-        f.pushToQueue(redisClient, config.annotationFinishQueue, json.dumps(result))
-        redisClient.zrem(config.annotationStartQueue, key)
-    except Exception as e:
-        print(e)
-
-
-def delaySchduled(inc, redisClient):
-    """Delay task method.
-         Args:
-            inc: scheduled task time.
-            redisClient: redis client.
-    """
-    try:
-        print("delay:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S"))
-        redisClient.eval(delay_script.delayTaskLua, 1, config.annotationStartQueue, delayId, int(time.time()))
-        schedule.enter(inc, 0, delaySchduled, (inc, redisClient))
-    except Exception as e:
-        print("delay error" + e)
-
-
-def delayKeyThread(redisClient):
-    """Delay task thread.
-        Args:
-             redisClient: redis client.
-    """
-    schedule.enter(0, 0, delaySchduled, (5, redisClient))
-    schedule.run()

dubhe_data_process/entrance/executor/text_classification.py

@@ -1,45 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-from entrance.executor import classify_by_textcnn as classify
-
-
-def _init():
-    print('init classify_obj')
-    global classify_obj
-    classify_obj = classify.TextCNNClassifier()  # label_log
-
-
-def _classification(text_path_list, id_list, label_list):
-    """Perform automatic text classification task."""
-    textnum = len(text_path_list)
-    batched_num = ((textnum - 1) // classify.BATCH_SIZE + 1) * classify.BATCH_SIZE
-    for i in range(batched_num - textnum):
-        text_path_list.append(text_path_list[0])
-        id_list.append(id_list[0])
-    annotations = classify_obj.inference(text_path_list, id_list, label_list)  #
-    return annotations[0:textnum]
-
-
-if __name__ == "__main__":
-    test_len = 22
-    _init()
-    ans = _classification(["dubhe-dev/dataset/2738/origin/32_3_ts1607326726114630.txt"] * test_len, [1] * test_len,
-                          [111, 112])
-    print(ans)
-    print(len(ans))

dubhe_data_process/entrance/executor/text_taskexecutor.py

@@ -1,94 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-
-import codecs
-import sched
-import sys
-import json
-import logging
-import time
-import common.RedisUtil as f
-import common.config as config
-from entrance.executor import text_classification as text_classification
-from datetime import datetime
-import luascript.delaytaskscript as delay_script
-
-logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
-                    level=logging.DEBUG)
-
-schedule = sched.scheduler(time.time, time.sleep)
-sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
-
-delayId = ""
-
-
-def textClassificationExecutor(redisClient, key):
-    """Annotation task method.
-                Args:
-                  redisClient: redis client.
-                  key: annotation task key.
-    """
-    global delayId
-    print('-------------process one-----------------')
-    try:
-        delayId = "\"" + eval(str(key, encoding="utf-8")) + "\""
-        logging.info('get element is  {0}'.format(key))
-        key = key.decode()
-        jsonStr = f.getByKey(redisClient, key.replace('"', ''))
-        print(jsonStr)
-        jsonObject = json.loads(jsonStr.decode('utf-8'))
-        text_path_list = []
-        id_list = []
-        label_list = jsonObject['labels']
-        for fileObject in jsonObject['files']:
-            text_path_list.append(fileObject['url'])
-            id_list.append(fileObject['id'])
-        print(text_path_list)
-        print(id_list)
-        print(label_list)
-        classifications = text_classification._classification(text_path_list, id_list, label_list)  # --------------
-        result = {"task": key, "classifications": classifications}  # --------------
-        f.pushToQueue(redisClient, config.textClassificationFinishQueue, json.dumps(result))
-        redisClient.zrem(config.textClassificationStartQueue, key)
-    except Exception as e:
-        print(e)
-
-
-def delaySchduled(inc, redisClient):
-    """Delay task method.
-         Args:
-            inc: scheduled task time.
-            redisClient: redis client.
-    """
-    try:
-        print("delay:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S"))
-        redisClient.eval(delay_script.delayTaskLua, 1, config.textClassificationStartQueue, delayId, int(time.time()))
-        schedule.enter(inc, 0, delaySchduled, (inc, redisClient))
-    except Exception as e:
-        print("delay error" + e)
-
-
-def delayKeyThread(redisClient):
-    """Delay task thread.
-        Args:
-             redisClient: redis client.
-    """
-    schedule.enter(0, 0, delaySchduled, (5, redisClient))
-    schedule.run()

dubhe_data_process/entrance/executor/track.py

@@ -1,93 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# -*- coding: utf-8 -*-
-import sched
-
-import common.config as config
-import luascript.delaytaskscript as delay_script
-from track_only.hog_track import *
-
-schedule = sched.scheduler(time.time, time.sleep)
-
-delayId = ""
-
-
-def trackProcess(task, key):
-    """Track task method.
-            Args:
-              task: dataset id.
-              key: video file path.
-            Returns:
-              True: track success
-              False: track failed
-    """
-    global delayId
-    delayId = "\"" + eval(str(key, encoding="utf-8")) + "\""
-    task = json.loads(task.decode('utf-8'))
-    image_list = []
-    label_list = []
-    images_data = task['images']
-    path = task['path']
-
-    for file in images_data:
-        filePath = path + "/origin/" + file
-        annotationPath = path + "/annotation/" + file.split('.')[0]
-        if not os.path.exists(filePath):
-            continue
-        if not os.path.exists(annotationPath):
-            continue
-        image_list.append(filePath)
-        label_list.append(annotationPath)
-    image_num = len(label_list)
-    track_det = Detector(
-        'xxx.avi',
-        min_confidence=0.35,
-        max_cosine_distance=0.2,
-        max_iou_distance=0.7,
-        max_age=30,
-        out_dir='results/')
-    track_det.write_img = False
-    RET = track_det.run_track(image_list, label_list)
-    if RET == 'OK':
-        return True
-    else:
-        return False
-
-
-def delaySchduled(inc, redisClient):
-    """Delay task method.
-        Args:
-            inc: scheduled task time.
-            redisClient: redis client.
-    """
-    try:
-        print("delay:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S"))
-        redisClient.eval(delay_script.delayTaskLua, 1, config.trackStartQueue, delayId, int(time.time()))
-        schedule.enter(inc, 0, delaySchduled, (inc, redisClient))
-    except Exception as e:
-        print("delay error" + e)
-
-
-def delayKeyThread(redisClient):
-    """Delay task thread.
-        Args:
-            redisClient: redis client.
-    """
-    schedule.enter(0, 0, delaySchduled, (5, redisClient))
-    schedule.run()

dubhe_data_process/entrance/executor/videosample.py

@@ -1,100 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-import json
-import os
-import sched
-import time
-from datetime import datetime
-
-import luascript.finishtaskscript as finish_script
-import luascript.failedtaskscript as failed_script
-import luascript.delaytaskscript as delay_script
-import common.config as config
-
-import cv2
-
-schedule = sched.scheduler(time.time, time.sleep)
-
-datasetIdKey = ""
-
-
-def sampleProcess(datasetId, path, frameList, redisClient):
-    """Video sampling method.
-        Args:
-          datasetId: dataset id.
-          path: video file path.
-          frameList: picture frame number list.
-          redisClient: redis client.
-    """
-    global datasetIdKey
-    datasetIdJson = {'datasetIdKey': datasetId}
-    datasetIdKey = json.dumps(datasetIdJson, separators=(',', ':'))
-    try:
-        videoName = path.split('/')[-1]
-        save_path = path.split(videoName)[0].replace("video", "origin")
-        is_exists = os.path.exists(save_path)
-        if not is_exists:
-            os.makedirs(save_path)
-            print('path of %s is build' % save_path)
-        else:
-            print('path of %s already exist and start' % save_path)
-        cap = cv2.VideoCapture(path)
-        for i in frameList:
-            cap.set(cv2.CAP_PROP_POS_FRAMES, i)
-            success, video_capture = cap.read()
-            # 保存图片
-            if success is True and video_capture is not None:
-                save_name = save_path + videoName.split('.')[0] + '_' + str(i) + '.jpg'
-                cv2.imwrite(save_name, video_capture)
-                redisClient.lpush("videoSample_pictures:" + datasetId,
-                                  '{' + '\"pictureName\":' + "\"" + save_name + "\"" + '}')
-                print('image of %s is saved' % save_name)
-        print('video is all read')
-        redisClient.eval(finish_script.finishTaskLua, 3, config.videoStartQueue, config.videoFinishQueue,
-                         "videoSample:" + str(datasetId),
-                         datasetIdKey, str(datasetIdKey))
-    except Exception as e:
-        print(e)
-        redisClient.eval(failed_script.failedTaskLua, 4, config.videoStartQueue, config.videoFailedQueue,
-                         "videoSample_pictures:" + datasetId,
-                         "videoSample:" + str(datasetId),
-                         datasetIdKey, str(datasetIdKey))
-
-
-def delaySchduled(inc, redisClient):
-    """Delay task method.
-        Args:
-          inc: scheduled task time.
-          redisClient: redis client.
-    """
-    try:
-        print("delay:" + datetime.now().strftime("B%Y-%m-%d %H:%M:%S"))
-        redisClient.eval(delay_script.delayTaskLua, 1, config.videoStartQueue, datasetIdKey, int(time.time()))
-        schedule.enter(inc, 0, delaySchduled, (inc, redisClient))
-    except Exception as e:
-        print("delay error" + e)
-
-
-def delayKeyThread(redisClient):
-    """Delay task thread.
-            Args:
-              redisClient: redis client.
-    """
-    schedule.enter(0, 0, delaySchduled, (5, redisClient))
-    schedule.run()

dubhe_data_process/luascript/delaytaskscript.py

@@ -1,26 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-
-delayTaskLua = """
-local element = redis.call('zscore', KEYS[1],ARGV[1])
-if element then
-redis.call('zadd',KEYS[1],ARGV[2],ARGV[1])
-end
-"""

dubhe_data_process/luascript/failedtaskscript.py

@@ -1,27 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-
-failedTaskLua = """
-redis.call('zrem',KEYS[1],ARGV[1])
-redis.call('lpush',KEYS[2],ARGV[2])
-redis.call('del',KEYS[3])
-redis.call('del',KEYS[4])
-return
-"""

dubhe_data_process/luascript/finishtaskscript.py

@@ -1,27 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-
-finishTaskLua = """
-local queues,values=KEYS,ARGV
-redis.call('zrem', queues[1], values[1])
-redis.call('lpush',queues[2],values[2])
-redis.call('del',KEYS[3])
-return
-"""

dubhe_data_process/luascript/gettaskscript.py

@@ -1,30 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-
-getTaskLua = """
-local queue = KEYS[1]
-local element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)
-print(element[0])
-if table.getn(element) > 0 then
-   print('delete this element')
-   redis.call('zrem', queue, element[1])
-end   
-return element
-"""

dubhe_data_process/luascript/starttaskscript.py

@@ -1,29 +0,0 @@
-"""
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
-"""
-# coding:utf-8
-
-startTaskLua = """
-local queue,value,time=KEYS[1],ARGV[1],ARGV[2]
-local element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)
-if table.getn(element)>0 then
-redis.call('zrem', queue, element[1])
-redis.call('zadd',value,time,element[1])
-end
-return element
-"""

dubhe_data_process/main.py

@@ -0,0 +1,34 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+
+import logging
+import sys
+from program.impl.config_actuator import ConfigActuator
+
+logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
+                    level=logging.DEBUG)
+
+if __name__ == '__main__':
+    """
+    Algorithm entry
+    """
+    algorithm = sys.argv[1]
+    actuator = ConfigActuator()
+    actuator.execute(algorithm)

dubhe_data_process/of_model/config.py

@@ -1,20 +1,21 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
 
 from __future__ import absolute_import

dubhe_data_process/of_model/imagenet1000_clsidx_to_labels.py

@@ -1,20 +1,21 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
 
 clsidx_2_labels = {

dubhe_data_process/of_model/of_develop_2_of_python.py

@@ -1,20 +1,21 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
 
 import os

dubhe_data_process/of_model/resnet_model.py

@@ -1,3 +1,6 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
 /**
 * Copyright 2020 Tianshu AI Platform. All Rights Reserved.

dubhe_data_process/program/abstract/actuator.py

@@ -0,0 +1,37 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+
+from abc import ABCMeta
+from abc import abstractmethod
+
+
+class Actuator(metaclass=ABCMeta):
+    """
+    Algorithm executor
+    """
+
+    @abstractmethod
+    def execute(self, algorithm):
+        """
+        Algorithm execution method
+        Parameter description:
+        algorithm: 表示当前执行的算法
+        """
+        pass

dubhe_data_process/program/abstract/algorithm.py

@@ -0,0 +1,33 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+# _*_ coding:utf-8 _*_
+
+from abc import ABCMeta
+from abc import abstractmethod
+
+
+class Algorithm(metaclass=ABCMeta):
+
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def execute(self, task):
+        pass

dubhe_data_process/program/abstract/storage.py

@@ -0,0 +1,55 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+
+from abc import ABCMeta
+from abc import abstractmethod
+
+
+class Storage(metaclass=ABCMeta):
+    """
+    algorithm task storage
+    """
+
+    @abstractmethod
+    def init_client(self):
+        """
+        init method
+        """
+        pass
+
+    @abstractmethod
+    def get_one_task(*args):
+        """
+        Get a task
+        Parameter description:
+        args[0]: Lua expression
+        args[1]: numkeys default 1
+        args[2]: Pending task queue
+        args[3]: Task queue in process
+        args[4]: time
+        """
+        pass
+
+    @abstractmethod
+    def save_result(*args):
+        """
+        Save the results
+        """
+        pass

dubhe_data_process/program/exec/annotation/annotation.py

@@ -0,0 +1,108 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+
+import codecs
+import os
+import sched
+import logging
+import time
+import sys
+
+from program.exec.annotation import predict_with_print_box as yolo_demo
+from common.config.log_config import setup_log
+from abc import ABC
+from program.abstract.algorithm import Algorithm
+
+logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
+                    level=logging.DEBUG)
+
+schedule = sched.scheduler(time.time, time.sleep)
+sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+
+label_log = setup_log('dev', 'label.log')
+
+
+class Annotation(Algorithm, ABC):
+
+    def __init__(self):
+        pass
+
+    def execute(task):
+        return Annotation.annotationExecutor(task)
+
+    def annotationExecutor(jsonObject):
+        """Annotation task method.
+                    Args:
+                      redisClient: redis client.
+                      key: annotation task key.
+        """
+        print('-------------process one-----------------')
+        try:
+            image_path_list = []
+            id_list = []
+            annotation_url_list = []
+            label_list = jsonObject['labels']
+            for fileObject in jsonObject['files']:
+                pic_url = '/nfs/' + fileObject['url']
+                image_path_list.append(pic_url)
+                annotation_url = pic_url.replace("origin/", "annotation/")
+                annotation_url_list.append(os.path.splitext(annotation_url)[0])
+                isExists = os.path.exists(os.path.dirname(annotation_url))
+                if not isExists:
+                    try:
+                        os.makedirs(os.path.dirname(annotation_url))
+                    except Exception as exception:
+                        logging.error(exception)
+                id_list.append(fileObject['id'])
+            print(image_path_list)
+            print(annotation_url_list)
+            print(label_list)
+            coco_flag = 0
+            if "labelType" in jsonObject:
+                label_type = jsonObject['labelType']
+                if label_type == 3:
+                    coco_flag = 80
+            annotations = Annotation._annotation(0, image_path_list, id_list, annotation_url_list, label_list,
+                                                 coco_flag);
+            finish_data = {"reTaskId": jsonObject["reTaskId"], "annotations": annotations}
+            return finish_data, True
+        except Exception as e:
+            print(e)
+            finish_data = {"reTaskId": jsonObject["reTaskId"], "annotations": annotations}
+            return finish_data, True
+
+    @staticmethod
+    def _init():
+        print('init yolo_obj')
+        global yolo_obj
+        yolo_obj = yolo_demo.YoloInference(label_log)
+
+    def _annotation(type_, image_path_list, id_list, annotation_url_list, label_list, coco_flag=0):
+        """Perform automatic annotation task."""
+        image_num = len(image_path_list)
+        if image_num < 16:
+            for i in range(16 - image_num):
+                image_path_list.append(image_path_list[0])
+                id_list.append(id_list[0])
+                annotation_url_list.append(annotation_url_list[0])
+        image_num = len(image_path_list)
+        annotations = yolo_obj.yolo_inference(type_, id_list, annotation_url_list, image_path_list, label_list,
+                                              coco_flag)
+        return annotations[0:image_num]

dubhe_data_process/program/exec/annotation/config.json

@@ -0,0 +1,90 @@
+{
+  "annotation": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "annotation_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "选择GPU",
+      "module": "common.util.public.select_gpu",
+      "class": "Select_gpu",
+      "method": "select_gpu",
+      "paramType": 0
+    },
+    {
+      "step": 3,
+      "desc": "初始化",
+      "module": "program.exec.annotation.annotation",
+      "class": "Annotation",
+      "method": "_init",
+      "paramType": 0
+    },
+    {
+      "step": 4,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 4
+    },
+    {
+      "step": 5,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "annotation_task_queue",
+        "annotation_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 4
+    },
+    {
+      "step": 6,
+      "desc": "执行任务",
+      "module": "program.exec.annotation.annotation",
+      "class": "Annotation",
+      "method": "execute",
+      "paramType": 1,
+      "param": [
+        5.2
+      ]
+    },
+    {
+      "step": 7,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "annotation_finished_queue",
+        "annotation_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        6.1,
+        6.2
+      ],
+      "jump": 4
+    }
+  ]
+}

dubhe_data_process/entrance/executor/predict_with_print_box.py → dubhe_data_process/program/exec/annotation/predict_with_print_box.py

@@ -1,20 +1,21 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
 import json
 import time
@@ -22,16 +23,14 @@ import time
 import cv2
 import numpy as np
 import oneflow_yolov3
-import sys
-sys.path.append("../../")
-from common.yolo_net import YoloPredictNet
+from common.util.algorithm.yolo_net import YoloPredictNet
 
 import oneflow as flow
 
 
 '''Init oneflow config'''
-model_load_dir = "../of_model/yolov3_model_python/"
-label_to_name_file = "../common/data/coco.names"
+model_load_dir = "of_model/yolov3_model_python/"
+label_to_name_file = "common/constant/coco.names"
 use_tensorrt = 0
 gpu_num_per_node = 1
 batch_size = 16
@@ -120,7 +119,7 @@ def yolo_show(image_path_list, batch_list):
             cv2.rectangle(img, (int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3])), (0, 255, 255), 1)
             cv2.putText(img, str(cls), (int(c[0]), int(c[1])), font, 1, (0, 0, 255), 1)
         result_name = img_path.split('/')[-1]
-        cv2.imwrite("data/results/" + result_name, img)
+        cv2.imwrite("constant/results/" + result_name, img)
 
 
 def resize_image(img, origin_h, origin_w, image_height, image_width):

dubhe_data_process/program/exec/imagenet/config.json

@@ -0,0 +1,90 @@
+{
+  "imagenet": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "imagenet_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "选择GPU",
+      "module": "common.util.public.select_gpu",
+      "class": "Select_gpu",
+      "method": "select_gpu",
+      "paramType": 0
+    },
+    {
+      "step": 3,
+      "desc": "初始化",
+      "module": "program.exec.imagenet.imagenet",
+      "class": "Imagenet",
+      "method": "_init",
+      "paramType": 0
+    },
+    {
+      "step": 4,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 4
+    },
+    {
+      "step": 5,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "imagenet_task_queue",
+        "imagenet_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 4
+    },
+    {
+      "step": 6,
+      "desc": "执行任务",
+      "module": "program.exec.imagenet.imagenet",
+      "class": "Imagenet",
+      "method": "execute",
+      "paramType": 1,
+      "param": [
+        5.2
+      ]
+    },
+    {
+      "step": 7,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "imagenet_finished_queue",
+        "imagenet_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        6.1,
+        6.2
+      ],
+      "jump": 4
+    }
+  ]
+}

dubhe_data_process/program/exec/imagenet/imagenet.py

@@ -0,0 +1,84 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+import os
+import sched
+
+import logging
+import time
+import json
+import common.util.algorithm.of_cnn_resnet as of_cnn_resnet
+import numpy as np
+from abc import ABC
+from program.abstract.algorithm import Algorithm
+
+schedule = sched.scheduler(time.time, time.sleep)
+
+base_path = "/nfs/"
+delayId = ""
+
+
+class Imagenet(Algorithm, ABC):
+
+    @staticmethod
+    def _init():
+        of_cnn_resnet.init_resnet()
+        logging.info('env init finished')
+
+    def __init__(self):
+        pass
+
+    def execute(task):
+        return Imagenet.process(task)
+
+    def process(task_dict):
+        """Imagenet task method.
+            Args:
+                task_dict: imagenet task details.
+                key: imagenet task key.
+        """
+        id_list = []
+        image_path_list = []
+        annotation_path_list = []
+        for file in task_dict["files"]:
+            id_list.append(file["id"])
+            image_path = base_path + file["url"]
+            image_path_list.append(image_path)
+            annotation_url = image_path.replace("origin/", "annotation/")
+            annotation_path_list.append(os.path.splitext(annotation_url)[0])
+            isExists = os.path.exists(os.path.dirname(annotation_url))
+            if not isExists:
+                try:
+                    os.makedirs(os.path.dirname(annotation_url))
+                except Exception as exception:
+                    logging.error(exception)
+        label_list = task_dict["labels"]
+        image_num = len(image_path_list)
+        annotations = []
+        for inds in range(len(image_path_list)):
+            temp = {}
+            temp['id'] = id_list[inds]
+            score, ca_id = of_cnn_resnet.resnet_inf(image_path_list[inds])
+            temp['annotation'] = [{'category_id': int(ca_id), 'score': np.float(score)}]
+            temp['annotation'] = json.dumps(temp['annotation'])
+            annotations.append(temp)
+            with open(annotation_path_list[inds], 'w') as w:
+                w.write(temp['annotation'])
+        finish_data = {"annotations": annotations, "reTaskId": task_dict["reTaskId"]}
+        return finish_data, True

dubhe_data_process/program/exec/imgprocess/config.json

@@ -0,0 +1,74 @@
+{
+  "imgprocess": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "imgprocess_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 3,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "imgProcess_task_queue",
+        "imgProcess_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 4,
+      "desc": "执行任务",
+      "module": "program.exec.imgprocess.imgprocess",
+      "class": "Imgprocess",
+      "method": "start_enhance_task",
+      "paramType": 1,
+      "param": [
+        3.2
+      ]
+    },
+    {
+      "step": 5,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "imgProcess_finished_queue",
+        "imgProcess_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        4.1,
+        4.2
+      ],
+      "jump": 2
+    }
+  ]
+}

dubhe_data_process/program/exec/imgprocess/imgprocess.py

@@ -0,0 +1,122 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+
+import logging
+import time
+import cv2
+import numpy as np
+import shutil
+import os
+from abc import ABC
+from program.abstract.algorithm import Algorithm
+from common.util.algorithm.ACE import ACE_color
+from common.util.algorithm.dehaze import deHaze, addHaze
+from common.util.algorithm.hist_equalize import adaptive_hist_equalize
+
+
+class Imgprocess(Algorithm, ABC):
+
+    def __init__(self):
+        pass
+
+    def execute(task):
+        return Imgprocess.start_enhance_task(task)
+
+    def start_enhance_task(taskParameters):
+        """
+        Enhance task method.
+        Args:
+            enhanceTaskId: enhance task id.
+            redisClient: redis client.
+        """
+        dataset_id = taskParameters['id']
+        img_save_path = taskParameters['enhanceFilePath']
+        ann_save_path = taskParameters["enhanceAnnotationPath"]
+        file_list = taskParameters['fileDtos']
+        nums_, img_path_list, ann_path_list = Imgprocess.img_ann_list_gen(file_list)
+        process_type = taskParameters['type']
+        re_task_id = taskParameters['reTaskId']
+        img_process_config = [dataset_id, img_save_path,
+                              ann_save_path, img_path_list,
+                              ann_path_list, process_type, re_task_id]
+        return Imgprocess.image_enhance_process(img_process_config)
+        logging.info(str(nums_) + ' images for augment')
+
+    def img_ann_list_gen(file_list):
+        """Analyze the json request and convert to list"""
+        nums_ = len(file_list)
+        img_list = []
+        ann_list = []
+        for i in range(nums_):
+            img_list.append(file_list[i]['filePath'])
+            ann_list.append(file_list[i]['annotationPath'])
+        return nums_, img_list, ann_list
+
+    def image_enhance_process(img_task):
+        """The implementation of image augmentation thread"""
+        global finish_key
+        global re_task_id
+        logging.info('img_process server start'.center(66, '-'))
+        result = True
+        try:
+            dataset_id = img_task[0]
+            img_save_path = img_task[1]
+            ann_save_path = img_task[2]
+            img_list = img_task[3]
+            ann_list = img_task[4]
+            method = img_task[5]
+            re_task_id = img_task[6]
+            suffix = '_enchanced_' + re_task_id
+            logging.info("dataset_id " + str(dataset_id))
+
+            finish_key = {"processKey": re_task_id}
+            finish_data = {"id": re_task_id,
+                           "suffix": suffix}
+            for j in range(len(ann_list)):
+                img_path = img_list[j]
+                ann_path = ann_list[j]
+                Imgprocess.img_process(suffix, img_path, ann_path,
+                            img_save_path, ann_save_path, method)
+
+            logging.info('suffix:' + suffix)
+            logging.info("End img_process of dataset:" + str(dataset_id))
+            return finish_data, result
+
+        except Exception as e:
+            result = False
+            return finish_data, result
+            logging.error("Error imgProcess")
+            logging.error(e)
+        time.sleep(0.01)
+
+    def img_process(suffix, img_path, ann_path, img_save_path, ann_save_path, method_ind):
+        """Process images and save in specified path"""
+        inds2method = {1: deHaze, 2: addHaze, 3: ACE_color, 4: adaptive_hist_equalize}
+        method = inds2method[method_ind]
+        img_raw = cv2.imdecode(np.fromfile(img_path.encode('utf-8'), dtype=np.uint8), 1)
+        img_suffix = os.path.splitext(img_path)[-1]
+        ann_name = os.path.basename(ann_path)
+        if method_ind <= 3:
+            processed_img = method(img_raw / 255.0) * 255
+        else:
+            processed_img = method(img_raw)
+        cv2.imwrite(img_save_path + "/" + ann_name + suffix + img_suffix,
+                    processed_img.astype(np.uint8))
+        shutil.copyfile(ann_path.encode('utf-8'), (ann_save_path + "/" + ann_name + suffix).encode('utf-8'))

dubhe_data_process/program/exec/lung_segmentation/config.json

@@ -0,0 +1,74 @@
+{
+  "lung_segmentation": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "dcm_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 3,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "dcm_task_queue",
+        "dcm_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 4,
+      "desc": "执行任务",
+      "module": "program.exec.lung_segmentation.lung_segmentation",
+      "class": "Lungsegmentation",
+      "method": "execute",
+      "paramType": 1,
+      "param": [
+        3.2
+      ]
+    },
+    {
+      "step": 5,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "dcm_finished_queue",
+        "dcm_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        4.1,
+        4.2
+      ],
+      "jump": 2
+    }
+  ]
+}

dubhe_data_process/program/exec/lung_segmentation/lung_segmentation.py

@@ -0,0 +1,155 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+import sched
+import sys
+
+import time
+import json
+import numpy as np
+from abc import ABC
+from program.abstract.algorithm import Algorithm
+from skimage.morphology import disk, binary_erosion, binary_closing
+from skimage.measure import label, regionprops, find_contours
+from skimage.filters import roberts
+from scipy import ndimage as ndi
+from skimage.segmentation import clear_border
+import pydicom as dicom
+import os
+import logging
+
+schedule = sched.scheduler(time.time, time.sleep)
+
+base_path = "/nfs/"
+delayId = ""
+
+
+class Lungsegmentation(Algorithm, ABC):
+
+    def __init__(self):
+        pass
+
+    def execute(task):
+        return Lungsegmentation.process(task)
+
+    def process(task_dict):
+        """Lung segmentation based on dcm task method.
+            Args:
+                task_dict: imagenet task details.
+                key: imagenet task key.
+        """
+        global delayId
+        base_path = task_dict["annotationPath"]
+        if not os.path.exists(base_path):
+            logging.info("make annotation path.")
+            os.makedirs(base_path)
+
+        for dcm in task_dict["dcms"]:
+            image, image_path = Lungsegmentation.preprocesss_dcm_image(dcm)
+            # segmentation and wirte coutours to result_path
+            result_path = os.path.join(base_path, image_path)
+            Lungsegmentation.contour(Lungsegmentation.segmentation(image), result_path)
+
+        logging.info("all dcms in one task are processed.")
+        finish_data = {"reTaskId": task_dict["reTaskId"]}
+        return finish_data, True
+
+    def preprocesss_dcm_image(path):
+        """Load and preprocesss dcm image.
+            Args:
+                path: dcm file path.
+        """
+        # result_path = os.path.basename(path).split(".", 1)[0] + ".json"
+        result_path = ".".join(os.path.basename(path).split(".")[0:-1]) + ".json"
+        dcm = dicom.dcmread(path)
+        image = dcm.pixel_array.astype(np.int16)
+
+        # Set outside-of-scan pixels to 0.
+        image[image == -2000] = 0
+
+        # Convert to Hounsfield units (HU)
+        intercept = dcm.RescaleIntercept
+        slope = dcm.RescaleSlope
+
+        if slope != 1:
+            image = slope * image.astype(np.float64)
+            image = image.astype(np.int16)
+
+        image += np.int16(intercept)
+        logging.info("preprocesss_dcm_image done.")
+        return np.array(image, dtype=np.int16), result_path
+
+    def segmentation(image):
+        """Segments the lung from the given 2D slice.
+            Args:
+                image: single image in one dcm.
+        """
+        # Step 1: Convert into a binary image.
+        binary = image < -350
+
+        # Step 2: Remove the blobs connected to the border of the image.
+        cleared = clear_border(binary)
+
+        # Step 3: Label the image.
+        label_image = label(cleared)
+
+        # Step 4: Keep the labels with 2 largest areas.
+        areas = [r.area for r in regionprops(label_image)]
+        areas.sort()
+        if len(areas) > 2:
+            for region in regionprops(label_image):
+                if region.area < areas[-2]:
+                    for coordinates in region.coords:
+                        label_image[coordinates[0], coordinates[1]] = 0
+        binary = label_image > 0
+
+        # Step 5: Erosion operation with a disk of radius 2. This operation is seperate the lung nodules attached to the blood vessels.
+        selem = disk(1)
+        binary = binary_erosion(binary, selem)
+
+        # Step 6: Closure operation with a disk of radius 10. This operation is to keep nodules attached to the lung wall.
+        selem = disk(16)
+        binary = binary_closing(binary, selem)
+
+        # Step 7: Fill in the small holes inside the binary mask of lungs.
+        for _ in range(3):
+            edges = roberts(binary)
+            binary = ndi.binary_fill_holes(edges)
+        logging.info("lung segmentation done.")
+        return binary
+
+    def contour(image, path):
+        """Get contours of segmentation.
+            Args:
+                seg: segmentation of lung.
+        """
+        result = []
+        contours = find_contours(image, 0.5)
+        if len(contours) > 2:
+            contours.sort(key=lambda x: int(x.shape[0]))
+            contours = contours[-2:]
+
+        for n, contour in enumerate(contours):
+            # result.append({"type":n, "annotation":contour.tolist()})
+            result.append({"type": n, "annotation": np.flip(contour, 1).tolist()})
+
+        # write json
+        with open(path, 'w') as f:
+            json.dump(result, f)
+        logging.info("write {} done.".format(path))

dubhe_data_process/program/exec/ofrecord/config.json

@@ -0,0 +1,74 @@
+{
+  "ofrecord": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "ofrecord_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 3,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "ofrecord_task_queue",
+        "ofrecord_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 4,
+      "desc": "执行任务",
+      "module": "program.exec.ofrecord.ofrecord",
+      "class": "Ofrecord",
+      "method": "execute",
+      "paramType": 1,
+      "param": [
+        3.2
+      ]
+    },
+    {
+      "step": 5,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "ofrecord_finished_queue",
+        "ofrecord_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        4.1,
+        4.2
+      ],
+      "jump": 2
+    }
+  ]
+}

dubhe_data_process/program/exec/ofrecord/ofrecord.py

@@ -0,0 +1,182 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+
+import logging
+import json
+import os
+import struct
+import cv2
+import sched
+import numpy as np
+import oneflow.core.record.record_pb2 as of_record
+import time
+from abc import ABC
+from program.abstract.algorithm import Algorithm
+
+schedule = sched.scheduler(time.time, time.sleep)
+
+delayId = ""
+basePath = '/nfs/'
+descPath = 'ofrecord/train'
+
+
+class ImageCoder(object):
+    """Helper class that provides image coding utilities."""
+
+    def __init__(self, size=None):
+        self.size = size
+
+    def _resize(self, image_data):
+        if self.size is not None and image_data.shape[:2] != self.size:
+            return cv2.resize(image_data, self.size)
+        return image_data
+
+    def image_to_jpeg(self, image_data):
+        image_data = cv2.imdecode(np.frombuffer(image_data, np.uint8), 1)
+        image_data = self._resize(image_data)
+        return cv2.imencode(".jpg", image_data)[1].tobytes(
+        ), image_data.shape[0], image_data.shape[1]
+
+
+class Ofrecord(Algorithm, ABC):
+
+    def __init__(self):
+        pass
+
+    def execute(task):
+        return Ofrecord.start_ofrecord(task)
+
+    def start_ofrecord(jsonStr):
+        label_map = {}
+        index = 0
+        for item in jsonStr["datasetLabels"].keys():
+            if index >= 0 and item != '@type':
+                label_map[item] = jsonStr["datasetLabels"][item]
+            index += 1
+        Ofrecord.executor(os.path.join(basePath, jsonStr["datasetPath"]),
+                         os.path.join(basePath, jsonStr["datasetPath"], descPath),
+                         label_map,
+                         jsonStr["files"],
+                         jsonStr["partNum"])
+        result = True
+        finish_data = {"reTaskId": jsonStr["reTaskId"]}
+        return finish_data, result
+
+    def _process_image(filename, coder):
+        """Process a single image file.
+            Args:
+                filename: string, path to an image file e.g., '/path/to/example.JPG'.
+                coder: instance of ImageCoder to provide image coding utils.
+            Returns:
+                image_buffer: string, JPEG encoding of RGB image.
+                height: integer, image height in pixels.
+                width: integer, image width in pixels.
+        """
+        # Read the image file.
+        with open(filename, 'rb') as f:
+            image_data = f.read()
+            image_data, height, width = coder.image_to_jpeg(image_data)
+
+            return image_data, height, width
+
+    def _bytes_feature(value):
+        """Wrapper for inserting bytes features into Example proto."""
+        return of_record.Feature(bytes_list=of_record.BytesList(value=[value]))
+
+    def dense_to_one_hot(labels_dense, num_classes):
+        """Convert class labels from scalars to one-hot vectors."""
+        num_labels = labels_dense.shape[0]
+        index_offset = np.arange(num_labels) * num_classes
+        labels_one_hot = np.zeros((num_labels, num_classes))
+        labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1
+        return labels_one_hot
+
+    def extract_img_label(names, path):
+        """Extract the images and labels into np array [index].
+        Args:
+          f: A file object that contain images and annotations.
+        Returns:
+          data: A 4D uint8 np array [index, h, w, depth].
+          labels: a 1D uint8 np array.
+          num_img: the number of images.
+        """
+        train_img = os.path.join(path, 'origin/')
+        train_label = os.path.join(path, 'annotation/')
+        num_imgs = len(names)
+        data = []
+        labels = []
+        print('^^^^^^^^^^ start img_set for sycle')
+        for i in names:
+            name = os.path.splitext(i)[0]
+            print(name)
+            coder = ImageCoder((224, 224))
+            image_buffer, height, width = Ofrecord._process_image(
+                os.path.join(train_img, i), coder)
+
+            data += [image_buffer]
+
+            if os.path.exists(os.path.join(train_label, name)):
+
+                with open(os.path.join(train_label, name), "r", encoding='utf-8') as jsonFile:
+                    la = json.load(jsonFile)
+                    if la:
+                        labels += [la[0]['category_id']]
+                    else:
+                        data.pop()
+                        num_imgs -= 1
+            else:
+                print('File is not found')
+        print('^^^^^^^^^ img_set for end')
+        data = np.array(data)
+        labels = np.array(labels)
+        print(data.shape, labels.shape)
+        return num_imgs, data, labels
+
+    def executor(src_path, desc, label_map, files, part_id):
+        """Execute ofrecord task method."""
+        global delayId
+        logging.info(part_id)
+        num_imgs, images, labels = Ofrecord.extract_img_label(files, src_path)
+        keys = sorted(list(map(int, label_map.keys())))
+        label_map_new = {}
+        for i in range(len(keys)):
+            label_map_new[label_map[str(keys[i])]] = i
+        if not num_imgs:
+            return False, 0, 0
+        try:
+            os.makedirs(desc)
+        except Exception as e:
+            print('{} exists.'.format(desc))
+        filename = 'part-{}'.format(part_id)
+        filename = os.path.join(desc, filename)
+        f = open(filename, 'wb')
+        print(filename)
+        for i in range(num_imgs):
+            img = images[i]
+            label = label_map_new[str(labels[i])]
+            sample = of_record.OFRecord(feature={
+                'class/label': of_record.Feature(int32_list=of_record.Int32List(value=[label])),
+                'encoded': Ofrecord._bytes_feature(img)
+            })
+            size = sample.ByteSize()
+            f.write(struct.pack("q", size))
+            f.write(sample.SerializeToString())
+        if f:
+            f.close()

dubhe_data_process/entrance/executor/classify_by_textcnn.py → dubhe_data_process/program/exec/text_classification/classify_by_textcnn.py

@@ -1,20 +1,21 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
 import json
 import re
@@ -23,7 +24,6 @@ import numpy as np
 from typing import Tuple
 # import requests # 在 nfs 没有挂载 时使用 url 访问
 import sys
-sys.path.append("../../")
 import oneflow as flow
 import oneflow.typing as tp
 
@@ -196,8 +196,8 @@ def predict_job(text: tp.Numpy.Placeholder((BATCH_SIZE, 150), dtype=flow.int32),
 class TextCNNClassifier:
     
     def __init__(self):
-        model_load_dir = "../of_model/textcnn_imdb_of_best_model/"
-        word_index_dir = "../of_model/imdb_word_index/imdb_word_index.json"
+        model_load_dir = "of_model/textcnn_imdb_of_best_model/"
+        word_index_dir = "of_model/imdb_word_index/imdb_word_index.json"
         
         checkpoint = flow.train.CheckPoint()
         checkpoint.init()

dubhe_data_process/program/exec/text_classification/config.json

@@ -0,0 +1,82 @@
+{
+  "text_classification": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "text_classification_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "初始化",
+      "module": "program.exec.text_classification.text_taskexecutor",
+      "class": "Text_classification",
+      "method": "_init",
+      "paramType": 0
+    },
+    {
+      "step": 3,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 3
+    },
+    {
+      "step": 4,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "text_classification_task_queue",
+        "text_classification_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 3
+    },
+    {
+      "step": 5,
+      "desc": "执行任务",
+      "module": "program.exec.text_classification.text_taskexecutor",
+      "class": "Text_classification",
+      "method": "execute",
+      "paramType": 1,
+      "param": [
+        4.2
+      ]
+    },
+    {
+      "step": 6,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "text_classification_finished_queue",
+        "text_classification_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        5.1,
+        5.2
+      ],
+      "jump": 3
+    }
+  ]
+}

dubhe_data_process/program/exec/text_classification/text_taskexecutor.py

@@ -0,0 +1,86 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+
+import codecs
+import sched
+import sys
+import logging
+import time
+from program.exec.text_classification import classify_by_textcnn as classify
+from abc import ABC
+from program.abstract.algorithm import Algorithm
+
+logging.basicConfig(format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s',
+                    level=logging.DEBUG)
+
+schedule = sched.scheduler(time.time, time.sleep)
+sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+
+delayId = ""
+
+
+class Text_classification(Algorithm, ABC):
+
+    def __init__(self):
+        pass
+
+    def execute(task):
+        return Text_classification.textClassificationExecutor(task)
+
+    def textClassificationExecutor(jsonObject):
+        """Annotation task method.
+                    Args:
+                      redisClient: redis client.
+                      key: annotation task key.
+        """
+        global delayId
+        result = True
+        print('-------------process one-----------------')
+        try:
+            text_path_list = []
+            id_list = []
+            label_list = jsonObject['labels']
+            for fileObject in jsonObject['files']:
+                text_path_list.append(fileObject['url'])
+                id_list.append(fileObject['id'])
+            print(text_path_list)
+            print(id_list)
+            print(label_list)
+            classifications = Text_classification._classification(text_path_list, id_list, label_list)  # --------------
+            finished_json = {"reTaskId": jsonObject['reTaskId'], "classifications": classifications}
+            return finished_json, result
+        except Exception as e:
+            print(e)
+
+    @staticmethod
+    def _init():
+        print('init classify_obj')
+        global classify_obj
+        classify_obj = classify.TextCNNClassifier()  # label_log
+
+    def _classification(text_path_list, id_list, label_list):
+        """Perform automatic text classification task."""
+        textnum = len(text_path_list)
+        batched_num = ((textnum - 1) // classify.BATCH_SIZE + 1) * classify.BATCH_SIZE
+        for i in range(batched_num - textnum):
+            text_path_list.append(text_path_list[0])
+            id_list.append(id_list[0])
+        annotations = classify_obj.inference(text_path_list, id_list, label_list)  #
+        return annotations[0:textnum]

dubhe_data_process/program/exec/track/config.json

@@ -0,0 +1,74 @@
+{
+  "track": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "track_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 3,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "track_task_queue",
+        "track_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 4,
+      "desc": "执行任务",
+      "module": "program.exec.track.track",
+      "class": "Track",
+      "method": "execute",
+      "paramType": 1,
+      "param": [
+        3.2
+      ]
+    },
+    {
+      "step": 5,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "track_finished_queue",
+        "track_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        4.1,
+        4.2
+      ],
+      "jump": 2
+    }
+  ]
+}

dubhe_data_process/program/exec/track/track.py

@@ -0,0 +1,79 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
+"""
+import sched
+
+from abc import ABC
+from program.abstract.algorithm import Algorithm
+from program.exec.track.track_only.hog_track import *
+
+schedule = sched.scheduler(time.time, time.sleep)
+
+delayId = ""
+
+
+class Track(Algorithm, ABC):
+
+    def __init__(self):
+        pass
+
+    def execute(task):
+        return Track.trackProcess(task)
+
+    def trackProcess(task):
+        """Track task method.
+                Args:
+                  task: dataset id.
+                  key: video file path.
+                Returns:
+                  True: track success
+                  False: track failed
+        """
+        global delayId
+        image_list = []
+        label_list = []
+        images_data = task['images']
+        path = task['path']
+        dataset_id = task['id']
+        result = True
+
+        for file in images_data:
+            filePath = path + "/origin/" + file
+            annotationPath = path + "/annotation/" + file.split('.')[0]
+            if not os.path.exists(filePath):
+                continue
+            if not os.path.exists(annotationPath):
+                continue
+            image_list.append(filePath)
+            label_list.append(annotationPath)
+        image_num = len(label_list)
+        track_det = Detector(
+            'xxx.avi',
+            min_confidence=0.35,
+            max_cosine_distance=0.2,
+            max_iou_distance=0.7,
+            max_age=30,
+            out_dir='results/')
+        track_det.write_img = False
+        RET = track_det.run_track(image_list, label_list)
+        finished_json = {'id': dataset_id}
+        if RET == 'OK':
+            return finished_json, result
+        else:
+            return finished_json, result

dubhe_data_process/track_only/feature/feature_extractor_batch.py → dubhe_data_process/program/exec/track/track_only/feature/feature_extractor_batch.py

@@ -1,20 +1,21 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
 import numpy as np
 import cv2

dubhe_data_process/track_only/hog_track.py → dubhe_data_process/program/exec/track/track_only/hog_track.py

@@ -1,22 +1,21 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
 import json
 import os
@@ -25,8 +24,8 @@ from datetime import datetime
 
 import cv2
 import numpy as np
-from track_only.mot_track_kc import KCTracker
-from track_only.util import draw_bboxes_conf_cls
+from program.exec.track.track_only.mot_track_kc import KCTracker
+from program.exec.track.track_only.util import draw_bboxes_conf_cls
 
 #将box四点坐标转换成左上角坐标和宽和高，并过滤低置信度的框
 def bbox_to_xywh_cls_conf(bbox_xyxyc, conf_thresh=0.5):

dubhe_data_process/track_only/mot_track_kc.py → dubhe_data_process/program/exec/track/track_only/mot_track_kc.py

@@ -22,12 +22,12 @@ import time
 
 import cv2
 import numpy as np
-from track_only.feature.feature_extractor_batch import Extractor
-from track_only.post_process import removeUnMoveLowConfObj, writeResult, removeSmallOrBigBbox
-from track_only.sort.detection import Detection
-from track_only.sort.iou_matching import iou
-from track_only.sort.nn_matching import NearestNeighborDistanceMetric
-from track_only.sort.tracker import Tracker
+from program.exec.track.track_only.feature.feature_extractor_batch import Extractor
+from program.exec.track.track_only.post_process import removeUnMoveLowConfObj, writeResult, removeSmallOrBigBbox
+from program.exec.track.track_only.sort.detection import Detection
+from program.exec.track.track_only.sort.iou_matching import iou
+from program.exec.track.track_only.sort.nn_matching import NearestNeighborDistanceMetric
+from program.exec.track.track_only.sort.tracker import Tracker
 
 
 class KCTracker(object):

dubhe_data_process/track_only/post_process.py → dubhe_data_process/program/exec/track/track_only/post_process.py

@@ -1,20 +1,21 @@
+# !/usr/bin/env python
+# -*- coding:utf-8 -*-
+
 """
-/**
-* Copyright 2020 Tianshu AI Platform. All Rights Reserved.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-* =============================================================
-*/
+Copyright 2020 Tianshu AI Platform. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=============================================================
 """
 import os
 from collections import defaultdict

dubhe_data_process/program/exec/videosample/config.json

@@ -0,0 +1,74 @@
+{
+  "videosample": [
+    {
+      "step": 1,
+      "desc": "启动延时线程",
+      "module": "program.thread.delay_schedule",
+      "class": "Start_thread",
+      "method": "start_thread",
+      "paramType": 0,
+      "paramLocal": [
+        "videoSample_processing_queue"
+      ]
+    },
+    {
+      "step": 2,
+      "desc": "加载",
+      "module": "common.util.public.json_util",
+      "class": "JsonUtil",
+      "method": "load_json",
+      "paramType": 0,
+      "paramLocal": [
+        "/root/algorithm/sign"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 3,
+      "desc": "获取任务",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "get_one_task",
+      "paramType": 0,
+      "paramLocal": [
+        "local queue,value=KEYS[1],ARGV[1]\nlocal a=redis.call('TIME')\nlocal time=(a[1]*1000000+a[2])/1000 \nlocal element = redis.call('zrangebyscore', queue, 0, 9999999999999, 'limit', 0, 1)\nif table.getn(element)>0 then\nredis.call('zrem', queue, element[1])\nredis.call('zadd',value,time,element[1])\nend\nreturn element",
+        1,
+        "videoSample_task_queue",
+        "videoSample_processing_queue",
+        "/root/algorithm/config.json"
+      ],
+      "judge": 0,
+      "jump": 2
+    },
+    {
+      "step": 4,
+      "desc": "执行任务",
+      "module": "program.exec.videosample.videosample",
+      "class": "Videosample",
+      "method": "execute",
+      "paramType": 1,
+      "param": [
+        3.2
+      ]
+    },
+    {
+      "step": 5,
+      "desc": "保存数据",
+      "module": "program.impl.redis_storage",
+      "class": "RedisStorage",
+      "method": "save_result",
+      "paramType": 1,
+      "paramLocal": [
+        "videoSample_finished_queue",
+        "videoSample_failed_queue",
+        "/root/algorithm/config.json"
+      ],
+      "param": [
+        4.1,
+        4.2
+      ],
+      "jump": 2
+    }
+  ]
+}