graphengine/third_party/fwkacllib/inc/ops/nn_detect_ops.h

/**
 * Copyright 2019-2020 Huawei Technologies Co., Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef GE_OP_NN_DETECT_OPS_H_
#define GE_OP_NN_DETECT_OPS_H_

#include "graph/operator_reg.h"
#include "graph/operator.h"

namespace ge {

/**
*@brief Generates bounding boxes based on "rois" and "deltas". It is a customized FasterRcnn operator.

*@par Inputs:
* Two inputs, including: \n
*@li rois: Region of interests (ROIs) generated by the region proposal network (RPN). A 2D Tensor of type float 32 with shape (N, 4). "N" indicates the number of ROIs, and the value "4" refers to "x0", "x1", "y0", and "y1".
*@li deltas: Absolute variation between the ROIs generated by the RPN and ground truth boxes. A 2D Tensor of type float32 with shape (N, 4). "N" indicates the number of errors, and 4 indicates "dx", "dy", "dw", and "dh".

*@par Attributes:
*@li means: An index of type int. Defaults to [0,0,0,0]. "deltas" = "deltas" x "stds" + "means".
*@li stds: An index of type int. Defaults to [0,0,0,0]. "deltas" = "deltas" x "stds" + "means".
*@li max_shape: Shape [h, w], specifying the size of the image transferred to the network. Used to ensure that the bbox shape after conversion does not exceed "max_shape".
*@li wh_ratio_clip: Defaults to "16/1000". The values of "dw" and "dh" fall within (-wh_ratio_clip, wh_ratio_clip).

*@par Outputs:
*bboxes: Bboxes generated based on "rois" and "deltas". Have the same format and type as "rois".
*/
REG_OP(BoundingBoxDecode)
    .INPUT(rois, TensorType({DT_FLOAT16, DT_FLOAT}))
    .INPUT(deltas, TensorType({DT_FLOAT16, DT_FLOAT}))
    .OUTPUT(bboxes, TensorType({DT_FLOAT16, DT_FLOAT}))
    .ATTR(means, ListFloat, {0.0, 0.0, 0.0, 0.0})
    .ATTR(stds, ListFloat, {1.0, 1.0, 1.0, 1.0})
    .REQUIRED_ATTR(max_shape, ListInt)
    .ATTR(wh_ratio_clip, Float, 0.016)
    .OP_END_FACTORY_REG(BoundingBoxDecode)

/**
*@brief Computes the coordinate variations between bboxes and ground truth boxes. It is a customized FasterRcnn operator.

*@par Inputs:
* Two inputs, including: \n
*@li anchor_box: Anchor boxes. A 2D Tensor of float32 with shape (N, 4). "N" indicates the number of bounding boxes, and the value "4" refers to "x0", "x1", "y0", and "y1".
*@li ground_truth_box: Ground truth boxes. A 2D Tensor of float32 with shape (N, 4). "N" indicates the number of bounding boxes, and the value "4" refers to "x0", "x1", "y0", and "y1".

*@par Attributes:
*@li means: An index of type int. Defaults to [0,0,0,0]. "deltas" = "deltas" x "stds" + "means".
*@li stds: An index of type int. Defaults to [0,0,0,0]. "deltas" = "deltas" x "stds" + "means".

*@par Outputs:
*delats: A 2D Tensor of type float32 with shape (N, 4), specifying the variations between all anchor boxes and ground truth boxes.
*/
REG_OP(BoundingBoxEncode)
    .INPUT(anchor_box, TensorType({DT_FLOAT16, DT_FLOAT}))
    .INPUT(ground_truth_box, TensorType({DT_FLOAT16, DT_FLOAT}))
    .OUTPUT(delats, TensorType({DT_FLOAT16, DT_FLOAT}))
    .ATTR(means, ListFloat, {0.0, 0.0, 0.0, 0.0})
    .ATTR(stds, ListFloat, {1.0, 1.0, 1.0, 1.0})
    .OP_END_FACTORY_REG(BoundingBoxEncode)

/**
*@brief Judges whether the bounding box is valid. It is a customized FasterRcnn operator.

*@par Inputs:
* Two inputs, including: \n
*@li bbox_tensor: Bounding box. A 2D Tensor of type float16 with shape (N, 4). "N" indicates the number of bounding boxes, the value "4" indicates "x0", "x1", "y0", and "y1".
*@li img_metas: Valid boundary value of the image. A 1D Tensor of type float16 with shape (16,)

*@par Outputs:
*valid_tensor: A bool with shape (N, 1), specifying whether an input anchor is in an image. "1" indicates valid, while "0" indicates invalid.

*@attention Constraints:
* 16 "img_metas" are input. The first three numbers (height, width, ratio) are valid, specifying the valid boundary (heights x ratio, weights x ratio).
*/
REG_OP(CheckValid)
    .INPUT(bbox_tensor, TensorType({DT_FLOAT16}))
    .INPUT(img_metas, TensorType({DT_FLOAT16}))
    .OUTPUT(valid_tensor, TensorType({DT_INT8}))
    .OP_END_FACTORY_REG(CheckValid)

/**
*@brief Computes the intersection over union (iou) or the intersection over foreground (iof) based on the ground-truth and predicted regions.

*@par Inputs:
* Two inputs, including: \n
*@li bboxes: Bounding boxes, a 2D Tensor of type float16 with shape (N, 4). "N" indicates the number of bounding boxes, and the value "4" refers to "x0", "x1", "y0", and "y1".
*@li gtboxes: Ground-truth boxes, a 2D Tensor of type float16 with shape (M, 4). "M" indicates the number of ground truth boxes, and the value "4" refers to "x0", "x1", "y0", and "y1".

*@par Attributes:
*mode: Computation mode, a character string with the value range of [iou, iof].

*@par Outputs:
*overlap: A 2D Tensor of type float16 with shape [M, N], specifying the IoU or IoF ratio.

*@attention Constraints:
* Only computation of float16 data is supported. To avoid overflow, the input length and width are scaled by 0.2 internally.
*/
REG_OP(Iou)
    .INPUT(bboxes, TensorType({DT_FLOAT16, DT_FLOAT}))
    .INPUT(gtboxes, TensorType({DT_FLOAT16, DT_FLOAT}))
    .OUTPUT(overlap, TensorType({DT_FLOAT16, DT_FLOAT}))
    .ATTR(mode, String, "iou")
    .OP_END_FACTORY_REG(Iou)

/**
*@brief Performs the backpropagation of ROIAlign for training scenarios.

*@par Inputs:
* Three inputs, including: \n
*@li ydiff: A 5HD gradient input of type float32.
*@li rois: ROI position. A 2D Tensor of float32 with shape (N, 5). "N" indicates the number of ROIs, the value "5" indicates the indexes of images where the ROIs are located, "x0", "x1", "y0", and "y1".
*@li rois_n: An optional input, specifying the number of valid ROIs. This parameter is reserved.

*@par Attributes:
*@li xdiff_shape: A required list of 4 ints, obtained based on the shape of "features" of ROIAlign.
*@li pooled_width: A required attribute of type int, specifying the W dimension.
*@li pooled_height: A required attribute of type int, specifying the H dimension.
*@li spatial_scale: A required attribute of type float, specifying the scaling ratio of "features" to the original image.
*@li sample_num: An optional attribute of type int, specifying the horizontal and vertical sampling frequency of each output. If this attribute is set to "0", the sampling frequency is equal to the rounded up value of "rois", which is a floating point number. Defaults to "2".

*@par Outputs:
*xdiff: Gradient added to input "features". Has the same 5HD shape as input "features".
*/
REG_OP(ROIAlignGrad)
    .INPUT(ydiff, TensorType({DT_FLOAT}))
    .INPUT(rois, TensorType({DT_FLOAT}))
    .OPTIONAL_INPUT(rois_n, TensorType({DT_INT32}))
    .OUTPUT(xdiff, TensorType({DT_FLOAT}))
    .REQUIRED_ATTR(xdiff_shape, ListInt)
    .REQUIRED_ATTR(pooled_width, Int)
    .REQUIRED_ATTR(pooled_height, Int)
    .REQUIRED_ATTR(spatial_scale, Float)
    .ATTR(sample_num, Int, 2)
    .OP_END_FACTORY_REG(ROIAlignGrad)

/**
*@brief Obtains the ROI feature matrix from the feature map. It is a customized FasterRcnn operator.

*@par Inputs:
* Three inputs, including: \n
*@li features: A 5HD Tensor of type float32.
*@li rois: ROI position. A 2D Tensor of float32 with shape (N, 5). "N" indicates the number of ROIs, the value "5" indicates the indexes of images where the ROIs are located, "x0", "x1", "y0", and "y1".
*@li rois_n: An optional input, specifying the number of valid ROIs. This parameter is reserved.

*@par Attributes:
*@li spatial_scale: A required attribute of type float, specifying the scaling ratio of "features" to the original image.
*@li pooled_height: A required attribute of type int, specifying the H dimension.
*@li pooled_width: A required attribute of type int, specifying the W dimension.
*@li sample_num: An optional attribute of type int, specifying the horizontal and vertical sampling frequency of each output. If this attribute is set to "0", the sampling frequency is equal to the rounded up value of "rois", which is a floating point number. Defaults to "2".

*@par Outputs:
*output: Outputs the feature sample of each ROI position. The format is 5HD. The axis N is the number of input ROIs. Axes H, W, and C are consistent with the values of "pooled_height", "pooled_width", and "features", respectively.
*/
REG_OP(ROIAlign)
    .INPUT(features, TensorType({DT_FLOAT16, DT_FLOAT}))
    .INPUT(rois, TensorType({DT_FLOAT16, DT_FLOAT}))
    .OPTIONAL_INPUT(rois_n, TensorType({DT_INT32}))
    .OUTPUT(y, TensorType({DT_FLOAT16, DT_FLOAT}))
    .REQUIRED_ATTR(spatial_scale, Float)
    .REQUIRED_ATTR(pooled_height, Int)
    .REQUIRED_ATTR(pooled_width, Int)
    .ATTR(sample_num, Int, 2)
    .ATTR(roi_end_mode, Int, 1)
    .OP_END_FACTORY_REG(ROIAlign)

/**
*@brief Performs SSD prior box detection.

*@par Inputs:
* Two inputs, including:
*@li x: An NC1HWC0 or NCHW feature map of type is float32 or float16.
*@li img: source image. Has the same type and format as "x".

*@par Attributes:
*@li min_size: A required float32, specifying the minimum edge length of a square prior box.
*@li max_size: A required float32, specifying the maximum edge length of a square prior box: sqrt(min_size * max_size)
*@li aspect_ratio: An required float32, specifying the aspect ratio for generated rectangle boxes. The height is min_size/sqrt(aspect_ratio), the width is min_size*sqrt(aspect_ratio). Defaults to "1.0".
*@li img_h: An optional int32, specifying the source image height. Defaults to "0".
*@li img_w: An optional int32, specifying the source image width. Defaults to "0".
*@li step_h: An optional float32, specifying the height step for mapping the center point from the feature map to the source image. Defaults to "0.0".
*@li step_w: An optional float32, specifying the width step for mapping the center point from the feature map to the source image. Defaults to "0.0".
*@li flip: An optional bool. If "True", "aspect_ratio" will be flipped. Defaults to "True".
*@li clip: An optional bool. If "True", a prior box is clipped to within [0, 1]. Defaults to "False".
*@li offset: An optional float32, specifying the offset. Defaults to "0.5".
*@li variance: An optional float32, specifying the variance of a prior box, either one or four variances. Defaults to "0.1" (one value).

*@par Outputs:
*y: An ND tensor of type float32 or float16, specifying the prior box information, including its coordinates and variance.

*@attention Constraints:\n
* This operator applies only to SSD networks.
*@see SSDDetectionOutput()
*/
 REG_OP(PriorBox)
     .INPUT(x, TensorType({DT_FLOAT16, DT_FLOAT}))
     .INPUT(img, TensorType({DT_FLOAT16, DT_FLOAT}))
     .OUTPUT(y, TensorType({DT_FLOAT16, DT_FLOAT}))
     .REQUIRED_ATTR(min_size, ListFloat)
     .REQUIRED_ATTR(max_size, ListFloat)
     .REQUIRED_ATTR(aspect_ratio, ListFloat)
     .ATTR(img_h, Int, 0)
     .ATTR(img_w, Int, 0)
     .ATTR(step_h, Float, 0.0)
     .ATTR(step_w, Float, 0.0)
     .ATTR(flip, Bool, true)
     .ATTR(clip, Bool, false)
     .ATTR(offset, Float, 0.5)
     .ATTR(variance, ListFloat, {0.1})
     .OP_END_FACTORY_REG(PriorBox);

/**
*@brief Performs SSD prior box detection, with four additional matrices and the "aspect_ratio" attribute deleted compared to PriorBox.

*@par Inputs:
* Six inputs, including:
*@li x: An NC1HWC0 or NCHW feature map of type is float32 or float16.
*@li img: source image. Has the same type and format as "x".
*@li data_h: An NC1HWC0 or NCHW tensor of type float32 or float16, specifying the matrix for indexing the feature map height.
*@li data_w: An NC1HWC0 or NCHW tensor of type float32 or float16, specifying the matrix for indexing the feature map width.
*@li box_height: An NC1HWC0 or NCHW tensor of type float32 or float16, specifying the height of each prior box.
*@li box_width: An NC1HWC0 or NCHW tensor of type float32 or float16, specifying the width of each prior box.

*@par Attributes:
*@li min_size: A required float32, specifying the minimum edge length of a square prior box.
*@li max_size: A required float32, specifying the maximum edge length of a square prior box: sqrt(min_size * max_size)
*@li img_h: An optional int32, specifying the height of the source image.
*@li img_w: An optional int32, specifying the width of the source image.
*@li step_h: An optional float32, specifying the height step for mapping the center point from the feature map to the source image.
*@li step_w: An optional float32, specifying the width step for mapping the center point from the feature map to the source image.
*@li flip: An optional bool. If "True", "aspect_ratio" will be flipped. Defaults to "True".
*@li clip: An optional bool. If "True", a prior box is clipped to within [0, 1]. Defaults to "False".
*@li offset: An optional float32, specifying the offset. Defaults to "0.5".
*@li variance: An optional float32, specifying the variance of a prior box, either one or four variances. Defaults to "0.1" (one value).

*@par Outputs:
*y: An ND tensor of type float32 or float16, specifying the prior box information, including its coordinates and variance.

*@attention Constraints:\n
* This operator applies only to SSD networks.
*@see SSDDetectionOutput()
*/
 REG_OP(PriorBoxD)
     .INPUT(x, TensorType({DT_FLOAT16, DT_FLOAT}))
     .INPUT(img, TensorType({DT_FLOAT16, DT_FLOAT}))
     .INPUT(data_h, TensorType({DT_FLOAT16, DT_FLOAT}))
     .INPUT(data_w, TensorType({DT_FLOAT16, DT_FLOAT}))
     .INPUT(box_height, TensorType({DT_FLOAT16, DT_FLOAT}))
     .INPUT(box_width, TensorType({DT_FLOAT16, DT_FLOAT}))
     .OUTPUT(y, TensorType({DT_FLOAT16, DT_FLOAT}))
     .REQUIRED_ATTR(min_size, ListFloat)
     .REQUIRED_ATTR(max_size, ListFloat)
     .ATTR(img_h, Int, 0)
     .ATTR(img_w, Int, 0)
     .ATTR(step_h, Float, 0.0)
     .ATTR(step_w, Float, 0.0)
     .ATTR(flip, Bool, true)
     .ATTR(clip, Bool, false)
     .ATTR(offset, Float, 0.5)
     .ATTR(variance, ListFloat, {0.1})
     .OP_END_FACTORY_REG(PriorBoxD);

/**
*@brief Performs Position Sensitive ROI Pooling.

*@par Inputs:
* Two inputs, including:
*@li x: An NC1HWC0 tensor of type float16 or float32, describing the feature
* map, dimension C1 must be equal to
* (int(output_dim+15)/C0))*group_size*group_size.
*@li rois: A tensor of type float16 or float32, with shape
* [batch, 5, rois_num], describing the ROIs, each ROI consists of five
* elements: "batch_id", "x1", "y1", "x2", and "y2", which "batch_id" indicates
* the index of the input feature map, "x1", "y1", "x2", or "y2" must be
* greater than or equal to "0.0".

*@par Attributes:
*@li output_dim: A required int32, specifying the number of output channels,
* must be greater than 0.
*@li group_size: A required int32, specifying the number of groups to encode
* position-sensitive score maps, must be within the range (0, 128).
*@li spatial_scale: A required scaling factor for mapping the input
* coordinates to the ROI coordinates.

*@par Outputs:
*y: An NC1HWC0 tensor of type float16 or float32, describing the result
* feature map.

*@attention Constraints:
* HC1HWC0: channel must be Group_size squared, rois_num is a multiple of 16
*/
REG_OP(PSROIPooling)
    .INPUT(x, TensorType({DT_FLOAT, DT_FLOAT16}))
    .INPUT(rois, TensorType({DT_FLOAT, DT_FLOAT16}))
    .ATTR(output_dim, Int, 0)
    .ATTR(group_size, Int, 0)
    .ATTR(spatial_scale, Float, 0.0625)
    .OUTPUT(y, TensorType({DT_FLOAT, DT_FLOAT16}))
    .OP_END_FACTORY_REG(PSROIPooling)
}  // namespace ge

#endif  // GE_OP_NN_DETECT_OPS_H_