From 9a867fb622c317ee28062ae2ecb06aaaf9a725d0 Mon Sep 17 00:00:00 2001 From: wxl Date: Fri, 11 Dec 2020 13:55:17 +0800 Subject: [PATCH 1/2] Bugfix: single op and ir build question bugfix --- ge/ir_build/ge_ir_build.cc | 48 ++++++++++++++++++++++++++++++++-- ge/offline/single_op_parser.cc | 19 ++++++++++++++ ge/offline/single_op_parser.h | 6 +++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/ge/ir_build/ge_ir_build.cc b/ge/ir_build/ge_ir_build.cc index 96ae9b24..c010a690 100644 --- a/ge/ir_build/ge_ir_build.cc +++ b/ge/ir_build/ge_ir_build.cc @@ -36,7 +36,6 @@ #include "model/ge_model.h" #include "graph/shape_refiner.h" #include "graph/opsproto_manager.h" -#include "graph/utils/type_utils.h" using std::string; using namespace std; @@ -50,6 +49,8 @@ const std::string IR_OPTION_LOG_LEVEL_DEFAULT = "default"; const std::string IR_OPTION_BUFFER_OPTIMIZE_DEFAULT = "l2_optimize"; const std::string IR_OPTION_DISABLE_REUSE_MEMORY_DEFAULT = "0"; const std::string IR_OPTION_ENABLE_COMPRESS_WEIGHT_DEFAULT = "false"; +const std::string kInputShape = "input_shape"; +const std::string kInputFormat = "input_format"; } // namespace static graphStatus CheckGlobalOptions(std::map &global_options) { @@ -227,6 +228,7 @@ class Impl { graphStatus CheckOptions(const std::map &options); graphStatus CreateInputsForIRBuild(const ge::Graph &graph, vector &inputs); graphStatus GetDefaultInputShape(const Graph &graph, string &default_shape); + graphStatus UpdateDataOpAttr(const Graph &graph); graphStatus Init(const Graph &graph, const std::map &options); graphStatus BuildModel(const Graph &graph, const std::map &options, ModelBufferData &ge_models); @@ -242,6 +244,40 @@ class Impl { OmgContext omg_context_; }; +graphStatus Impl::UpdateDataOpAttr(const Graph &graph) { + GELOGD("Enter Update Data Attr Process!"); + if (options_.find(kInputShape) == options_.end()) { + return GRAPH_SUCCESS; + } + unordered_map> shape_map; + vector>> user_shape_map; + GE_CHK_BOOL_EXEC(ParseInputShape(options_[kInputShape], shape_map, user_shape_map, true), + return GRAPH_PARAM_INVALID, "parse input shape failed!"); + auto compute_graph = ge::GraphUtils::GetComputeGraph(graph); + GE_CHECK_NOTNULL(compute_graph); + for (ge::NodePtr &input_node : compute_graph->GetDirectNode()) { + GE_CHECK_NOTNULL(input_node); + ge::OpDescPtr op = input_node->GetOpDesc(); + GE_CHECK_NOTNULL(op); + if (op->GetType() == DATA) { + auto tensor_input = op->MutableInputDesc(0); + auto tensor_output = op->MutableOutputDesc(0); + GE_CHECK_NOTNULL(tensor_input); + GE_CHECK_NOTNULL(tensor_output); + string data_op_name = op->GetName(); + auto iter = shape_map.find(data_op_name); + if (iter != shape_map.end()) { + tensor_input->SetShape(ge::GeShape(iter->second)); + tensor_output->SetShape(ge::GeShape(iter->second)); + GELOGD("update input [%s] shape info", data_op_name.c_str()); + } else { + GELOGI("no need update input [%s] attr because not found from input_shape.", data_op_name.c_str()); + } + } + } + return GRAPH_SUCCESS; +} + graphStatus Impl::CheckOptions(const std::map &options) { for (auto &ele : options) { auto it = ge::ir_option::ir_builder_suppported_options.find(ele.first); @@ -277,6 +313,11 @@ graphStatus Impl::CheckOptions(const std::map &options return GRAPH_PARAM_INVALID; } } + // Check option EXEC_DISABLE_REUSED_MEMORY + it = options_.find(ge::ir_option::EXEC_DISABLE_REUSED_MEMORY); + if (it != options_.end() && (CheckDisableReuseMemoryParamValid(it->second) != GRAPH_SUCCESS)) { + return GRAPH_PARAM_INVALID; + } return GRAPH_SUCCESS; } @@ -323,7 +364,10 @@ graphStatus Impl::Init(const Graph &graph, const std::map &dict, string &key, T default_val) { } void from_json(const Json &j, SingleOpTensorDesc &desc) { + bool is_tensor_valid = true; desc.dims = j.at(kKeyShape).get>(); auto it = j.find(kKeyShapeRange); if (it != j.end()) { @@ -189,9 +190,12 @@ void from_json(const Json &j, SingleOpTensorDesc &desc) { string type_str = j.at(kKeyType).get(); desc.format = GetValue(kFormatDict, format_str, FORMAT_RESERVED); desc.type = GetValue(kDataTypeDict, type_str, DT_UNDEFINED); + is_tensor_valid = is_tensor_valid && ge::TypeUtils::IsFormatValid(format_str); + is_tensor_valid = is_tensor_valid && ge::TypeUtils::IsDataTypeValid(type_str); it = j.find(kKeyOriginFormat); if (it != j.end()) { string origin_format_str = j.at(kKeyOriginFormat).get(); + is_tensor_valid = is_tensor_valid && ge::TypeUtils::IsFormatValid(origin_format_str); desc.ori_format = GetValue(kFormatDict, origin_format_str, FORMAT_RESERVED); } auto tensor_name = j.find(kKeyName); @@ -202,6 +206,9 @@ void from_json(const Json &j, SingleOpTensorDesc &desc) { if (dynamic_input_name != j.end()) { desc.dynamic_input_name = dynamic_input_name->get(); } + if (!is_tensor_valid) { + desc.SetValidFlag(is_tensor_valid); + } } void from_json(const Json &j, SingleOpAttr &attr) { @@ -305,6 +312,12 @@ bool SingleOpParser::Validate(const SingleOpDesc &op_desc) { int index = 0; for (auto &tensor_desc : op_desc.input_desc) { + if (!tensor_desc.GetValidFlag()) { + ErrorManager::GetInstance().ATCReportErrMessage("E10027", {"input", "type", "index"}, + {"intput", "datatype or format", std::to_string(index)}); + GELOGE(PARAM_INVALID, "Input's dataType or format is invalid when the index is %d", index); + return false; + } if ((tensor_desc.type == DT_UNDEFINED && tensor_desc.format != FORMAT_RESERVED) || (tensor_desc.type != DT_UNDEFINED && tensor_desc.format == FORMAT_RESERVED)){ ErrorManager::GetInstance().ATCReportErrMessage("E10027", {"input", "type", "index"}, @@ -317,6 +330,12 @@ bool SingleOpParser::Validate(const SingleOpDesc &op_desc) { index = 0; for (auto &tensor_desc : op_desc.output_desc) { + if (!tensor_desc.GetValidFlag()) { + ErrorManager::GetInstance().ATCReportErrMessage("E10027", {"input", "type", "index"}, + {"output", "datatype", std::to_string(index)}); + GELOGE(PARAM_INVALID, "Output's dataType is invalid when the index is %d", index); + return false; + } if (tensor_desc.type == DT_UNDEFINED) { ErrorManager::GetInstance().ATCReportErrMessage("E10027", {"input", "type", "index"}, {"output", "datatype", std::to_string(index)}); diff --git a/ge/offline/single_op_parser.h b/ge/offline/single_op_parser.h index 19879a32..71aa58bb 100644 --- a/ge/offline/single_op_parser.h +++ b/ge/offline/single_op_parser.h @@ -28,6 +28,10 @@ namespace ge { struct SingleOpTensorDesc { +public: + bool GetValidFlag() const { return is_valid_; } + void SetValidFlag(bool is_valid) { is_valid_ = is_valid; } +public: std::string name; std::vector dims; std::vector ori_dims; @@ -36,6 +40,8 @@ struct SingleOpTensorDesc { ge::Format ori_format = ge::FORMAT_RESERVED; ge::DataType type = ge::DT_UNDEFINED; std::string dynamic_input_name; +private: + bool is_valid_ = true; }; struct SingleOpAttr { From 2c9c9474bfbd2baa044898394a90806225337d9b Mon Sep 17 00:00:00 2001 From: wxl Date: Fri, 11 Dec 2020 13:59:54 +0800 Subject: [PATCH 2/2] Bugfix: single op and ir build question bugfix --- ge/offline/single_op_parser.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/ge/offline/single_op_parser.cc b/ge/offline/single_op_parser.cc index 09001515..b1e0da6d 100644 --- a/ge/offline/single_op_parser.cc +++ b/ge/offline/single_op_parser.cc @@ -27,6 +27,7 @@ #include "common/ge_inner_error_codes.h" #include "framework/common/util.h" #include "graph/utils/tensor_utils.h" +#include "graph/utils/type_utils.h" #include "graph/utils/op_desc_utils.h" #include "graph/operator_factory_impl.h"