From da11b53751d251fef76e4bdd4264f80c66d08d6c Mon Sep 17 00:00:00 2001 From: zhaozhixuan Date: Wed, 12 May 2021 14:25:32 +0800 Subject: [PATCH 01/48] Set storage shape to single_op executor. --- ge/hybrid/executor/hybrid_model_executor.cc | 2 +- ge/single_op/single_op.cc | 22 +++++++++++++++++++++ tests/ut/ge/single_op/single_op_unittest.cc | 7 +++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ge/hybrid/executor/hybrid_model_executor.cc b/ge/hybrid/executor/hybrid_model_executor.cc index 2ab4ed5d..60db0d33 100755 --- a/ge/hybrid/executor/hybrid_model_executor.cc +++ b/ge/hybrid/executor/hybrid_model_executor.cc @@ -50,7 +50,7 @@ Status HybridModelExecutor::Execute(HybridModelExecutor::ExecuteArgs &args) { auto root_graph_item = model_->GetRootGraphItem(); GE_CHECK_NOTNULL(root_graph_item); - if (root_graph_item->IsDynamic()) { + if (root_graph_item->IsDynamic() && !model_->IsSingleOp()) { GE_CHK_STATUS_RET(CheckInputShapeByShapeRange(root_graph_item, args), "[%s] check input node shape by shape range failed.", root_graph_item->GetName().c_str()); diff --git a/ge/single_op/single_op.cc b/ge/single_op/single_op.cc index 4b3f17cf..b9cd4e85 100755 --- a/ge/single_op/single_op.cc +++ b/ge/single_op/single_op.cc @@ -113,6 +113,27 @@ Status UpdateInputsBufferAddr(StreamResource *stream_resource, rtStream_t stream return SUCCESS; } +Status ModifyTensorDesc(GeTensorDesc &tensor) { + int64_t storage_format_val = static_cast(FORMAT_RESERVED); + (void)AttrUtils::GetInt(tensor, ge::ATTR_NAME_STORAGE_FORMAT, storage_format_val); + auto storage_format = static_cast(storage_format_val); + if (storage_format != FORMAT_RESERVED) { + std::vector storage_shape; + if (!AttrUtils::GetListInt(tensor, ge::ATTR_NAME_STORAGE_SHAPE, storage_shape)) { + GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Get][storage_shape]failed while storage_format was set."); + REPORT_INNER_ERROR("E19999", "Get storage_shape failed while storage_format was set."); + return ACL_ERROR_GE_INTERNAL_ERROR; + } + + GELOGD("Storage format set. update shape to [%s], and original shape to [%s]", + GeShape(storage_shape).ToString().c_str(), tensor.GetShape().ToString().c_str()); + tensor.SetShape(GeShape(std::move(storage_shape))); + tensor.SetFormat(std::move(storage_format)); + } + + return SUCCESS; +} + Status InitHybridModelArgs(const std::vector &input_buffers, const std::vector &output_buffers, const std::vector &inputs_desc, @@ -126,6 +147,7 @@ Status InitHybridModelArgs(const std::vector &input_buffers, for (auto &tensor_desc : inputs_desc) { auto desc = MakeShared(tensor_desc); GE_CHECK_NOTNULL(desc); + GE_CHK_STATUS_RET_NOLOG(ModifyTensorDesc(*desc)); args.input_desc.emplace_back(desc); } return SUCCESS; diff --git a/tests/ut/ge/single_op/single_op_unittest.cc b/tests/ut/ge/single_op/single_op_unittest.cc index 8c2f6e51..bdd6401d 100644 --- a/tests/ut/ge/single_op/single_op_unittest.cc +++ b/tests/ut/ge/single_op/single_op_unittest.cc @@ -159,5 +159,12 @@ TEST_F(UtestSingleOp, test_singleop_execute_async2) { single_op.hybrid_model_executor_.reset(new (std::nothrow)hybrid::HybridModelExecutor(single_op.hybrid_model_.get(), 0, stream)); EXPECT_EQ(single_op.running_param_->mem_base, nullptr); EXPECT_EQ(single_op.tasks_.size(), 0); + + GeTensorDesc tensor; + int64_t storage_format_val = static_cast(FORMAT_NCHW); + AttrUtils::SetInt(tensor, "storage_format", storage_format_val); + std::vector storage_shape{1, 1, 1, 1}; + AttrUtils::SetListInt(tensor, "storage_shape", storage_shape); + single_op.inputs_desc_.emplace_back(tensor); EXPECT_EQ(single_op.ExecuteAsync(input_buffers, output_buffers), PARAM_INVALID); } \ No newline at end of file From 775d53be2ced3ae0cee4492f703070efd10bcd8c Mon Sep 17 00:00:00 2001 From: y00500818 Date: Thu, 13 May 2021 10:55:47 +0800 Subject: [PATCH 02/48] add infer format for single op --- ge/generator/ge_generator.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ge/generator/ge_generator.cc b/ge/generator/ge_generator.cc index a12f3cf7..fa6d8fa8 100644 --- a/ge/generator/ge_generator.cc +++ b/ge/generator/ge_generator.cc @@ -843,6 +843,9 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector &in Graph graph; GE_CHK_STATUS(BuildSingleOpGraph(op_desc, inputs, outputs, name, graph), "[Build][Graph] for single op:%s fail.", op_desc->GetName().c_str()); + auto op = OpDescUtils::CreateOperatorFromOpDesc(op_desc); + GE_CHK_STATUS_RET(op_desc->CallInferFormatFunc(op), + "[Call][InferFormatFunc] for single op:%s fail.", op_desc->GetName().c_str()); // 2. check engine type when compile online if (model_file_name == kFileNameSuffix) { From 97b47eb57c2c3c2e05d8ce0fd073dda1f0af56e1 Mon Sep 17 00:00:00 2001 From: "gengchao4@huawei.com" Date: Thu, 13 May 2021 16:27:40 +0800 Subject: [PATCH 03/48] bugfix for var addr update --- ge/graph/build/memory/graph_mem_assigner.cc | 24 +++++++++++++++++---- ge/graph/load/model_manager/model_utils.cc | 14 ++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/ge/graph/build/memory/graph_mem_assigner.cc b/ge/graph/build/memory/graph_mem_assigner.cc index a45fb239..0ac58fe2 100755 --- a/ge/graph/build/memory/graph_mem_assigner.cc +++ b/ge/graph/build/memory/graph_mem_assigner.cc @@ -1482,6 +1482,12 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector< output_list = last_peer_out_op_desc->GetOutputOffset(); auto out_index = static_cast(peer_out_anchor->GetIdx()); if (output_list.size() > static_cast(out_index)) { + int64_t peer_out_inner_offset; + if (ge::AttrUtils::GetInt(last_peer_out_op_desc->MutableOutputDesc(out_index), ATTR_NAME_INNER_OFFSET, + peer_out_inner_offset)) { + (void)ge::AttrUtils::SetInt(tmp_op_desc->MutableInputDesc(anchor->GetIdx()), ATTR_NAME_INNER_OFFSET, + peer_out_inner_offset); + } bool is_l1_type = false; int64_t input_offset = output_list.at(out_index); if (has_mem_type_attr && !origin_input_list.empty()) { @@ -1496,8 +1502,11 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector< GE_ERRORLOG_AND_ERRORMSG(ge::FAILED, error.c_str()); return ge::FAILED; } - GELOGD("Node[%s] input[%d] has origin offset[%ld]", tmp_op_desc->GetName().c_str(), anchor->GetIdx(), - origin_input_list[valid_input_index]); + int64_t inner_offset; + (void)ge::AttrUtils::GetInt(tmp_op_desc->MutableInputDesc(anchor->GetIdx()), ATTR_NAME_INNER_OFFSET, + inner_offset); + GELOGD("Node[%s] input[%d] has origin offset[%ld] origin_inner_offset[%ld]", tmp_op_desc->GetName().c_str(), + anchor->GetIdx(), origin_input_list[valid_input_index], inner_offset); // L1 keep original input_offset is_l1_type = (memory_type[valid_input_index] == RT_MEMORY_L1); if (is_l1_type) { @@ -1505,6 +1514,8 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector< } else { // hbm input_offset = original input_offset + output_offset input_offset = origin_input_list[valid_input_index] + output_list.at(out_index); + (void)ge::AttrUtils::SetInt(tmp_op_desc->MutableInputDesc(anchor->GetIdx()), ATTR_NAME_INNER_OFFSET, + origin_input_list[valid_input_index] + inner_offset); } } const auto &in_node = GetKnownInputNode(peer_out_anchor->GetOwnerNode()); @@ -1532,6 +1543,8 @@ ge::Status GraphMemoryAssigner::UpdateRefOpOutputOffset(const NodePtr &node, con const int ref_in, const int64_t input_offset) const { auto opdesc = node->GetOpDesc(); GE_CHECK_NOTNULL(opdesc); + int64_t inner_offset; + bool has_inner_offset = ge::AttrUtils::GetInt(opdesc->MutableInputDesc(ref_in), ATTR_NAME_INNER_OFFSET, inner_offset); for (const auto &out2in : out2ins) { auto out_i = out2in.first; auto in_i = out2in.second; @@ -1545,8 +1558,11 @@ ge::Status GraphMemoryAssigner::UpdateRefOpOutputOffset(const NodePtr &node, con } origin_output_list[out_i] = input_offset; opdesc->SetOutputOffset(origin_output_list); - GELOGI("Node[%s] output[%d] is updated from reuse input index[%d] to offset[%ld]", opdesc->GetName().c_str(), - out_i, ref_in, input_offset); + if (has_inner_offset) { + (void)ge::AttrUtils::SetInt(opdesc->MutableOutputDesc(out_i), ATTR_NAME_INNER_OFFSET,inner_offset); + } + GELOGI("Node[%s] output[%d] is updated from reuse input index[%d] to offset[%ld], inner_offset[%ld]", opdesc->GetName().c_str(), + out_i, ref_in, input_offset, inner_offset); } } return ge::SUCCESS; diff --git a/ge/graph/load/model_manager/model_utils.cc b/ge/graph/load/model_manager/model_utils.cc index 058a538f..f872791e 100755 --- a/ge/graph/load/model_manager/model_utils.cc +++ b/ge/graph/load/model_manager/model_utils.cc @@ -340,9 +340,12 @@ vector ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co int64_t input_offset = v_input_offset[non_const_index]; non_const_index++; - GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(input_offset), + int64_t inner_offset; + (void)ge::AttrUtils::GetInt(op_desc->MutableInputDesc(i), ATTR_NAME_INNER_OFFSET, inner_offset); + GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(input_offset - inner_offset), uint8_t *variable_addr = nullptr; - GE_CHK_STATUS_EXEC(GetVarAddr(model_param, op_desc, input_offset, variable_addr), return {}); + GE_CHK_STATUS_EXEC(GetVarAddr(model_param, op_desc, input_offset - inner_offset, variable_addr), return {}); + variable_addr += inner_offset; v_input_data_addr.push_back(variable_addr); GELOGI("[IMAS]GetInputDataAddrs graph_%u type[V] name[%s] input[%lu] memaddr[%p]", model_param.graph_id, op_desc->GetName().c_str(), i, variable_addr); @@ -450,9 +453,12 @@ vector ModelUtils::GetOutputDataAddrs(const RuntimeParam &model_param, C GELOGD("%s is an optional output, the address don't need to be saved.", tensor_desc->GetName().c_str()); continue; } - GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(v_output_offset[i]), + int64_t inner_offset; + (void)ge::AttrUtils::GetInt(op_desc->MutableOutputDesc(i), ATTR_NAME_INNER_OFFSET, inner_offset); + GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(v_output_offset[i] - inner_offset), uint8_t *variable_addr = nullptr; - GE_CHK_STATUS_EXEC(GetVarAddr(model_param, op_desc, v_output_offset[i], variable_addr), return {}); + GE_CHK_STATUS_EXEC(GetVarAddr(model_param, op_desc, v_output_offset[i] - inner_offset, variable_addr), return {}); + variable_addr += inner_offset; v_output_data_addr.push_back(variable_addr); GELOGI("[IMAS]GetOutputDataAddrs graph_%u type[V] name[%s] output[%zu] memaddr[%p]", model_param.graph_id, op_desc->GetName().c_str(), i, variable_addr); From aecb1431d7aefc2aef037bf6d403c2987a978610 Mon Sep 17 00:00:00 2001 From: zhaozhixuan Date: Fri, 14 May 2021 14:38:34 +0800 Subject: [PATCH 04/48] Set storage shape to single_op executor. --- ge/single_op/single_op.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ge/single_op/single_op.cc b/ge/single_op/single_op.cc index b9cd4e85..ec26958b 100755 --- a/ge/single_op/single_op.cc +++ b/ge/single_op/single_op.cc @@ -117,7 +117,8 @@ Status ModifyTensorDesc(GeTensorDesc &tensor) { int64_t storage_format_val = static_cast(FORMAT_RESERVED); (void)AttrUtils::GetInt(tensor, ge::ATTR_NAME_STORAGE_FORMAT, storage_format_val); auto storage_format = static_cast(storage_format_val); - if (storage_format != FORMAT_RESERVED) { + auto format = tensor.GetFormat(); + if (storage_format != FORMAT_RESERVED && storage_format != format) { std::vector storage_shape; if (!AttrUtils::GetListInt(tensor, ge::ATTR_NAME_STORAGE_SHAPE, storage_shape)) { GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Get][storage_shape]failed while storage_format was set."); @@ -127,8 +128,10 @@ Status ModifyTensorDesc(GeTensorDesc &tensor) { GELOGD("Storage format set. update shape to [%s], and original shape to [%s]", GeShape(storage_shape).ToString().c_str(), tensor.GetShape().ToString().c_str()); - tensor.SetShape(GeShape(std::move(storage_shape))); - tensor.SetFormat(std::move(storage_format)); + tensor.SetOriginShape(tensor.GetShape()); + tensor.SetOriginFormat(format); + tensor.SetShape(GeShape(storage_shape)); + tensor.SetFormat(storage_format); } return SUCCESS; From 7337e3ceb2299c38af1af2c2caac54f9f04df5ee Mon Sep 17 00:00:00 2001 From: zhangxiaokun Date: Mon, 17 May 2021 20:25:47 +0800 Subject: [PATCH 05/48] Fix DSP for v1 control flow --- ge/graph/common/omg_util.cc | 20 ++++-- ge/graph/common/omg_util.h | 3 +- ge/graph/manager/graph_manager.cc | 3 - ge/graph/partition/dynamic_shape_partition.cc | 60 +++++++++++------- ge/graph/partition/dynamic_shape_partition.h | 9 ++- .../mark_force_unknown_for_cond_pass.cc | 63 ++++++++++++++----- ge/graph/passes/merge_to_stream_merge_pass.cc | 7 ++- ge/graph/passes/next_iteration_pass.cc | 25 ++++---- ge/graph/passes/next_iteration_pass.h | 3 +- .../passes/switch_to_stream_switch_pass.cc | 13 ++-- ge/graph/preprocess/graph_preprocess.cc | 13 ++++ ge/graph/preprocess/graph_preprocess.h | 1 + ge/hybrid/executor/node_state.cc | 41 ++++++++++-- ge/hybrid/executor/node_state.h | 2 + 14 files changed, 190 insertions(+), 73 deletions(-) diff --git a/ge/graph/common/omg_util.cc b/ge/graph/common/omg_util.cc index 1dba8c51..15fa3c47 100644 --- a/ge/graph/common/omg_util.cc +++ b/ge/graph/common/omg_util.cc @@ -272,20 +272,32 @@ bool IsUnknownShapeTensor(const GeTensorDesc &tensor_desc) { /// @brief Set Op _force_unknown_shape flag /// @param [in] node /// @param [in] force_unknown, set attribute if true +/// @param [in] group_index, condition group index of node. /// @return /// -void MarkForceUnknownShape(const NodePtr &node, bool force_unknown) { - GE_RT_VOID_CHECK_NOTNULL(node); +void MarkForceUnknownShape(const NodePtr &node, bool force_unknown, int64_t group_index) { if (!force_unknown) { return; } - GELOGD("[%s] mark as force unknown shape node", node->GetName().c_str()); - if (!AttrUtils::SetBool(node->GetOpDesc(), ATTR_NAME_FORCE_UNKNOWN_SHAPE, force_unknown)) { + GE_RT_VOID_CHECK_NOTNULL(node); + const auto &op_desc = node->GetOpDesc(); + GE_RT_VOID_CHECK_NOTNULL(op_desc); + + // op_desc as AttrHolderAdapter valid, Set attribute always success, just log for check. + GELOGD("Mark [%s] as force unknown shape node, group index: %ld", node->GetName().c_str(), group_index); + if (!AttrUtils::SetBool(op_desc, ATTR_NAME_FORCE_UNKNOWN_SHAPE, force_unknown)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_FORCE_UNKNOWN_SHAPE.c_str(), node->GetName().c_str(), node->GetType().c_str()); GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_FORCE_UNKNOWN_SHAPE.c_str(), node->GetName().c_str(), node->GetType().c_str()); } + + if (!AttrUtils::SetInt(op_desc, ATTR_NAME_CONTROL_FLOW_GROUP, group_index)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_CONTROL_FLOW_GROUP.c_str(), + node->GetName().c_str(), node->GetType().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_CONTROL_FLOW_GROUP.c_str(), + node->GetName().c_str(), node->GetType().c_str()); + } } } // namespace ge diff --git a/ge/graph/common/omg_util.h b/ge/graph/common/omg_util.h index c84da7f8..fdb0e138 100644 --- a/ge/graph/common/omg_util.h +++ b/ge/graph/common/omg_util.h @@ -129,9 +129,10 @@ bool IsUnknownShapeTensor(const GeTensorDesc &tensor_desc); /// @brief Set Op _force_unknown_shape flag /// @param [in] node /// @param [in] force_unknown, set attribute if true +/// @param [in] group_index, condition group index of node. /// @return /// -void MarkForceUnknownShape(const NodePtr &node, bool force_unknown); +void MarkForceUnknownShape(const NodePtr &node, bool force_unknown, int64_t group_index); } // namespace ge #endif // GE_GRAPH_COMMON_OMG_UTIL_H_ diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index 465ae749..69c84f6f 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -65,7 +65,6 @@ #include "graph/passes/merge_pass.h" #include "graph/passes/merge_input_memcpy_pass.h" #include "graph/passes/merge_to_stream_merge_pass.h" -#include "graph/passes/mark_force_unknown_for_cond_pass.h" #include "graph/passes/multi_batch_pass.h" #include "graph/passes/next_iteration_pass.h" #include "graph/passes/permute_pass.h" @@ -2582,8 +2581,6 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::Migration", new (std::nothrow) SubgraphConstMigrationPass)); GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::ArgsClean", new (std::nothrow) UnusedArgsCleanPass)); GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::PrunePass", new (std::nothrow) PrunePass)); - auto mark_force_unknown_pass = new (std::nothrow) MarkForceUnknownForCondPass; - GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::MarkForceUnknownForCondPass", mark_force_unknown_pass)); GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::NextIterationPass", new (std::nothrow) NextIterationPass)) GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::ControlTriggerPass", new (std::nothrow) ControlTriggerPass)) GE_CHK_STATUS_RET( diff --git a/ge/graph/partition/dynamic_shape_partition.cc b/ge/graph/partition/dynamic_shape_partition.cc index 516d06d1..8fee1eb5 100755 --- a/ge/graph/partition/dynamic_shape_partition.cc +++ b/ge/graph/partition/dynamic_shape_partition.cc @@ -46,11 +46,6 @@ #define REQUIRE_GRAPH_SUCCESS(cond, ...) REQUIRE(((cond) == GRAPH_SUCCESS), __VA_ARGS__) namespace ge { -namespace { -const std::set kControlFlowOps{ - STREAMACTIVE, STREAMSWITCH, STREAMMERGE, ENTER, REFENTER, LOOPCOND, NEXTITERATION, REFNEXTITERATION, EXIT, REFEXIT -}; -} using Cluster = DynamicShapePartitioner::Cluster; using ClusterPtr = std::shared_ptr; @@ -279,9 +274,17 @@ Status DynamicShapePartitioner::InitClusters() { auto cluster = MakeShared(rank++, type, node, this); REQUIRE_NOT_NULL(cluster, "Failed new memory for cluster."); node_2_cluster_[node] = cluster; - if (cluster->IsUnknownShape() && !cluster->IsControlFlow()) { + if (cluster->IsUnknownShape()) { ordered_cluster_.push_back(cluster); } + + int64_t group_index = -1; + if (AttrUtils::GetInt(node->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, group_index)) { + GELOGD("[%s] is rts control flow Op, group index: %ld", node->GetName().c_str(), group_index); + auto &control_cluster = control_clusters_[group_index]; + control_cluster.emplace_back(cluster); + } + // Already sorted topologically, so access to the parent cluster is safe for (const auto &parent : node->GetInAllNodes()) { cluster->AddInput(node_2_cluster_[parent]); @@ -350,14 +353,38 @@ static std::string ToString(const std::vector &clusters) { } } +void DynamicShapePartitioner::MergeClustersControlFlow() { + for (const auto &item : control_clusters_) { + const auto &control_cluster = item.second; + auto rit = control_cluster.rbegin(); + if (rit == control_cluster.rend()) { + GELOGW("Invalid empty control flow cluster."); + continue; + } + + const auto &cluster = *rit; + for (++rit; rit != control_cluster.rend(); ++rit) { + const auto &cluster_from = *rit; + auto merged_clusters = cluster->MergeAllPathFrom(cluster_from); + GELOGD("Merge all path cluster from %lu to %lu %s.", cluster_from->Id(), cluster->Id(), + ToString(merged_clusters).c_str()); + for (const auto &merged_cluster : merged_clusters) { + for (const auto &node : merged_cluster->Nodes()) { + node_2_cluster_[node] = cluster; + } + } + } + } +} + void DynamicShapePartitioner::MergeClustersUnknownShape() { // Merge unknown shape clusters for (const auto &cluster : ordered_cluster_) { - if (cluster->IsIndependent() || cluster->IsControlFlow()) { + if (cluster->IsIndependent()) { continue; } for (const auto &in_cluster : cluster->Inputs()) { - if (!in_cluster->IsUnknownShape() || in_cluster->IsControlFlow()) { + if (!in_cluster->IsUnknownShape()) { continue; } auto merged_clusters = cluster->MergeAllPathFrom(in_cluster); @@ -419,6 +446,7 @@ void DynamicShapePartitioner::MergeClustersInputData() { } Status DynamicShapePartitioner::MergeClusters() { + MergeClustersControlFlow(); MergeClustersUnknownShape(); REQUIRE_SUCCESS(TopologicalSortClusters(), "Failed topological sort clusters after merge unknown shape clusters."); MergeClustersKnownShape(); @@ -608,13 +636,6 @@ bool Cluster::IsRefVariable() const { return false; } -bool Cluster::IsControlFlow() const { - const auto &op_desc = nodes_[0]->GetOpDesc(); - bool is_ctrl_flow = kControlFlowOps.count(op_desc->GetType()) > 0 && op_desc->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE); - GELOGD("[%s] %s rts control flow Op ", op_desc->GetName().c_str(), is_ctrl_flow ? "Is" : "Not"); - return is_ctrl_flow; -} - void Cluster::AddInput(ClusterPtr in) { if (std::find(in_clusters_.begin(), in_clusters_.end(), in) != in_clusters_.end()) return; in_clusters_.insert(in_clusters_.end(), in); @@ -694,10 +715,7 @@ std::vector Cluster::MergeAllPathFrom(ClusterPtr other) { if (other->IsIndependent()) { return path_clusters; } - if (std::find(other->out_clusters_.begin(), other->out_clusters_.end(), shared_from_this()) == - other->out_clusters_.end()) { - return path_clusters; - } + path_clusters.push_back(other); forward_reached_queue.push(other); backward_reached_queue.push(shared_from_this()); @@ -761,7 +779,7 @@ InControlAnchorPtr Cluster::GetFrameInControlAnchor() { return partition_node_-> OutControlAnchorPtr Cluster::GetFrameOutControlAnchor() { return partition_node_->GetOutControlAnchor(); }; Status Cluster::BuildFrame() { - if ((IsUnknownShape() || IsKnownShape() || IsInputNode()) && !IsControlFlow()) { + if (IsUnknownShape() || IsKnownShape() || IsInputNode()) { return BuildPartitionFrame(); } else { auto node = nodes_.front(); @@ -896,7 +914,7 @@ Status Cluster::CombinePartitionFrame() { } Status Cluster::BuildPartitionSubgraph() { - if (IsData() || IsNetOutput() || IsIndependent() || IsControlFlow()) { + if (IsData() || IsNetOutput() || IsIndependent()) { return SUCCESS; } int64_t parent_node_index = 0; diff --git a/ge/graph/partition/dynamic_shape_partition.h b/ge/graph/partition/dynamic_shape_partition.h index 93f86d82..f1d711eb 100644 --- a/ge/graph/partition/dynamic_shape_partition.h +++ b/ge/graph/partition/dynamic_shape_partition.h @@ -47,7 +47,6 @@ class DynamicShapePartitioner { bool IsUnknownShape() const; bool IsIndependent() const; bool IsNetOutput() const; - bool IsControlFlow() const; std::vector> Inputs() const; std::vector> Outputs() const; bool IsInputNode() const; @@ -126,13 +125,15 @@ class DynamicShapePartitioner { // and there's only one path between the two clusters , merge the two clusters // 3) Iterate through the INPUT_DATA clusters, merge all INPUT_DATA Status MergeClusters(); + // Merge clusters step0 + void MergeClustersControlFlow(); // Merge clusters step1 void MergeClustersUnknownShape(); // Merge clusters step2 void MergeClustersKnownShape(); // Merge clusters step3 void MergeClustersInputData(); - // Topological sort clusters after merge unknow shape clusters. + // Topological sort clusters after merge unknown shape clusters. Status TopologicalSortClusters(); // Deduplicate merged clusters void PruneUniqueClusters(); @@ -140,7 +141,7 @@ class DynamicShapePartitioner { Status BuildPartitionFrame(); // Establish connection between corresponding partitioned of clusters Status CombinePartitionFrame(); - // Convert the nodes in cluster into a complete ComputeGraoh + // Convert the nodes in cluster into a complete ComputeGraph Status BuildPartitionSubgraph(); // Clear resource and break circular dependency void ClearResource(); @@ -155,6 +156,8 @@ class DynamicShapePartitioner { Status CtrlEdgeTransfer(); ge::ComputeGraphPtr root_graph_; // The original graph to partition std::unordered_map> node_2_cluster_; // Record nodes and the cluster it belongs to + // V1 control flow cluster, need merge to one Graph. + std::unordered_map>> control_clusters_; // topological sorted clusters, this field will change with the splitting. // When partitioning UNKNOWN_SHAPE cluster, it is a collection of all topological sorted UNKNOWN_SHAPE clusters // When partitioning KNOWN_SHAPE cluster, it is a collection of all topological sorted KNOWN_SHAPE clusters diff --git a/ge/graph/passes/mark_force_unknown_for_cond_pass.cc b/ge/graph/passes/mark_force_unknown_for_cond_pass.cc index 6729a647..f6c87d58 100644 --- a/ge/graph/passes/mark_force_unknown_for_cond_pass.cc +++ b/ge/graph/passes/mark_force_unknown_for_cond_pass.cc @@ -18,20 +18,25 @@ #include +#include "graph/utils/node_utils.h" #include "graph/common/omg_util.h" namespace ge { namespace { -const std::set kMergeOpTypes{ MERGE, REFMERGE }; +inline bool IsMergeInLoop(const NodePtr &node) { + const static std::set kLoopMergeInputs{ ENTER, REFENTER, NEXTITERATION, REFNEXTITERATION }; -const std::set kSwitchOpTypes{ SWITCH, REFSWITCH }; + std::string node_type; + (void)GetOriginalType(node, node_type); + return kLoopMergeInputs.count(node_type) > 0; +} -const std::set kLoopMergeInputs{ ENTER, REFENTER, NEXTITERATION, REFNEXTITERATION }; +inline bool IsSwitchInLoop(const NodePtr &node) { + const static std::set kLoopSwitchInputs{ MERGE, REFMERGE, LOOPCOND }; -inline bool IsMergeInLoop(const NodePtr &node) { std::string node_type; (void)GetOriginalType(node, node_type); - return kLoopMergeInputs.count(node_type) > 0; + return kLoopSwitchInputs.count(node_type) > 0; } } @@ -103,7 +108,13 @@ void MarkForceUnknownForCondPass::MarkUnknownForSwitch(const NodePtr &node, std: if (dst_span > 0) { search_queue.push({in_node, dst_span - 1}); } else { - switch_group.emplace_back(in_node); + const auto &all_in_nodes = in_node->GetInDataNodes(); + if (std::any_of(all_in_nodes.begin(), all_in_nodes.end(), IsSwitchInLoop)) { + GELOGW("Travel node: %s, %s node: %s, Skip LoopCond switch", dst_node->GetName().c_str(), node_type.c_str(), + in_node->GetName().c_str()); + } else { + switch_group.emplace_back(in_node); + } } } else if (kMergeOpTypes.count(node_type) > 0) { // Merge input node. search_queue.push({in_node, dst_span + 1}); @@ -121,19 +132,37 @@ void MarkForceUnknownForCondPass::MarkUnknownForSwitch(const NodePtr &node, std: /// void MarkForceUnknownForCondPass::MarkUnknownForSwitch(const std::map> &switch_groups) { std::function callback = [](const NodePtr &n) { - return n->GetOpDesc()->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE); + return n->GetOpDesc()->HasAttr(ATTR_NAME_CONTROL_FLOW_GROUP); }; - for (const auto &group : switch_groups) { - const auto &node = group.first; - const auto &switch_group = group.second; - const auto &op_desc = node->GetOpDesc(); - if (IsUnknownShapeTensor(op_desc->GetOutputDesc(0)) || op_desc->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE) || - std::any_of(switch_group.begin(), switch_group.end(), callback)) { - GELOGI("Mark [%s] as force unknown shape", node->GetName().c_str()); - MarkForceUnknownShape(node, true); - for (const auto &n : switch_group) { - MarkForceUnknownShape(n, true); + for (auto it1 = switch_groups.begin(); it1 != switch_groups.end(); ++it1) { + const auto &op_node1 = it1->first; + const auto &op_desc1 = op_node1->GetOpDesc(); + if (op_desc1->HasAttr(ATTR_NAME_CONTROL_FLOW_GROUP)) { + continue; + } + + if (IsUnknownShapeTensor(op_desc1->GetOutputDesc(0))) { + int64_t group_index = op_desc1->GetId(); + GELOGI("Mark %s as unknown shape control flow, group index: %ld", op_desc1->GetName().c_str(), group_index); + MarkForceUnknownShape(op_node1, true, group_index); + for (const auto &n : it1->second) { + MarkForceUnknownShape(n, true, group_index); + } + + for (auto it2 = switch_groups.begin(); it2 != switch_groups.end(); ++it2) { + const auto &op_node2 = it2->first; + const auto &op_desc2 = op_node2->GetOpDesc(); + if (op_desc2->HasAttr(ATTR_NAME_CONTROL_FLOW_GROUP)) { + continue; + } + + if (std::any_of(it2->second.begin(), it2->second.end(), callback)) { + MarkForceUnknownShape(op_node2, true, group_index); + for (const auto &n : it2->second) { + MarkForceUnknownShape(n, true, group_index); + } + } } } } diff --git a/ge/graph/passes/merge_to_stream_merge_pass.cc b/ge/graph/passes/merge_to_stream_merge_pass.cc index f3a437a6..4c1ad1ae 100644 --- a/ge/graph/passes/merge_to_stream_merge_pass.cc +++ b/ge/graph/passes/merge_to_stream_merge_pass.cc @@ -84,8 +84,9 @@ Status MergeToStreamMergePass::AddActiveNodes(const ComputeGraphPtr &graph, cons GE_CHK_BOOL_EXEC(node != nullptr, REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); return FAILED, "Param of pre node is null."); - bool force_unknown = node->GetOpDesc()->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE); - MarkForceUnknownShape(node, force_unknown); + int64_t group_index = -1; + bool force_unknown = AttrUtils::GetInt(node->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, group_index); + MarkForceUnknownShape(node, force_unknown, group_index); for (const InDataAnchorPtr &in_data_anchor : node->GetAllInDataAnchors()) { OutDataAnchorPtr peer_out_anchor = in_data_anchor->GetPeerOutAnchor(); GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue); @@ -102,7 +103,7 @@ Status MergeToStreamMergePass::AddActiveNodes(const ComputeGraphPtr &graph, cons GELOGE(FAILED, "SetActiveLabelList for node %s failed.", active_node->GetName().c_str()); return FAILED; } - MarkForceUnknownShape(active_node, force_unknown); + MarkForceUnknownShape(active_node, force_unknown, group_index); } return SUCCESS; diff --git a/ge/graph/passes/next_iteration_pass.cc b/ge/graph/passes/next_iteration_pass.cc index 5f4fc4d0..7128b3dc 100644 --- a/ge/graph/passes/next_iteration_pass.cc +++ b/ge/graph/passes/next_iteration_pass.cc @@ -18,6 +18,7 @@ #include "common/ge/ge_util.h" #include "graph/common/omg_util.h" +#include "graph/utils/node_utils.h" using std::string; @@ -203,6 +204,7 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { for (const auto &loop_cond_iter : loop_group_map_) { const LoopCondGroup &loop_group = *loop_cond_iter.second; const std::string &cond_name = loop_cond_iter.second->loop_cond->GetName(); + const int64_t group_index = loop_group.loop_cond->GetOpDesc()->GetId(); GELOGI("Handle while group, LoopCond node: %s.", cond_name.c_str()); // Create Active node, Enter->Active->Merge, NextIteration->Active->Merge @@ -223,7 +225,7 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { enter_active->GetName().c_str()); return INTERNAL_ERROR; } - MarkForceUnknownShape(enter_node, loop_group.is_unknown_shape); + MarkForceUnknownShape(enter_node, loop_group.is_unknown_shape, group_index); } for (const auto &pair : loop_cond_iter.second->merge_next_pairs) { @@ -253,8 +255,8 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { return INTERNAL_ERROR; } - MarkForceUnknownShape(next_node, loop_group.is_unknown_shape); - MarkForceUnknownShape(merge_node, loop_group.is_unknown_shape); + MarkForceUnknownShape(next_node, loop_group.is_unknown_shape, group_index); + MarkForceUnknownShape(merge_node, loop_group.is_unknown_shape, group_index); } if ((SetActiveLabelList(enter_active, {cond_name}) != SUCCESS) || @@ -263,10 +265,10 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { return INTERNAL_ERROR; } - MarkForceUnknownShape(loop_group.loop_cond, loop_group.is_unknown_shape); - MarkForceUnknownShape(enter_active, loop_group.is_unknown_shape); - MarkForceUnknownShape(next_active, loop_group.is_unknown_shape); - HandleSwitchExitNodes(loop_group); + MarkForceUnknownShape(loop_group.loop_cond, loop_group.is_unknown_shape, group_index); + MarkForceUnknownShape(enter_active, loop_group.is_unknown_shape, group_index); + MarkForceUnknownShape(next_active, loop_group.is_unknown_shape, group_index); + HandleSwitchExitNodes(loop_group, group_index); } return SUCCESS; @@ -275,20 +277,21 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { /// /// @brief Mark force unknown for Exit node /// @param [in] group of LoopCond +/// @param [in] index of LoopCond Node /// @return void /// -void NextIterationPass::HandleSwitchExitNodes(const LoopCondGroup &loop_group) { +void NextIterationPass::HandleSwitchExitNodes(const LoopCondGroup &loop_group, int64_t group_index) { if (!loop_group.is_unknown_shape) { return; } for (const auto &switch_node : loop_group.switch_nodes) { - MarkForceUnknownShape(switch_node, loop_group.is_unknown_shape); + MarkForceUnknownShape(switch_node, loop_group.is_unknown_shape, group_index); for (const auto &node : switch_node->GetOutDataNodes()) { std::string node_type; (void)GetOriginalType(node, node_type); - if (node_type == EXIT || node_type == REFEXIT) { - MarkForceUnknownShape(node, loop_group.is_unknown_shape); + if (kExitOpTypes.count(node_type) > 0) { + MarkForceUnknownShape(node, loop_group.is_unknown_shape, group_index); } } } diff --git a/ge/graph/passes/next_iteration_pass.h b/ge/graph/passes/next_iteration_pass.h index e8786516..b6a0846d 100755 --- a/ge/graph/passes/next_iteration_pass.h +++ b/ge/graph/passes/next_iteration_pass.h @@ -96,9 +96,10 @@ class NextIterationPass : public GraphPass { /// /// @brief Mark force unknown for Exit node /// @param [in] group of LoopCond + /// @param [in] index of LoopCond Node /// @return void /// - void HandleSwitchExitNodes(const LoopCondGroup &loop_group); + void HandleSwitchExitNodes(const LoopCondGroup &loop_group, int64_t group_index); // map std::unordered_map loop_group_map_; diff --git a/ge/graph/passes/switch_to_stream_switch_pass.cc b/ge/graph/passes/switch_to_stream_switch_pass.cc index 949fff41..66a60ab9 100644 --- a/ge/graph/passes/switch_to_stream_switch_pass.cc +++ b/ge/graph/passes/switch_to_stream_switch_pass.cc @@ -369,7 +369,9 @@ NodePtr SwitchToStreamSwitchPass::CreateStreamSwitchNode(const ComputeGraphPtr & GE_CHK_STATUS(GraphUtils::AddEdge(peer_cond_anchor, stream_switch->GetInDataAnchor(0)), "StreamSwitch node add cond edge failed."); - MarkForceUnknownShape(stream_switch, switch_node->GetOpDesc()->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE)); + int64_t group_index = -1; + bool force_unknown = AttrUtils::GetInt(switch_node->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, group_index); + MarkForceUnknownShape(stream_switch, force_unknown, group_index); return stream_switch; } @@ -488,11 +490,12 @@ Status SwitchToStreamSwitchPass::CombineSwitchNode(const ComputeGraphPtr &graph) return FAILED; } - std::function callback = [](const NodePtr &n) { - return n->GetOpDesc()->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE); + int64_t group_index = -1; + std::function callback = [&group_index](const NodePtr &n) { + return AttrUtils::GetInt(n->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, group_index); }; bool is_unknown_shape = std::any_of(same_cond_switch.begin(), same_cond_switch.end(), callback); - MarkForceUnknownShape(active_node, is_unknown_shape); + MarkForceUnknownShape(active_node, is_unknown_shape, group_index); const std::string &cond_group = cond_node->GetName(); for (uint32_t i = 0; i < SWITCH_OUTPUT_NUM; ++i) { @@ -522,7 +525,7 @@ Status SwitchToStreamSwitchPass::CombineSwitchNode(const ComputeGraphPtr &graph) GE_CHK_STATUS(GraphUtils::AddEdge(cast_node->GetOutDataAnchor(0), stream_switch->GetInDataAnchor(0)), "Cast add data edge failed."); - MarkForceUnknownShape(stream_switch, is_unknown_shape); + MarkForceUnknownShape(stream_switch, is_unknown_shape, group_index); for (const NodePtr &node : switch_list) { GE_IF_BOOL_EXEC(node != stream_switch, { GE_CHK_STATUS(GraphUtils::RemoveEdge(peer_cond_anchor, node->GetInDataAnchor(0)), diff --git a/ge/graph/preprocess/graph_preprocess.cc b/ge/graph/preprocess/graph_preprocess.cc index 4e9046e4..8597cc61 100644 --- a/ge/graph/preprocess/graph_preprocess.cc +++ b/ge/graph/preprocess/graph_preprocess.cc @@ -74,6 +74,7 @@ #include "graph/passes/unused_const_pass.h" #include "graph/passes/var_is_initialized_op_pass.h" #include "graph/passes/variable_prepare_op_pass.h" +#include "graph/passes/mark_force_unknown_for_cond_pass.h" #include "graph/preprocess/insert_op/util_insert_aipp_op.h" #include "graph/utils/type_utils.h" #include "inc/pass_manager.h" @@ -1675,6 +1676,7 @@ Status GraphPrepare::PrepareDynShape(const GraphNodePtr &graph_node, const std:: PP_RUN_AND_DUMP("InsertAipp", TryDoAipp); PP_RUN_AND_DUMP("ProcessBeforeInfershape", ProcessBeforeInfershape); PP_RUN_AND_DUMP("InferFormatAndShape", FormatAndShapeProcess); + PP_RUN_AND_DUMP("CtrlFlowPreProcess", CtrlFlowPreProcess); PP_RUN_AND_DUMP("GetDynamicOutputShape", multibatch::GetDynamicOutputShape, compute_graph_); PP_RUN_AND_DUMP("ProcessAippStage2", InsertNewOpUtil::Instance().UpdateDataNodeByAipp, compute_graph_); PP_RUN("SaveOriginalGraphToOmModel", SaveOriginalGraphToOmModel); @@ -1683,6 +1685,17 @@ Status GraphPrepare::PrepareDynShape(const GraphNodePtr &graph_node, const std:: return SUCCESS; } +Status GraphPrepare::CtrlFlowPreProcess() { + PassManager graph_pass; + + // After InferShape Mark v1 control flow for unknown shape. + auto mark_force_unknown_pass = new (std::nothrow) MarkForceUnknownForCondPass; + GE_CHK_STATUS_RET(graph_pass.AddPass("PreRun::MarkForceUnknownForCondPass", mark_force_unknown_pass)); + + GE_CHK_STATUS_RET(graph_pass.Run(compute_graph_)); + return SUCCESS; +} + Status GraphPrepare::RecordAIPPInfo(ge::ComputeGraphPtr &compute_graph) { PP_RUN("RecordAIPPInfo", InsertNewOpUtil::Instance().RecordAIPPInfoToData, compute_graph_); return SUCCESS; diff --git a/ge/graph/preprocess/graph_preprocess.h b/ge/graph/preprocess/graph_preprocess.h index 9dc3e679..3eb5e03a 100755 --- a/ge/graph/preprocess/graph_preprocess.h +++ b/ge/graph/preprocess/graph_preprocess.h @@ -79,6 +79,7 @@ class GraphPrepare { Status ProcessNetOutput(); Status ProcessBeforeInfershape(); Status UpdateInputOutputByOptions(); + Status CtrlFlowPreProcess(); bool IsTansDataOpData(const ge::NodePtr &var_node); diff --git a/ge/hybrid/executor/node_state.cc b/ge/hybrid/executor/node_state.cc index aaa7801f..9ec5431a 100644 --- a/ge/hybrid/executor/node_state.cc +++ b/ge/hybrid/executor/node_state.cc @@ -104,11 +104,47 @@ void ShapeInferenceState::UpdateInputShapeFuture(int idx, ShapeFuture &&future) } } +Status ShapeInferenceState::UpdateInputForMerge(const GraphExecutionContext &context) { + int merge_index = -1; + const auto &guard = node_item.MutexGuard("UpdateInputForMerge"); + if (!AttrUtils::GetInt(node_item.op_desc, ATTR_NAME_MERGE_INPUT_INDEX, merge_index)) { + GELOGE(FAILED, "[%s] Get attr %s failed", node_item.NodeName().c_str(), ATTR_NAME_MERGE_INPUT_INDEX.c_str()); + return FAILED; + } + + if (merge_index < 0 || static_cast(merge_index) >= input_tensor_desc.size()) { + GELOGE(FAILED, "[%s] merge index: %d invalid, should in range[0, %zu)", + node_item.NodeName().c_str(), merge_index, input_tensor_desc.size()); + return FAILED; + } + + auto dst_tensor_desc = node_item.MutableInputDesc(merge_index); + GE_CHECK_NOTNULL(dst_tensor_desc); + + int64_t tensor_size = -1; + auto &tensor_desc = input_tensor_desc[merge_index]; + (void)TensorUtils::GetSize(tensor_desc, tensor_size); + + dst_tensor_desc->SetShape(tensor_desc.MutableShape()); + dst_tensor_desc->SetOriginShape(tensor_desc.GetOriginShape()); + (void)TensorUtils::SetSize(*dst_tensor_desc, tensor_size); + (void)guard; + GELOGD("[%s] Update input shape [%u] with shape: [%s] and ori_shape: [%s], tensor size = %ld", + node_item.NodeName().c_str(), merge_index, dst_tensor_desc->GetShape().ToString().c_str(), + dst_tensor_desc->GetOriginShape().ToString().c_str(), tensor_size); + + return SUCCESS; +} + Status ShapeInferenceState::AwaitShapesReady(const GraphExecutionContext &context) { if (!node_item.is_dynamic) { return SUCCESS; } std::unique_lock lk(mu_); + if (node_item.IsMergeOp()) { + return UpdateInputForMerge(context); + } + if (num_pending_shapes_ > 0) { GELOGD("[%s] Await pending shape or shape future start.", node_item.NodeName().c_str()); int try_count = 0; @@ -169,7 +205,7 @@ Status ShapeInferenceState::AwaitShapesReady(const GraphExecutionContext &contex int64_t tensor_size = -1; (void) TensorUtils::GetSize(*src_tensor_desc, tensor_size); - GELOGD("[%s] Update input shape [%u] with shape: [%s] and ori_shape: [%s], index = %zu", + GELOGD("[%s] Update input shape [%u] with shape: [%s] and ori_shape: [%s], tensor size = %ld", node_item.NodeName().c_str(), idx, src_tensor_desc->GetShape().ToString().c_str(), @@ -283,11 +319,8 @@ void NodeState::ResetContext(int group) { } switch_index_ = -1; - const auto &guard = node_item_->MutexGuard("ResetContext"); - shape_inference_state_.InitShapeState(); subgraph_context_->ResetContext(node_item_->node); GELOGD("Node[%s] in while loop, current loop: %lu, merge index: %d", GetName().c_str(), loop_count_, merge_index_); - (void)guard; } void NodeState::ResetSchedule() { diff --git a/ge/hybrid/executor/node_state.h b/ge/hybrid/executor/node_state.h index 49861611..d3f176ce 100644 --- a/ge/hybrid/executor/node_state.h +++ b/ge/hybrid/executor/node_state.h @@ -67,6 +67,8 @@ struct ShapeInferenceState { const NodeItem &node_item; private: + Status UpdateInputForMerge(const GraphExecutionContext &context); + friend struct NodeState; std::vector> shape_futures; // do not directly update op_desc, in case race condition across pipelines From ebd8c7979e700d54e75c9ab498efd2fbf790f08f Mon Sep 17 00:00:00 2001 From: zhangxiaokun Date: Mon, 17 May 2021 23:05:34 +0800 Subject: [PATCH 06/48] Add UT for DSP --- tests/ut/ge/CMakeLists.txt | 2 +- .../dynamic_shape_partition_unittest.cc | 34 +++++- .../preprocess/graph_preprocess_unittest.cc | 13 +++ .../ge/hybrid/executor/node_state_unittest.cc | 106 ++++++++++++++++++ 4 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 tests/ut/ge/hybrid/executor/node_state_unittest.cc diff --git a/tests/ut/ge/CMakeLists.txt b/tests/ut/ge/CMakeLists.txt index 2b68d8fe..96da50d1 100755 --- a/tests/ut/ge/CMakeLists.txt +++ b/tests/ut/ge/CMakeLists.txt @@ -823,7 +823,7 @@ set(PROFILING_MNG_TEST_FILES set(HYBRID_TEST_FILES "hybrid/ge_hybrid_unittest.cc" "hybrid/known_node_executor_unittest.cc" - "hybrid/executor/worker/execution_engine_unittest.cc" + "hybrid/executor/node_state_unittest.cc" "hybrid/executor/subgraph_executor_unittest.cc" "hybrid/executor/worker/execution_engine_unittest.cc" "hybrid/model/hybrid_model_builder_unittest.cc" diff --git a/tests/ut/ge/graph/partition/dynamic_shape_partition_unittest.cc b/tests/ut/ge/graph/partition/dynamic_shape_partition_unittest.cc index b60e0ddd..c8abadb5 100644 --- a/tests/ut/ge/graph/partition/dynamic_shape_partition_unittest.cc +++ b/tests/ut/ge/graph/partition/dynamic_shape_partition_unittest.cc @@ -15,20 +15,17 @@ */ #include + +#define private public +#define protected public #include "graph/partition/dynamic_shape_partition.h" #include "compute_graph.h" #include "inc/framework/common/types.h" #include "utils/graph_utils.h" #include "graph/debug/ge_attr_define.h" - -#define private public -#define protected public - namespace ge { - namespace { - GeTensorDescPtr CreateTensorDesc(std::initializer_list shape, Format format = FORMAT_NCHW, DataType data_type = DT_FLOAT) { GeShape ge_shape{vector(shape)}; @@ -94,4 +91,29 @@ TEST_F(UtestDynamicShapePartition, single_op_scene_success) { DynamicShapePartitioner partitioner(graph); EXPECT_EQ(partitioner.Partition(), SUCCESS); } + +TEST_F(UtestDynamicShapePartition, merge_control_flow_group) { + ComputeGraphPtr graph = std::make_shared("default"); + AttrUtils::SetStr(*graph, ATTR_NAME_SESSION_GRAPH_ID, "session_graph_id"); + + NodePtr data1 = NodeBuilder("data1", DATA).AddInputDesc({1}).AddOutputDesc({1}).Build(graph); + NodePtr data2 = NodeBuilder("data2", DATA).AddInputDesc({1}).AddOutputDesc({1}).Build(graph); + NodePtr merge = NodeBuilder("node2", MERGE).AddInputDesc({1}).AddInputDesc({1}) + .AddOutputDesc({1}).AddOutputDesc({}).Build(graph); + + GraphUtils::AddEdge(data1->GetOutDataAnchor(0), merge->GetInDataAnchor(0)); + GraphUtils::AddEdge(data2->GetOutDataAnchor(0), merge->GetInDataAnchor(1)); + + (void)AttrUtils::SetBool(data1->GetOpDesc(), ATTR_NAME_FORCE_UNKNOWN_SHAPE, true); + (void)AttrUtils::SetInt(data1->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, 3); + (void)AttrUtils::SetBool(data2->GetOpDesc(), ATTR_NAME_FORCE_UNKNOWN_SHAPE, true); + (void)AttrUtils::SetInt(data2->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, 3); + (void)AttrUtils::SetBool(merge->GetOpDesc(), ATTR_NAME_FORCE_UNKNOWN_SHAPE, true); + (void)AttrUtils::SetInt(merge->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, 3); + + EXPECT_EQ(graph->sub_graph_.size(), 0); + DynamicShapePartitioner partitioner(graph); + EXPECT_EQ(partitioner.Partition(), SUCCESS); + EXPECT_EQ(graph->sub_graph_.size(), 1); +} } // namespace ge \ No newline at end of file diff --git a/tests/ut/ge/graph/preprocess/graph_preprocess_unittest.cc b/tests/ut/ge/graph/preprocess/graph_preprocess_unittest.cc index 6c5babfc..8ece7564 100644 --- a/tests/ut/ge/graph/preprocess/graph_preprocess_unittest.cc +++ b/tests/ut/ge/graph/preprocess/graph_preprocess_unittest.cc @@ -223,4 +223,17 @@ TEST_F(UtestGraphPreproces, test_update_dtype_mbatch_case) { auto data1_output = data1_desc->MutableOutputDesc(0); EXPECT_EQ(data1_output->GetDataType(), 1); } + +TEST_F(UtestGraphPreproces, test_prepare_dyn_shape) { + ComputeGraphPtr compute_graph = BuildGraph5(); + GraphPtr graph_ptr = std::make_shared(GraphUtils::CreateGraphFromComputeGraph(compute_graph)); + + GraphNodePtr graph_node = make_shared(0); + graph_node->SetComputeGraph(compute_graph); + graph_node->SetGraph(graph_ptr); + + std::vector user_input; + GraphPrepare graph_prepare; + EXPECT_EQ(graph_prepare.PrepareDynShape(graph_node, user_input, compute_graph, 0), SUCCESS); +} } \ No newline at end of file diff --git a/tests/ut/ge/hybrid/executor/node_state_unittest.cc b/tests/ut/ge/hybrid/executor/node_state_unittest.cc new file mode 100644 index 00000000..f33fc601 --- /dev/null +++ b/tests/ut/ge/hybrid/executor/node_state_unittest.cc @@ -0,0 +1,106 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#define private public +#define protected public +#include "hybrid/executor/node_state.h" +#include "hybrid/executor/subgraph_context.h" +#include "hybrid/model/graph_item.h" +#include "graph/utils/graph_utils.h" + +using namespace std; +using namespace testing; + +namespace ge { +using namespace hybrid; + +class UtestNodeState : public testing::Test { + protected: + void SetUp() { + } + void TearDown() { + } +}; + +static NodePtr CreateNode(ComputeGraph &graph, const string &name, const string &type, int in_num, int out_num) { + OpDescPtr op_desc = std::make_shared(name, type); + op_desc->SetStreamId(0); + static int32_t index = 0; + op_desc->SetId(index++); + + GeTensorDesc tensor(GeShape(), FORMAT_ND, DT_INT64); + TensorUtils::SetSize(tensor, 64); + vector input_offset; + for (int i = 0; i < in_num; i++) { + op_desc->AddInputDesc(tensor); + input_offset.emplace_back(index * 64 + i * 64); + } + op_desc->SetInputOffset(input_offset); + + vector output_offset; + for (int i = 0; i < out_num; i++) { + op_desc->AddOutputDesc(tensor); + output_offset.emplace_back(index * 64 + in_num * 64 + i * 64); + } + op_desc->SetOutputOffset(output_offset); + + op_desc->SetWorkspace({}); + op_desc->SetWorkspaceBytes({}); + op_desc->SetOpKernelLibName("DNN_VM_RTS_OP_STORE"); + + return graph.AddNode(op_desc); +} + +TEST_F(UtestNodeState, merge_await_shapes_ready) { + ComputeGraphPtr graph = std::make_shared("test"); + + const auto data0 = CreateNode(*graph, "data", DATA, 1, 1); + const auto data1 = CreateNode(*graph, "data1", DATA, 1, 1); + const auto merge1 = CreateNode(*graph, "merge", STREAMMERGE, 2, 2); + const auto output1 = CreateNode(*graph, "net_output", NETOUTPUT, 1, 1); + + GraphUtils::AddEdge(data0->GetOutDataAnchor(0), merge1->GetInDataAnchor(0)); + GraphUtils::AddEdge(data1->GetOutDataAnchor(0), merge1->GetInDataAnchor(1)); + GraphUtils::AddEdge(merge1->GetOutDataAnchor(0), output1->GetInDataAnchor(0)); + + GraphItem graph_item; + GraphExecutionContext graph_context; + SubgraphContext subgraph_context(&graph_item, &graph_context); + + std::unique_ptr node_item; + NodeItem::Create(merge1, node_item); + NodeState node_state(*node_item, &subgraph_context); + + // Not dynamic. + ASSERT_EQ(node_state.shape_inference_state_.AwaitShapesReady(graph_context), SUCCESS); + + // Not set merge index. + node_item->is_dynamic = true; + ASSERT_EQ(node_state.shape_inference_state_.AwaitShapesReady(graph_context), FAILED); + + // merge index out of bound. + AttrUtils::SetInt(merge1->GetOpDesc(), ATTR_NAME_MERGE_INPUT_INDEX, 3); + ASSERT_EQ(node_state.shape_inference_state_.AwaitShapesReady(graph_context), FAILED); + + AttrUtils::SetInt(merge1->GetOpDesc(), ATTR_NAME_MERGE_INPUT_INDEX, 1); + ASSERT_EQ(node_state.shape_inference_state_.AwaitShapesReady(graph_context), SUCCESS); +} + +} // namespace ge \ No newline at end of file From 3ed7ada7532d06d03f9887cbf1c39f25ebebbdee Mon Sep 17 00:00:00 2001 From: zhengyuanhua Date: Fri, 14 May 2021 09:59:48 +0800 Subject: [PATCH 07/48] input shape rane --- ge/ir_build/ge_ir_build.cc | 130 ++++++++++++++++-- inc/framework/omg/omg_inner_types.h | 1 + .../ge/graph/build/model_builder_unittest.cc | 5 +- tests/ut/ge/graph_ir/ge_ir_build_unittest.cc | 123 +++++++++++++++++ 4 files changed, 245 insertions(+), 14 deletions(-) diff --git a/ge/ir_build/ge_ir_build.cc b/ge/ir_build/ge_ir_build.cc index c9dfac07..bd6a2d3a 100644 --- a/ge/ir_build/ge_ir_build.cc +++ b/ge/ir_build/ge_ir_build.cc @@ -251,17 +251,24 @@ class Impl { omg_context_.dynamic_batch_size.clear(); omg_context_.dynamic_image_size.clear(); omg_context_.dynamic_dims.clear(); + omg_context_.user_attr_index_valid = false; }; ~Impl() { (void)generator_.Finalize(); }; graphStatus CheckOptions(const std::map &options); graphStatus CreateInputsForIRBuild(const ge::Graph &graph, vector &inputs); graphStatus UpdateDataOpAttr(const Graph &graph); + graphStatus CheckDataOpAttrIndexValid(const Graph &graph, const std::string &input_shape_range); graphStatus Init(const Graph &graph, const std::map &options); graphStatus BuildModel(const Graph &graph, const std::map &options, ModelBufferData &ge_models); graphStatus InitDomiOmgContext(const string &input_shape, const string &input_format, const string &net_format, bool is_dynamic_input); + graphStatus GetInputShapeRange(const string &input_shape_range, + std::map>> &name_shape_range_map, + std::vector>> &index_shape_range_map); static graphStatus InferShapePrepare(const ComputeGraphPtr &compute_graph); + bool GetUsrAttrIndexValidFlag(); + bool IsAttrIndexSetByUser(const ComputeGraphPtr &compute_graph, size_t &data_num, vector &attr_index); void SetRtSocVersion(); void UpdateThreadContext(); void LoadOpsProto(); @@ -288,11 +295,113 @@ graphStatus Impl::InferShapePrepare(const ComputeGraphPtr &compute_graph) { return GRAPH_SUCCESS; } +bool Impl::GetUsrAttrIndexValidFlag() { + return omg_context_.user_attr_index_valid; +} + +bool Impl::IsAttrIndexSetByUser(const ComputeGraphPtr &compute_graph, + size_t &data_num, + vector &attr_index) { + bool all_zero_flag = true; + for (ge::NodePtr &input_node : compute_graph->GetDirectNode()) { + GE_CHECK_NOTNULL(input_node); + ge::OpDescPtr op = input_node->GetOpDesc(); + GE_CHECK_NOTNULL(op); + if (op->GetType() == DATA) { + data_num++; + GeAttrValue::INT index = 0; + if (AttrUtils::GetInt(op, ATTR_NAME_INDEX, index)) { + if (index != 0) { + all_zero_flag = false; + } + attr_index.push_back(index); + } else { + GELOGW("[Get][AttrIndex] Get index[%ld] failed for op[%s].", index, op->GetName().c_str()); + } + } + } + if (data_num > 1 && attr_index.size() == data_num && all_zero_flag) { + GELOGI("Attr indexes are not set by user."); + return false; + } + return true; +} + +graphStatus Impl::GetInputShapeRange(const string &input_shape_range, + std::map>> &name_shape_range_map, + std::vector>> &index_shape_range_map) { + if (input_shape_range.empty()) { + GELOGI("Input shape range is empty."); + return GRAPH_SUCCESS; + } + Status ret = GRAPH_PARAM_INVALID; + if (input_shape_range.find(":") != string::npos) { + ret = ParseInputShapeRange(input_shape_range, name_shape_range_map); + } else { + ret = ParseInputShapeRange(input_shape_range, index_shape_range_map); + } + if (ret != SUCCESS) { + GELOGE(GRAPH_PARAM_INVALID, "[Parse][InputShapeRange] parse shape range[%s] failed.", input_shape_range.c_str()); + return GRAPH_PARAM_INVALID; + } + return GRAPH_SUCCESS; +} + +graphStatus Impl::CheckDataOpAttrIndexValid(const Graph &graph, const std::string &input_shape_range) { + auto compute_graph = ge::GraphUtils::GetComputeGraph(graph); + GE_CHECK_NOTNULL(compute_graph); + + // when set input shape range by index, user must set data attr index, eg. "[1, 3, 3, -1],[1, 3~5, 6, -1]" + bool index_input_shape_range_flag = !input_shape_range.empty() && (input_shape_range.find(":") == string::npos); + size_t data_num = 0; + vector attr_index; + if (!IsAttrIndexSetByUser(compute_graph, data_num, attr_index)) { + if (index_input_shape_range_flag) { + std::string situation = "Data op index"; + std::string reason = "it must be set by user, total data op num[" + std::to_string(data_num) + "], " + "when set input shape range by index."; + REPORT_INPUT_ERROR("E19025", std::vector({"situation", "reason"}), + std::vector({situation, reason})); + GELOGE(GRAPH_FAILED, "[Check][AttrIndex] Data op index is not set, total data op num[%ld], " + "when set input shape range by index.", data_num); + return GRAPH_FAILED; + } + return GRAPH_SUCCESS; + } + + omg_context_.user_attr_index_valid = true; + for (size_t i = 0; i < data_num; ++i) { + if (std::find(attr_index.begin(), attr_index.end(), i) == attr_index.end()) { + omg_context_.user_attr_index_valid = false; + if (index_input_shape_range_flag) { + std::string situation = "Data op index[" + std::to_string(i) + "]"; + std::string reason = "it must be set by user, total data op num[" + std::to_string(data_num) + "], " + "when set input shape range by index"; + REPORT_INPUT_ERROR("E19025", std::vector({"situation", "reason"}), + std::vector({situation, reason})); + GELOGE(GRAPH_FAILED, "[Check][AttrIndex] Attr index [%ld] is not set, total data op num[%ld], " + "when set input shape range by index", i, data_num); + return GRAPH_FAILED; + } else { + GELOGW("[Check][AttrIndex] Attr index [%ld] is not set, total data op num[%ld].", i, data_num); + } + } + } + GELOGI("Data op attr indexes are set by user and valid."); + return GRAPH_SUCCESS; +} + graphStatus Impl::UpdateDataOpAttr(const Graph &graph) { GELOGD("Enter Update Data Attr Process!"); std::string input_shape = (options_.find(kInputShape) == options_.end()) ? "" : options_[kInputShape]; std::string input_shape_range = (options_.find(kInputShapeRange) == options_.end()) ? "" : options_[kInputShapeRange]; + graphStatus ret = CheckDataOpAttrIndexValid(graph, input_shape_range); + if (ret != GRAPH_SUCCESS) { + GELOGE(GRAPH_FAILED, "[Check][DataOpAttrIndex] fail, shape range[%s].", input_shape_range.c_str()); + return GRAPH_FAILED; + } + map> shape_map; vector>> user_shape_map; if (!input_shape.empty()) { @@ -301,20 +410,13 @@ graphStatus Impl::UpdateDataOpAttr(const Graph &graph) { } std::map>> name_shape_range_map; std::vector>> index_shape_range_map; - if (!input_shape_range.empty()) { - Status ret = GRAPH_PARAM_INVALID; - if (input_shape_range.find(":") != string::npos) { - ret = ParseInputShapeRange(input_shape_range, name_shape_range_map); - } else { - ret = ParseInputShapeRange(input_shape_range, index_shape_range_map); - } - if (ret != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "[Parse][InputShapeRange] parse shape range[%s] failed.", input_shape_range.c_str()); - return GRAPH_PARAM_INVALID; - } - } auto compute_graph = ge::GraphUtils::GetComputeGraph(graph); GE_CHECK_NOTNULL(compute_graph); + ret = GetInputShapeRange(input_shape_range, name_shape_range_map, index_shape_range_map); + if (ret != GRAPH_SUCCESS) { + GELOGE(GRAPH_FAILED, "[Get][InputShapeRange] fail, shape range[%s].", input_shape_range.c_str()); + return GRAPH_FAILED; + } for (ge::NodePtr &input_node : compute_graph->GetDirectNode()) { GE_CHECK_NOTNULL(input_node); ge::OpDescPtr op = input_node->GetOpDesc(); @@ -495,7 +597,9 @@ graphStatus Impl::CreateInputsForIRBuild(const ge::Graph &graph, vectorGetOpDesc(); GE_CHECK_NOTNULL(op); if (op->GetType() == DATA) { - (void)AttrUtils::SetInt(op, ATTR_NAME_INDEX, index++); + if (!GetUsrAttrIndexValidFlag()) { + (void)AttrUtils::SetInt(op, ATTR_NAME_INDEX, index++); + } GELOGD("Data op inputDesc size: %zu", op->GetAllInputsDesc().size()); ge::GeTensorDesc tensor = op->GetInputDesc(0); string data_op_name = op->GetName(); diff --git a/inc/framework/omg/omg_inner_types.h b/inc/framework/omg/omg_inner_types.h index 1024f7e6..0b799bf2 100644 --- a/inc/framework/omg/omg_inner_types.h +++ b/inc/framework/omg/omg_inner_types.h @@ -125,6 +125,7 @@ struct OmgContext { std::vector getnext_nosink_nodes; bool fuzz_compile_flag = false; std::string atc_cmdline; + bool user_attr_index_valid = false; }; } // namespace ge diff --git a/tests/ut/ge/graph/build/model_builder_unittest.cc b/tests/ut/ge/graph/build/model_builder_unittest.cc index 628d0fda..d544e1a3 100644 --- a/tests/ut/ge/graph/build/model_builder_unittest.cc +++ b/tests/ut/ge/graph/build/model_builder_unittest.cc @@ -17,6 +17,7 @@ #include #include +#include "graph/common/local_context.h" #include "graph/anchor.h" #include "graph/attr_value.h" #include "graph/debug/ge_attr_define.h" @@ -165,7 +166,9 @@ void MakeSessionScopeReuseGraph(ge::ComputeGraphPtr graph) { } protected: - void SetUp() {} + void SetUp() { + SetLocalOmgContext(domi::GetContext()); + } void TearDown() { GetContext().out_nodes_map.clear(); } }; diff --git a/tests/ut/ge/graph_ir/ge_ir_build_unittest.cc b/tests/ut/ge/graph_ir/ge_ir_build_unittest.cc index ec7b9488..fb4a5a8d 100644 --- a/tests/ut/ge/graph_ir/ge_ir_build_unittest.cc +++ b/tests/ut/ge/graph_ir/ge_ir_build_unittest.cc @@ -18,6 +18,9 @@ #include "ir_build/option_utils.h" #include "graph/testcase/ge_graph/graph_builder_utils.h" #include "graph/debug/ge_attr_define.h" +#include "graph/utils/graph_utils.h" +#include "ge/ge_ir_build.h" +#include "graph/ops_stub.h" #define protected public #define private public @@ -37,6 +40,13 @@ class UtestIrCommon : public testing::Test { void TearDown() {} }; +class UtestIrBuild : public testing::Test { + protected: + void SetUp() {} + + void TearDown() {} +}; + static ge::OpDescPtr CreateOpDesc(const std::string &name, const std::string &type) { OpDescPtr op_desc = std::make_shared(name, type); ge::GeTensorDesc ge_tensor_desc; @@ -60,6 +70,59 @@ static ComputeGraphPtr BuildComputeGraph() { return builder.GetGraph(); } +// data not set attr index; +// but becasue of op proto, register attr index. so all data index is zero; +static Graph BuildIrGraph() { + auto data1 = op::Data("data1"); + auto data2 = op::Data("data2"); + auto data3 = op::Data("data3"); + std::vector inputs {data1, data2, data3}; + std::vector outputs; + + Graph graph("test_graph"); + graph.SetInputs(inputs).SetOutputs(outputs); + return graph; +} + +// data set attr index, but is not valid +static Graph BuildIrGraph1() { + auto data1 = op::Data("data1").set_attr_index(0); + auto data2 = op::Data("data2").set_attr_index(1); + auto data3 = op::Data("data3"); + std::vector inputs {data1, data2, data3}; + std::vector outputs; + + Graph graph("test_graph"); + graph.SetInputs(inputs).SetOutputs(outputs); + return graph; +} + +// data set attr index, but is not valid +static Graph BuildIrGraph2() { + auto data1 = op::Data("data1").set_attr_index(0); + auto data2 = op::Data("data2"); + auto data3 = op::Data("data3").set_attr_index(2); + std::vector inputs {data1, data2, data3}; + std::vector outputs; + + Graph graph("test_graph"); + graph.SetInputs(inputs).SetOutputs(outputs); + return graph; +} + +// data set attr index +static Graph BuildIrGraph3() { + auto data1 = op::Data("data1").set_attr_index(0); + auto data2 = op::Data("data2").set_attr_index(1); + auto data3 = op::Data("data3").set_attr_index(2); + std::vector inputs {data1, data2, data3}; + std::vector outputs; + + Graph graph("test_graph"); + graph.SetInputs(inputs).SetOutputs(outputs); + return graph; +} + TEST(UtestIrCommon, update_data_op_shape) { ge::OpDescPtr op_desc = CreateOpDesc("Data", "Data"); map> shape_map; @@ -227,3 +290,63 @@ TEST(UtestIrCommon, check_param_failed) { ret = CheckLogParamValidAndSetLogLevel(param_invalid); } + +// Get attr index failed, when set input shape range +TEST(UtestIrBuild, check_data_op_attr_index_invalid_0) { + ComputeGraphPtr compute_graph = BuildComputeGraph(); + Graph graph = GraphUtils::CreateGraphFromComputeGraph(compute_graph); + const map build_options = { + {"input_shape_range", "[1, 2~3, -1],[4~5, 3~5, 10],[1, 2~3, -1]"} + }; + ModelBufferData model; + graphStatus ret = aclgrphBuildModel(graph, build_options, model); + EXPECT_EQ(ret, GRAPH_FAILED); +} + +// not set attr index, when set input shape range +TEST(UtestIrBuild, check_data_op_attr_index_invalid_1) { + Graph graph = BuildIrGraph(); + const map build_options = { + {"input_shape_range", "[1, 2~3, -1],[4~5, 3~5, 10],[1, 2~3, -1]"} + }; + ModelBufferData model; + graphStatus ret = aclgrphBuildModel(graph, build_options, model); + EXPECT_EQ(ret, GRAPH_FAILED); +} + +// set attr index, but not valid, when set input shape range +TEST(UtestIrBuild, check_data_op_attr_index_invalid_2) { + Graph graph = BuildIrGraph1(); + const map build_options = { + {"input_shape_range", "[1, 2~3, -1],[4~5, 3~5, 10],[1, 2~3, -1]"} + }; + ModelBufferData model; + graphStatus ret = aclgrphBuildModel(graph, build_options, model); + EXPECT_EQ(ret, GRAPH_FAILED); + + Graph graph2 = BuildIrGraph2(); + ret = aclgrphBuildModel(graph2, build_options, model); + EXPECT_EQ(ret, GRAPH_FAILED); +} + +// set attr index valid, when set input shape range +// only check data op attr index valid func. +TEST(UtestIrBuild, check_data_op_attr_index_valid) { + Graph graph = BuildIrGraph3(); + const map build_options = { + {"input_shape_range", "[1, 2~3, -1],[4~5, 3~5, 10],[1, 2~3, -1]"} + }; + ModelBufferData model; + graphStatus ret = aclgrphBuildModel(graph, build_options, model); + EXPECT_EQ(ret, GE_GENERATOR_GRAPH_MANAGER_BUILD_GRAPH_FAILED); +} + +// set attr index invalid, when not set input shape range +// only check data op attr index valid func. +TEST(UtestIrBuild, check_data_attr_index_succ_no_input_range) { + Graph graph = BuildIrGraph1(); + const map build_options; + ModelBufferData model; + graphStatus ret = aclgrphBuildModel(graph, build_options, model); + EXPECT_EQ(ret, GE_GENERATOR_GRAPH_MANAGER_BUILD_GRAPH_FAILED); +} \ No newline at end of file From 061dfe4d0c03ae91ae7ecd4791599f04aab3a4c1 Mon Sep 17 00:00:00 2001 From: wangzhengjun Date: Tue, 18 May 2021 15:47:08 +0800 Subject: [PATCH 08/48] ref cascade --- ge/graph/build/memory/graph_mem_assigner.cc | 28 +++++++++++++++---- ge/single_op/single_op_model.cc | 2 +- .../ge/graph/build/mem_assigner_unittest.cc | 28 +++++++++++++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/ge/graph/build/memory/graph_mem_assigner.cc b/ge/graph/build/memory/graph_mem_assigner.cc index 8becd90e..b098a5f5 100755 --- a/ge/graph/build/memory/graph_mem_assigner.cc +++ b/ge/graph/build/memory/graph_mem_assigner.cc @@ -69,6 +69,10 @@ int64_t GetSymbolOutputOffset(const std::map &anchor_t } return ge::kInvalidOffset; } + +bool isVariableMemoryNode(const ge::NodePtr &node) { + return (node->GetType() == ge::VARIABLE) || (node->GetType() == ge::CONSTANTOP); +} } // namespace namespace ge { Status VariableMemoryAssigner::Assign() { @@ -447,22 +451,31 @@ bool IsContinuousInputConflict(const ge::NodePtr &node, const OpDescPtr &peer_op /// op1 -> node -> op2 /// return true when node is ref from input, and op1 or op2 is reuse input from output bool GraphMemoryAssigner::IsRefFromInputOpCascade(const NodePtr &node) { - bool ref_from_input = false; + std::unordered_set ref_input_index; int32_t reuse_in_index = -1; for (const auto &out_anchor : node->GetAllOutDataAnchors()) { - ref_from_input = GraphUtils::IsRefFromInput(out_anchor, reuse_in_index); - if (ref_from_input) { + bool reuse_input = GraphUtils::IsRefFromInput(out_anchor, reuse_in_index); + if (reuse_input) { GELOGD("IsRefFromInputOpCascade: cur node:%s:%d is ref", node->GetName().c_str(), reuse_in_index); - break; + ref_input_index.insert(reuse_in_index); } } + bool ref_from_input = !ref_input_index.empty(); + if (!ref_from_input) { + return false; + } for (const auto &in_anchor : node->GetAllInDataAnchors()) { const auto &peer_out_anchor = in_anchor->GetPeerOutAnchor(); GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue); + auto in_node = peer_out_anchor->GetOwnerNode(); + if (isVariableMemoryNode(in_node) && (ref_input_index.count(in_anchor->GetIdx()) > 0)) { + GELOGD("Reuse variable memory, input node:%s, type:%s.", in_node->GetName().c_str(), in_node->GetType().c_str()); + return false; + } if (ref_from_input && GraphUtils::IsRefFromInput(peer_out_anchor, reuse_in_index)) { GELOGD("IsRefFromInputOpCascade: in node[%s] is ref, reuse index is:%d", - peer_out_anchor->GetOwnerNode()->GetName().c_str(), reuse_in_index); + in_node->GetName().c_str(), reuse_in_index); return true; } } @@ -500,6 +513,11 @@ Status GraphMemoryAssigner::UpdateRefOpOffsetReverse(const NodePtr &node) { GE_CHECK_NOTNULL(peer_out_anchor); auto peer_node = peer_out_anchor->GetOwnerNode(); GE_CHECK_NOTNULL(peer_node); + if (isVariableMemoryNode(peer_node)) { + GELOGW("Peer node to update is %s, skip it. Node name:%s.", + peer_node->GetType().c_str(), peer_node->GetName().c_str()); + continue; + } auto peer_op_desc = peer_node->GetOpDesc(); GE_CHECK_NOTNULL(peer_op_desc); vector peer_output_list = peer_op_desc->GetOutputOffset(); diff --git a/ge/single_op/single_op_model.cc b/ge/single_op/single_op_model.cc index 6959c6b3..5b5f24a2 100755 --- a/ge/single_op/single_op_model.cc +++ b/ge/single_op/single_op_model.cc @@ -613,7 +613,7 @@ Status SingleOpModel::BuildDynamicOp(StreamResource &resource, DynamicSingleOp & single_op.num_inputs_ = data_ops_.size(); single_op.num_outputs_ = netoutput_op_->GetAllInputsSize(); GE_CHK_STATUS_RET_NOLOG(InitModelMem(resource)); - model_params_.memory_size = UINT_MAX; + model_params_.memory_size = UINT64_MAX; model_params_.graph_is_dynamic = true; auto ge_model = model_helper_.GetGeModel(); diff --git a/tests/ut/ge/graph/build/mem_assigner_unittest.cc b/tests/ut/ge/graph/build/mem_assigner_unittest.cc index c9b0b579..785af2ef 100644 --- a/tests/ut/ge/graph/build/mem_assigner_unittest.cc +++ b/tests/ut/ge/graph/build/mem_assigner_unittest.cc @@ -525,6 +525,34 @@ TEST_F(UtestMemoryAssignerTest, graph_memory_assign_update_ref_op_offset_reverse EXPECT_EQ(memoryAssigner.UpdateRefOpOffsetReverse(add), SUCCESS); } +TEST_F(UtestMemoryAssignerTest, graph_memory_assign_var_input_ref_cascade_false) { + ge::ut::GraphBuilder builder("graph"); + auto var = builder.AddNode("var", VARIABLE, 1, 1); + auto broadcast = builder.AddNode("broadcast", HCOMBROADCAST, 1, 1); + auto assign = builder.AddNode("assign", "Assign", 2, 1); + // add link + builder.AddDataEdge(var, 0, assign, 0); + builder.AddDataEdge(var, 0, broadcast, 0); + builder.AddDataEdge(broadcast, 0, assign, 1); + + int reuse_input_index = 0; + auto broadcast_desc = broadcast->GetOpDesc()->MutableOutputDesc(0); + ge::TensorUtils::SetReuseInput(*broadcast_desc, true); + ge::TensorUtils::SetReuseInputIndex(*broadcast_desc, reuse_input_index); + + ge::ComputeGraphPtr graph = builder.GetGraph(); + + GraphMemoryAssigner memory_assigner(graph); + bool ref_cascade = memory_assigner.IsRefFromInputOpCascade(broadcast); + EXPECT_EQ(ref_cascade, false); + ref_cascade = memory_assigner.IsRefFromInputOpCascade(assign); + EXPECT_EQ(ref_cascade, false); + auto ret = memory_assigner.UpdateRefOpOffsetReverse(broadcast); + EXPECT_EQ(ret, SUCCESS); + ret = memory_assigner.UpdateRefOpOffsetReverse(assign); + EXPECT_EQ(ret, SUCCESS); +} + TEST_F(UtestMemoryAssignerTest, graph_memory_assign_atomic_output_and_workspace) { ge::ut::GraphBuilder builder("graph"); auto data_input = builder.AddNode("data", "Data", 1, 1); From e3fc3f3a49e77e3d1fe3bf65dd5c7fcf4917cb20 Mon Sep 17 00:00:00 2001 From: medivh-x Date: Tue, 18 May 2021 20:41:01 +0800 Subject: [PATCH 09/48] model parser now support serialized proto --- inc/framework/omg/parser/model_parser.h | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/inc/framework/omg/parser/model_parser.h b/inc/framework/omg/parser/model_parser.h index 8fae5556..67f9338d 100644 --- a/inc/framework/omg/parser/model_parser.h +++ b/inc/framework/omg/parser/model_parser.h @@ -37,6 +37,9 @@ using Status = domi::Status; namespace domi { using GetGraphCallback = std::function( const google::protobuf::Message *root_proto, const std::string &graph)>; + +using GetGraphCallbackV2 = std::function; + class GE_FUNC_VISIBILITY ModelParser { public: ModelParser() {} @@ -117,6 +120,30 @@ class GE_FUNC_VISIBILITY ModelParser { virtual ge::DataType ConvertToGeDataType(const uint32_t type) = 0; virtual Status ParseAllGraph(const google::protobuf::Message *root_proto, ge::ComputeGraphPtr &root_graph) = 0; + + /** + * @ingroup domi_omg + * @brief Analyze network model data + * @param [in] proto serialized network model + * @param [in|out] graph Save the network information after analysis + * @return SUCCESS + * @return Others failed + */ + virtual Status ParseProto(const std::string &serialized_proto, ge::ComputeGraphPtr &graph) { return UNSUPPORTED; } + + /** + * @ingroup domi_omg + * @brief Analyze callback model data in subgraph + * @param [in] proto serialized network model + * @param [in] callback callback of subgraph + * @param [in|out] graph Save the network information after analysis + * @return SUCCESS + * @return Others failed + */ + virtual Status ParseProtoWithSubgraph(const std::string &serialized_proto, GetGraphCallbackV2 callback, + ge::ComputeGraphPtr &graph) { + return UNSUPPORTED; + } }; } // namespace domi From eb65640805db8cec97428079985de42b345682a8 Mon Sep 17 00:00:00 2001 From: liudingyan Date: Wed, 12 May 2021 15:04:03 +0800 Subject: [PATCH 10/48] modify message of GELOGE --- ge/common/model_parser/model_parser.cc | 41 +- ge/graph/load/graph_loader.cc | 60 +- ge/graph/load/model_manager/model_manager.cc | 369 ++++++------ ge/graph/load/model_manager/model_manager.h | 2 +- ge/graph/load/model_manager/model_utils.cc | 28 +- .../load/model_manager/tbe_handle_store.cc | 17 +- ge/graph/load/model_manager/ts_mem_mall.h | 8 +- .../load/model_manager/zero_copy_offset.cc | 24 +- ge/graph/load/model_manager/zero_copy_task.cc | 11 +- ge/graph/manager/graph_caching_allocator.cc | 39 +- ge/graph/manager/graph_context.cc | 19 +- ge/graph/manager/graph_manager.cc | 530 +++++++++--------- ge/graph/manager/graph_manager_utils.cc | 15 +- ge/graph/manager/graph_mem_allocator.cc | 18 +- ge/graph/manager/graph_var_manager.cc | 85 +-- ge/graph/manager/host_mem_allocator.cc | 8 +- ge/graph/manager/host_mem_manager.cc | 22 +- ge/graph/manager/memory_api.cc | 18 +- .../manager/model_manager/event_manager.cc | 2 +- ge/graph/manager/rdma_pool_allocator.cc | 21 +- ge/graph/manager/trans_var_data_utils.cc | 132 +++-- ge/graph/manager/util/debug.cc | 10 +- ge/graph/manager/util/hcom_util.cc | 115 ++-- .../manager/util/variable_accelerate_ctrl.cc | 2 +- 24 files changed, 818 insertions(+), 778 deletions(-) diff --git a/ge/common/model_parser/model_parser.cc b/ge/common/model_parser/model_parser.cc index ce654887..5c68eea8 100644 --- a/ge/common/model_parser/model_parser.cc +++ b/ge/common/model_parser/model_parser.cc @@ -62,7 +62,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelParserBase::LoadFro char *data = new (std::nothrow) char[len]; if (data == nullptr) { - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Load model From file failed, bad memory allocation occur. (need:%u)", len); GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Load][ModelFromFile]Failed, " "bad memory allocation occur(need %u), file %s", len, model_path); REPORT_CALL_ERROR("E19999", "Load model from file %s failed, " @@ -90,33 +89,45 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelParserBase::ParseMo GE_CHECK_NOTNULL(model.model_data); // Model length too small - GE_CHK_BOOL_RET_STATUS(model.model_len >= sizeof(ModelFileHeader), ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID, - "Invalid model. Model data size %u must be greater than or equal to %zu.", model.model_len, - sizeof(ModelFileHeader)); + GE_CHK_BOOL_EXEC(model.model_len >= sizeof(ModelFileHeader), + REPORT_INPUT_ERROR("E10003", std::vector({"parameter", "value", "reason"}), + std::vector({"om", model.om_name.c_str(), "invalid om file"})); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID, + "[Check][Param] Invalid model. Model data size %u must be greater than or equal to %zu.", + model.model_len, sizeof(ModelFileHeader)); + return ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID;); // Get file header auto file_header = reinterpret_cast(model.model_data); // Determine whether the file length and magic number match - GE_CHK_BOOL_RET_STATUS( - file_header->length == model.model_len - sizeof(ModelFileHeader) && file_header->magic == MODEL_FILE_MAGIC_NUM, - ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID, - "Invalid model. file_header->length[%u] + sizeof(ModelFileHeader)[%zu] != model->model_len[%u] || " - "MODEL_FILE_MAGIC_NUM[%u] != file_header->magic[%u]", - file_header->length, sizeof(ModelFileHeader), model.model_len, MODEL_FILE_MAGIC_NUM, file_header->magic); - + GE_CHK_BOOL_EXEC(file_header->length == model.model_len - sizeof(ModelFileHeader) && + file_header->magic == MODEL_FILE_MAGIC_NUM, + REPORT_INPUT_ERROR("E10003", std::vector({"parameter", "value", "reason"}), + std::vector({"om", model.om_name.c_str(), "invalid om file"})); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID, + "[Check][Param] Invalid model, file_header->length[%u] + sizeof(ModelFileHeader)[%zu] != " + "model->model_len[%u] || MODEL_FILE_MAGIC_NUM[%u] != file_header->magic[%u]", + file_header->length, sizeof(ModelFileHeader), model.model_len, + MODEL_FILE_MAGIC_NUM, file_header->magic); + return ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID;); Status res = SUCCESS; // Get data address uint8_t *data = reinterpret_cast(model.model_data) + sizeof(ModelFileHeader); if (file_header->is_encrypt == ModelEncryptType::UNENCRYPTED) { // Unencrypted model - GE_CHK_BOOL_RET_STATUS(model.key.empty(), ACL_ERROR_GE_PARAM_INVALID, - "Invalid param. model is unencrypted, but key is not empty."); - + if (!model.key.empty()) { + REPORT_INPUT_ERROR("E10003", std::vector({"parameter", "value", "reason"}), + std::vector({"om", model.om_name.c_str(), "invalid om file"})); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, + "[Check][Param] Invalid param, model is unencrypted, but key is not empty."); + return ACL_ERROR_GE_PARAM_INVALID; + } model_data = data; model_len = file_header->length; GELOGD("Model_len is %u, model_file_head_len is %zu.", model_len, sizeof(ModelFileHeader)); } else { GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Param]Invalid, model encrypt type not supported"); - REPORT_CALL_ERROR("E19999","Invalid model, encrypt type not supported"); + REPORT_INPUT_ERROR("E10003", std::vector({"parameter", "value", "reason"}), + std::vector({"om", model.om_name.c_str(), "invalid om file"})); res = ACL_ERROR_GE_PARAM_INVALID; } diff --git a/ge/graph/load/graph_loader.cc b/ge/graph/load/graph_loader.cc index e4904614..7ee74d1d 100755 --- a/ge/graph/load/graph_loader.cc +++ b/ge/graph/load/graph_loader.cc @@ -33,12 +33,12 @@ Status GraphLoader::UnloadModel(uint32_t model_id) { Status ret = model_manager->Stop(model_id); if (ret != SUCCESS) { - GELOGE(ret, "UnloadModel: Stop failed. model id:%u", model_id); + GELOGE(ret, "[Stop][Model] failed. model id:%u", model_id); } ret = model_manager->Unload(model_id); if (ret != SUCCESS) { - GELOGE(ret, "UnloadModel: Unload failed. model id:%u", model_id); + GELOGE(ret, "[Unload][Model] failed. model id:%u", model_id); return ret; } GELOGI("UnLoad model success, model id:%u.", model_id); @@ -50,14 +50,13 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptrLoadModelOnline(model_id, ge_root_model_ptr, listener); if (ret != SUCCESS) { - GELOGE(ret, "LoadModel: Load failed. ret = %u", ret); + GELOGE(ret, "[Load][Model] Online failed. ret = %u, model_id:%u", ret, model_id); rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); } return ret; } @@ -81,31 +80,31 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptrStart(model_id); if (ret != SUCCESS) { if (model_manager->Unload(model_id) != SUCCESS) { - GELOGE(ret, "LoadModel: Unload failed while trying to unload after a failed start."); + GELOGE(ret, "[Unload][Model] failed while trying to unload after a failed start, model_id:%u.", model_id); } rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); } - GELOGE(ret, "LoadModel: Start failed."); + GELOGE(ret, "[Start][Model] failed, model_id:%u.", model_id); return ret; } rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); return RT_FAILED; } GELOGI("Load model online success, model_id:%u.", model_id); @@ -118,7 +117,7 @@ Status GraphLoader::GetMaxUsedMemory(uint32_t model_id, uint64_t &max_size) { GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->GetMaxUsedMemory(model_id, max_size); if (ret != SUCCESS) { - GELOGE(ret, "GetMaxUsedMemory: GetMaxUsedMemory failed."); + GELOGE(ret, "[Call][GetMaxUsedMemory] failed, model_id:%u.", model_id); return ret; } return SUCCESS; @@ -127,21 +126,20 @@ Status GraphLoader::GetMaxUsedMemory(uint32_t model_id, uint64_t &max_size) { Status GraphLoader::LoadDataFromFile(const std::string &path, const std::string &key_path, int32_t priority, ModelData &model_data) { if (!CheckInputPathValid(path)) { - GELOGE(ACL_ERROR_GE_EXEC_MODEL_PATH_INVALID, "model path is invalid: %s", path.c_str()); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_PATH_INVALID, "[Check][Param] model path is invalid:%s", path.c_str()); return ACL_ERROR_GE_EXEC_MODEL_PATH_INVALID; } GELOGI("Load model begin, model path is: %s", path.c_str()); if (!key_path.empty() && !CheckInputPathValid(key_path)) { - REPORT_INNER_ERROR("E19999", "Param key_path:%s empty or invalid", - key_path.c_str()); - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "decrypt_key path is invalid: %s", key_path.c_str()); + REPORT_INNER_ERROR("E19999", "Param key_path:%s empty or invalid", key_path.c_str()); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Param] decrypt_key path is invalid:%s", key_path.c_str()); return ACL_ERROR_GE_PARAM_INVALID; } Status ret = ModelParserBase::LoadFromFile(path.c_str(), key_path.c_str(), priority, model_data); if (ret != SUCCESS) { - GELOGE(ret, "LoadModelFromFile: Load failed. ret = %u", ret); + GELOGE(ret, "[Call][LoadFromFile] failed. ret = %u, path:%s, key path:%s", ret, path.c_str(), key_path.c_str()); if (model_data.model_data != nullptr) { delete[] static_cast(model_data.model_data); model_data.model_data = nullptr; @@ -156,18 +154,19 @@ Status GraphLoader::CommandHandle(const Command &command) { GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->HandleCommand(command); if (ret != SUCCESS) { - GELOGE(ret, "CommandHandle: Command Handle failed."); + GELOGE(ret, "[Handle][Command] failed, module_index:%lu.", command.module_index); return ret; } } catch (std::bad_alloc &) { REPORT_INNER_ERROR("E19999", "Bad memory allocation occur"); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Command handle failed, bad memory allocation occur !"); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Handle][Command] failed, " + "bad memory allocation occur, module_index:%lu.", command.module_index); return ACL_ERROR_GE_MEMORY_ALLOCATION; } catch (...) { REPORT_INNER_ERROR("E19999", "Some exceptions occur"); - GELOGE(FAILED, "Command handle failed, some exceptions occur !"); + GELOGE(FAILED, "[Handle][Command] failed, some exceptions occur, module_index:%lu.", command.module_index); return FAILED; } @@ -184,7 +183,7 @@ Status GraphLoader::LoadModelFromData(uint32_t &model_id, const ModelData &model Status ret = model_manager->LoadModelOffline( model_id, model_data, nullptr, dev_ptr, mem_size, weight_ptr, weight_size); if (ret != SUCCESS) { - GELOGE(ret, "Load model failed, model_id:%u.", model_id); + GELOGE(ret, "[Load][Model] failed, model_id:%u.", model_id); return ret; } GELOGI("Load model success, model_id:%u.", model_id); @@ -210,7 +209,7 @@ Status GraphLoader::LoadModelWithQ(uint32_t &model_id, const ModelData &model_da GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->LoadModelWithQ(model_id, model_data, input_queue_ids, output_queue_ids); if (ret != SUCCESS) { - GELOGE(ret, "Load model with queue failed, model_id:%u.", model_id); + GELOGE(ret, "[Load][Model] with queue failed, model_id:%u.", model_id); return ret; } @@ -237,7 +236,7 @@ Status GraphLoader::ExecuteModel(uint32_t model_id, rtStream_t stream, bool asyn Status ret = model_manager->ExecuteModel(model_id, stream, async_mode, input_data, input_desc, output_data, output_desc); if (ret != SUCCESS) { - GELOGE(ret, "Execute model failed, model_id:%u.", model_id); + GELOGE(ret, "[Execute][Model] failed, model_id:%u.", model_id); return ret; } @@ -250,7 +249,7 @@ Status GraphLoader::GetMemoryInfo(int64_t &free) { if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtSetDevice] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); return RT_FAILED; } size_t total_mem = 0; @@ -258,14 +257,14 @@ Status GraphLoader::GetMemoryInfo(int64_t &free) { rt_ret = rtMemGetInfo(&free_mem, &total_mem); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemGetInfo failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemGetInfo] failed, ret:0x%X", rt_ret); return RT_FAILED; } rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); return RT_FAILED; } // Add small page memory size @@ -280,7 +279,8 @@ Status GraphLoader::DestroyAicpuKernel(uint64_t session_id, uint32_t model_id, u GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->DestroyAicpuKernel(session_id, model_id, sub_model_id); if (ret != SUCCESS) { - GELOGE(ret, "Destroy aicpu kernel failed."); + GELOGE(ret, "[Destroy][AicpuKernel] failed, session_id:%lu, model_id:%u, sub_model_id:%u.", + session_id, model_id, sub_model_id); return ret; } return SUCCESS; @@ -291,7 +291,7 @@ Status GraphLoader::DestroyAicpuSessionForInfer(uint32_t model_id) { GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->DestroyAicpuSessionForInfer(model_id); if (ret != SUCCESS) { - GELOGE(ret, "Destroy aicpu serrion for infer failed."); + GELOGE(ret, "[Call][DestroyAicpuSessionForInfer] failed, model_id:%u.", model_id); return ret; } return SUCCESS; diff --git a/ge/graph/load/model_manager/model_manager.cc b/ge/graph/load/model_manager/model_manager.cc index 47d104f4..770fdb08 100755 --- a/ge/graph/load/model_manager/model_manager.cc +++ b/ge/graph/load/model_manager/model_manager.cc @@ -102,16 +102,14 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u auto kernel_size = sizeof(uint64_t) * (v_aicpu_kernel.size()); rtError_t rt_ret = rtMalloc(&aicpu_kernel_addr, kernel_size, RT_MEMORY_HBM); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret: 0x%X", - kernel_size, rt_ret); - GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", kernel_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%zu, ret:0x%X", kernel_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(aicpu_kernel_addr, kernel_size, v_aicpu_kernel.data(), kernel_size, RT_MEMCPY_HOST_TO_DEVICE); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret: 0x%X", - kernel_size, rt_ret); - GELOGE(RT_FAILED, "rtMemcpy to input_output_addr_ error: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", kernel_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", kernel_size, rt_ret); GE_CHK_RT(rtFree(aicpu_kernel_addr)); return RT_ERROR_TO_GE_STATUS(rt_ret);) uint64_t kernel_id_addr = static_cast(reinterpret_cast(aicpu_kernel_addr)); param_base.fwkKernelBase.fwk_kernel.kernelID = kernel_id_addr; @@ -122,9 +120,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u rtError_t rt_ret = rtMalloc(&(devicebase), sizeof(STR_FWK_OP_KERNEL), RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret: 0x%X", - sizeof(STR_FWK_OP_KERNEL), rt_ret); - GELOGE(RT_FAILED, "malloc device memory failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed. size:%zu, ret:0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -132,9 +129,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u rt_ret = rtMemcpy(devicebase, sizeof(STR_FWK_OP_KERNEL), ¶m_base, sizeof(STR_FWK_OP_KERNEL), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret: 0x%X", - sizeof(STR_FWK_OP_KERNEL), rt_ret); - GELOGE(RT_FAILED, "memory copy to device failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); GE_CHK_RT(rtFree(devicebase)); return RT_ERROR_TO_GE_STATUS(rt_ret); @@ -143,8 +139,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u rtStream_t stream = nullptr; rt_ret = rtStreamCreate(&stream, 0); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamCreate failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "create stream failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamCreate failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Create][Stream] failed. ret:0x%X", rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); GE_CHK_RT(rtFree(devicebase)); return RT_ERROR_TO_GE_STATUS(rt_ret); @@ -152,8 +148,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u rt_ret = rtKernelLaunchEx(devicebase, sizeof(STR_FWK_OP_KERNEL), 0, stream); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchEx failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "rtKernelLaunchEx failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchEx failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtKernelLaunchEx] failed. ret:0x%X", rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); GE_CHK_RT(rtFree(devicebase)); GE_CHK_RT(rtStreamDestroy(stream)); @@ -161,9 +157,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u } rt_ret = rtStreamSynchronize(stream); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize failed, ret: 0x%X", - rt_ret); - GELOGE(RT_FAILED, "rtStreamSynchronize failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtStreamSynchronize] failed. ret:0x%X", rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); GE_CHK_RT(rtFree(devicebase)); GE_CHK_RT(rtStreamDestroy(stream)); @@ -172,8 +167,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u if (aicpu_kernel_addr != nullptr) { rt_ret = rtFree(aicpu_kernel_addr); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "free memory failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Free][Memory] failed. ret:0x%X", rt_ret); GE_CHK_RT(rtFree(devicebase)); GE_CHK_RT(rtStreamDestroy(stream)); return RT_ERROR_TO_GE_STATUS(rt_ret); @@ -181,15 +176,15 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u } rt_ret = rtFree(devicebase); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "free memory failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Free][Memory] failed. ret:0x%X", rt_ret); GE_CHK_RT(rtStreamDestroy(stream)); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtStreamDestroy(stream); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamDestroy failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "rtStreamDestroy failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamDestroy failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtStreamDestroy] failed. ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } return SUCCESS; @@ -235,9 +230,8 @@ ge::Status ModelManager::DestroyAicpuSessionForInfer(uint32_t model_id) { auto it = model_map_.find(model_id); if (it == model_map_.end()) { - REPORT_INNER_ERROR("E19999", "Param model_id:%u can't find in model_map, check invalid", - model_id); - GELOGE(ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, "model id %u does not exists.", model_id); + REPORT_INNER_ERROR("E19999", "Param model_id:%u can't find in model_map, check invalid", model_id); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, "[Check][Param] model id %u does not exists.", model_id); return ACL_ERROR_GE_EXEC_MODEL_ID_INVALID; } uint64_t session_id = it->second->GetSessionId(); @@ -256,7 +250,8 @@ ge::Status ModelManager::DestroyAicpuKernel(uint64_t session_id, uint32_t model_ if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Call KernelLaunchEx fail, model_id:%u, sub_model_id:%u, session_id:%lu", model_id, sub_model_id, session_id); - GELOGE(FAILED, "Destroy aicpu kernel failed."); + GELOGE(FAILED, "[Call][KernelLaunchEx] fail, model_id:%u, sub_model_id:%u, session_id:%lu", + model_id, sub_model_id, session_id); return FAILED; } } @@ -304,7 +299,7 @@ ge::Status ModelManager::DoLoadHybridModelOnline(uint32_t model_id, const string hybrid_model->SetModelId(model_id); hybrid_model->SetDeviceId(GetContext().DeviceId()); hybrid_model->SetOmName(om_name); - GE_CHK_STATUS_RET(hybrid_model->Init(), "Failed to init hybrid model. model_id = %u", model_id); + GE_CHK_STATUS_RET(hybrid_model->Init(), "[Init][HybridModel] failed. model_id = %u", model_id); auto shared_model = std::shared_ptr(hybrid_model.release()); InsertModel(model_id, shared_model); return SUCCESS; @@ -315,7 +310,8 @@ bool ModelManager::IsNeedHybridLoad(ge::GeRootModel &ge_root_model) { if (root_graph == nullptr) { REPORT_INNER_ERROR("E19999", "root graph in param ge_root_model is nullptr, model_id:%u, " "check invalid", ge_root_model.GetModelId()); - GELOGE(FAILED, "no model on root model"); + GELOGE(FAILED, "[Check][Param] root graph in param ge_root_model is nullptr, model_id:%u", + ge_root_model.GetModelId()); return false; } bool is_shape_unknown = root_graph->GetGraphUnknownFlag(); @@ -354,9 +350,11 @@ Status ModelManager::UpdateSessionId(uint32_t model_id, GeModelPtr ge_model, std::shared_ptr &davinci_model, uint64_t &session_id) { uint64_t new_session_id; Status ret = GenSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Generate session_id for infer failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, + "[Generate][SessionId] for infer failed, model_id:%u.", model_id); ret = davinci_model->UpdateSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Update session_id for infer failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, + "[Update][SessionId] for infer failed, model_id:%u.", model_id); ge_model->InsertSessionMap(model_id, new_session_id); GELOGD("Update new session id: %lu.", new_session_id); session_id = new_session_id; @@ -382,7 +380,7 @@ bool ModelManager::HasVarNode(ComputeGraphPtr &compute_graph) const { /// Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr &ge_root_model, std::shared_ptr listener) { - GE_CHK_BOOL_RET_STATUS(listener.get() != nullptr, PARAM_INVALID, "Param incorrect, listener is null"); + GE_CHK_BOOL_RET_STATUS(listener.get() != nullptr, PARAM_INVALID, "[Check][Param] Param incorrect, listener is null"); if (model_id == INVALID_MODEL_ID) { GenModelId(&model_id); GELOGD("Generate new model_id:%u", model_id); @@ -444,13 +442,13 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr &davinci_model) { - GE_CHK_BOOL_EXEC(davinci_model != nullptr, return, "davinci_model ptr is null, id: %u", model_id); + GE_CHK_BOOL_EXEC(davinci_model != nullptr, return, "[Check][Param] davinci_model ptr is null, id:%u", model_id); std::lock_guard lock(map_mutex_); model_map_[model_id] = davinci_model; } void ModelManager::InsertModel(uint32_t model_id, shared_ptr &hybrid_model) { - GE_CHK_BOOL_EXEC(hybrid_model != nullptr, return, "hybrid_model ptr is null, id: %u", model_id); + GE_CHK_BOOL_EXEC(hybrid_model != nullptr, return, "[Check][Param] hybrid_model ptr is null, id:%u", model_id); std::lock_guard lock(map_mutex_); hybrid_model_map_[model_id] = hybrid_model; } @@ -502,7 +500,7 @@ std::shared_ptr ModelManager::GetHybridModel(uint32_ } Status ModelManager::Unload(uint32_t model_id) { - GE_CHK_STATUS_RET(DeleteModel(model_id), "failed to unload model id: %u", model_id); + GE_CHK_STATUS_RET(DeleteModel(model_id), "[Delete][Model] failed, model id:%u", model_id); if (device_count > 0) { device_count--; GELOGI("Unload model %u success.", model_id); @@ -515,7 +513,7 @@ Status ModelManager::Unload(uint32_t model_id) { } Status ModelManager::UnloadModeldef(uint32_t model_id) { - GE_CHK_STATUS_RET(DeleteModel(model_id), "failed to unload modeldef id: %u", model_id); + GE_CHK_STATUS_RET(DeleteModel(model_id), "[Delete][Model] failed, model id: %u", model_id); return SUCCESS; } @@ -526,8 +524,8 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d Status status = data_wrap->Init(input_data, output_data); if (status != SUCCESS) { - REPORT_CALL_ERROR("E19999", "Init InputDataWrapper failed, input data index: %u", input_data.index); - GELOGE(domi::PUSH_DATA_FAILED, "Init InputDataWrapper failed, input data index: %u.", input_data.index); + REPORT_CALL_ERROR("E19999", "Init InputDataWrapper failed, input data index:%u", input_data.index); + GELOGE(domi::PUSH_DATA_FAILED, "[Init][InputDataWrapper] failed, input data index:%u.", input_data.index); return domi::PUSH_DATA_FAILED; } @@ -536,7 +534,8 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d std::shared_ptr model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(model != nullptr, PARAM_INVALID, "Invalid model id %u in InputData! ", model_id); + GE_CHK_BOOL_RET_STATUS(model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id %u in InputData!", model_id); GE_IF_BOOL_EXEC(model->GetDataInputTid() == 0, model->SetDataInputTid(mmGetTid())); @@ -544,7 +543,7 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d GE_CHECK_NOTNULL(inputer); if (inputer->Push(data_wrap) != SUCCESS) { REPORT_CALL_ERROR("E19999", "DataInputer queue is full, please call again later, model_id %u", model_id); - GELOGE(domi::DATA_QUEUE_ISFULL, "Data queue is full, please call again later, model_id %u ", model_id); + GELOGE(domi::DATA_QUEUE_ISFULL, "[Call][Push] Data queue is full, please call again later, model_id %u ", model_id); return domi::DATA_QUEUE_ISFULL; } GELOGD("Data input success, model id:%u", model_id); @@ -558,10 +557,9 @@ Status ModelManager::GetCurDynamicDims(const vector> &user_real_ GELOGD("Start get cur dynamic dims."); if (user_real_input_dims.size() != user_input_dims.size()) { REPORT_INNER_ERROR("E19999", "Param user_real_input_dims.size:%zu != user_input_dims.size:%zu, " - "check invalid", - user_real_input_dims.size(), user_input_dims.size()); + "check invalid", user_real_input_dims.size(), user_input_dims.size()); GELOGE(INTERNAL_ERROR, - "The input count of user: %zu should be equal to the data count of graph: %zu", + "[Check][Param] The input count of user:%zu should be equal to the data count of graph:%zu", user_real_input_dims.size(), user_input_dims.size()); return INTERNAL_ERROR; } @@ -571,8 +569,8 @@ Status ModelManager::GetCurDynamicDims(const vector> &user_real_ REPORT_INNER_ERROR("E19999", "Param user_real_input_dims[%zu].size:%zu != user_input_dims[%zu].size:%zu, " "check invalid", i, user_real_input_dims[i].size(), i, user_input_dims[i].second.size()); - GELOGE(INTERNAL_ERROR, - "The shape size: %zu of dynamic input: %s should be equal to the shape size of input shape: %zu.", + GELOGE(INTERNAL_ERROR, "[Check][Param] The shape size:%zu of dynamic input:%s " + "should be equal to the shape size of input shape:%zu.", user_real_input_dims[i].size(), user_input_dims[i].first.c_str(), user_input_dims[i].second.size()); return INTERNAL_ERROR; } @@ -594,7 +592,7 @@ Status ModelManager::GetCurDynamicDims(const vector> &user_real_ if (!cur_dynamic_dims_valid) { REPORT_INNER_ERROR("E19999", "cur dynamic dims is %s, not exist in options, check invalid", formats::JoinToString(cur_dynamic_dims).c_str()); - GELOGE(INTERNAL_ERROR, "Cur dynamic dims is %s, not exist in options.", + GELOGE(INTERNAL_ERROR, "[Check][Param] Cur dynamic dims is %s, not exist in options.", formats::JoinToString(cur_dynamic_dims).c_str()); return INTERNAL_ERROR; } @@ -632,15 +630,16 @@ Status ModelManager::DataInputTensor(uint32_t model_id, const std::vector(cur_dynamic_dims.size() * sizeof(int32_t)); - GE_CHK_BOOL_EXEC(memcpy_s(data.data, length, cur_dynamic_dims.data(), length) == EOK, return INTERNAL_ERROR, - "Failed to memcpy data."); + GE_CHK_BOOL_EXEC(memcpy_s(data.data, length, cur_dynamic_dims.data(), length) == EOK, + REPORT_CALL_ERROR("E19999", "memcpy data failed, size:%u", length); + return INTERNAL_ERROR, "[Memcpy][Data] failed, size:%u.", length); data.length = length; input_data.blobs.push_back(data); } @@ -654,21 +653,22 @@ Status ModelManager::DataInputTensor(uint32_t model_id, const std::vectorInit(input_data, output_data), return domi::PUSH_DATA_FAILED, - "Init InputDataWrapper failed,input data model_id is : %u.", model_id); + "[Init][InputDataWrapper] failed, input data model_id:%u.", model_id); if (hybrid_model != nullptr) { - GE_CHK_STATUS_RET(hybrid_model->EnqueueData(data_wrap), "Data queue is full, please call again later, model_id %u ", - model_id); + GE_CHK_STATUS_RET(hybrid_model->EnqueueData(data_wrap), + "[Enqueue][Data] Data queue is full, please call again later, model_id:%u", model_id); return SUCCESS; } - GE_CHK_BOOL_RET_STATUS(model != nullptr, PARAM_INVALID, "Invalid model id %u in InputData! ", model_id); + GE_CHK_BOOL_RET_STATUS(model != nullptr, PARAM_INVALID, + "[Check][Param] Invalid model id %u in InputData!", model_id); DataInputer *inputer = model->GetDataInputer(); GE_CHECK_NOTNULL(inputer); GE_CHK_STATUS_EXEC(inputer->Push(data_wrap), return domi::DATA_QUEUE_ISFULL, - "Data queue is full, please call again later, model_id %u ", model_id); + "[Call][Push] Data queue is full, please call again later, model_id %u ", model_id); GELOGD("Data input success, model id:%u", model_id); @@ -691,7 +691,8 @@ Status ModelManager::Start(uint32_t model_id) { std::shared_ptr davinci_model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, "Invalid model id %u to start! ", model_id); + GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id %u to start! ", model_id); Status status = davinci_model->ModelRunStart(); if (status == SUCCESS) { @@ -718,7 +719,8 @@ Status ModelManager::Stop(uint32_t model_id) { } std::shared_ptr davinci_model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, "Invalid model id %u to stop!", model_id); + GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id %u to stop!", model_id); Status status = davinci_model->ModelRunStop(); if (status == SUCCESS) { @@ -746,9 +748,8 @@ Status ModelManager::HandleCommand(const Command &command) { auto iter = cmds.find(command.cmd_type); if (iter == cmds.end()) { - REPORT_INNER_ERROR("E19999", "Unsupported command:%s check", - command.cmd_type.c_str()); - GELOGE(PARAM_INVALID, "Unsupported command: %s", command.cmd_type.c_str()); + REPORT_INNER_ERROR("E19999", "Unsupported command:%s check", command.cmd_type.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Unsupported command:%s", command.cmd_type.c_str()); return PARAM_INVALID; } else { return iter->second(command); @@ -761,8 +762,8 @@ Status ModelManager::GetModelByCmd(const Command &command, REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu < kCmdParSize:%u, command_type:%s, " "check invalid", command.cmd_params.size(), kCmdParSize, command.cmd_type.c_str()); - GELOGE(PARAM_INVALID, "When the cmd_type is '%s', the size of cmd_params must larger than 2.", - command.cmd_type.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] When the cmd_type is '%s', the size of cmd_params must larger than 2.", + command.cmd_type.c_str()); return PARAM_INVALID; } @@ -773,19 +774,16 @@ Status ModelManager::GetModelByCmd(const Command &command, try { model_id = std::stoi(value); } catch (std::invalid_argument &) { - REPORT_INNER_ERROR("E19999", "%s param:%s, check invalid", PROFILE_MODEL_ID.c_str(), - value.c_str()); - GELOGE(PARAM_INVALID, "Model id: %s is invalid.", value.c_str()); + REPORT_INNER_ERROR("E19999", "%s param:%s, check invalid", PROFILE_MODEL_ID.c_str(), value.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Model id:%s is invalid.", value.c_str()); return PARAM_INVALID; } catch (std::out_of_range &) { - REPORT_INNER_ERROR("E19999", "%s param:%s, check out of range", PROFILE_MODEL_ID.c_str(), - value.c_str()); - GELOGE(PARAM_INVALID, "Model id: %s is out of range.", value.c_str()); + REPORT_INNER_ERROR("E19999", "%s param:%s, check out of range", PROFILE_MODEL_ID.c_str(), value.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Model id:%s is out of range.", value.c_str()); return PARAM_INVALID; } catch (...) { - REPORT_INNER_ERROR("E19999", "%s param:%s, check cannot change to int", - PROFILE_MODEL_ID.c_str(), value.c_str()); - GELOGE(FAILED, "Model id: %s cannot change to int.", value.c_str()); + REPORT_INNER_ERROR("E19999", "%s param:%s, check cannot change to int", PROFILE_MODEL_ID.c_str(), value.c_str()); + GELOGE(FAILED, "[Check][Param] Model id:%s cannot change to int.", value.c_str()); return FAILED; } @@ -793,15 +791,13 @@ Status ModelManager::GetModelByCmd(const Command &command, GE_CHECK_NOTNULL(model_manager); davinci_model = model_manager->GetModel(static_cast(model_id)); if (davinci_model == nullptr) { - REPORT_INNER_ERROR("E19999", "GetModel from model_manager fail, model_id:%u", - model_id); - GELOGE(FAILED, "Model id: %d is invaild or model is not loaded.", model_id); + REPORT_INNER_ERROR("E19999", "GetModel from model_manager fail, model_id:%u", model_id); + GELOGE(FAILED, "[Get][Model] failed, Model id:%d is invaild or model is not loaded.", model_id); return FAILED; } } else { - REPORT_INNER_ERROR("E19999", "Fisrt cmd_param not %s, check invalid", - PROFILE_MODEL_ID.c_str()); - GELOGE(FAILED, "The model_id parameter is not found in the command."); + REPORT_INNER_ERROR("E19999", "Fisrt cmd_param not %s, check invalid", PROFILE_MODEL_ID.c_str()); + GELOGE(FAILED, "[Check][Param] The model_id parameter is not found in the command."); return FAILED; } @@ -817,7 +813,8 @@ Status ModelManager::HandleProfModelSubscribeCommand(const Command &command) { if (ProfilingManager::Instance().ProfModelSubscribe(command.module_index, static_cast(davinci_model.get())) != SUCCESS) { - GELOGE(FAILED, "Handle prof model subscribe failed."); + GELOGE(FAILED, "[Handle][ProfModelSubscribe] failed, module_index:%lu.", + command.module_index); return FAILED; } @@ -832,7 +829,7 @@ Status ModelManager::HandleProfModelUnsubscribeCommand(const Command &command) { } if (ProfilingManager::Instance().ProfModelUnsubscribe(static_cast(davinci_model.get())) != SUCCESS) { - GELOGE(FAILED, "Handle prof model unsubscribe failed."); + GELOGE(FAILED, "[Handle][ProfModelUnsubscribe] failed."); return FAILED; } @@ -842,7 +839,7 @@ Status ModelManager::HandleProfModelUnsubscribeCommand(const Command &command) { Status ModelManager::HandleProfInitCommand(const Command &command) { uint64_t module_index = command.module_index; if (ProfilingManager::Instance().ProfInit(module_index) != SUCCESS) { - GELOGE(FAILED, "Handle prof init failed."); + GELOGE(FAILED, "[Handle][ProfInit] failed, module_index:%lu.", module_index); return FAILED; } return SUCCESS; @@ -850,7 +847,7 @@ Status ModelManager::HandleProfInitCommand(const Command &command) { Status ModelManager::HandleProfFinalizeCommand(const Command &command) { if (ProfilingManager::Instance().ProfFinalize() != SUCCESS) { - GELOGE(FAILED, "Handle prof finalize failed."); + GELOGE(FAILED, "[Handle][ProfFinalize] failed."); return FAILED; } return SUCCESS; @@ -866,13 +863,14 @@ Status ModelManager::HandleProfStartCommand(const Command &command) { if (command.cmd_params.size() < kProfStartCmdParaSize) { REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu < %zu, check invalid", command.cmd_params.size(), kProfStartCmdParaSize); - GELOGE(PARAM_INVALID, "When the cmd_type is 'profile start', the size of cmd_params must larger than 2."); + GELOGE(PARAM_INVALID, "[Check][Param] When the cmd_type is 'profile start', " + "the size:%zu of cmd_params must larger than 2.", command.cmd_params.size()); return PARAM_INVALID; } if (command.cmd_params.size() > kProfCmdParaMaxSize) { REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu > %zu, check invalid", command.cmd_params.size(), kProfCmdParaMaxSize); - GELOGE(PARAM_INVALID, "Command para size[%zu] larger than max[1000].", command.cmd_params.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Command param size[%zu] larger than max[1000].", command.cmd_params.size()); return PARAM_INVALID; } @@ -886,7 +884,7 @@ Status ModelManager::HandleProfStartCommand(const Command &command) { } uint64_t module_index = command.module_index; if (ProfilingManager::Instance().ProfStartProfiling(module_index, cmd_params_map) != SUCCESS) { - GELOGE(FAILED, "Handle prof start failed."); + GELOGE(FAILED, "[Handle][ProfStartProfiling] failed, module_index:%lu.", module_index); return FAILED; } return SUCCESS; @@ -896,13 +894,14 @@ Status ModelManager::HandleProfStopCommand(const Command &command) { if (command.cmd_params.size() < kProfStartCmdParaSize) { REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu < %zu, check invalid", command.cmd_params.size(), kProfStartCmdParaSize); - GELOGE(PARAM_INVALID, "When the cmd_type is 'profile stop', the size of cmd_params must larger than 2."); + GELOGE(PARAM_INVALID, "[Check][Param] When the cmd_type is 'profile stop', " + "the size:%zu of cmd_params must larger than 2.", command.cmd_params.size()); return PARAM_INVALID; } if (command.cmd_params.size() > kProfCmdParaMaxSize) { REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu > %zu, check invalid", command.cmd_params.size(), kProfCmdParaMaxSize); - GELOGE(PARAM_INVALID, "Command para size[%zu] larger than max[1000].", command.cmd_params.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Command param size[%zu] larger than max[1000].", command.cmd_params.size()); return PARAM_INVALID; } @@ -916,7 +915,7 @@ Status ModelManager::HandleProfStopCommand(const Command &command) { } uint64_t module_index = command.module_index; if (ProfilingManager::Instance().ProfStopProfiling(module_index, cmd_params_map) != SUCCESS) { - GELOGE(FAILED, "Handle prof finalize failed."); + GELOGE(FAILED, "[Handle][ProfStopProfiling] failed, module_index:%lu.", module_index); return FAILED; } return SUCCESS; @@ -927,9 +926,8 @@ static Status ParserPara(const Command &command, const string &dump_key, string if (iter != command.cmd_params.end()) { ++iter; if (iter == command.cmd_params.end()) { - REPORT_INNER_ERROR("E19999", "dump_key:%s can't find in command.param, check invalid", - dump_key.c_str()); - GELOGE(PARAM_INVALID, "Invalid access."); + REPORT_INNER_ERROR("E19999", "dump_key:%s can't find in command.param, check invalid", dump_key.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] dump_key:%s can't find in command.param, check invalid", dump_key.c_str()); return PARAM_INVALID; } dump_value = *iter; @@ -939,9 +937,9 @@ static Status ParserPara(const Command &command, const string &dump_key, string Status ModelManager::HandleDumpCommand(const Command &command) { if (command.cmd_params.size() % kDumpCmdPairSize != 0) { - REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu MOD 2 != 0, check invalid", - command.cmd_params.size()); - GELOGE(PARAM_INVALID, "When the cmd_type is 'dump', the size of cmd_params must be a even number."); + REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu MOD 2 != 0, check invalid", command.cmd_params.size()); + GELOGE(PARAM_INVALID, "[Check][Param] When the cmd_type is 'dump', " + "the size:%zu of cmd_params must be a even number.", command.cmd_params.size()); return PARAM_INVALID; } @@ -953,14 +951,14 @@ Status ModelManager::HandleDumpCommand(const Command &command) { auto ret = ParserPara(command, DUMP_STATUS, dump_status); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "parser dump status failed"); + GELOGE(PARAM_INVALID, "[Parser][DumpStatus] failed, ret:%d", ret); return FAILED; } GELOGI("dump status = %s.", dump_status.c_str()); ret = ParserPara(command, DUMP_MODEL, dump_model); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "parser dump model failed"); + GELOGE(PARAM_INVALID, "[Parser][DumpModel] failed, ret:%d", ret); return FAILED; } GELOGI("dump model = %s.", dump_model.c_str()); @@ -979,7 +977,7 @@ Status ModelManager::HandleDumpCommand(const Command &command) { ret = ParserPara(command, DUMP_FILE_PATH, dump_path); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "parser dump path failed"); + GELOGE(PARAM_INVALID, "[Parser][DumpPath] failed, ret:%d", ret); return FAILED; } if (!dump_path.empty() && dump_path[dump_path.size() - 1] != '/') { @@ -990,7 +988,7 @@ Status ModelManager::HandleDumpCommand(const Command &command) { ret = ParserPara(command, DUMP_MODE, dump_mode); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "parser dump mode failed"); + GELOGE(PARAM_INVALID, "[Parser][DumpMode] failed, ret:%d", ret); return FAILED; } GELOGI("dump mode = %s", dump_mode.c_str()); @@ -1010,8 +1008,8 @@ Status ModelManager::GetMaxUsedMemory(const uint32_t model_id, uint64_t &max_siz } std::shared_ptr davinci_model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, "GetMaxUsedMemory Failed, Invalid model id %u!", - model_id); + GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id:%u!", model_id); max_size = davinci_model->TotalMemSize(); return SUCCESS; @@ -1020,8 +1018,8 @@ Status ModelManager::GetMaxUsedMemory(const uint32_t model_id, uint64_t &max_siz Status ModelManager::GetInputOutputDescInfo(const uint32_t model_id, vector &input_desc, vector &output_desc) { std::shared_ptr davinci_model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, "GetInputOutputDescInfo Failed, Invalid model id %u!", - model_id); + GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id %u!", model_id); return davinci_model->GetInputOutputDescInfo(input_desc, output_desc); } @@ -1038,7 +1036,7 @@ Status ModelManager::GetInputOutputDescInfo(const uint32_t model_id, vector davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetInputOutputDescInfo Failed, Invalid model id %u!", model_id); + "[Get][Model] Failed, Invalid model id %u!", model_id); return davinci_model->GetInputOutputDescInfo(input_desc, output_desc, inputFormats, outputFormats, new_model_desc); } @@ -1059,7 +1057,7 @@ Status ModelManager::GetDynamicBatchInfo(const uint32_t model_id, std::vector davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetDynamicBatchInfo failed, Invalid model id %u!", model_id); + "[Get][Model] failed, Invalid model id %u!", model_id); return davinci_model->GetDynamicBatchInfo(batch_info, dynamic_type); } @@ -1074,7 +1072,7 @@ Status ModelManager::GetDynamicBatchInfo(const uint32_t model_id, std::vector> &batch_info) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetCombinedDynamicDims Failed, Invalid Model ID %u!", model_id); + "[Get][Model] Failed, Invalid Model ID %u!", model_id); davinci_model->GetCombinedDynamicDims(batch_info); return SUCCESS; @@ -1097,7 +1095,7 @@ Status ModelManager::GetUserDesignateShapeOrder(const uint32_t model_id, auto davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetUserDesignateShapeOrder Failed, Invalid Model ID %u!", model_id) + "[Get][Model] Failed, Invalid Model ID %u!", model_id) davinci_model->GetUserDesignateShapeOrder(user_input_shape_order); return SUCCESS; } @@ -1105,7 +1103,7 @@ Status ModelManager::GetUserDesignateShapeOrder(const uint32_t model_id, Status ModelManager::GetCurShape(const uint32_t model_id, std::vector &batch_info, int32_t &dynamic_type) { auto davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetCurShape Failed, Invalid Model ID %u!", model_id); + "[Get][Model] Failed, Invalid Model ID %u!", model_id); davinci_model->GetCurShape(batch_info, dynamic_type); return SUCCESS; } @@ -1134,7 +1132,7 @@ Status ModelManager::GetModelAttr(uint32_t model_id, std::vector &dynami std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetModelAttr Failed, Invalid Model ID %u!", model_id); + "[Get][Model] Failed, Invalid Model ID %u!", model_id); davinci_model->GetModelAttr(dynamic_output_shape_info); return SUCCESS; } @@ -1150,14 +1148,14 @@ Status ModelManager::GetModelAttr(uint32_t model_id, std::vector &dynami Status ModelManager::GetAippInfo(const uint32_t model_id, uint32_t index, AippConfigInfo &aipp_info) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetAIPPInfo failed, invalid model_id is %u.", model_id); + "[Get][Model] failed, invalid model_id is %u.", model_id); return davinci_model->GetAippInfo(index, aipp_info); } Status ModelManager::GetAippType(uint32_t model_id, uint32_t index, InputAippType &type, size_t &aipp_index) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetAIPPInfo failed, invalid model_id is %u.", model_id); + "[Get][Model] failed, invalid model_id is %u.", model_id); return davinci_model->GetAippType(index, type, aipp_index); } @@ -1173,7 +1171,7 @@ Status ModelManager::GenSessionId(uint64_t &session_id) { mmTimeval tv; if (mmGetTimeOfDay(&tv, nullptr) != 0) { REPORT_CALL_ERROR("E19999", "Call mmGetTimeOfDay fail. errmsg:%s", strerror(errno)); - GELOGE(INTERNAL_ERROR, "Failed to get current time."); + GELOGE(INTERNAL_ERROR, "[Call][MmGetTimeOfDay] fail. errmsg:%s", strerror(errno)); return INTERNAL_ERROR; } uint64_t timestamp = static_cast(tv.tv_sec * kTimeSpecMiro + tv.tv_usec); // 1000000us @@ -1192,7 +1190,8 @@ Status ModelManager::GenSessionId(uint64_t &session_id) { Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model, shared_ptr listener, void *dev_ptr, size_t mem_size, void *weight_ptr, size_t weight_size) { GE_CHK_BOOL_RET_STATUS(model.key.empty() || mmAccess2(model.key.c_str(), M_F_OK) == EN_OK, - ACL_ERROR_GE_PARAM_INVALID, "Input key file path %s is invalid, %s", model.key.c_str(), strerror(errno)); + ACL_ERROR_GE_PARAM_INVALID, + "[Check][Param] Input key file path %s is invalid, %s", model.key.c_str(), strerror(errno)); GenModelId(&model_id); mmTimespec timespec = mmGetTickCount(); @@ -1200,14 +1199,14 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model ModelHelper model_helper; Status ret = model_helper.LoadRootModel(model); if (ret != SUCCESS) { - GELOGE(ret, "load model failed."); + GELOGE(ret, "[Load][RootModel] failed, ret:%d, model_id:%u.", ret, model_id); return ret; } if (model_helper.GetModelType()) { bool is_shape_unknown = false; GE_CHK_STATUS_RET(model_helper.GetGeRootModel()->CheckIsUnknownShape(is_shape_unknown), - "CheckIsUnknownShape failed, model id:%u", model_id); + "[Check][IsUnknownShape] failed, model id:%u", model_id); if (is_shape_unknown || GetContext().GetHostExecFlag()) { return DoLoadHybridModelOnline(model_id, model.om_name, model_helper.GetGeRootModel(), listener); } @@ -1218,7 +1217,7 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model shared_ptr davinci_model = MakeShared(model.priority, listener); if (davinci_model == nullptr) { REPORT_CALL_ERROR("E19999", "New DavinciModel fail"); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Make shared failed"); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[New][DavinciModel] fail"); return ACL_ERROR_GE_MEMORY_ALLOCATION; } davinci_model->SetProfileTime(MODEL_LOAD_START, (timespec.tv_sec * kTimeSpecNano + @@ -1234,7 +1233,7 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model rtError_t rt_ret = rtGetDevice(&device_id); if (rt_ret != RT_ERROR_NONE || device_id < 0) { REPORT_CALL_ERROR("E19999", "Call rtGetDevice failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "Call rtGetDevice failed, ret = 0x%X, device_id = %d.", rt_ret, device_id); + GELOGE(rt_ret, "[Call][RtGetDevice] failed, ret = 0x%X, device_id = %d.", rt_ret, device_id); return RT_ERROR_TO_GE_STATUS(rt_ret); } davinci_model->SetDeviceId(device_id); @@ -1250,12 +1249,13 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model /// Update session_id for infer in load model to avoid the same session_id. uint64_t new_session_id; ret = GenSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Generate session_id for inference failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "[Generate][SessionId] for inference failed, ret:%d.", ret); ret = davinci_model->UpdateSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Update session_id for inference failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, + "[Update][SessionId] for inference failed, session id:%lu.", new_session_id); ret = davinci_model->Init(dev_ptr, mem_size, weight_ptr, weight_size); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "DavinciInit failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "[Init][DavinciModel] failed, ret:%d.", ret); InsertModel(model_id, davinci_model); @@ -1280,26 +1280,27 @@ Status ModelManager::LoadModelWithQ(uint32_t &model_id, const ModelData &model_d const std::vector &input_queue_ids, const std::vector &output_queue_ids) { GE_CHK_BOOL_RET_STATUS(model_data.key.empty() || mmAccess2(model_data.key.c_str(), M_F_OK) == EN_OK, - ACL_ERROR_GE_PARAM_INVALID, "input key file path %s is not valid, %s", + ACL_ERROR_GE_PARAM_INVALID, + "[Check][Param] input key file path %s is not valid, %s", model_data.key.c_str(), strerror(errno)); ModelHelper model_helper; Status ret = model_helper.LoadModel(model_data); if (ret != SUCCESS) { - GELOGE(ret, "load model failed."); + GELOGE(ret, "[Load][Model] failed."); return ret; } shared_ptr davinci_model = MakeShared(model_data.priority, nullptr); if (davinci_model == nullptr) { REPORT_CALL_ERROR("E19999", "New DavinciModel fail"); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "create model failed."); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Create][Model] failed."); return ACL_ERROR_GE_MEMORY_ALLOCATION; } ret = davinci_model->Assign(model_helper.GetGeModel()); if (ret != SUCCESS) { - GELOGE(ret, "assign model failed."); + GELOGE(ret, "[Assign][Model] failed, ret:%d.", ret); return ret; } @@ -1308,15 +1309,17 @@ Status ModelManager::LoadModelWithQ(uint32_t &model_id, const ModelData &model_d /// Update session_id for infer in load model to avoid the same session_id. uint64_t new_session_id; ret = GenSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Generate session_id for infer failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, + "[Generate][SessionId] for infer failed, ret:%d.", ret); ret = davinci_model->UpdateSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Update session_id for infer failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, + "[Update][SessionId] for infer failed, SessionId:%lu.", new_session_id); GenModelId(&model_id); davinci_model->SetId(model_id); ret = davinci_model->SetQueIds(input_queue_ids, output_queue_ids); if (ret != SUCCESS) { - GELOGE(ret, "set model queue ids failed."); + GELOGE(ret, "[Set][Ids] for model queue failed, ret:%d, model_id:%u.", ret, model_id); return ret; } @@ -1324,7 +1327,7 @@ Status ModelManager::LoadModelWithQ(uint32_t &model_id, const ModelData &model_d ret = davinci_model->Init(); if (ret != SUCCESS) { - GELOGE(ret, "init model failed."); + GELOGE(ret, "[Init][Model] failed, ret:%d, model_id:%u.", ret, model_id); return ret; } @@ -1362,7 +1365,7 @@ Status ModelManager::ExecuteModel(uint32_t model_id, rtStream_t stream, bool asy std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "Invalid model id %u, check whether model has been loaded or not.", model_id); + "[Get][Model] Invalid model id %u, check whether model has been loaded or not.", model_id); if (davinci_model->NeedDestroyAicpuKernel()) { GELOGI("Start to destroy specified aicpu kernel."); @@ -1413,9 +1416,8 @@ Status ModelManager::LoadCustAicpuSo(const OpDescPtr &op_desc, const string &so_ rtContext_t rt_cur_ctx = nullptr; auto rt_error = rtCtxGetCurrent(&rt_cur_ctx); if (rt_error != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X", - rt_error); - GELOGE(RT_FAILED, "get current context failed, runtime result is %d", static_cast(rt_error)); + REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X", rt_error); + GELOGE(RT_FAILED, "[Call][RtCtxGetCurrent] failed, runtime result is %d", static_cast(rt_error)); return RT_FAILED; } @@ -1450,9 +1452,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { rtContext_t rt_cur_ctx = nullptr; auto rt_error = rtCtxGetCurrent(&rt_cur_ctx); if (rt_error != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X", - rt_error); - GELOGE(RT_FAILED, "get current context failed, runtime result is %d", static_cast(rt_error)); + REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X", rt_error); + GELOGE(RT_FAILED, "[Call][RtCtxGetCurrent] failed, runtime result is %d", static_cast(rt_error)); return RT_FAILED; } uintptr_t resource_id = reinterpret_cast(rt_cur_ctx); @@ -1477,17 +1478,15 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { status = rtMalloc(&d_aicpu_data, aicpu_data_length, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret = 0x%X", - aicpu_data_length, status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret = 0x%X", aicpu_data_length, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret = 0x%X", aicpu_data_length, status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_aicpu_data); status = rtMalloc(&d_so_name, so_name.size(), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", - so_name.size(), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", so_name.size(), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%zu, ret = 0x%X", so_name.size(), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_so_name); @@ -1509,9 +1508,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { uint32_t args_size = sizeof(CustAicpuSoBuf) * v_cust_so.size(); status = rtMalloc(&args, args_size, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", - args_size, status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", args_size, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%u, ret = 0x%X", args_size, status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(args); @@ -1525,9 +1523,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { uint32_t batch_args_size = sizeof(BatchLoadOpFromBufArgs); status = rtMalloc(&batch_args, batch_args_size, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", - batch_args_size, status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", batch_args_size, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%u, ret = 0x%X", batch_args_size, status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(batch_args); @@ -1539,9 +1536,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { status = rtStreamSynchronize(stream); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X", - status); - GELOGE(RT_FAILED, "Call rt stream sync failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X", status); + GELOGE(RT_FAILED, "[Call][RtStreamSynchronize] fail, ret = 0x%X", status); return RT_ERROR_TO_GE_STATUS(status); } std::function callback = [&]() { @@ -1556,12 +1552,14 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { } Status ModelManager::ClearAicpuSo() { - GE_CHK_STATUS_RET(LaunchKernelCustAicpuSo(kDeleteCustOp), "delete cust op so failed."); + GE_CHK_STATUS_RET(LaunchKernelCustAicpuSo(kDeleteCustOp), + "[Call][LaunchKernelCustAicpuSo] delete cust op so failed."); return SUCCESS; } Status ModelManager::LaunchCustAicpuSo() { - GE_CHK_STATUS_RET(LaunchKernelCustAicpuSo(kBatchLoadBuf), "launch cust op so failed."); + GE_CHK_STATUS_RET(LaunchKernelCustAicpuSo(kBatchLoadBuf), + "[Call][LaunchKernelCustAicpuSo] launch cust op so failed."); return SUCCESS; } @@ -1577,21 +1575,21 @@ Status ModelManager::GetModelMemAndWeightSize(const ModelData &model, size_t &me uint8_t *model_data = nullptr; uint32_t model_len = 0; Status ret = ModelParserBase::ParseModelContent(model, model_data, model_len); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ACL_ERROR_GE_PARAM_INVALID, "parse model content failed!"); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ACL_ERROR_GE_PARAM_INVALID, "[Parse][ModelContent] failed!"); OmFileLoadHelper om_file_helper; ret = om_file_helper.Init(model_data, model_len); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "om file helperInit failed!"); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "[Init][OmFileHelper] failed, ret:%d", ret); auto partition_table = reinterpret_cast(model_data); if (partition_table->num == 1) { REPORT_INNER_ERROR("E19999", "partition_table num in model_data is 1, check invalid"); - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "om model is error,please use executable om model"); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Param] om model is error, please use executable om model"); return ACL_ERROR_GE_PARAM_INVALID; } ModelPartition task_partition; if (om_file_helper.GetModelPartition(ModelPartitionType::TASK_INFO, task_partition) != SUCCESS) { - GELOGE(ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED, "get task model partition failed."); + GELOGE(ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED, "[Get][ModelPartition] failed."); return ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED; } @@ -1601,7 +1599,7 @@ Status ModelManager::GetModelMemAndWeightSize(const ModelData &model, size_t &me } if (task_partition.size != 0) { if (!ReadProtoFromArray(task_partition.data, static_cast(task_partition.size), model_task_def.get())) { - GELOGE(ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED, "ReadProtoFromArray failed."); + GELOGE(ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED, "[Read][Proto] From Array failed."); return ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED; } } @@ -1609,7 +1607,7 @@ Status ModelManager::GetModelMemAndWeightSize(const ModelData &model, size_t &me ModelPartition partition_weight; ret = om_file_helper.GetModelPartition(ModelPartitionType::WEIGHTS_DATA, partition_weight); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ACL_ERROR_GE_EXEC_LOAD_WEIGHT_PARTITION_FAILED, - "Get weight partition failed. ret = %u", ret); + "[Get][ModelPartition] failed. ret = %u", ret); mem_size = model_task_def->memory_size(); weight_size = partition_weight.size; @@ -1627,8 +1625,7 @@ void ModelManager::GenModelId(uint32_t *id) { Status ModelManager::GetOrigInputInfo(uint32_t model_id, uint32_t index, OriginInputInfo &orig_input_info) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetOrigInputInfo failed, invalid model_id is %u.", - model_id); + "[Get][Model] failed, invalid model_id is %u.", model_id); return davinci_model->GetOrigInputInfo(index, orig_input_info); } @@ -1638,7 +1635,7 @@ Status ModelManager::GetAllAippInputOutputDims(uint32_t model_id, uint32_t index std::vector &output_dims) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetAllAippInputOutputDims failed, invalid model_id is %u.", model_id); + "[Get][Model] failed, invalid model_id is %u.", model_id); return davinci_model->GetAllAippInputOutputDims(index, input_dims, output_dims); } @@ -1653,7 +1650,7 @@ ge::Status ModelManager::SyncExecuteModel(uint32_t model_id, const vector &options dump_exception_flag_ = true; rtError_t rt_ret = rtSetTaskFailCallback(reinterpret_cast(ExceptionCallback)); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtSetTaskFailCallback fail, ret = 0x%X", - rt_ret); - GELOGE(RT_FAILED, "rtSetTaskFailCallback failed"); + REPORT_CALL_ERROR("E19999", "Call rtSetTaskFailCallback fail, ret = 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtSetTaskFailCallback] fail, ret = 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } else { @@ -1743,9 +1739,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op // malloc sysOpInfoList in SysOpCheckInfo status = rtMalloc(&d_req_op_list, op_nums * sizeof(SysOpInfo), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", - op_nums * sizeof(SysOpInfo), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", op_nums * sizeof(SysOpInfo), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%zu, ret = 0x%X", op_nums * sizeof(SysOpInfo), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_req_op_list); @@ -1753,9 +1748,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op // malloc sysOpInfoList in SysOpCheckResp status = rtMalloc(&d_res_op_list, op_nums * sizeof(SysOpInfo), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", - op_nums * sizeof(SysOpInfo), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", op_nums * sizeof(SysOpInfo), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%zu, ret = 0x%X", op_nums * sizeof(SysOpInfo), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_res_op_list); @@ -1763,9 +1757,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op // malloc returnCodeList in SysOpCheckResp status = rtMalloc(&d_ret_code_list, op_nums * sizeof(ReturnCode), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", - op_nums * sizeof(ReturnCode), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", op_nums * sizeof(ReturnCode), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%zu, ret = 0x%X", op_nums * sizeof(ReturnCode), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_ret_code_list); @@ -1776,9 +1769,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op void *d_op_type_name = nullptr; status = rtMalloc(&d_op_type_name, op_type.length(), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X", - op_type.length(), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X", op_type.length(), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%lu, ret = 0x%X", op_type.length(), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_op_type_name); @@ -1795,9 +1787,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op void *d_op_type_name = nullptr; status = rtMalloc(&d_op_type_name, op_type.size(), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X", - op_type.length(), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X", op_type.length(), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%lu, ret = 0x%X", op_type.size(), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_op_type_name); @@ -1825,9 +1816,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op uint32_t args_size = sizeof(SysOpCheckInfo) + sizeof(SysOpCheckResp); status = rtMalloc(&args, args_size, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", - args_size, status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", args_size, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%u, ret = 0x%X", args_size, status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(args); @@ -1842,9 +1832,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op status = rtStreamSynchronize(stream); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X", - status); - GELOGE(RT_FAILED, "Call rt stream sync failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X", status); + GELOGE(RT_FAILED, "[Call][RtStreamSynchronize] failed, ret:0x%X", status); GE_CHK_RT(rtStreamDestroy(stream)); return RT_ERROR_TO_GE_STATUS(status); } @@ -1879,7 +1868,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op REPORT_INNER_ERROR("E19999", "res_ret_code_list.size:%zu res_aicpu_op_info_list.size:%zu res_op_nums:%lu " "not equal, check invalid", res_ret_code_list.size(), res_aicpu_op_info_list.size(), res_op_nums); - GELOGE(FAILED, "Number of retcode is not equal to number of op type."); + GELOGE(FAILED, "[Check][Param] Number:%zu of retcode is not equal to number:%zu of op type or not equal %lu.", + res_ret_code_list.size(), res_aicpu_op_info_list.size(), res_op_nums); GE_CHK_RT(rtStreamDestroy(stream)); return FAILED; } @@ -1902,9 +1892,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op "<0: op_type, 1: format, 2: datatype> \n"; } fail_reason += "not support."; - REPORT_INNER_ERROR("E19999", "Check aicpu op_type failed, details:%s", - fail_reason.c_str()); - GELOGE(FAILED, "Check aicpu op_type failed. details: %s", fail_reason.c_str()); + REPORT_INNER_ERROR("E19999", "Check aicpu op_type failed, details:%s", fail_reason.c_str()); + GELOGE(FAILED, "[Check][Param] Check aicpu op_type failed. details:%s", fail_reason.c_str()); GE_CHK_RT(rtStreamDestroy(stream)); return FAILED; } @@ -1924,7 +1913,7 @@ Status ModelManager::CheckAicpuOpList(GeModelPtr ge_model) { return SUCCESS; } GE_CHK_STATUS_RET(LaunchKernelCheckAicpuOp(aicpu_optype_list, aicpu_tf_optype_list), - "Launch check aicpu op type failed."); + "[Call][LaunchKernelCheckAicpuOp] failed."); return SUCCESS; } diff --git a/ge/graph/load/model_manager/model_manager.h b/ge/graph/load/model_manager/model_manager.h index c0f14934..efba54ec 100755 --- a/ge/graph/load/model_manager/model_manager.h +++ b/ge/graph/load/model_manager/model_manager.h @@ -310,7 +310,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager { std::lock_guard lock(exeception_infos_mutex_); auto instance = ModelManager::GetInstance(); if (instance == nullptr) { - GELOGE(FAILED, "Instance is nullptr"); + GELOGE(FAILED, "[Get][Instance] failed, as ret is nullptr"); return; } instance->AddExceptionInfo(*rt_exception_info); diff --git a/ge/graph/load/model_manager/model_utils.cc b/ge/graph/load/model_manager/model_utils.cc index f6ff591a..f593f67b 100755 --- a/ge/graph/load/model_manager/model_utils.cc +++ b/ge/graph/load/model_manager/model_utils.cc @@ -26,10 +26,10 @@ #define VALIDATE_MEM_RANGE(OP, SIZE, OFFSET) \ do { \ if (SIZE <= static_cast(OFFSET)) { \ - REPORT_INNER_ERROR("E19999", \ - "Node:%s(%s) offset:%ld out of range size:%lu, check invalid", \ + REPORT_INNER_ERROR("E19999", "Node:%s(%s) offset:%ld out of range size:%lu, check invalid", \ OP->GetName().c_str(), OP->GetType().c_str(), OFFSET, SIZE); \ - GELOGE(OUT_OF_MEMORY, "Node: %s, memory out of range[%lu: %ld]", OP->GetName().c_str(), SIZE, OFFSET); \ + GELOGE(OUT_OF_MEMORY, "[Check][Param]Node: %s, memory out of range[%lu: %ld]", \ + OP->GetName().c_str(), SIZE, OFFSET); \ return {}; \ } \ } while (0) @@ -312,8 +312,9 @@ vector ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co REPORT_INNER_ERROR("E19999", "Attr:%s, memory_type.size:%zu != input_desc.size:%zu, op:%s(%s), check invalid", ATTR_NAME_INPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), inputs_size, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "Fusion: check input size failed, op: %s, input v_memory_type size: %zu input numbers: %zu", - op_desc->GetName().c_str(), v_memory_type.size(), inputs_size); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s, memory_type.size:%zu != input_desc.size:%zu, op:%s(%s)", + ATTR_NAME_INPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), inputs_size, + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return v_input_data_addr; } for (size_t i = 0; i < op_desc->GetAllInputsSize(); ++i) { @@ -392,8 +393,7 @@ Status ModelUtils::GetVarAddr(const RuntimeParam &model_param, const ConstOpDesc case RT_MEMORY_RDMA_HBM: if (offset < 0) { REPORT_INNER_ERROR("E19999", "Param offset:%ld < 0, check invalid", offset); - GELOGE(PARAM_INVALID, "rdma var addr is invalid, addr=%p", - reinterpret_cast(static_cast(offset))); + GELOGE(PARAM_INVALID, "[Check][Param] Param offset:%ld cannot be negative", offset); return PARAM_INVALID; } var_addr = reinterpret_cast(static_cast(offset)); @@ -403,9 +403,9 @@ Status ModelUtils::GetVarAddr(const RuntimeParam &model_param, const ConstOpDesc var_addr = model_param.var_base + offset - model_param.logic_var_base; break; default: - REPORT_INNER_ERROR("E19999", "Get mem_type:%d for offset:%ld is unsupported, check invalid", - mem_type, offset); - GELOGE(PARAM_INVALID, "unsupported memory type %u", mem_type); + REPORT_INNER_ERROR("E19999", "Get mem_type:%d for offset:%ld is unsupported, check invalid", mem_type, offset); + GELOGE(PARAM_INVALID, "[Check][Param] Get mem_type:%d for offset:%ld is unsupported, check invalid", + mem_type, offset); return PARAM_INVALID; } GE_CHECK_NOTNULL(var_addr); @@ -433,9 +433,9 @@ vector ModelUtils::GetOutputDataAddrs(const RuntimeParam &model_param, C REPORT_INNER_ERROR("E19999", "Attr:%s, memory_type.size:%zu != output_desc.size:%zu, op:%s(%s), check invalid", ATTR_NAME_OUTPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), outputs_size, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, - "Fusion: check output size failed, op: %s, output v_memory_type size: %lu output numbers: %zu", - op_desc->GetName().c_str(), v_memory_type.size(), outputs_size); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s, memory_type.size:%zu != output_desc.size:%zu, op:%s(%s)", + ATTR_NAME_OUTPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), outputs_size, + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return v_output_data_addr; } for (size_t i = 0; i < outputs_size; ++i) { @@ -594,7 +594,7 @@ Status ModelUtils::GetRtAddress(const RuntimeParam ¶m, uintptr_t logic_addr, } else if (logic_addr != 0) { mem_addr = nullptr; REPORT_INNER_ERROR("E19999", "Check param logic addr:0x%lx abnormal", logic_addr); - GELOGE(PARAM_INVALID, "The logic addr:0x%lx is abnormal", logic_addr); + GELOGE(PARAM_INVALID, "[Check][Param] The logic addr:0x%lx is abnormal", logic_addr); return PARAM_INVALID; } diff --git a/ge/graph/load/model_manager/tbe_handle_store.cc b/ge/graph/load/model_manager/tbe_handle_store.cc index 6efb6190..36207aa2 100755 --- a/ge/graph/load/model_manager/tbe_handle_store.cc +++ b/ge/graph/load/model_manager/tbe_handle_store.cc @@ -24,7 +24,7 @@ namespace ge { void TbeHandleInfo::used_inc(uint32_t num) { if (used_ > std::numeric_limits::max() - num) { REPORT_INNER_ERROR("E19999", "Used:%u reach numeric max", used_); - GELOGE(INTERNAL_ERROR, "Used[%u] reach numeric max.", used_); + GELOGE(INTERNAL_ERROR, "[Check][Param] Used[%u] reach numeric max.", used_); return; } @@ -34,7 +34,7 @@ void TbeHandleInfo::used_inc(uint32_t num) { void TbeHandleInfo::used_dec(uint32_t num) { if (used_ < std::numeric_limits::min() + num) { REPORT_INNER_ERROR("E19999", "Used:%u reach numeric min", used_); - GELOGE(INTERNAL_ERROR, "Used[%u] reach numeric min.", used_); + GELOGE(INTERNAL_ERROR, "[Check][Param] Used[%u] reach numeric min.", used_); return; } @@ -107,9 +107,8 @@ void TBEHandleStore::ReferTBEHandle(const std::string &name) { std::lock_guard lock(mutex_); auto it = kernels_.find(name); if (it == kernels_.end()) { - REPORT_INNER_ERROR("E19999", "Kernel:%s not found in stored check invalid", - name.c_str()); - GELOGE(INTERNAL_ERROR, "Kernel[%s] not found in stored.", name.c_str()); + REPORT_INNER_ERROR("E19999", "Kernel:%s not found in stored check invalid", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Kernel[%s] not found in stored.", name.c_str()); return; } @@ -128,9 +127,8 @@ void TBEHandleStore::EraseTBEHandle(const std::map &names for (auto &item : names) { auto it = kernels_.find(item.first); if (it == kernels_.end()) { - REPORT_INNER_ERROR("E19999", "Kernel:%s not found in stored check invalid", - item.first.c_str()); - GELOGE(INTERNAL_ERROR, "Kernel[%s] not found in stored.", item.first.c_str()); + REPORT_INNER_ERROR("E19999", "Kernel:%s not found in stored check invalid", item.first.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Kernel[%s] not found in stored.", item.first.c_str()); continue; } @@ -142,7 +140,8 @@ void TBEHandleStore::EraseTBEHandle(const std::map &names if (rt_ret != RT_ERROR_NONE) { REPORT_INNER_ERROR("E19999", "Call rtDevBinaryUnRegister failed for Kernel:%s fail, ret:0x%X", item.first.c_str(), rt_ret); - GELOGE(INTERNAL_ERROR, "Kernel[%s] UnRegister handle fail:%u.", item.first.c_str(), rt_ret); + GELOGE(INTERNAL_ERROR, "[Call][RtDevBinaryUnRegister] Kernel[%s] UnRegister handle fail:%u.", + item.first.c_str(), rt_ret); } kernels_.erase(it); } diff --git a/ge/graph/load/model_manager/ts_mem_mall.h b/ge/graph/load/model_manager/ts_mem_mall.h index 74ce5a16..986b3101 100644 --- a/ge/graph/load/model_manager/ts_mem_mall.h +++ b/ge/graph/load/model_manager/ts_mem_mall.h @@ -43,7 +43,7 @@ class TsMemMall { for (auto it : mem_store_size_) { rtError_t ret = rtFree(it.second); if (ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtFree failed, ret: 0x%X", ret); + GELOGE(RT_FAILED, "[Call][RtFree] failed, ret:0x%X", ret); } } mem_store_size_.clear(); @@ -52,7 +52,7 @@ class TsMemMall { void *Acquire(int64_t offset, uint64_t size) { if (size == 0) { - GELOGE(RT_FAILED, "Acquire mem block failed, size: %lu", size); + GELOGE(RT_FAILED, "[Check][Param] Acquire mem block failed, size:%lu", size); return nullptr; } @@ -71,7 +71,7 @@ class TsMemMall { void *addr = nullptr; rtError_t rt_ret = rtMalloc(&addr, bytes, mem_type_); if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%lu, ret:0x%X", bytes, rt_ret); return nullptr; } @@ -94,7 +94,7 @@ class TsMemMall { mem_store_addr_.erase(it); rtError_t ret = rtFree(addr); if (ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtFree failed, ret: 0x%X", ret); + GELOGE(RT_FAILED, "[Call][RtFree] failed, ret:0x%X", ret); } } diff --git a/ge/graph/load/model_manager/zero_copy_offset.cc b/ge/graph/load/model_manager/zero_copy_offset.cc index 9d6f4e4f..4a57a899 100644 --- a/ge/graph/load/model_manager/zero_copy_offset.cc +++ b/ge/graph/load/model_manager/zero_copy_offset.cc @@ -38,8 +38,13 @@ Status ZeroCopyOffset::InitInputDataInfo(int64_t output_size, void *virtual_addr op_name_ = op_desc->GetName(); (void)ge::AttrUtils::GetListInt(op_desc, ATTR_ZERO_COPY_BASIC_OFFSET, zero_copy_basic_offset_); (void)ge::AttrUtils::GetListInt(op_desc, ATTR_ZERO_COPY_RELATIVE_OFFSET, zero_copy_relative_offset_); - GE_CHK_BOOL_EXEC(zero_copy_basic_offset_.size() == zero_copy_relative_offset_.size(), return PARAM_INVALID, - "basic_offset_size should be equal to relative_offset_size"); + GE_CHK_BOOL_EXEC(zero_copy_basic_offset_.size() == zero_copy_relative_offset_.size(), + REPORT_INNER_ERROR("E19999", "basic_offset_size:%zu not equal to relative_offset_size:%zu, " + "check invalid", zero_copy_basic_offset_.size(), + zero_copy_relative_offset_.size()); + return PARAM_INVALID, + "[Check][Param] basic_offset_size:%zu should be equal to relative_offset_size:%zu", + zero_copy_basic_offset_.size(), zero_copy_relative_offset_.size()); GELOGD("[ZCPY] zero_copy_basic_offset size is %zu", zero_copy_basic_offset_.size()); int64_t virtual_addr_offset = op_desc->GetOutputOffset().at(kDataIndex); @@ -78,7 +83,8 @@ Status ZeroCopyOffset::InitOutputDataInfo(const vector &input_size_list if (TensorUtils::GetTensorSizeInBytes(*tensor_desc, size) != GRAPH_SUCCESS) { REPORT_INNER_ERROR("E19999", "Get input TensorSize in op:%s(%s) failed, input_index:%zu", op_desc->GetName().c_str(), op_desc->GetType().c_str(), idx); - GELOGE(FAILED, "GetTensorSizeInBytes failed!"); + GELOGE(FAILED, "[Get][InputTensorSize] in op:%s(%s) failed, input_index:%zu", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), idx); return FAILED; } @@ -88,8 +94,13 @@ Status ZeroCopyOffset::InitOutputDataInfo(const vector &input_size_list op_name_ = op_desc->GetName(); (void)ge::AttrUtils::GetListInt(op_desc, ATTR_ZERO_COPY_BASIC_OFFSET, zero_copy_basic_offset_); (void)ge::AttrUtils::GetListInt(op_desc, ATTR_ZERO_COPY_RELATIVE_OFFSET, zero_copy_relative_offset_); - GE_CHK_BOOL_EXEC(zero_copy_basic_offset_.size() == zero_copy_relative_offset_.size(), return PARAM_INVALID, - "basic_offset_size should be equal to relative_offset_size"); + GE_CHK_BOOL_EXEC(zero_copy_basic_offset_.size() == zero_copy_relative_offset_.size(), + REPORT_INNER_ERROR("E19999", "basic_offset_size:%zu not equal to relative_offset_size:%zu, " + "check invalid", + zero_copy_basic_offset_.size(), zero_copy_relative_offset_.size()); + return PARAM_INVALID, + "[Check][Param] basic_offset_size:%zu should be equal to relative_offset_size:%zu", + zero_copy_basic_offset_.size(), zero_copy_relative_offset_.size()); int64_t virtual_addr_offset = op_desc->GetInputOffset().at(idx); IsL2Fusion(zero_copy_basic_offset_, virtual_addr_offset, fusion_flag); @@ -194,7 +205,8 @@ void ZeroCopyOffset::SetOutsideAddrsValue(ZeroCopyTask &zero_copy_task, void *ou for (uint32_t out_count = 0; out_count < GetAddrCount(); ++out_count) { auto args_addrs = outside_addrs_[out_count].find(outside_addr); if (args_addrs != outside_addrs_[out_count].end()) { - GE_CHK_STATUS(zero_copy_task.SetTaskArgsOffset(addr_val, offset), "Input args invalid."); + GE_CHK_STATUS(zero_copy_task.SetTaskArgsOffset(addr_val, offset), + "[Set][TaskArgsOffset] failed, Input args invalid, offset:%zu.", offset); void *args_val = static_cast(args) + offset; args_addrs->second.push_back(args_val); GELOGD("[ZCPY] set copy input: virtual_addr: 0x%lx, task_addr: %p, args: %p, offset: %zu.", addr_val, args_val, diff --git a/ge/graph/load/model_manager/zero_copy_task.cc b/ge/graph/load/model_manager/zero_copy_task.cc index c96dd8b7..4957f8ea 100755 --- a/ge/graph/load/model_manager/zero_copy_task.cc +++ b/ge/graph/load/model_manager/zero_copy_task.cc @@ -36,9 +36,9 @@ ZeroCopyTask::~ZeroCopyTask() { args_addr_ = nullptr; } */ Status ZeroCopyTask::SetTaskArgsOffset(uintptr_t addr, size_t offset) { if (offset + sizeof(uintptr_t) > args_size_) { - REPORT_INNER_ERROR("E19999", "Param offset:%zu + 8 > args_size_:%zu, check invalid", - offset, args_size_); - GELOGE(FAILED, "[ZCPY] %s set task args failed, args size: %zu, offset: %zu", name_.c_str(), args_size_, offset); + REPORT_INNER_ERROR("E19999", "Param offset:%zu + 8 > args_size_:%zu, check invalid", offset, args_size_); + GELOGE(FAILED, "[Check][Param] [ZCPY] %s set task args failed, args size:%zu, offset:%zu", + name_.c_str(), args_size_, offset); return FAILED; // unexpected error, need fix. } @@ -118,9 +118,8 @@ Status ZeroCopyTask::DistributeParam(bool async_mode, rtStream_t stream) { } if (rt_err != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpyAsync or rtMemcpy failed, size:%zu, ret: 0x%X", - args_size_, rt_err); - GELOGE(RT_FAILED, "[ZCPY] %s distribute task param failed, error=0x%x", name_.c_str(), rt_err); + REPORT_CALL_ERROR("E19999", "Call rtMemcpyAsync or rtMemcpy failed, size:%zu, ret:0x%X", args_size_, rt_err); + GELOGE(RT_FAILED, "[Distribute][TaskParam] for %s failed, error = 0x%x", name_.c_str(), rt_err); return RT_ERROR_TO_GE_STATUS(rt_err); } diff --git a/ge/graph/manager/graph_caching_allocator.cc b/ge/graph/manager/graph_caching_allocator.cc index 75aa5c01..cdb1d131 100644 --- a/ge/graph/manager/graph_caching_allocator.cc +++ b/ge/graph/manager/graph_caching_allocator.cc @@ -112,7 +112,7 @@ Status CachingAllocator::Initialize(uint32_t device_id) { auto bin_ptr = new (std::nothrow) BlockBin(BlockComparator); if (bin_ptr == nullptr) { REPORT_CALL_ERROR("E19999", "New BlockBin fail, device_id:%u", device_id); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc BlockBin failed."); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Alloc][BlockBin] failed, device_id:%u", device_id); return ACL_ERROR_GE_MEMORY_ALLOCATION; } free_block_bins_[i] = bin_ptr; @@ -147,9 +147,8 @@ uint8_t *CachingAllocator::Malloc(size_t size, uint8_t *org_ptr, uint32_t device ptr = block->ptr; } if (ptr == nullptr) { - REPORT_INNER_ERROR("E19999", "FindFreeBlock fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(FAILED, "Malloc failed device id = %u, size= %zu", device_id, size); + REPORT_INNER_ERROR("E19999", "FindFreeBlock fail, size:%zu, device_id:%u", size, device_id); + GELOGE(FAILED, "[Check][Param] FindFreeBlock failed device id = %u, size= %zu", device_id, size); } return ptr; } @@ -157,18 +156,16 @@ uint8_t *CachingAllocator::Malloc(size_t size, uint8_t *org_ptr, uint32_t device Status CachingAllocator::Free(uint8_t *ptr, uint32_t device_id) { GELOGI("Free device id = %u", device_id); if (ptr == nullptr) { - REPORT_INNER_ERROR("E19999", "Param ptr is nullptr, device_id:%u, check invalid", - device_id); - GELOGE(PARAM_INVALID, "Invalid memory pointer"); + REPORT_INNER_ERROR("E19999", "Param ptr is nullptr, device_id:%u, check invalid", device_id); + GELOGE(PARAM_INVALID, "[Check][Param] Invalid memory pointer, device_id:%u", device_id); return ge::PARAM_INVALID; } std::lock_guard lock(mutex_); auto it = allocated_blocks_.find(ptr); if (it == allocated_blocks_.end()) { - REPORT_INNER_ERROR("E19999", "Param ptr not allocated before, device_id:%u, check invalid", - device_id); - GELOGE(PARAM_INVALID, "Invalid memory pointer: %p", ptr); + REPORT_INNER_ERROR("E19999", "Param ptr not allocated before, device_id:%u, check invalid", device_id); + GELOGE(PARAM_INVALID, "[Check][Param] Param ptr not allocated before, device_id:%u", device_id); return ge::PARAM_INVALID; } Block *block = it->second; @@ -225,9 +222,8 @@ Block *CachingAllocator::FindFreeBlock(size_t size, uint8_t *org_ptr, uint32_t d Block key(device_id, size, org_ptr); BlockBin *bin = GetBlockBin(size); if (bin == nullptr) { - REPORT_INNER_ERROR("E19999", "GetBlockBin fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(ge::FAILED, "Get block bin failed size = %zu", size); + REPORT_INNER_ERROR("E19999", "GetBlockBin fail, size:%zu, device_id:%u", size, device_id); + GELOGE(ge::FAILED, "[Get][BlockBin] failed, size:%zu, device_id:%u", size, device_id); return nullptr; } std::lock_guard lock(mutex_); @@ -258,9 +254,8 @@ Block *CachingAllocator::SplitBlock(Block *block, size_t size, BlockBin &bin, ui Block *remaining = block; Block *new_block = new (std::nothrow) Block(device_id, size, &bin, block->ptr); if (new_block == nullptr) { - REPORT_CALL_ERROR("E19999", "New Block fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(ge::FAILED, "Alloc block failed size = %zu", size); + REPORT_CALL_ERROR("E19999", "New Block fail, size:%zu, device_id:%u", size, device_id); + GELOGE(ge::FAILED, "[Alloc][Block] failed, size:%zu, device_id:%u", size, device_id); return block; } new_block->prev = remaining->prev; @@ -285,7 +280,7 @@ Status CachingAllocator::TryExtendCache(size_t size, uint32_t device_id) { size_t free_cached_memory_size = FreeCachedBlocks(); memory_addr = memory_allocator_->MallocMemory(purpose, memory_size, device_id); if (memory_addr == nullptr) { - GELOGE(ge::FAILED, "TryExtendCache failed, no enough memory for size = %zu, device_id = %u", memory_size, + GELOGE(ge::FAILED, "[Malloc][Memory] failed, no enough memory for size = %zu, device_id = %u", memory_size, device_id); return ge::FAILED; } @@ -304,16 +299,14 @@ Status CachingAllocator::TryExtendCache(size_t size, uint32_t device_id) { Status CachingAllocator::AddToBlockBin(uint8_t *ptr, size_t size, uint32_t device_id) { BlockBin *bin = GetBlockBin(size); if (bin == nullptr) { - REPORT_INNER_ERROR("E19999", "GetBlockBin fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(ge::FAILED, "Get block bin failed size = %zu", size); + REPORT_INNER_ERROR("E19999", "GetBlockBin fail, size:%zu, device_id:%u", size, device_id); + GELOGE(ge::FAILED, "[Get][BlockBin] failed, size:%zu, device_id:%u", size, device_id); return ge::FAILED; } Block *block = new (std::nothrow) Block(device_id, size, bin, nullptr); if (block == nullptr) { - REPORT_CALL_ERROR("E19999", "New Block fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(ge::FAILED, "Alloc block failed size = %zu", size); + REPORT_CALL_ERROR("E19999", "New Block fail, size:%zu, device_id:%u", size, device_id); + GELOGE(ge::FAILED, "[Alloc][Block] failed, size:%zu, device_id:%u", size, device_id); return ge::FAILED; } diff --git a/ge/graph/manager/graph_context.cc b/ge/graph/manager/graph_context.cc index 3a705ad9..6d202cef 100644 --- a/ge/graph/manager/graph_context.cc +++ b/ge/graph/manager/graph_context.cc @@ -33,7 +33,7 @@ GraphContext::GraphContext(const GraphNodePtr &graph_node) { if (compute_graph_ == nullptr) { std::shared_ptr graph = graph_node->GetGraph(); if (graph == nullptr) { - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "compute_graph by graphNode is NULL!"); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Get][Graph] failed, compute_graph by graphNode is NULL!"); return; } @@ -45,7 +45,7 @@ GraphContext::GraphContext(const GraphNodePtr &graph_node) { Status GraphContext::SetComputeGraph(const GraphNodePtr &graph_node) { if (graph_node == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph_node is nullptr, check invalid"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "graphNode is NULL!"); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] graphNode is NULL!"); return GE_GRAPH_PARAM_NULLPTR; } @@ -56,7 +56,7 @@ Status GraphContext::SetComputeGraph(const GraphNodePtr &graph_node) { std::shared_ptr graph = graph_node->GetGraph(); if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph in graph_node is nullptr, check invalid"); - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "compute_graph by graphNode is NULL!"); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Get][Graph] failed, compute_graph by graphNode is NULL!"); return GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL; } @@ -73,14 +73,15 @@ Status GraphContext::Finalize() const { return SUCCESS; } Status GraphContext::GetVariableTensor(const std::string &var_data_name, GeTensor &returned_tensor) { if (var_data_name.empty()) { REPORT_INNER_ERROR("E19999", "Param var_data_name is empty, check invalid"); - GELOGE(GE_GRAPH_EMPTY_STRING_NAME, "Variable data name is empty!"); + GELOGE(GE_GRAPH_EMPTY_STRING_NAME, "[Check][Param] Variable data name is empty!"); return GE_GRAPH_EMPTY_STRING_NAME; } if (GetVarNodeTensorTable().empty()) { REPORT_INNER_ERROR("E19999", "VarNodeTensorTable is empty, var_data_name:%s, check invalid", var_data_name.c_str()); - GELOGE(GE_GRAPH_EMPTY_VARIABLE_TENSOR_TABLE, "VarNodeTensorTable is empty!"); + GELOGE(GE_GRAPH_EMPTY_VARIABLE_TENSOR_TABLE, "[Check][Param] VarNodeTensorTable is empty, var_data_name:%s", + var_data_name.c_str()); return GE_GRAPH_EMPTY_VARIABLE_TENSOR_TABLE; } for (auto &var_record : GetVarNodeTensorTable()) { @@ -88,9 +89,8 @@ Status GraphContext::GetVariableTensor(const std::string &var_data_name, GeTenso returned_tensor.SetTensorDesc(var_record.second.GetTensorDesc()); auto ret = returned_tensor.SetData(var_record.second.GetData()); if (ret != SUCCESS) { - REPORT_INNER_ERROR("E19999", "SetData to tensor fail, var_data_name:%s", - var_data_name.c_str()); - GELOGE(ret, "Set Tensor data failed!"); + REPORT_INNER_ERROR("E19999", "SetData to tensor fail, var_data_name:%s", var_data_name.c_str()); + GELOGE(ret, "[Set][Data] to Tensor failed, var_data_name:%s", var_data_name.c_str()); return ret; } @@ -100,7 +100,8 @@ Status GraphContext::GetVariableTensor(const std::string &var_data_name, GeTenso REPORT_INNER_ERROR("E19999", "VarRecord with data_name:%s does not exist, check invalid", var_data_name.c_str()); - GELOGE(GE_GRAPH_VARIABLE_DOES_NOT_EXIST, "VarRecord with data_name %s does NOT exist!", var_data_name.c_str()); + GELOGE(GE_GRAPH_VARIABLE_DOES_NOT_EXIST, "[Check][Param] VarRecord with data_name %s does NOT exist!", + var_data_name.c_str()); return GE_GRAPH_VARIABLE_DOES_NOT_EXIST; } diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index 69c84f6f..dc1142d0 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -149,7 +149,8 @@ ge::Status CheckFpCeilingMode() { if (ret == ge::GRAPH_SUCCESS) { if (kValidFpCeilingMode.count(mode) == 0) { REPORT_INNER_ERROR("E19999", "Option ge.fpCeilingMode is invalid, value:%s", mode.c_str()); - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "The fp_ceiling_mode %s is invalid, options are 0, 1, and 2.", mode.c_str()); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Get][Option] The fp_ceiling_mode %s is invalid, options are 0, 1, and 2.", + mode.c_str()); return ge::GE_GRAPH_OPTIONS_INVALID; } GELOGI("The parameter fp_ceiling_mode is set to %s.", mode.c_str()); @@ -178,33 +179,33 @@ Status GraphManager::Initialize(const std::map &options) { graph_run_listener_ = MakeShared(sync_run_mutex_, condition_); if (graph_run_listener_ == nullptr) { REPORT_CALL_ERROR("E19999", "New GraphModelListener fail"); - GELOGE(MEMALLOC_FAILED, "Make shared failed"); + GELOGE(MEMALLOC_FAILED, "[New][GraphModelListener] failed"); return MEMALLOC_FAILED; } // graph context graph_context_ = MakeShared(); if (graph_context_ == nullptr) { - REPORT_CALL_ERROR("E19999", "New GraphModelListener fail"); - GELOGE(MEMALLOC_FAILED, "Make shared failed."); + REPORT_CALL_ERROR("E19999", "New GraphContext fail"); + GELOGE(MEMALLOC_FAILED, "[New][GraphContext] failed."); return MEMALLOC_FAILED; } // parse option parameters Status ret = ParseOptions(options); if (ret != SUCCESS) { - GELOGE(ret, "[Initialize] parse options failed."); + GELOGE(ret, "[Parse][Options] failed."); return ret; } ret = CheckFpCeilingMode(); if (ret != SUCCESS) { - GELOGE(ret, "[Initialize] Check fp-ceiling-mode options failed."); + GELOGE(ret, "[Check][FpCeilingMode] failed."); return ret; } ret = graph_context_->Initialize(options); if (ret != SUCCESS) { - GELOGE(ret, "[Initialize] GraphContext initialize failed."); + GELOGE(ret, "[Initialize][GraphContext] failed."); return ret; } @@ -302,7 +303,7 @@ Status GraphManager::Finalize() { if (graph_context_ != nullptr) { Status ret_final = graph_context_->Finalize(); if (ret_final != SUCCESS) { - GELOGE(ret_final, "[GraphManager] graph context Finalize failed!"); + GELOGE(ret_final, "[Finalize][GraphContext] failed!"); unload_model_ret = ret_final; } } @@ -321,9 +322,8 @@ Status GraphManager::InitDynamicParams(ComputeGraphPtr &compute_graph) { std::string op_type; auto ret = GetOriginalType(node, op_type); if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", "GetOriginalType from op:%s fail", - node->GetName().c_str()); - GELOGE(FAILED, "Failed to get node %s original type.", node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "GetOriginalType from op:%s fail", node->GetName().c_str()); + GELOGE(FAILED, "[Get][OriginalType] from op:%s failed.", node->GetName().c_str()); return FAILED; } if ((op_desc->GetType() == DATA) || (op_type == kGetNextName)) { @@ -335,7 +335,7 @@ Status GraphManager::InitDynamicParams(ComputeGraphPtr &compute_graph) { if (!options_.input_shape.empty() && !options_.dynamic_dims.empty()) { if (!ge::ParseInputShape(options_.input_shape, GetLocalOmgContext().input_dims, GetLocalOmgContext().user_input_dims, true)) { - GELOGE(GRAPH_PARAM_INVALID, "Failed to parse input shape: %s.", options_.input_shape.c_str()); + GELOGE(GRAPH_PARAM_INVALID, "[Parse][InputShape] %s failed.", options_.input_shape.c_str()); return GRAPH_PARAM_INVALID; } GetLocalOmgContext().dynamic_dims = options_.dynamic_dims; @@ -380,7 +380,7 @@ void GraphManager::RemoveAddGraphCondition(GraphId graph_id) { Status GraphManager::CheckRepeatAdd(uint32_t graph_id, bool &is_added) { uint32_t count = 0; if (GetGraphCount(graph_id, count) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Get graph [id:%u] count failed, graph might have not been added.", graph_id); + GELOGE(INTERNAL_ERROR, "[Get][GraphCount] failed, graph[id:%u] might have not been added.", graph_id); return INTERNAL_ERROR; } // previous thread owns same graph_id has been in the middle of the AddGraph procession @@ -393,7 +393,7 @@ Status GraphManager::CheckRepeatAdd(uint32_t graph_id, bool &is_added) { GraphNodePtr graph_node; Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS) { - GELOGE(ret, "[AddGraph] GetGraphNode failed, graph_id = %u.", graph_id); + GELOGE(ret, "[Get][GraphNode] failed, graph_id = %u.", graph_id); return ret; } is_added = true; @@ -418,7 +418,7 @@ void GraphManager::SetSessionGraphId(ComputeGraphPtr compute_graph, uint32_t gra Status GraphManager::NotifyWaittingGraph(uint32_t graph_id) { uint32_t count = 0; if (GetGraphCount(graph_id, count) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Get graph [id:%u] count failed, graph might have not been added.", graph_id); + GELOGE(INTERNAL_ERROR, "[Get][GraphCount] failed, graph[id:%u] might have not been added.", graph_id); return INTERNAL_ERROR; } GELOGD("Add graph finished, graph_id:%u", graph_id); @@ -433,15 +433,13 @@ Status GraphManager::CreateGraphNode(uint32_t graph_id, const Graph &graph, const std::map &options) { GraphNodePtr graph_node = MakeShared(graph_id); GE_IF_BOOL_EXEC(graph_node == nullptr, - REPORT_CALL_ERROR("E19999", "New GraphNode fail, graph_id:%u", - graph_id); - GELOGE(FAILED, "GraphNode make shared failed"); + REPORT_CALL_ERROR("E19999", "New GraphNode fail, graph_id:%u", graph_id); + GELOGE(FAILED, "[New][GraphNode] fail, graph_id:%u", graph_id); return FAILED); std::shared_ptr graph_ptr = MakeShared(graph); GE_IF_BOOL_EXEC(graph_ptr == nullptr, - REPORT_CALL_ERROR("E19999", "New Graph fail, graph_id:%u", - graph_id); - GELOGE(FAILED, "GraphPtr make shared failed"); + REPORT_CALL_ERROR("E19999", "New Graph fail, graph_id:%u", graph_id); + GELOGE(FAILED, "[New][Graph] fail, graph_id:%u", graph_id); return FAILED); // update option about tuning graph ParseOption(options, BUILD_MODE, options_.build_mode); @@ -459,7 +457,7 @@ Status GraphManager::SetStagesOptions(uint32_t graph_id, const GraphManagerOptio stages.preparer.SetOptions(options_); Status status = stages.optimizer.SetOptions(options_); if (status != SUCCESS) { - GELOGE(status, "Graph optimizer set options failed."); + GELOGE(status, "[Set][Options] for Graph optimizer failed, graph id:%u.", graph_id); return status; } stages.builder.SetOptions(options_); @@ -517,7 +515,8 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, if (GetAddGraphCondition(graph_id) == kDoneAdded) { GraphNodePtr graph_node; if (GetGraphNode(graph_id, graph_node) != SUCCESS) { - GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "Graph not exist while done adding previously, graph_id = %u.", graph_id); + GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[Get][GraphNode] failed, Graph not exist while done adding previously, " + "graph_id = %u.", graph_id); return GE_GRAPH_GRAPH_NOT_EXIST; } graph_node->IncreaseLoadCount(); @@ -528,7 +527,7 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, // done adding graph of the former graph, avoiding repeatively adding same graph. bool is_added = false; if (CheckRepeatAdd(graph_id, is_added) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "CheckRepeatAdd for graph[id:%u] failed.", graph_id); + GELOGE(INTERNAL_ERROR, "[Check][RepeatAdd] for graph[id:%u] failed.", graph_id); return INTERNAL_ERROR; } // The former graph (from different thread) owns same graph id has been successfully added. @@ -538,7 +537,7 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, // Do add graph SetAddGraphCondition(graph_id, kStartAdd); if (CheckGraphAdded(graph_id, graph) != SUCCESS) { - GELOGE(FAILED, "AddGraph failed."); + GELOGE(FAILED, "[Check][GraphAdded] failed, graph id:%u.", graph_id); return FAILED; } GE_CHK_STATUS_RET(ModifyDataIndex(graph, options)); @@ -548,7 +547,7 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, SetSessionGraphId(compute_graph, graph_id); if (CreateGraphNode(graph_id, graph, options) != SUCCESS) { - GELOGE(FAILED, "Failed to create graph_node."); + GELOGE(FAILED, "[Create][GraphNode] failed, graph id:%u.", graph_id); return FAILED; } @@ -557,12 +556,12 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, GetLocalOmgContext().output_type = options_.output_datatype; } if (InitDynamicParams(compute_graph) != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "Failed to init params when online infer is dynamic."); + GELOGE(GRAPH_PARAM_INVALID, "[Init][Params] failed, when online infer is dynamic, graph id:%u.", graph_id); return GRAPH_PARAM_INVALID; } if (SetStagesOptions(graph_id, options_) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Set stage options failed."); + GELOGE(INTERNAL_ERROR, "[Set][StagesOptions] failed, graph id:%u.", graph_id); return INTERNAL_ERROR; } @@ -570,7 +569,7 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, SetAddGraphCondition(graph_id, kDoneAdded); // There are threads waitting for adding same graph if (NotifyWaittingGraph(graph_id) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "NotifyWaittingGraph failed."); + GELOGE(INTERNAL_ERROR, "[Notify][WaittingGraph] failed, graph id:%u.", graph_id); return INTERNAL_ERROR; } return SUCCESS; @@ -585,14 +584,13 @@ Status GraphManager::CheckGraphAdded(const GraphId &graph_id, const Graph &graph && graph_has_been_added) { REPORT_INNER_ERROR("E19999", "Get Attr:%s from graph:%u fail.", ATTR_NAME_GRAPH_HAS_BEEN_ADDED.c_str(), graph_id); - GELOGE(GE_GRAPH_GRAPH_ALREADY_EXIST, - "[GraphManager] same graph object can not be added again, graph_id = %u.", graph_id); + GELOGE(GE_GRAPH_GRAPH_ALREADY_EXIST, "[Get][Attr] %s from graph:%u fail.", + ATTR_NAME_GRAPH_HAS_BEEN_ADDED.c_str(), graph_id); return GE_GRAPH_GRAPH_ALREADY_EXIST; } } else { - REPORT_INNER_ERROR("E19999", "compute_graph from graph:%u is nullptr, check invalid", - graph_id); - GELOGE(FAILED, "compute graph is null"); + REPORT_INNER_ERROR("E19999", "compute_graph from graph:%u is nullptr, check invalid", graph_id); + GELOGE(FAILED, "[Get][ComputeGraph] failed, compute graph from graph:%u is nullptr", graph_id); return FAILED; } return SUCCESS; @@ -602,11 +600,11 @@ Status GraphManager::AddGraphWithCopy(const GraphId &graph_id, const Graph &grap const std::map &options, const OmgContext &omg_context) { if (HasGraphNode(graph_id)) { - GELOGE(GE_GRAPH_GRAPH_ALREADY_EXIST, "[GraphManager] graph exists, graph_id = %u", graph_id); + GELOGE(GE_GRAPH_GRAPH_ALREADY_EXIST, "[Has][GraphNode] graph exists, graph_id = %u", graph_id); return GE_GRAPH_GRAPH_ALREADY_EXIST; } if (CheckGraphAdded(graph_id, graph) != SUCCESS) { - GELOGE(FAILED, "AddGraphWithCopy failed."); + GELOGE(FAILED, "[Check][GraphAdded] failed, graph_id = %u", graph_id); return FAILED; } IncreaseGraphCount(graph_id); @@ -620,7 +618,7 @@ Status GraphManager::AddGraphWithCopy(const GraphId &graph_id, const Graph &grap SetSessionGraphId(new_compute_graph, graph_id); std::shared_ptr new_graph_ptr = GraphUtils::CreateGraphPtrFromComputeGraph(new_compute_graph); if (CreateGraphNode(graph_id, *new_graph_ptr, options) != SUCCESS) { - GELOGE(FAILED, "Failed to create graph_node."); + GELOGE(FAILED, "[Create][GraphNode] failed, graph_id = %u", graph_id); return FAILED; } @@ -629,12 +627,12 @@ Status GraphManager::AddGraphWithCopy(const GraphId &graph_id, const Graph &grap GetLocalOmgContext().output_type = options_.output_datatype; } if (InitDynamicParams(new_compute_graph) != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "Failed to init params when online infer is dynamic."); + GELOGE(GRAPH_PARAM_INVALID, "[Init][Params] failed, when online infer is dynamic, graph_id = %u", graph_id); return GRAPH_PARAM_INVALID; } if (SetStagesOptions(graph_id, options_) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Set stage options failed."); + GELOGE(INTERNAL_ERROR, "[Set][StagesOptions] failed, graph_id = %u", graph_id); return INTERNAL_ERROR; } @@ -655,9 +653,9 @@ Status GraphManager::MergeSubGraph(ComputeGraphPtr &compute_graph, const ge::Com Status ret_topo = compute_graph->TopologicalSorting(); if (ret_topo != SUCCESS) { - REPORT_CALL_ERROR("E19999", "TopologicalSorting fail, graph_id:%u", - compute_graph->GetGraphID()); - GELOGE(ret_topo, "[GraphManager]: TopologicalSorting the merged graph failed."); + REPORT_CALL_ERROR("E19999", "TopologicalSorting fail, graph_id:%u", compute_graph->GetGraphID()); + GELOGE(ret_topo, "[Call][TopologicalSorting] for the merged graph failed, graph_id:%u", + compute_graph->GetGraphID()); return ret_topo; } } else { @@ -692,16 +690,16 @@ Status GraphManager::CopySubGraphAndMarkFusion(const ComputeGraphPtr &compute_gr std::vector output_nodes; ComputeGraphPtr new_compute_graph = GraphUtils::CloneGraph(old_compute_graph, "", input_nodes, output_nodes); if (new_compute_graph == nullptr) { - REPORT_CALL_ERROR("E19999", "CloneGraph fail, graph_id:%u", - compute_graph->GetGraphID()); - GELOGE(INTERNAL_ERROR, "Clone graph failed."); + REPORT_CALL_ERROR("E19999", "CloneGraph fail, graph_id:%u", compute_graph->GetGraphID()); + GELOGE(INTERNAL_ERROR, "[Clone][Graph] failed, graph_id:%u", compute_graph->GetGraphID()); return INTERNAL_ERROR; } copy_graphs.emplace(old_compute_graph->GetName(), new_compute_graph); if (!AttrUtils::SetBool(old_compute_graph, ATTR_NAME_NEED_LX_FUSION, true)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s to graph:%u fail", ATTR_NAME_NEED_LX_FUSION.c_str(), old_compute_graph->GetGraphID()); - GELOGE(INTERNAL_ERROR, "Set attr lx_fusion to graph failed."); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to graph:%u failed.", + ATTR_NAME_NEED_LX_FUSION.c_str(), old_compute_graph->GetGraphID()); return INTERNAL_ERROR; } } @@ -739,7 +737,7 @@ Status GraphManager::OptimizeSubGraphWithMultiThreads(ComputeGraphPtr compute_gr ErrorManager::GetInstance().GetErrorManagerContext(), GetThreadLocalContext()); if (!f.valid()) { - GELOGE(FAILED, "Future is invalid"); + GELOGE(FAILED, "[Call][Commit] failed, Future is invalid, session_id:%lu", session_id); return FAILED; } vector_future.emplace_back(std::move(f)); @@ -756,7 +754,7 @@ Status GraphManager::OptimizeSubGraphWithMultiThreads(ComputeGraphPtr compute_gr ErrorManager::GetInstance().GetErrorManagerContext(), GetThreadLocalContext()); if (!f.valid()) { - GELOGE(FAILED, "Future is invalid"); + GELOGE(FAILED, "[Call][Commit] failed, Future is invalid, session_id:%lu", session_id); return FAILED; } vector_future.emplace_back(std::move(f)); @@ -767,7 +765,7 @@ Status GraphManager::OptimizeSubGraphWithMultiThreads(ComputeGraphPtr compute_gr Status ret_status = vector_future[i].get(); if (ret_status != SUCCESS) { REPORT_CALL_ERROR("E19999", "subgraph %zu optimize failed", i); - GELOGE(ret_status, "subgraph %zu optimize failed", i); + GELOGE(ret_status, "[Check][Param] subgraph %zu optimize failed", i); return ret_status; } } @@ -778,7 +776,7 @@ bool GraphManager::CheckAllFusionOptimizeSuccess(const ComputeGraphPtr &compute_ Graph2SubGraphInfoList &sub_graph_map) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "Input param compute_graph is nullptr."); + GELOGE(PARAM_INVALID, "[Check][Param] Input param compute_graph is nullptr."); return false; } @@ -819,7 +817,8 @@ Status GraphManager::ReplaceSubgraphWithOriGraph(const ComputeGraphPtr &compute_ if (iter == copy_graphs.end()) { REPORT_INNER_ERROR("E19999", "Can not find subgraph:%s in copy graphs, check invalid", subgraph->GetSubGraph()->GetName().c_str()); - GELOGE(FAILED, "Can not find subgraph:%s in copy graphs.", subgraph->GetSubGraph()->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Can not find subgraph:%s in copy graphs.", + subgraph->GetSubGraph()->GetName().c_str()); return FAILED; } subgraph->SetSubGraph(iter->second); @@ -832,7 +831,8 @@ Status GraphManager::ReplaceSubgraphWithOriGraph(const ComputeGraphPtr &compute_ if (iter == copy_graphs.end()) { REPORT_INNER_ERROR("E19999", "Can not find subgraph:%s in copy graphs, check invalid", subgraph->GetSubGraph()->GetName().c_str()); - GELOGE(FAILED, "Can not find subgraph:%s in copy graphs.", subgraph->GetSubGraph()->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Can not find subgraph:%s in copy graphs.", + subgraph->GetSubGraph()->GetName().c_str()); return FAILED; } subgraph->SetSubGraph(iter->second); @@ -850,7 +850,7 @@ Status GraphManager::SetSubgraph(uint64_t session_id, ComputeGraphPtr compute_gr options_.build_step.c_str()); Status ret = OptimizeSubGraphWithMultiThreads(compute_graph, sub_graph_map, session_id); if (ret != SUCCESS) { - GELOGE(ret, "Multiply optimize subgraph failed"); + GELOGE(ret, "[Call][OptimizeSubGraphWithMultiThreads] failed, ret:%d, session_id:%lu", ret, session_id); return ret; } return SUCCESS; @@ -892,7 +892,8 @@ Status GraphManager::PreRunOptimizeOriginalGraph(const GraphNodePtr &graph_node, GE_CHK_STATUS_RET(graph_pass.AddPass("PreRun::CtrlEdgeTransferPass", new (std::nothrow) CtrlEdgeTransferPass)) GE_CHK_STATUS_RET(graph_pass.Run(compute_graph)); - GE_CHK_STATUS_RET(stages.optimizer.IdentifyReference(compute_graph), "Identify reference failed."); + GE_CHK_STATUS_RET(stages.optimizer.IdentifyReference(compute_graph), + "[Identify][Reference] failed, graph:%s.", compute_graph->GetName().c_str()); GELOGD("PreRun:PreRunOptimizeOriginalGraph success."); return SUCCESS; } @@ -930,9 +931,8 @@ Status GraphManager::PreRunAfterOptimizeSubGraph(const GraphNodePtr &graph_node, Status ret = compute_graph->TopologicalSorting(); if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", "TopologicalSorting fail, graph_id:%u", - compute_graph->GetGraphID()); - GELOGE(ret, "Graph topological sort failed, ret:%d.", ret); + REPORT_CALL_ERROR("E19999", "TopologicalSorting fail, graph_id:%u", compute_graph->GetGraphID()); + GELOGE(ret, "[Call][TopologicalSorting] fail, graph_id:%u", compute_graph->GetGraphID()); return ret; } @@ -949,14 +949,14 @@ Status GraphManager::SetRtContext(rtContext_t rt_context, rtCtxMode_t mode, uint if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxCreate faileded, session_id:%lu, graph_id:%u, mode:%d", session_id, graph_id, mode); - GELOGE(FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(FAILED, "[Call][RtCtxCreate] faileded, session_id:%lu, graph_id:%u, mode:%d", session_id, graph_id, mode); return FAILED; } rt_ret = rtCtxSetCurrent(rt_context); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, session_id:%lu, graph_id:%u, mode:%d", session_id, graph_id, mode); - GELOGE(FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(FAILED, "[Call][RtCtxSetCurrent] failed, session_id:%lu, graph_id:%u, mode:%d", session_id, graph_id, mode); return FAILED; } RtContextUtil::GetInstance().AddRtContext(session_id, graph_id, rt_context); @@ -970,7 +970,7 @@ Status GraphManager::RunCustomPass(const GraphNodePtr &graph_node) { GE_TIMESTAMP_START(RunCustomPass); GraphPtr graph = std::const_pointer_cast(const_graph); - GE_CHK_STATUS_RET(CustomPassHelper::Instance().Run(graph), "Graph[%s] run custom pass fail.", + GE_CHK_STATUS_RET(CustomPassHelper::Instance().Run(graph), "[Call][Run] for Graph[%s] fail.", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(RunCustomPass, "GraphBuilder::RunCustomPass"); return SUCCESS; @@ -986,7 +986,7 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorSetSessionID(session_id); auto analyzer_instance = Analyzer::GetInstance(); GE_CHK_STATUS_RET(analyzer_instance->BuildJsonObject(session_id, compute_graph->GetGraphID()), - "BuildJsonObject Failed") + "[Build][JsonObject] Failed, session_id:%lu", session_id) GEEVENT("PreRun start: graph node size %zu, session id %lu, graph id %u, graph name %s.", compute_graph->GetDirectNodesSize(), session_id, compute_graph->GetGraphID(), @@ -995,7 +995,7 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorGetGraphID()); if (ret != SUCCESS) { - GELOGE(ret, "Set rt context failed."); + GELOGE(ret, "[Set][RtContext] failed, session_id:%lu, graph_id:%u.", session_id, compute_graph->GetGraphID()); return ret; } @@ -1009,17 +1009,20 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorGetName().c_str()); + GELOGE(ret, "[Run][PreRunOptimizeOriginalGraph] failed for graph:%s, session_id:%lu", + compute_graph->GetName().c_str(), session_id); return ret; } } ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kSubGraphOptimize); // set fuzz compile flag after origin graph optimize - GE_CHK_STATUS_RET(SetFuzzCompileFlag(compute_graph), "Set fuzz compile flag failed."); + GE_CHK_STATUS_RET(SetFuzzCompileFlag(compute_graph), + "[Set][FuzzCompileFlag] failed for graph:%s.", compute_graph->GetName().c_str()); ret = PreRunOptimizeSubGraph(graph_node, compute_graph, session_id); if (ret != SUCCESS) { - GELOGE(ret, "Run PreRunOptimizeSubGraph failed for graph:%s.", compute_graph->GetName().c_str()); + GELOGE(ret, "[Run][PreRunOptimizeSubGraph] failed for graph:%s, session_id:%lu.", + compute_graph->GetName().c_str(), session_id); return ret; } @@ -1033,7 +1036,8 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorGetName().c_str()); + GELOGE(ret, "[Run][PreRunAfterOptimizeSubGraph] failed for graph:%s, session_id:%lu.", + compute_graph->GetName().c_str(), session_id); return ret; } } @@ -1058,7 +1062,7 @@ Status GraphManager::SetFuzzCompileFlag(ComputeGraphPtr &compute_graph) { GE_CHECK_NOTNULL(op_desc); GELOGD("Fuzz compile flag is %d.", GetLocalOmgContext().fuzz_compile_flag); if (!AttrUtils::SetBool(op_desc, ATTR_NAME_FUZZ_BUILD, GetLocalOmgContext().fuzz_compile_flag)) { - GELOGE(FAILED, "[Set][ATTR_NAME_FUZZ_BUILD]Failed to set fuzz build attr to %s.", op_desc->GetName().c_str()); + GELOGE(FAILED, "[Set][ATTR] %s to %s failed.", ATTR_NAME_FUZZ_BUILD.c_str(), op_desc->GetName().c_str()); return FAILED; } } @@ -1074,7 +1078,7 @@ Status GraphManager::SubexpressionMigration(ComputeGraphPtr &compute_graph) { auto ret = pass_manager.Run(compute_graph); GE_TIMESTAMP_END(SubexpressionMigrationPass, "GraphManager::SubexpressionMigration"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run SubexpressionMigrationPass failed, ret:%u.", ret); + GELOGE(ret, "[Run][SubexpressionMigrationPass] failed, ret:%u.", ret); return ret; } @@ -1091,7 +1095,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: REPORT_INNER_ERROR("E19999", "Graph:%u has not build before, can't run directly, " "check invalid", graph_node->GetGraphId()); GELOGE(PARAM_INVALID, - "The graph %u need to re-build, you should remove it from GE " + "[Get][BuildFlag] The graph %u need to re-build, you should remove it from GE " "first, then AddGraph again and rebuild it.", graph_node->GetGraphId()); return PARAM_INVALID; @@ -1104,7 +1108,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: // release rts generate context RtContextUtil::GetInstance().DestroyRtContexts(session_id, graph_node->GetGraphId()); if (ret != SUCCESS) { - GELOGE(ret, "PreRun Failed, graph_id:%u.", graph_node->GetGraphId()); + GELOGE(ret, "[Call][PreRun] Failed, graph_id:%u, session_id:%lu.", graph_node->GetGraphId(), session_id); return ret; } } @@ -1115,7 +1119,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: ret = LoadGraphAsync(ge_root_model, graph_node); } if (ret != SUCCESS) { - GELOGE(ret, "LoadGraph Failed."); + GELOGE(ret, "[Load][Graph] Failed, graph_id:%u.", graph_node->GetGraphId()); return ret; } graph_node->SetBuildFlag(true); @@ -1129,7 +1133,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: ret = LoadGraphAsync(ge_root_model_ptr, graph_node); } if (ret != SUCCESS) { - GELOGE(ret, "LoadGraph Failed."); + GELOGE(ret, "[Load][Graph] Failed, graph_id:%u.", graph_node->GetGraphId()); return ret; } } @@ -1160,7 +1164,7 @@ Status GraphManager::LoadGraph(const GeRootModelPtr &ge_root_model, const GraphN Status ret = GraphLoader::LoadModelOnline(model_id_info.model_id, ge_root_model, model_listener); GE_TIMESTAMP_EVENT_END(LoadGraph, "GraphManager::LoadGraph"); if (ret != SUCCESS) { - GELOGE(ret, "[StartForRunGraph] LoadGraph Failed"); + GELOGE(ret, "[Load][Model] failed, ret:%d", ret); graph_node->SetRunFlag(false); return ret; } @@ -1260,7 +1264,7 @@ Status GraphManager::InnerRunGraph(GraphNodePtr &graph_node, const GraphId &grap const std::vector &inputs, std::vector &outputs) { Status ret = graph_executor_.SetCondition(&sync_run_mutex_, &condition_, graph_run_listener_); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_RUNGRAPH_FAILED, "[RunGraph] set condition failed, graph_id = %u.", graph_id); + GELOGE(GE_GRAPH_RUNGRAPH_FAILED, "[Set][Condition] failed, graph_id = %u.", graph_id); graph_node->SetRunFlag(false); return GE_GRAPH_RUNGRAPH_FAILED; } @@ -1273,7 +1277,7 @@ Status GraphManager::InnerRunGraph(GraphNodePtr &graph_node, const GraphId &grap graph_node->SetRunFlag(false); if (ret != SUCCESS) { - GELOGE(ret, "[RunGraph] execute graph failed, graph_id = %u.", graph_id); + GELOGE(ret, "[Execute][Graph] failed, graph_id = %u.", graph_id); return ret; } return SUCCESS; @@ -1283,8 +1287,7 @@ Status GraphManager::InnerRunGraphWithStream(GraphNodePtr &graph_node, const Gra const std::vector &inputs, std::vector &outputs) { auto ret = graph_executor_.SetCondition(&sync_run_mutex_, &condition_, graph_run_listener_); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_RUNGRAPH_FAILED, "[Run][GraphWithStreamAsync] set condition failed, " - "graph id = %u, stream = %p.", graph_id, stream); + GELOGE(GE_GRAPH_RUNGRAPH_FAILED, "[Set][Condition] failed, graph id = %u, stream = %p.", graph_id, stream); graph_node->SetRunFlag(false); return GE_GRAPH_RUNGRAPH_FAILED; } @@ -1293,7 +1296,7 @@ Status GraphManager::InnerRunGraphWithStream(GraphNodePtr &graph_node, const Gra graph_node->SetRunFlag(false); graph_node->SetIsSpecificStream(false); if (ret != SUCCESS) { - GELOGE(ret, "[Run][GraphWithStreamAsync] execute graph failed, graph id = %u, stream = %p.", graph_id, stream); + GELOGE(ret, "[Execute][Graph] With Stream failed, graph id = %u, stream = %p.", graph_id, stream); return ret; } GELOGI("[Run][GraphWithStreamAsync] run graph success, graph id = %u, stream = %p.", graph_id, stream); @@ -1315,18 +1318,20 @@ Status GraphManager::RunGraphWithStreamAsync(const GraphId &graph_id, rtStream_t Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS) { REPORT_INNER_ERROR("E19999", "graph id = %u not exist in graph_map, check invalid.", graph_id); - GELOGE(ret, "Run graph with stream async graph not exist, graph id = %u.", graph_id); + GELOGE(ret, "[Get][GraphNode] failed, Run graph with stream async, graph not exist, graph id = %u.", graph_id); return ret; } if (graph_node == nullptr) { REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph id = %u, check invalid.", graph_id); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Run graph with stream async graph node is NULL, graph id = %u.", graph_id); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Check][Param] Run graph with stream async, graph node is NULL, " + "graph id = %u.", graph_id); return GE_GRAPH_GRAPH_NODE_NULL; } if (graph_node->GetRunFlag()) { REPORT_INNER_ERROR("E19999", "Graph is already running, can't be run again, graph id = %u, " "check invalid.", graph_id); - GELOGE(GE_GRAPH_ALREADY_RUNNING, "Run graph with stream async graph already running, graph id = %u.", graph_id); + GELOGE(GE_GRAPH_ALREADY_RUNNING, "[Get][RunFlag] Run graph with stream async graph already running, " + "graph id = %u.", graph_id); return GE_GRAPH_ALREADY_RUNNING; } @@ -1344,7 +1349,7 @@ Status GraphManager::RunGraphWithStreamAsync(const GraphId &graph_id, rtStream_t GeRootModelPtr ge_root_model = nullptr; ret = StartForRunGraph(graph_node, inputs, ge_root_model, session_id); if (ret != SUCCESS) { - GELOGE(ret, "[Run][GraphWithStreamAsync] StartForRunGraph failed!"); + GELOGE(ret, "[Call][StartForRunGraph] failed, session_id:%lu", session_id); graph_node->SetRunFlag(false); return ret; } @@ -1365,23 +1370,20 @@ Status GraphManager::RunGraph(const GraphId &graph_id, const std::vectorGetRunFlag()) { - REPORT_INNER_ERROR("E19999", "Graph is already running, can't be run again, graph_id:%u, " - "check invalid", graph_id); - GELOGE(GE_GRAPH_ALREADY_RUNNING, "[RunGraph] graph already running, graph id = %u", graph_id); + REPORT_INNER_ERROR("E19999", "Graph is already running, can't be run again, graph_id:%u, check invalid", graph_id); + GELOGE(GE_GRAPH_ALREADY_RUNNING, "[Get][RunFlag] graph already running, graph id = %u", graph_id); return GE_GRAPH_ALREADY_RUNNING; } @@ -1395,8 +1397,8 @@ Status GraphManager::RunGraph(const GraphId &graph_id, const std::vectorSetRunFlag(false); return ret; } @@ -1425,7 +1427,7 @@ Status GraphManager::RunGraph(const GraphId &graph_id, const std::vectorIsSummaryGraph()) { ret = SummaryHandle(graph_id, outputs); if (ret != SUCCESS) { - GELOGE(ret, "[RunGraph] SummaryHandle failed!"); + GELOGE(ret, "[Call][SummaryHandle] failed, graph_id:%u", graph_id); } } @@ -1436,7 +1438,7 @@ Status GraphManager::RunGraph(const GraphId &graph_id, const std::vectorInitFlag()) { - REPORT_INNER_ERROR("E19999", "GELib is not init before, graph_id:%u, check invalid", - graph_id); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized"); + REPORT_INNER_ERROR("E19999", "GELib is not init before, graph_id:%u, check invalid", graph_id); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Get][GELib] GELib is not init before, graph_id:%u", graph_id); return GE_CLI_GE_NOT_INITIALIZED; } @@ -1524,7 +1523,9 @@ Status GraphManager::BuildGraphForUnregisteredOp(const GraphId &graph_id, const REPORT_INNER_ERROR("E19999", "GetOpsKernelInfoStore fail for op:%s(%s), kernel_lib_name:%s, graph_id:%u, " "check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_desc->GetOpKernelLibName().c_str(), graph_id); - GELOGE(FAILED, "Get op kernel info store failed"); + GELOGE(FAILED, "[Get][OpsKernelInfoStore] fail for op:%s(%s), kernel_lib_name:%s, graph_id:%u", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), + op_desc->GetOpKernelLibName().c_str(), graph_id); return FAILED; } @@ -1533,8 +1534,7 @@ Status GraphManager::BuildGraphForUnregisteredOp(const GraphId &graph_id, const REPORT_CALL_ERROR("E19999", "Call CompileOp fail for op:%s(%s), kernel_lib_name:%s, graph_id:%u, " "check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_desc->GetOpKernelLibName().c_str(), graph_id); - GELOGE(FAILED, "Get op kernel info store failed"); - GELOGE(ret, "Compile op failed, op = %s, graph_id = %u.", op_desc->GetName().c_str(), graph_id); + GELOGE(ret, "[Compile][Op] failed, op = %s, graph_id = %u.", op_desc->GetName().c_str(), graph_id); return ret; } } @@ -1557,23 +1557,21 @@ Status GraphManager::BuildGraph(const GraphId &graph_id, const std::vectorGetRunFlag()) { REPORT_INNER_ERROR("E19999", "Graph is already running, can't be run again, graph_id:%u, " "check invalid", graph_id); - GELOGE(GE_GRAPH_ALREADY_RUNNING, "[BuildGraph] graph already running, graph id = %u", graph_node->GetGraphId()); + GELOGE(GE_GRAPH_ALREADY_RUNNING, "[Get][RunFlag] graph already running, graph id = %u", graph_node->GetGraphId()); return GE_GRAPH_ALREADY_RUNNING; } @@ -1586,7 +1584,7 @@ Status GraphManager::BuildGraph(const GraphId &graph_id, const std::vectorSetRunFlag(false); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_PRERUN_FAILED, "[BuildGraph] StartForRunGraph failed! graph_id:%u.", graph_id); + GELOGE(GE_GRAPH_PRERUN_FAILED, "[Call][StartForRunGraph] failed! graph_id:%u.", graph_id); return GE_GRAPH_PRERUN_FAILED; } @@ -1606,18 +1604,18 @@ Status GraphManager::BuildGraph(const GraphId &graph_id, const std::vector &attrs, const std::vector &inputs, const std::vector &outputs) { - GE_CHK_BOOL_EXEC(ge::AttrUtils::SetStr(&model, "ATTR_MODEL_OP_TYPE", type), return FAILED, "Set Op[%s] type fail", - type.c_str()); + GE_CHK_BOOL_EXEC(ge::AttrUtils::SetStr(&model, "ATTR_MODEL_OP_TYPE", type), return FAILED, + "[Set][Str] model type[%s] fail", type.c_str()); for (const auto &it : attrs) { GE_CHK_BOOL_EXEC(model.SetAttr("ATTR_MODEL_" + it.first, it.second) == GRAPH_SUCCESS, return FAILED, - "Set OpDesc attribute[%s] fail", it.first.c_str()); + "[Set][Attr] OpDesc attribute[%s] fail", it.first.c_str()); } GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListTensor(&model, "ATTR_MODEL_TENSOR_INPUTS", inputs), return FAILED, - "Set Inputs tensor list fail"); + "[Set][InputsTensor] list fail"); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListTensor(&model, "ATTR_MODEL_TENSOR_OUTPUTS", outputs), return FAILED, - "Set Outputs tensor list fail"); + "[Set][OutputsTensor] list fail"); return SUCCESS; } @@ -1644,9 +1642,8 @@ Status GraphManager::RemoveGraph(const GraphId &graph_id) { GraphNodePtr graph_node = nullptr; Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS || graph_node == nullptr) { - REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid when GraphManager %s", - graph_id, __FUNCTION__); - GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[GraphManager] Id %u does not exists.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", graph_id); + GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[Get][GraphNode] Id %u does not exists.", graph_id); return GE_GRAPH_GRAPH_NOT_EXIST; } if (graph_node->GetRunFlag()) { @@ -1671,7 +1668,7 @@ Status GraphManager::RemoveGraph(const GraphId &graph_id) { if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, graph_id:%u", GetContext().DeviceId(), graph_id); - GELOGE(RT_FAILED, "[GraphManager:] rtSetDevice failed, modelId=%u, graphId=%u.", ge_root_model->GetModelId(), + GELOGE(RT_FAILED, "[Call][RtSetDevice] failed, modelId=%u, graphId=%u.", ge_root_model->GetModelId(), graph_id); return FAILED; } @@ -1679,16 +1676,15 @@ Status GraphManager::RemoveGraph(const GraphId &graph_id) { // unload them respectively. middle_ret = UnloadModel(ge_root_model, graph_id); if (middle_ret != SUCCESS) { - REPORT_INNER_ERROR("E19999", "UnloadModel for graph:%u failed, check unload detail in GraphLoader %s", - graph_id, __FUNCTION__); - GELOGE(middle_ret, "[GraphManager:] unload model failed, graph_id=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "UnloadModel for graph:%u failed, check invalid", graph_id); + GELOGE(middle_ret, "[Unload][Model] model failed, graph_id=%u.", graph_id); ret = middle_ret; } rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, graph_id:%u, when GraphManager %s", - GetContext().DeviceId(), graph_id, __FUNCTION__); - GELOGE(RT_FAILED, "[GraphManager:] rtDeviceReset failed, graphId=%u.", graph_id); + REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, graph_id:%u", + GetContext().DeviceId(), graph_id); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, graph_id:%u", GetContext().DeviceId(), graph_id); ret = FAILED; } } @@ -1697,7 +1693,7 @@ Status GraphManager::RemoveGraph(const GraphId &graph_id) { RemoveGraphCount(graph_id); RemoveAddGraphCondition(graph_id); - GE_CHK_STATUS_RET(ret, "[GraphManager:] Remove graph failed, graph_id=%u.", graph_id); + GE_CHK_STATUS_RET(ret, "[Remove][Graph] failed, graph_id=%u.", graph_id); GELOGI("[GraphManager] remove graph success, graph_id=%u.", graph_id); return SUCCESS; } @@ -1710,8 +1706,7 @@ Status GraphManager::ParseOptions(const std::map &opti ret = ParseOption(options, STREAM_MAX_PARALLEL_NUM, options_.stream_max_parallel_num); if (ret != SUCCESS) { GELOGE(GE_GRAPH_OPTIONS_INVALID, - "parse Key:%s value failed, it must be same format as " - "DNN_V100:2,DNN_HCCL:3", + "[Parse][Option] %s value failed, it must be same format as DNN_V100:2,DNN_HCCL:3", STREAM_MAX_PARALLEL_NUM.c_str()); return GE_GRAPH_OPTIONS_INVALID; } @@ -1719,23 +1714,23 @@ Status GraphManager::ParseOptions(const std::map &opti // get stream num ret = ParseOption(options, STREAM_NUM, options_.stream_num); if ((ret != SUCCESS) || (options_.stream_num == 0)) { - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.stream_num, its value %d is invalid, must be not equal zero.", - options_.stream_num); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.stream_num, its value %d is invalid, " + "must be not equal zero.", options_.stream_num); return GE_GRAPH_OPTIONS_INVALID; } // get perf level, its value please see enum PerfLevel ret = ParseOption(options, PERF_LEVEL, options_.perf_level); if ((ret != SUCCESS) || IsPerfLevelInvalid(options_.perf_level)) { - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.perfLevel, its value %d is invalid, must be enum PerfLevel type.", - options_.perf_level); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.perfLevel, its value %d is invalid, " + "must be enum PerfLevel type.", options_.perf_level); return GE_GRAPH_OPTIONS_INVALID; } // get encrypt mode ret = ParseOption(options, ENCRYPT_MODE, options_.encrypt_mode); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.encryptMode value invalid."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.encryptMode value invalid."); return GE_GRAPH_OPTIONS_INVALID); // get ek file @@ -1775,7 +1770,8 @@ Status GraphManager::ParseOptions(const std::map &opti // get weight compress flag ret = ParseOption(options, COMPRESS_FLAG, options_.compress_flag); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.compressFlag value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.compressFlag value is invalid, " + "must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); // Set Build model and step ParseOption(options, BUILD_MODE, options_.build_mode); @@ -1786,21 +1782,22 @@ Status GraphManager::ParseOptions(const std::map &opti options_.run_graph_flag = true; ret = ParseOption(options, RUN_FLAG, options_.run_graph_flag); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.runFlag value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.runFlag value is invalid, must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); // ge.graphType ret = ParseTrainGraphFlag(options_.run_graph_flag, options_.train_graph_flag); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.runFlag value is invalid"); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][TrainGraphFlag] Key:ge.runFlag value is invalid"); return GE_GRAPH_OPTIONS_INVALID); // parse FmkOp options_.local_fmk_op_flag = false; ret = ParseOption(options, LOCAL_FMKOP_FLAG, options_.local_fmk_op_flag); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.localFmkopFlag value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.localFmkopFlag value is invalid, " + "must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); options_.enable_print_op_pass = true; ret = ParseOption(options, ENABLE_PRINT_OP_PASS, options_.enable_print_op_pass); @@ -1808,13 +1805,15 @@ Status GraphManager::ParseOptions(const std::map &opti options_.is_single_op = false; ret = ParseOption(options, SINGLE_OP_FLAG, options_.is_single_op); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.enablePrintOpPass value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.enablePrintOpPass value is invalid, " + "must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); // parse hcom parallel options_.hcom_parallel = false; ret = ParseOption(options, HCOM_PARALLEL, options_.hcom_parallel); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.hcomParallel value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.hcomParallel value is invalid, " + "must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); // net output node dataType ParseOption(options, OUTPUT_DATATYPE, options_.output_datatype); @@ -1874,10 +1873,9 @@ Status GraphManager::ParseOption(const std::map &optio } else if (flag == "1") { option = true; } else { - REPORT_INNER_ERROR("E19999", "Option:%s value:%s must be 0 or 1, check invalid", - key.c_str(), flag.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:%s, its value %s is invalid, it must be 0 or 1.", key.c_str(), - flag.c_str()); + REPORT_INNER_ERROR("E19999", "Option:%s value:%s must be 0 or 1, check invalid", key.c_str(), flag.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] Key:%s, its value %s is invalid, it must be 0 or 1.", + key.c_str(), flag.c_str()); return GE_GRAPH_OPTIONS_INVALID; } } @@ -1894,8 +1892,8 @@ Status GraphManager::ParseOption(const std::map &optio if (ptr != nullptr && *ptr != '\0') { REPORT_INNER_ERROR("E19999", "Option:%s value:%s must be int32_t type, check invalid", key.c_str(), iter->second.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:%s, its value %s is invalid, must be int32_t type.", key.c_str(), - iter->second.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] Key:%s, its value %s is invalid, must be int32_t type.", + key.c_str(), iter->second.c_str()); return GE_GRAPH_OPTIONS_INVALID; } } @@ -1939,10 +1937,8 @@ Status GraphManager::ParseOption(const std::map &optio if (pos == string::npos) { REPORT_INNER_ERROR("E19999", "Option:%s, value:%s, engine and num must be connected by :, check invalid", key.c_str(), engine_parallel.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, - "engine and num must be connected by :, " - "while your input is %s", - engine_parallel.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] engine and num must be connected by :, " + "while your input is %s", engine_parallel.c_str()); return GE_GRAPH_OPTIONS_INVALID; } std::string engine_name = engine_parallel.substr(0, pos); @@ -1952,14 +1948,14 @@ Status GraphManager::ParseOption(const std::map &optio Status ret = CheckEngineName(engine_name, key, option); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_OPTIONS_INVALID, "check engine name : %s failed, ", engine_name.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][EngineName] %s failed", engine_name.c_str()); return GE_GRAPH_OPTIONS_INVALID; } int num = 0; ret = ParseParallelNum(parallel_num, key, num); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parse parallel num failed"); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][ParallelNum] %s failed", parallel_num.c_str()); return GE_GRAPH_OPTIONS_INVALID; } @@ -1974,7 +1970,7 @@ Status GraphManager::CheckEngineName(const std::string &engine_name, const std:: if (engine_name.empty()) { REPORT_INNER_ERROR("E19999", "Option:%s, param engine_name:%s is empty, check invalid", key.c_str(), engine_name.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "engine name of %s is empty", key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] engine name of %s is empty", key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } // judge whether exist in engine list @@ -1986,7 +1982,7 @@ Status GraphManager::CheckEngineName(const std::string &engine_name, const std:: if (it_stream_repeat != option.end()) { REPORT_INNER_ERROR("E19999", "Option:%s, param engine_name:%s is repeated, check invalid", key.c_str(), engine_name.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "engine : %s of %s is repeated", engine_name.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] engine:%s of %s is repeated", engine_name.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } return SUCCESS; @@ -1996,14 +1992,15 @@ Status GraphManager::ParseParallelNum(const std::string ¶llel_num, const std if (parallel_num.empty()) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is empty, check invalid", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num of %s is empty", key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num of %s is empty", key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } for (char c : parallel_num) { if (!isdigit(c)) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is not digit, check invalid", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "%s input is invalid ", key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] Option:%s, param parallel num:%s is not digit, check invalid", + key.c_str(), parallel_num.c_str()); return GE_GRAPH_OPTIONS_INVALID; } } @@ -2013,24 +2010,28 @@ Status GraphManager::ParseParallelNum(const std::string ¶llel_num, const std } catch (std::invalid_argument &) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is invalid argument, check", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num : %s of %s is invalid argument", parallel_num.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num:%s of %s is invalid argument", + parallel_num.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } catch (std::out_of_range &) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is out of range, check", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num : %s of %s is out of range", parallel_num.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num:%s of %s is out of range", + parallel_num.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } catch (...) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is invalid argument, check", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num : %s of %s is invalid argument", parallel_num.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num:%s of %s is invalid argument", + parallel_num.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } if (num < 1) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s < 1, check invalid", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num : %s of %s must bigger than 0", parallel_num.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num:%s of %s must bigger than 0", + parallel_num.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } return SUCCESS; @@ -2057,9 +2058,8 @@ Status GraphManager::GetGraphNode(const GraphId &graph_id, GraphNodePtr &out) { auto iter = graph_map_.find(graph_id); if (iter == graph_map_.end()) { out = nullptr; - REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", - graph_id); - GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[GraphManager] graph not exist, graph_id= %u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", graph_id); + GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[Check][Param] graph not exist, graph_id= %u.", graph_id); return GE_GRAPH_GRAPH_NOT_EXIST; } out = iter->second; @@ -2080,7 +2080,7 @@ Status GraphManager::SummaryHandle(const GraphId &graph_id, std::vector &summary_output_indexes = whole_summary_output_indexes.at(graph_id); @@ -2125,9 +2125,8 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap } } if (netoutput == nullptr) { - REPORT_INNER_ERROR("E19999", "No netoutput node in graph:%u, check invalid", - graph_id); - GELOGE(FAILED, "Netoutput is null."); + REPORT_INNER_ERROR("E19999", "No netoutput node in graph:%u, check invalid", graph_id); + GELOGE(FAILED, "[Check][Param] No netoutput node in graph:%u", graph_id); return FAILED; } for (const auto &in : netoutput->GetAllInDataAnchors()) { @@ -2135,9 +2134,9 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap auto out_anchor = in->GetPeerOutAnchor(); if (out_anchor == nullptr) { REPORT_INNER_ERROR("E19999", "Peer anchor of op:%s(%s), in_index:%u is nullptr, graph_id:%u, check invalid", - netoutput->GetName().c_str(), netoutput->GetType().c_str(), - in->GetIdx(), graph_id); - GELOGE(FAILED, "out_anchor is null."); + netoutput->GetName().c_str(), netoutput->GetType().c_str(), in->GetIdx(), graph_id); + GELOGE(FAILED, "[Get][PeerOutAnchor] Peer anchor of op:%s(%s), in_index:%u is nullptr, graph_id:%u", + netoutput->GetName().c_str(), netoutput->GetType().c_str(), in->GetIdx(), graph_id); return FAILED; } ge::NodePtr peer_node = out_anchor->GetOwnerNode(); @@ -2146,7 +2145,8 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap if (peer_node->GetAllInDataAnchors().size() != 1) { REPORT_INNER_ERROR("E19999", "More than one prior nodes of peer_node:%s(%s) in checkpoint Graph:%u, " "check invalid", peer_node->GetName().c_str(), peer_node->GetType().c_str(), graph_id); - GELOGE(FAILED, "More than one prior nodes of peer_node %s in checkpoint Graph.", peer_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] More than one prior nodes of peer_node:%s(%s) in checkpoint Graph:%u.", + peer_node->GetName().c_str(), peer_node->GetType().c_str(), graph_id); return FAILED; } auto peer_node_in = peer_node->GetAllInDataAnchors().at(0); @@ -2160,9 +2160,9 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap } if (peer_node == nullptr) { REPORT_INNER_ERROR("E19999", "Peer anchor node of op:%s(%s), in_index:%u is nullptr, graph_id:%u, check invalid", - netoutput->GetName().c_str(), netoutput->GetType().c_str(), - in->GetIdx(), graph_id); - GELOGE(FAILED, "No variable op found in one branch, checkpoint graph illegal."); + netoutput->GetName().c_str(), netoutput->GetType().c_str(), in->GetIdx(), graph_id); + GELOGE(FAILED, "[Check][Param] Peer anchor node of op:%s(%s), in_index:%u is nullptr, graph_id:%u", + netoutput->GetName().c_str(), netoutput->GetType().c_str(), in->GetIdx(), graph_id); return FAILED; } desc_name = peer_node->GetName(); @@ -2171,7 +2171,8 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap REPORT_INNER_ERROR("E19999", "in index:%u of op:%s(%s) is out of outputs.size:%zu range, graph_id:%u, " "check invalid", in->GetIdx(), netoutput->GetName().c_str(), netoutput->GetType().c_str(), outputs.size(), graph_id); - GELOGE(FAILED, "variable index out of range."); + GELOGE(FAILED, "[Check][Param] in index:%u of op:%s(%s) is out of outputs.size:%zu range, graph_id:%u", + in->GetIdx(), netoutput->GetName().c_str(), netoutput->GetType().c_str(), outputs.size(), graph_id); return FAILED; } save_results.emplace(desc_name, TensorAdapter::AsTensor(outputs.at(in->GetIdx()))); @@ -2217,9 +2218,8 @@ Status GraphManager::PushSummaryData2ME(const GraphId &graph_id, } return iter->second(graph_id, tmp_summary_data); } - REPORT_INNER_ERROR("E19999", "No summary callback found, graph_id:%u, check invalid", - graph_id); - GELOGE(FAILED, "[GraphManager] PushSummaryData2ME failed, not found summary callback."); + REPORT_INNER_ERROR("E19999", "No summary callback found, graph_id:%u, check invalid", graph_id); + GELOGE(FAILED, "[Check][Param] No summary callback found, graph_id:%u", graph_id); return FAILED; } return itr->second(graph_id, summary_data); @@ -2239,9 +2239,8 @@ Status GraphManager::PushSaveData2ME(const GraphId &graph_id, const std::mapsecond(graph_id, tmp_save_data); } - REPORT_INNER_ERROR("E19999", "No checkpoint callback found, graph_id:%u, check invalid", - graph_id); - GELOGE(FAILED, "[GraphManager] PushSaveData2ME failed, not found checkpoint callback."); + REPORT_INNER_ERROR("E19999", "No checkpoint callback found, graph_id:%u, check invalid", graph_id); + GELOGE(FAILED, "[Check][Param] No checkpoint callback found, graph_id:%u", graph_id); return FAILED; } return itr->second(graph_id, save_data); @@ -2271,7 +2270,8 @@ bool GraphManager::CheckVariableForCheckpointGraph(NodePtr &node) { if (out == nullptr) { REPORT_INNER_ERROR("E19999", "anchor index:0 of op:%s(%s) is nullptr, check invalid", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "out is nullptr."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Get][OutDataAnchor] anchor index:0 of op:%s(%s) is nullptr", + node->GetName().c_str(), node->GetType().c_str()); return false; } auto peer_out = out->GetPeerInDataAnchors(); @@ -2304,7 +2304,7 @@ static inline bool CheckConstanOpForCheckpointGraph(NodePtr &node) { return node bool GraphManager::IsCheckpointGraph(ComputeGraphPtr &compute_graph) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[IsCheckpointGraph] computeGraph is nullptr."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] computeGraph is nullptr."); return false; } for (auto &node : compute_graph->GetAllNodes()) { @@ -2438,9 +2438,8 @@ Status GraphManager::RemoveIsolatedConstInThisGraph(ge::ComputeGraphPtr &compute if (n->GetOutAllNodes().empty() && n->GetInAllNodes().empty()) { // it is an isolated constant, just remove it if (GraphUtils::RemoveJustNode(compute_graph, n) != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Remove constant op:%s(%s) failed", - n->GetName().c_str(), n->GetType().c_str()); - GELOGE(FAILED, "remove constant %s failed.", n->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "Remove constant op:%s(%s) failed", n->GetName().c_str(), n->GetType().c_str()); + GELOGE(FAILED, "[Call][RemoveJustNode] remove constant %s failed.", n->GetName().c_str()); return FAILED; } } @@ -2519,7 +2518,7 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { auto ret = after_merge_passes.Run(compute_graph); GE_TIMESTAMP_END(after_merge_passes, "GraphManager::OptimizeStage1_1"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run passes when OptimizeStage1_1 failed, ret:%u.", ret); + GELOGE(ret, "[Run][Passes] when OptimizeStage1_1 failed, ret:%u.", ret); return ret; } @@ -2558,7 +2557,7 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { ret = GEPass(compute_graph).Run(names_to_passes); GE_TIMESTAMP_END(names_to_passes, "GraphManager::OptimizeStage1_2"); if (ret != SUCCESS) { - GELOGE(ret, "Run passes when OptimizeStage1_2 failed, ret:%u.", ret); + GELOGE(ret, "[Run][Passes] when OptimizeStage1_2 failed, ret:%u.", ret); return ret; } // Calculate Op/Fe constantfolding cost @@ -2614,7 +2613,7 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { ret = graph_pass.Run(compute_graph); GE_TIMESTAMP_END(graph_pass, "GraphManager::OptimizeStage1_3"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run passes when OptimizeStage1_3 failed, ret:%u.", ret); + GELOGE(ret, "[Run][Passes] when OptimizeStage1_3 failed, ret:%u.", ret); return ret; } NamesToPass node_pass; @@ -2624,7 +2623,7 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { ret = GEPass(compute_graph).Run(node_pass); GE_TIMESTAMP_END(node_pass, "GraphPrepare::node_pass"); if (ret != SUCCESS) { - GELOGE(ret, "Run identity remove pass for preprocess failed, ret:%u.", ret); + GELOGE(ret, "[Run][Identity] remove pass for preprocess failed, ret:%u.", ret); return ret; } return SUCCESS; @@ -2644,7 +2643,7 @@ Status GraphManager::OptimizeStage2(ge::ComputeGraphPtr &compute_graph) { auto ret = after_merge_passes.Run(compute_graph); GE_TIMESTAMP_END(after_merge_passes, "OptimizeStage2::AfterMergePasses"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run passes after merge sub graph failed, ret:%d.", ret); + GELOGE(ret, "[Run][Passes] after merge sub graph failed, ret:%d.", ret); return ret; } SetAttrForHcomBroadCastOp(compute_graph); @@ -2668,13 +2667,13 @@ Status GraphManager::OptimizeStage2(ge::ComputeGraphPtr &compute_graph) { ret = GEPass(compute_graph).Run(names_to_passes); GE_TIMESTAMP_END(names_to_passes, "OptimizeStage2::MergedGraphNameToPasses"); if (ret != SUCCESS) { - GELOGE(ret, "Run ge_passes optimize for OptimizeAfterMergeSubGraph failed, ret:%d.", ret); + GELOGE(ret, "[Run][GEPasses] optimize for OptimizeAfterMergeSubGraph failed, ret:%d.", ret); return ret; } ret = RemoveIsolatedConst(compute_graph); if (ret != SUCCESS) { - GELOGE(ret, "Remove isolated Constant failed, ret:%d.", ret); + GELOGE(ret, "[Remove][IsolatedConst] failed, ret:%d.", ret); return ret; } @@ -2725,32 +2724,33 @@ Status GraphManager::OptimizeStage2(ge::ComputeGraphPtr &compute_graph) { ret = pass_for_control_attr_optimize.Run(compute_graph); GE_TIMESTAMP_END(pass_for_control_attr_optimize, "OptimizeStage2::ControlAttrOptimize"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run passes when optimize stage 2 failed"); + GELOGE(ret, "[Run][Passes] when optimize stage 2 failed"); return ret; } // Assign functional op labels. GE_TIMESTAMP_START(AssignFunctionalLabels); LabelAllocator label_allocator(compute_graph); - GE_CHK_STATUS_RET(label_allocator.AssignFunctionalLabels(), "Assign label failed."); + GE_CHK_STATUS_RET(label_allocator.AssignFunctionalLabels(), "[Assign][Label] failed."); GE_TIMESTAMP_END(AssignFunctionalLabels, "ModelBuilder::AssignFunctionalLabels"); // Add memcpy addr asynchronous node. GE_TIMESTAMP_START(AddMemcpyAddrAsyncNode); MemcpyAddrAsyncPass memcpy_addr; - GE_CHK_STATUS_RET(memcpy_addr.Run(compute_graph), "Add memcpy_addr_async node failed."); + GE_CHK_STATUS_RET(memcpy_addr.Run(compute_graph), "[Call][Run] Add memcpy_addr_async node failed."); GE_TIMESTAMP_END(AddMemcpyAddrAsyncNode, "MemcpyAddrAsyncPass::Run."); // Process offset and dependency for buffer pool memory assigner. GE_TIMESTAMP_START(BufferPoolMemoryPass); BufferPoolMemoryPass buffer_pool_mem_pass; - GE_CHK_STATUS_RET(buffer_pool_mem_pass.Run(compute_graph), "Failed to process for buffer pool allocator."); + GE_CHK_STATUS_RET(buffer_pool_mem_pass.Run(compute_graph), + "[Call][Run] Failed to process for buffer pool allocator."); GE_TIMESTAMP_END(BufferPoolMemoryPass, "BufferPoolMemoryPass::Run."); // Handle parallel group . GE_TIMESTAMP_START(ParallelGroup); ParallelGroupPass parallel_group_pass; - GE_CHK_STATUS_RET(parallel_group_pass.Run(compute_graph), "Handle parallel group failed."); + GE_CHK_STATUS_RET(parallel_group_pass.Run(compute_graph), "[Handle][ParallelGroup] failed."); GE_TIMESTAMP_END(ParallelGroup, "ParallelGroupPass::Run."); // After while sub graph handle, mark all node rw type @@ -2803,7 +2803,7 @@ Status GraphManager::LoadGraphAsync(const GeRootModelPtr &ge_root_model, const G Status ret = GraphLoader::LoadModelOnline(model_id_info.model_id, ge_root_model, listener); GE_TIMESTAMP_EVENT_END(LoadGraph, "GraphManager::LoadGraphAsync"); if (ret != SUCCESS) { - GELOGE(ret, "[LoadGraphAsync] LoadGraphAsync Failed"); + GELOGE(ret, "[Load][ModelOnline] Failed, model_id:%u", model_id_info.model_id); graph_node->SetRunFlag(false); return ret; } @@ -2818,9 +2818,8 @@ void GraphManager::ReleaseMemory(const GeModelPtr &ge_model, GraphNodePtr &graph const std::vector &model_ids, uint32_t graph_id, uint64_t session_id) { rtError_t rt_ret = rtSetDevice(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, when GraphManager %s", - GetContext().DeviceId(), __FUNCTION__); - GELOGE(RT_FAILED, "[GraphManager:] rtSetDevice failed, graphId=%u.", graph_id); + REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u", GetContext().DeviceId()); + GELOGE(RT_FAILED, "[Call][RtSetDevice] failed, device_id=%u.", GetContext().DeviceId()); return; } for (auto model_id : model_ids) { @@ -2862,9 +2861,8 @@ void GraphManager::ReleaseMemory(const GeModelPtr &ge_model, GraphNodePtr &graph ge_root_model->ClearAllModelId(); rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, when GraphManager %s", - GetContext().DeviceId(), __FUNCTION__); - GELOGE(RT_FAILED, "[GraphManager:] rtDeviceReset failed, graphId=%u.", graph_id); + REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u", GetContext().DeviceId()); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u.", GetContext().DeviceId()); return; } } @@ -2892,7 +2890,8 @@ Status GraphManager::CheckAndReleaseMemory(const GeModelPtr &ge_model, const Gra if (ge::CheckInt64AddOverflow(memory_size, weight_size) != SUCCESS) { REPORT_INNER_ERROR("E19999", "memory_size:%ld and weight_size:%ld will overflow after add, check invalid", memory_size, weight_size); - GELOGE(INTERNAL_ERROR, "The sum of Memory size and weight size exceeds INT64_MAX"); + GELOGE(INTERNAL_ERROR, "[Check][Param] memory_size:%ld and weight_size:%ld will overflow after add", + memory_size, weight_size); return INTERNAL_ERROR; } if (free_memory >= (memory_size + weight_size)) { @@ -2952,23 +2951,25 @@ Status GraphManager::ProcessSubGraphWithMultiThreads(GraphManager *graph_manager GE_DUMP(compute_graph_tmp, "OptimizeSubGraphBefore"); GE_CHECK_NOTNULL(compute_graph_tmp); if (!AttrUtils::SetInt(*compute_graph_tmp, ATTR_NAME_ROOT_GRAPH_ID, root_graph_id)) { - REPORT_CALL_ERROR("E19999", "Set Attr:%s to graph:%u", ATTR_NAME_ROOT_GRAPH_ID.c_str(), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to graph:%u failed", ATTR_NAME_ROOT_GRAPH_ID.c_str(), compute_graph_tmp->GetGraphID()); - GELOGE(FAILED, "Failed to set attr ATTR_NAME_ROOT_GRAPH_ID for subgraph, graph_id: %u.", root_graph_id); + GELOGE(FAILED, "[Set][Attr] %s to graph:%u failed", ATTR_NAME_ROOT_GRAPH_ID.c_str(), + compute_graph_tmp->GetGraphID()); return FAILED; } if (!AttrUtils::SetStr(*compute_graph_tmp, ATTR_NAME_ROOT_GRAPH_NAME, root_graph_name)) { - REPORT_CALL_ERROR("E19999", "Set Attr:%s to graph:%u", ATTR_NAME_ROOT_GRAPH_NAME.c_str(), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to graph:%u failed", ATTR_NAME_ROOT_GRAPH_NAME.c_str(), compute_graph_tmp->GetGraphID()); - GELOGE(FAILED, "Failed to set attr ATTR_NAME_ROOT_GRAPH_NAME for subgraph, \ - root_graph_name: %s.", root_graph_name.c_str()); + GELOGE(FAILED, "[Set][Attr] %s to graph:%u failed", ATTR_NAME_ROOT_GRAPH_NAME.c_str(), + compute_graph_tmp->GetGraphID()); return FAILED; } compute_graph_tmp->SetSessionID(session_id); Status ret = graph_manager->GetCompilerStages(root_graph_id).optimizer.OptimizeSubGraph(compute_graph_tmp, engine_name); if (ret != SUCCESS) { - GELOGE(ret, "SubGraph optimize Failed %s", engine_name.c_str()); + GELOGE(ret, "[Optimize][SubGraph] Failed, engine:%s, graph:%s", + engine_name.c_str(), compute_graph_tmp->GetName().c_str()); return ret; } else { GELOGD("SubGraph optimize success %s", engine_name.c_str()); @@ -2980,7 +2981,7 @@ Status GraphManager::ProcessSubGraphWithMultiThreads(GraphManager *graph_manager pthread_self()); } else { REPORT_INNER_ERROR("E19999", "Param sub_graph_info_ptr or graph_manager is nullptr"); - GELOGE(FAILED, "graph_manager or sub_graph_info_ptr is nullptr"); + GELOGE(FAILED, "[Check][Param] graph_manager or sub_graph_info_ptr is nullptr"); return FAILED; } @@ -2997,7 +2998,7 @@ Status GraphManager::RunGraphAsync(const GraphId &graph_id, const std::vectorGetGraphCount(args.graph_id, count) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Get graph [id:%u] count failed.", args.graph_id); + GELOGE(INTERNAL_ERROR, "[Get][GraphCount] failed, graph id:%u.", args.graph_id); return; } // Avoid repeatively prerun for graphs owns same graph_id in online inference concurrency @@ -3215,7 +3216,8 @@ Status GraphManager::ParseInputsDimsForGetNexNosinkAndData(const vector if (!(AttrUtils::GetInt(op_desc, ATTR_NAME_INDEX, index))) { REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) fail", ATTR_NAME_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "Get index from attr failed"); + GELOGE(PARAM_INVALID, "[Get][Attr] %s from op:%s(%s) fail", ATTR_NAME_INDEX.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } if (static_cast(index) > input_tensor.size()) { @@ -3223,7 +3225,9 @@ Status GraphManager::ParseInputsDimsForGetNexNosinkAndData(const vector "check invalid", ATTR_NAME_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), index, input_tensor.size()); - GELOGE(PARAM_INVALID, "The count of input tensor should be equal to the count of data."); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s in op:%s(%s) value:%ld > param input_tensor.size:%zu", + ATTR_NAME_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), + index, input_tensor.size()); return PARAM_INVALID; } @@ -3253,7 +3257,7 @@ Status GraphManager::ParseInputsDims(const std::vector &input_tensor } else { // data+getnext_nosink, but only need to get shape_dims of data if (ParseInputsDimsForGetNexNosinkAndData(data_nodes, input_tensor) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to parse dims from data, when data coexist with getnext nosink."); + GELOGE(PARAM_INVALID, "[Parse][Dims] from data failed, when data coexist with getnext nosink."); return PARAM_INVALID; } } @@ -3269,7 +3273,7 @@ Status GraphManager::ParseInputsDims(const std::vector &input_tensor } else { // getnext_nosink + data, but only need to get shape_dims of getnext_nosink if (ParseInputsDimsForGetNexNosinkAndData(getnext_nosink_nodes, input_tensor) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to parse dims from getnext nosink, when data coexist with getnext nosink"); + GELOGE(PARAM_INVALID, "[Parse][Dims] from getnext nosink failed, when data coexist with getnext nosink"); return PARAM_INVALID; } } @@ -3376,9 +3380,8 @@ void GraphManager::ReturnError(GraphManager *graph_manager, GraphNodePtr &graph_ std::vector outputs; auto compute_graph = GraphUtils::GetComputeGraph(*graph_node->GetGraph()); if (graph_manager == nullptr || compute_graph == nullptr) { - REPORT_INNER_ERROR("E19999", "Param graph_manager or compute_graph in graph_node is nullptr, " - "check invalid"); - GELOGE(GRAPH_FAILED, "[Analyze Mode] compute graph is null!"); + REPORT_INNER_ERROR("E19999", "Param graph_manager or compute_graph in graph_node is nullptr, check invalid"); + GELOGE(GRAPH_FAILED, "[Check][Param] compute graph or graph manager is nullptr"); callback(GRAPH_FAILED, outputs); return; } @@ -3401,7 +3404,8 @@ void GraphManager::ReturnError(GraphManager *graph_manager, GraphNodePtr &graph_ REPORT_INNER_ERROR("E19999", "InputIndex:%zu ShapeSize:%ld of op:%s(%s) < 0, unknown shape is not support, " "check invalid", i, len, node->GetName().c_str(), node->GetType().c_str()); - GELOGE(GRAPH_FAILED, "Analyze Mode does not support GEOP output unknown shape!"); + GELOGE(GRAPH_FAILED, "[Check][Param] InputIndex:%zu ShapeSize:%ld of op:%s(%s) < 0, " + "unknown shape is not support", i, len, node->GetName().c_str(), node->GetType().c_str()); callback(GRAPH_FAILED, outputs); return; } else if (len == 0) { @@ -3411,8 +3415,8 @@ void GraphManager::ReturnError(GraphManager *graph_manager, GraphNodePtr &graph_ auto length = GetSizeInBytes(len, input_desc->GetDataType()); auto aligned_ptr = MakeShared(length, kAlignment); if (aligned_ptr == nullptr) { - REPORT_INNER_ERROR("E19999", "Aligned_ptr is nullptr"); - GELOGE(GRAPH_FAILED, "[Analyze Mode] Aligned_ptr is nullptr"); + REPORT_CALL_ERROR("E19999", "New AlignedPtr failed, len:%ld", length); + GELOGE(GRAPH_FAILED, "[Create][AlignedPtr] failed, len:%ld", length); return; } ge_tensor.SetData(aligned_ptr, length); @@ -3434,16 +3438,14 @@ bool GraphManager::IsGraphNeedRebuild(uint32_t graph_id) { GraphNodePtr graph_node = nullptr; Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS) { - REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", - graph_id); - GELOGE(ret, "[RunGraph] graph not exist, graph_id=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", graph_id); + GELOGE(ret, "[Get][GraphNode] failed, graph not exist, graph_id:%u.", graph_id); return true; } if (graph_node == nullptr) { - REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph_id:%u, check invalid", - graph_id); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[RunGraph] graph node is NULL, graphId=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph_id:%u, check invalid", graph_id); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Check][Param] graph node is NULL, graph_id:%u.", graph_id); return true; } @@ -3457,16 +3459,14 @@ const map *GraphManager::GetGraphOptions(uint32_t grap GraphNodePtr graph_node = nullptr; Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS) { - REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", - graph_id); - GELOGE(ret, "[RunGraph] graph not exist, graph_id=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", graph_id); + GELOGE(ret, "[Get][GraphNode] failed, graph not exist, graph_id:%u.", graph_id); return nullptr; } if (!graph_node) { - REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph_id:%u, check invalid", - graph_id); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[RunGraph] graph node is NULL, graph_id=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph_id:%u, check invalid", graph_id); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Check][Param] graph node is NULL, graph_id:%u.", graph_id); return nullptr; } return &(graph_node->GetOptions()); @@ -3482,7 +3482,7 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra StagePartitioner stage_partitioner(compute_graph); auto ret = stage_partitioner.Partition(); if (ret != SUCCESS) { - GELOGE(ret, "Graph partition by stage Failed"); + GELOGE(ret, "[Call][Partition] for Graph:%s by stage Failed", compute_graph->GetName().c_str()); return ret; } GE_TIMESTAMP_EVENT_END(StagePartition, "OptimizeSubgraph::StagePartition"); @@ -3491,14 +3491,16 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra DynamicShapePartitioner dynamic_shape_partitioner(compute_graph); ret = dynamic_shape_partitioner.Partition(); if (ret != SUCCESS) { - GELOGE(ret, "Graph partition by dynamic shape Failed"); + GELOGE(ret, "[Call][Partition] for Graph:%s by dynamic shape Failed", compute_graph->GetName().c_str()); return ret; } bool dynamic_shape_partitioned = false; if (!AttrUtils::GetBool(*compute_graph, ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED, dynamic_shape_partitioned)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s from graph:%u fail", - ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetGraphID()); - GELOGE(FAILED, "failed get dynamic shape partitioned flag on partitioned graph."); + REPORT_INNER_ERROR("E19999", "Get Attr:%s from graph:%s(id:%u) fail", + ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetName().c_str(), + compute_graph->GetGraphID()); + GELOGE(FAILED, "[Get][Attr] %s from graph:%u failed", + ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetGraphID()); return FAILED; } GE_TIMESTAMP_EVENT_END(GraphPartitionDynamicShape, "OptimizeSubgraph::GraphPartitionDynamicShape"); @@ -3507,14 +3509,14 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra GraphPartitioner &partitioner = GetCompilerStages(graph_node->GetGraphId()).partitioner; ret = partitioner.Partition(compute_graph, GraphPartitioner::kPartitioning); if (ret != SUCCESS) { - GELOGE(ret, "Graph partition Failed"); + GELOGE(ret, "[Call][Partition] for Graph:%s Failed", compute_graph->GetName().c_str()); return ret; } GE_TIMESTAMP_EVENT_END(GraphPartition, "OptimizeSubgraph::Partition1"); GE_TIMESTAMP_START(SetSubgraph); ret = SetSubgraph(session_id, compute_graph, partitioner); if (ret != SUCCESS) { - GELOGE(ret, "Graph set subgraph Failed"); + GELOGE(ret, "[Set][Subgraph] failed for graph:%s, session_id:%lu", compute_graph->GetName().c_str(), session_id); return ret; } GE_TIMESTAMP_EVENT_END(SetSubgraph, "OptimizeSubgraph::SetSubGraph"); @@ -3527,7 +3529,7 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra Status ret = ConvertGraphToFile(compute_graph, partitioner, tuning_path, (options_.build_step == BUILD_STEP_AFTER_BUILDER)); if (ret != SUCCESS) { - GELOGE(ret, "Convert graph[%s] to file failed", compute_graph->GetName().c_str()); + GELOGE(ret, "[Convert][Graph] [%s] to file failed", compute_graph->GetName().c_str()); return ret; } GE_TIMESTAMP_EVENT_END(ConvertGraphToFile, "OptimizeSubgraph::ConvertGraphToFile"); @@ -3540,7 +3542,8 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra GE_TIMESTAMP_START(MergeSubgraph); ret = MergeSubGraph(merged_compute_graph, compute_graph, graph_node->GetGraphId()); if (ret != SUCCESS) { - GELOGE(ret, "Merge SubGraph Failed"); + GELOGE(ret, "[Merge][SubGraph] Failed, graph:%s(id:%u)", + compute_graph->GetName().c_str(), graph_node->GetGraphId()); return ret; } GE_CHECK_NOTNULL(merged_compute_graph); @@ -3557,8 +3560,8 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra if (!AttrUtils::SetBool(merged_compute_graph, ATTR_NAME_OFF_SUPERKERNEL_ATTR, off_superkernel)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s to graph:%u fail", ATTR_NAME_OFF_SUPERKERNEL_ATTR.c_str(), compute_graph->GetGraphID()); - GELOGE(FAILED, "Compute graph %s set superkernel flag %d failed", merged_compute_graph->GetName().c_str(), - off_superkernel); + GELOGE(FAILED, "[Set][Attr] %s to graph:%u fail", + ATTR_NAME_OFF_SUPERKERNEL_ATTR.c_str(), compute_graph->GetGraphID()); return FAILED; } } @@ -3568,7 +3571,8 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra if (!AttrUtils::SetBool(*compute_graph, ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED, dynamic_shape_partitioned)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s to graph:%u fail", ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetGraphID()); - GELOGE(FAILED, "failed set dynamic shape partitioned flag on partitioned graph."); + GELOGE(FAILED, "[Set][Attr] %s to graph:%u fail", + ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetGraphID()); return FAILED; } return SUCCESS; @@ -3628,7 +3632,7 @@ Status GraphManager::Build(const GraphNodePtr &graph_node, ComputeGraphPtr &comp auto ret = GetCompilerStages(graph_node->GetGraphId()).builder.Build(compute_graph, ge_root_model, session_id); if (ret != SUCCESS) { - GELOGE(ret, "SubGraph build Failed."); + GELOGE(ret, "[Call][Build] failed, session_id:%lu.", session_id); return ret; } @@ -3666,7 +3670,7 @@ Status GraphManager::GenCheckPointGraph(const std::mapAddOutputDesc(iter->second)); NodePtr var_node = compute_graph->AddNode(var_desc); GE_CHK_STATUS(GraphUtils::AddEdge(var_node->GetOutDataAnchor(0), save_node->GetInDataAnchor(index)), - "Add edge[%s->%s] fail.", var_node->GetName().c_str(), save_node->GetName().c_str()); + "[Add][Edge][%s->%s] fail.", var_node->GetName().c_str(), save_node->GetName().c_str()); index++; } compute_graph->Dump(); @@ -3677,13 +3681,12 @@ Status GraphManager::GenCheckPointGraph(const std::map &var_names, const std::vector &outputs, std::vector &var_values) { map var_results; - GE_CHK_STATUS_RET(SaveCheckPointResult(graph, outputs, var_results), "Save check point result failed."); + GE_CHK_STATUS_RET(SaveCheckPointResult(graph, outputs, var_results), "[Save][CheckPointResult] failed."); if (!var_names.empty()) { for (const auto &var_name : var_names) { if (var_results.count(var_name) == 0) { - REPORT_INNER_ERROR("E19999", "Fetch Var:%s result value fail", - var_name.c_str()); - GELOGE(FAILED, "Fetch var[%s] value failed.", var_name.c_str()); + REPORT_INNER_ERROR("E19999", "Fetch Var:%s result value fail", var_name.c_str()); + GELOGE(FAILED, "[Check][Param] Fetch var[%s] value failed.", var_name.c_str()); return FAILED; } else { auto var_tensor = var_results[var_name].GetTensorDesc(); @@ -3724,7 +3727,8 @@ Status GraphManager::SaveCheckPointResult(const Graph &graph, const std::vector< REPORT_INNER_ERROR("E19999", "peer node:%s(%s) of netoutput has more than 1 input in checkpoint Graph, " "check invalid", peer_node->GetName().c_str(), peer_node->GetType().c_str()); - GELOGE(FAILED, "peer_node [%s] has more than 1 input in checkpoint Graph.", peer_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] peer_node [%s] has more than 1 input in checkpoint Graph.", + peer_node->GetName().c_str()); return FAILED; } auto peer_node_in_anchor = peer_node->GetAllInDataAnchors().at(0); @@ -3740,7 +3744,7 @@ Status GraphManager::SaveCheckPointResult(const Graph &graph, const std::vector< REPORT_INNER_ERROR("E19999", "peer node:%s(%s) of netoutput is not variable in checkpoint Graph, " "check invalid", peer_node->GetName().c_str(), peer_node->GetType().c_str()); - GELOGE(FAILED, " peer_node %s is not variable in checkpoint Graph.", peer_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] peer_node %s is not variable in checkpoint Graph.", peer_node->GetName().c_str()); return FAILED; } auto var_name = peer_node->GetName(); @@ -3748,7 +3752,7 @@ Status GraphManager::SaveCheckPointResult(const Graph &graph, const std::vector< if (in->GetIdx() >= static_cast(outputs.size())) { REPORT_INNER_ERROR("E19999", "In index:%u of netoutput is out of outputs.size:%zu range in checkpoint Graph, " "check invalid", in->GetIdx(), outputs.size()); - GELOGE(FAILED, "variable index[%d] out of range[%zu].", in->GetIdx(), outputs.size()); + GELOGE(FAILED, "[Check][Param] variable index[%d] out of range[%zu].", in->GetIdx(), outputs.size()); return FAILED; } var_results.emplace(var_name, outputs.at(in->GetIdx())); diff --git a/ge/graph/manager/graph_manager_utils.cc b/ge/graph/manager/graph_manager_utils.cc index 0f93654c..a70b15a6 100644 --- a/ge/graph/manager/graph_manager_utils.cc +++ b/ge/graph/manager/graph_manager_utils.cc @@ -46,7 +46,7 @@ GraphNode::GraphNode(GraphId graph_id) sem_(1) { graph_run_async_listener_ = MakeShared(); if (graph_run_async_listener_ == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make shared failed"); + GELOGE(MEMALLOC_FAILED, "[New][RunAsyncListener] failed"); } } @@ -82,7 +82,8 @@ SubGraphInfo::~SubGraphInfo() { rt_ret = rtFreeHost(buffer_addr); buffer_addr = nullptr; if (rt_ret != RT_ERROR_NONE) { - GELOGE(rt_ret, "[GraphManager] subgraph free buffer failed, modelId = %u", model_id_info_.model_id); + GELOGE(rt_ret, "[Call][RtFreeHost] subgraph free buffer failed, modelId = %u", + model_id_info_.model_id); } } } @@ -94,8 +95,8 @@ Status SubGraphInfo::FreeInOutBuffer() { rtError_t rt_ret; rt_ret = rtFreeHost(*iter); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtFreeHost fail"); - GELOGE(rt_ret, "[GraphManager] subgraph free buffer failed, modelId = %u", model_id_info_.model_id); + REPORT_CALL_ERROR("E19999", "Call rtFreeHost fail, ret:%d", rt_ret); + GELOGE(rt_ret, "[Call][RtFreeHost] subgraph free buffer failed, modelId = %u", model_id_info_.model_id); buffer_addr_.erase(buffer_addr_.begin(), iter); return GE_GRAPH_FREE_FAILED; } @@ -131,7 +132,7 @@ Status GraphModelListener::OnComputeDone(uint32_t model_id, uint32_t task_id, ui uint32_t GraphModelListener::GetResultCode() const { if (!is_finished_) { REPORT_CALL_ERROR("E19999", "Model not run finish"); - GELOGE(INTERNAL_ERROR, "[GraphManager] model not run finish."); + GELOGE(INTERNAL_ERROR, "[Check][Param] model not run finish."); return INTERNAL_ERROR; } return result_code_; @@ -170,7 +171,9 @@ bool HasCalcOp(const ComputeGraphPtr &graph) { for (const auto &node : graph->GetAllNodes()) { OpDescPtr op_desc = node->GetOpDesc(); - GE_IF_BOOL_EXEC(op_desc == nullptr, GELOGE(FAILED, "Node GetOpDesc is nullptr"); return false); + GE_IF_BOOL_EXEC(op_desc == nullptr, + REPORT_INNER_ERROR("E19999", "GetOpDesc failed, Node GetOpDesc is nullptr"); + GELOGE(FAILED, "[Get][OpDesc] failed, Node GetOpDesc is nullptr"); return false); if (calc_op_type.find(op_desc->GetType()) != calc_op_type.end()) { return true; } diff --git a/ge/graph/manager/graph_mem_allocator.cc b/ge/graph/manager/graph_mem_allocator.cc index 0cccaf99..dd38274e 100755 --- a/ge/graph/manager/graph_mem_allocator.cc +++ b/ge/graph/manager/graph_mem_allocator.cc @@ -50,9 +50,7 @@ uint8_t *MemoryAllocator::MallocMemory(const string &purpose, size_t memory_size if (rtMalloc(reinterpret_cast(&memory_addr), memory_size, memory_type_) != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, purpose:%s, size:%zu, device_id:%u", purpose.c_str(), memory_size, device_id); - GELOGE(ge::INTERNAL_ERROR, - "MemoryAllocator::MallocMemory device_id = %u," - " size= %lu", + GELOGE(ge::INTERNAL_ERROR, "[Malloc][Memory] failed, device_id = %u, size= %lu", device_id, memory_size); return nullptr; @@ -68,7 +66,7 @@ Status MemoryAllocator::FreeMemory(uint8_t *memory_addr, uint32_t device_id) con auto rtRet = rtFree(memory_addr); if (rtRet != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtFree fail, device_id:%u", device_id); - GELOGE(rtRet, "MemoryAllocator::MallocMemory device_id = %u", device_id); + GELOGE(rtRet, "[Call][RtFree] failed, device_id = %u", device_id); return RT_ERROR_TO_GE_STATUS(rtRet); } memory_addr = nullptr; @@ -88,10 +86,8 @@ uint8_t *MemoryAllocator::MallocMemory(const string &purpose, const string &memo if (memory_addr == nullptr) { REPORT_CALL_ERROR("E19999", "Malloc Memory fail, purpose:%s, memory_key:%s, memory_size:%zu, device_id:%u", purpose.c_str(), memory_key.c_str(), memory_size, device_id); - GELOGE(ge::INTERNAL_ERROR, - "MemoryAllocator::MallocMemory failed," - " memory_key[%s], size = %lu.", - memory_key.c_str(), memory_size); + GELOGE(ge::INTERNAL_ERROR, "[Malloc][Memory] failed, memory_key[%s], size = %lu, device_id:%u.", + memory_key.c_str(), memory_size, device_id); return nullptr; } @@ -126,10 +122,8 @@ Status MemoryAllocator::FreeMemory(const string &memory_key, uint32_t device_id) if (FreeMemory(it->second.memory_addr_, device_id) != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Free Memory fail, memory_key:%s, device_id:%u", memory_key.c_str(), device_id); - GELOGE(ge::INTERNAL_ERROR, - "MemoryAllocator::FreeMemory rtFree failed," - " memory_key[%s]", - memory_key.c_str()); + GELOGE(ge::INTERNAL_ERROR, "[Free][Memory] failed, memory_key[%s], device_id:%u", + memory_key.c_str(), device_id); return ge::INTERNAL_ERROR; } diff --git a/ge/graph/manager/graph_var_manager.cc b/ge/graph/manager/graph_var_manager.cc index 5f7586da..64918aab 100755 --- a/ge/graph/manager/graph_var_manager.cc +++ b/ge/graph/manager/graph_var_manager.cc @@ -40,7 +40,8 @@ ge::Status VarResource::GetVarAddr(const std::string &var_name, const ge::GeTens if (dev_ptr == nullptr) { REPORT_INNER_ERROR("E19999", "Param dev_ptr is nullptr, var_name:%s, session_id:%lu, " "check invalid", var_name.c_str(), session_id_); - GELOGE(FAILED, "[GetVarAddr] dev_ptr is null!"); + GELOGE(FAILED, "[Check][Param] Param dev_ptr is nullptr, var_name:%s, session_id:%lu", + var_name.c_str(), session_id_); return FAILED; } std::string var_key = VarKey(var_name, tensor_desc); @@ -51,7 +52,8 @@ ge::Status VarResource::GetVarAddr(const std::string &var_name, const ge::GeTens REPORT_INNER_ERROR("E19999", "var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, " "check invalid", var_key.c_str(), var_name.c_str(), session_id_); - GELOGE(FAILED, "VarResource::GetVarAddr failed, var_key %s", var_key.c_str()); + GELOGE(FAILED, "[Check][Param] var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu", + var_key.c_str(), var_name.c_str(), session_id_); return FAILED; } @@ -109,7 +111,8 @@ ge::Status VarResource::SaveVarAddr(const std::string &var_name, const ge::GeTen REPORT_INNER_ERROR("E19999", "var_key:%s conflict in var_addr_mgr_map_, var_name:%s, session_id:%lu, " "check invalid", var_key.c_str(), var_name.c_str(), session_id_); - GELOGE(FAILED, "VarResource::SaveVarAddr, var_key %s save addr conflict", var_key.c_str()); + GELOGE(FAILED, "[Check][Param] var_key:%s conflict in var_addr_mgr_map_, var_name:%s, session_id:%lu", + var_key.c_str(), var_name.c_str(), session_id_); return FAILED; } @@ -145,14 +148,15 @@ ge::Status VarResource::RenewCurVarDesc(const std::string &var_name, const ge::O if (op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "Param op_desc is nullptr, var_name:%s, session_id:%lu, check invalid", var_name.c_str(), session_id_); - GELOGE(FAILED, "[RenewCurVarDesc] renew var desc fail! input opdesc is null!"); + GELOGE(FAILED, "[Check][Param] input opdesc is nullptr, var_name:%s, session_id:%lu", + var_name.c_str(), session_id_); return FAILED; } ge::GeTensorDesc curr_desc; ge::Status ret = GetCurVarDesc(var_name, curr_desc); if (ret != SUCCESS) { - GELOGE(FAILED, "[RenewCurVarDesc] Get var desc fail!"); + GELOGE(FAILED, "[Get][CurVarDesc] fail, var_name:%s, session_id:%lu", var_name.c_str(), session_id_); return FAILED; } std::string key = VarKey(var_name, curr_desc); @@ -164,7 +168,8 @@ ge::Status VarResource::RenewCurVarDesc(const std::string &var_name, const ge::O REPORT_INNER_ERROR("E19999", "var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, op:%s(%s), " "check invalid", key.c_str(), var_name.c_str(), session_id_, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "[RenewCurVarDesc] can't find ele with key [%s]", key.c_str()); + GELOGE(FAILED, "[Check][Param] var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, op:%s(%s)", + key.c_str(), var_name.c_str(), session_id_, op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } auto val = iter->second; @@ -285,14 +290,15 @@ Status HbmMemResource::AssignVarMem(const std::string &var_name, uint64_t size, if (total_size_ < var_mem_size_) { REPORT_INNER_ERROR("E19999", "VarMemMaxSize:%lu < var_mem_size_:%lu, var_size:%lu, var_name:%s, check invalid" "", total_size_, var_mem_size_, size, var_name.c_str()); - GELOGE(PARAM_INVALID, "total_size_: %lu is smaller than var_mem_size_: %lu", total_size_, var_mem_size_); + GELOGE(PARAM_INVALID, "[Check][Param] total_size_:%lu is smaller than var_mem_size_:%lu, var_name:%s", + total_size_, var_mem_size_, var_name.c_str()); return PARAM_INVALID; } uint64_t free_size = total_size_ - var_mem_size_; if (free_size < (size + kSessionMemAlignSize * kSessionMemAlignUnit)) { REPORT_INNER_ERROR("E19999", "free_size:%lu not enough, var_align_size:%lu, var_name:%s, check invalid", free_size, size, var_name.c_str()); - GELOGE(PARAM_INVALID, "Out of memory : current var size[%lu] exceeds total var size[%lu]", + GELOGE(PARAM_INVALID, "[Check][Param] Out of memory: current var size[%lu] exceeds total var size[%lu]", size + kSessionMemAlignSize * kSessionMemAlignUnit + var_mem_size_, total_size_); return PARAM_INVALID; } @@ -317,7 +323,7 @@ Status RdmaMemResource::AssignVarMem(const std::string &var_name, uint64_t size, if (buffer == nullptr) { REPORT_CALL_ERROR("E19999", "malloc rdma memory fail, var_size:%lu, var_name:%s", size, var_name.c_str()); - GELOGE(MEMALLOC_FAILED, "Failed to malloc rdma memory for node %s, size = %lu", var_name.c_str(), size); + GELOGE(MEMALLOC_FAILED, "[Malloc][RdmaMemory] for node %s failed, size = %lu", var_name.c_str(), size); return MEMALLOC_FAILED; } address = static_cast(reinterpret_cast(buffer)); @@ -468,7 +474,8 @@ int64_t VarManager::GetVarMemSize(rtMemType_t memory_type) { if (mem_resource == nullptr) { REPORT_INNER_ERROR("E19999", "Find no mem_resource in map, memory_type:%d, session_id:%lu", memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid."); + GELOGE(ge::INTERNAL_ERROR, "[Check][Param] MemResource is invalid, memory_type:%d, session_id:%lu", + memory_type, session_id_); return 0; } return mem_resource->GetVarMemSize(); @@ -483,7 +490,8 @@ Status VarManager::UpdateVarMemSize(rtMemType_t memory_type, int64_t mem_size) { if (mem_resource == nullptr) { REPORT_CALL_ERROR("E19999", "memory_type:%d invalid or New MemResource fail, session_id:%lu", memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "Alloc MemResource failed, memory_type = %u.", memory_type); + GELOGE(ge::INTERNAL_ERROR, "[Alloc][MemResource] failed, memory_type:%u, session_id:%lu", + memory_type, session_id_); return ge::INTERNAL_ERROR; } else { mem_resource_map_[memory_type] = mem_resource; @@ -495,7 +503,8 @@ Status VarManager::UpdateVarMemSize(rtMemType_t memory_type, int64_t mem_size) { if (mem_resource == nullptr) { REPORT_INNER_ERROR("E19999", "MemResource is invalid, memory_type:%d, session_id:%lu", memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid."); + GELOGE(ge::INTERNAL_ERROR, "[Check][Param] MemResource is invalid, memory_type:%u, session_id:%lu", + memory_type, session_id_); return FAILED; } mem_resource->UpdateVarMemSize(mem_size); @@ -515,7 +524,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen if (result != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Get size from tensor fail, var_name:%s, memory_type:%d, session_id:%lu", var_name.c_str(), memory_type, session_id_); - GELOGE(result, "get size from TensorDesc failed"); + GELOGE(result, "[Get][Size] from tensor fail, var_name:%s, memory_type:%u, session_id:%lu", + var_name.c_str(), memory_type, session_id_); return result; } @@ -526,7 +536,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen if (mem_resource == nullptr) { REPORT_CALL_ERROR("E19999", "memory_type:%d invalid or New MemResource fail, session_id:%lu", memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "Alloc MemResource failed, memory_type = %u.", memory_type); + GELOGE(ge::INTERNAL_ERROR, "[Alloc][MemResource] failed, memory_type:%u, session_id:%lu.", + memory_type, session_id_); return ge::INTERNAL_ERROR; } else { mem_resource_map_[memory_type] = mem_resource; @@ -538,7 +549,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen if (mem_resource == nullptr) { REPORT_INNER_ERROR("E19999", "MemResource is invalid, memory_type:%d, session_id:%lu", memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid, memory_type = %u.", memory_type); + GELOGE(ge::INTERNAL_ERROR, "[Check][Param] MemResource is invalid, memory_type:%u, session_id:%lu.", + memory_type, session_id_); return ge::INTERNAL_ERROR; } @@ -567,14 +579,15 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen if (can_not_reuse_old_memory) { result = mem_resource->AssignVarMem(var_name, tensor_desc_size, session_id_, mem_offset); if (result != SUCCESS) { - GELOGE(ge::INTERNAL_ERROR, "AssignVarMem by offset failed."); + GELOGE(ge::INTERNAL_ERROR, "[Assign][VarMem] by offset failed, session_id:%lu.", session_id_); return ge::INTERNAL_ERROR; } result = var_resource_->SaveVarAddr( var_name, tensor_desc, reinterpret_cast(static_cast(mem_offset)), memory_type); if (result != SUCCESS) { - GELOGE(ge::INTERNAL_ERROR, "AssignVarMem by offset failed."); + GELOGE(ge::INTERNAL_ERROR, "[Save][VarAddr] by offset failed, memory type:%u, session_id:%lu.", + memory_type, session_id_); return ge::INTERNAL_ERROR; } } @@ -681,7 +694,8 @@ ge::Status VarManager::RenewCurVarDesc(const std::string &var_name, ge::OpDescPt REPORT_INNER_ERROR("E19999", "VarManager has not been init, op:%s(%s), session_id:%lu, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), session_id_); - GELOGE(ge::INTERNAL_ERROR, "VarManager has not been init."); + GELOGE(ge::INTERNAL_ERROR, "[Check][Param] VarManager has not been init, op:%s(%s), session_id:%lu", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), session_id_); return ge::INTERNAL_ERROR; } return var_resource_->RenewCurVarDesc(var_name, std::move(op_desc)); @@ -729,10 +743,8 @@ ge::Status VarManager::MallocVarMemory(size_t memory_size) { const string purpose("variables and constant op memory in training network."); var_mem_base = MemManager::Instance().MemInstance(RT_MEMORY_HBM).MallocMemory(purpose, memory_key, var_memory_size); if (var_mem_base == nullptr) { - GELOGE(ge::INTERNAL_ERROR, - "VarManager::MallocVarMemory failed " - "session_id = %s", - memory_key.c_str()); + GELOGE(ge::INTERNAL_ERROR, "[Malloc][VarMemory] failed, size:%zu, session_id:%s", + var_memory_size, memory_key.c_str()); return ge::INTERNAL_ERROR; } return SUCCESS; @@ -812,7 +824,7 @@ Status VarManager::SetMemoryMallocSize(const map &options) { string graph_memory_manager_malloc_max_size = it->second; ge::Status ret = ParseMemoryMallocSize(graph_memory_manager_malloc_max_size, graph_mem_max_size_); if (ret != SUCCESS) { - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "Parse graph memory manager malloc max size failed."); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Call][ParseMemoryMallocSize] failed, session id:%lu.", session_id_); return ge::GE_GRAPH_OPTIONS_INVALID; } GELOGI("The max size for graph mem is set to %zu", graph_mem_max_size_); @@ -825,7 +837,7 @@ Status VarManager::SetMemoryMallocSize(const map &options) { string memory_var_manager_malloc_size = it->second; ge::Status ret = ParseMemoryMallocSize(memory_var_manager_malloc_size, var_mem_max_size_); if (ret != SUCCESS) { - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "Parse memory var manager malloc size failed."); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Call][ParseMemoryMallocSize] failed, session id:%lu.", session_id_); return ge::GE_GRAPH_OPTIONS_INVALID; } } @@ -834,8 +846,8 @@ Status VarManager::SetMemoryMallocSize(const map &options) { if (var_mem_logic_base_ > kMaxMemorySize) { REPORT_INNER_ERROR("E19999", "var_login_base:%zu can not exeed limit:%zu, session_id:%lu, check invalid", var_mem_logic_base_, kMaxMemorySize, session_id_); - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "kMemoryVarLogicBase : %zu can not exceed max memory size : %zu.", - var_mem_logic_base_, kMaxMemorySize); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Check][Param] kMemoryVarLogicBase:%zu can not exceed " + "max memory size:%zu, session_id:%lu.", var_mem_logic_base_, kMaxMemorySize, session_id_); return ge::GE_GRAPH_OPTIONS_INVALID; } @@ -843,8 +855,8 @@ Status VarManager::SetMemoryMallocSize(const map &options) { if (use_max_mem_size_ > kMaxMemorySize) { REPORT_INNER_ERROR("E19999", "all mem_use size:%zu can not exeed limit:%zu, session_id:%lu, check invalid", use_max_mem_size_, kMaxMemorySize, session_id_); - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "kUseMaxMemorySize : %zu can not exceed max memory size : %zu.", - use_max_mem_size_, kMaxMemorySize); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Check][Param] kUseMaxMemorySize:%zu can not exceed " + "max memory size:%zu, session_id:%lu.", use_max_mem_size_, kMaxMemorySize, session_id_); return ge::GE_GRAPH_OPTIONS_INVALID; } GELOGI("Set memory malloc size successfully"); @@ -855,7 +867,7 @@ Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) { if (memory_size.empty()) { REPORT_INNER_ERROR("E19999", "Param memory_size is empty, session_id:%lu, check invalid", session_id_); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Memory malloc size input is empty."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] Memory malloc size input is empty, session_id:%lu.", session_id_); return GE_GRAPH_OPTIONS_INVALID; } // split string by '*' @@ -882,7 +894,9 @@ Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) { if (!isdigit(c)) { REPORT_INNER_ERROR("E19999", "Param memory_size:%s contains non digit, session_id:%lu, check invalid", memory_size.c_str(), session_id_); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Memory malloc size input contains non digit."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, + "[Check][Param] Memory malloc size:%s input contains non digit, session_id:%lu.", + memory_size.c_str(), session_id_); return GE_GRAPH_OPTIONS_INVALID; } } @@ -891,13 +905,15 @@ Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) { REPORT_INNER_ERROR("E19999", "Param memory_size:%s will overflow after multi all, session_id:%lu, " "check invalid", memory_size.c_str(), session_id_); - GELOGE(FAILED, "Input memory size is out of range."); + GELOGE(FAILED, "[Check][Param] Param memory_size:%s will overflow after multi all, session_id:%lu", + memory_size.c_str(), session_id_); return FAILED); if ((num > kMaxMemorySize) || (result * static_cast(num) > kMaxMemorySize)) { REPORT_INNER_ERROR("E19999", "Param memory_size:%s after multi will exceed limit:%lu, session_id:%lu, " "check invalid", memory_size.c_str(), kMaxMemorySize, session_id_); - GELOGE(FAILED, "Input memory size can not exceed max memory size : %zu.", kMaxMemorySize); + GELOGE(FAILED, "[Check][Param] Input memory size can not exceed max memory size:%zu, session_id:%lu.", + kMaxMemorySize, session_id_); return FAILED; } result *= static_cast(num); @@ -1001,10 +1017,7 @@ VarManager *VarManagerPool::GetVarManager(uint64_t session_id) { VarManager *var_manager = new (std::nothrow) VarManager(session_id); if (var_manager == nullptr) { REPORT_INNER_ERROR("E19999", "New VarManager fail, session_id:%lu", session_id); - GELOGE(INTERNAL_ERROR, - "VarManager::Instance find session by " - "session_id[%lu] failed.", - session_id); + GELOGE(INTERNAL_ERROR, "[New][VarManager] fail, session_id:%lu", session_id); static VarManager new_var_manager(0); return &new_var_manager; } diff --git a/ge/graph/manager/host_mem_allocator.cc b/ge/graph/manager/host_mem_allocator.cc index 98f9a313..588d08a2 100644 --- a/ge/graph/manager/host_mem_allocator.cc +++ b/ge/graph/manager/host_mem_allocator.cc @@ -34,8 +34,8 @@ uint8_t *HostMemAllocator::Malloc(size_t size) { std::lock_guard lock(mutex_); std::shared_ptr aligned_ptr = MakeShared(size); if (aligned_ptr == nullptr) { - REPORT_INNER_ERROR("E19999", "New AlignedPtr fail"); - GELOGE(INTERNAL_ERROR, "make shared_ptr for AlignedPtr failed"); + REPORT_INNER_ERROR("E19999", "New AlignedPtr fail, size:%zu", size); + GELOGE(INTERNAL_ERROR, "[Call][MakeShared] for AlignedPtr failed, size:%zu", size); return nullptr; } allocated_blocks_[aligned_ptr->Get()] = { size, aligned_ptr }; @@ -46,7 +46,7 @@ uint8_t *HostMemAllocator::Malloc(size_t size) { Status HostMemAllocator::Free(const void *memory_addr) { if (memory_addr == nullptr) { REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, check invalid"); - GELOGE(GE_GRAPH_FREE_FAILED, "Invalid memory pointer"); + GELOGE(GE_GRAPH_FREE_FAILED, "[Check][Param] Invalid memory pointer"); return GE_GRAPH_FREE_FAILED; } @@ -54,7 +54,7 @@ Status HostMemAllocator::Free(const void *memory_addr) { auto it = allocated_blocks_.find(memory_addr); if (it == allocated_blocks_.end()) { REPORT_INNER_ERROR("E19999", "Memory_addr is not alloc before, check invalid"); - GELOGE(PARAM_INVALID, "Invalid memory pointer"); + GELOGE(PARAM_INVALID, "[Check][Param] Invalid memory pointer:%p", memory_addr); return PARAM_INVALID; } it->second.second.reset(); diff --git a/ge/graph/manager/host_mem_manager.cc b/ge/graph/manager/host_mem_manager.cc index 2908df39..63265bf4 100644 --- a/ge/graph/manager/host_mem_manager.cc +++ b/ge/graph/manager/host_mem_manager.cc @@ -39,9 +39,8 @@ Status SharedMemAllocator::Allocate(SharedMemInfo &mem_info) { rtMallocHostSharedMemoryOut output_para; rtError_t rt_ret = rtMallocHostSharedMemory(&input_para, &output_para); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMallocHostSharedMemory fail, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api(rtMallocHostSharedMemory) failed, devid:[%u].", device_id); + REPORT_CALL_ERROR("E19999", "Call rtMallocHostSharedMemory fail, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMallocHostSharedMemory] failed, devid:[%u].", device_id); return GE_GRAPH_MEMORY_ALLOC_FAILED; } mem_info.fd = output_para.fd; @@ -60,9 +59,8 @@ Status SharedMemAllocator::DeAllocate(SharedMemInfo &mem_info) { mem_info.host_aligned_ptr->MutableGet(), mem_info.device_address}; rtError_t rt_ret = rtFreeHostSharedMemory(&free_para); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtFreeHostSharedMemory fail, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api(rtFreeHostSharedMemory) failed, ret: 0x%X.", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtFreeHostSharedMemory fail, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtFreeHostSharedMemory] failed, ret:0x%X.", rt_ret); return RT_FAILED; } return ge::SUCCESS; @@ -78,7 +76,7 @@ Status HostMemManager::Initialize() { allocator_ = std::unique_ptr(new (std::nothrow) SharedMemAllocator()); if (allocator_ == nullptr) { REPORT_CALL_ERROR("E19999", "New SharedMemAllocator fail"); - GELOGE(GE_GRAPH_MALLOC_FAILED, "Shared memory allocator init failed!"); + GELOGE(GE_GRAPH_MALLOC_FAILED, "[New][SharedMemAllocator] failed!"); return GE_GRAPH_MALLOC_FAILED; } return SUCCESS; @@ -98,9 +96,8 @@ Status HostMemManager::MallocSharedMemory(SharedMemInfo &mem_info) { std::lock_guard lock(mutex_); auto iter = var_memory_base_map_.find(mem_info.op_name); if (iter != var_memory_base_map_.end()) { - REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_", - mem_info.op_name.c_str()); - GELOGE(FAILED, "Host shared memory for op %s has been malloced", mem_info.op_name.c_str()); + REPORT_INNER_ERROR("E19999", "Host shared memory for op %s has been malloced", mem_info.op_name.c_str()); + GELOGE(FAILED, "[Check][Param] Host shared memory for op %s has been malloced", mem_info.op_name.c_str()); return FAILED; } mem_info.shm_name = OpNameToShmName(mem_info.op_name); @@ -113,9 +110,8 @@ Status HostMemManager::MallocSharedMemory(SharedMemInfo &mem_info) { Status HostMemManager::QueryVarMemInfo(const string &op_name, uint64_t &base_addr, uint64_t &data_size) { std::lock_guard lock(mutex_); if (var_memory_base_map_.find(op_name) == var_memory_base_map_.end()) { - REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_", - op_name.c_str()); - GELOGE(INTERNAL_ERROR, "Find host base base_addr failed,node name:%s!", op_name.c_str()); + REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_", op_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Find host base base_addr failed, node name:%s!", op_name.c_str()); return INTERNAL_ERROR; } base_addr = static_cast(reinterpret_cast(var_memory_base_map_[op_name].device_address)); diff --git a/ge/graph/manager/memory_api.cc b/ge/graph/manager/memory_api.cc index 8e737021..8ea0594b 100644 --- a/ge/graph/manager/memory_api.cc +++ b/ge/graph/manager/memory_api.cc @@ -50,9 +50,8 @@ Status RdmaRemoteRegister(const std::vector &var_info, rtMemType_t path.append(file_name); string canonical_path = RealPath(path.c_str()); if (canonical_path.empty()) { - REPORT_INNER_ERROR("E19999", "canonical_path:%s is empty, check invalid", - canonical_path.c_str()); - GELOGE(FAILED, "Failed to get realpath of %s", path.c_str()); + REPORT_INNER_ERROR("E19999", "canonical_path:%s is empty, check invalid", canonical_path.c_str()); + GELOGE(FAILED, "[Call][RealPath] Failed to get realpath of %s", path.c_str()); return FAILED; } GELOGI("FileName:%s, Path:%s.", file_name.c_str(), canonical_path.c_str()); @@ -69,15 +68,14 @@ Status RdmaRemoteRegister(const std::vector &var_info, rtMemType_t if (hcom_remote_mem_register == nullptr) { REPORT_CALL_ERROR("E19999", "Symbol HcomRegRemoteAccessMem can't find in %s, check invalid", canonical_path.c_str()); - GELOGE(FAILED, "Failed to invoke hcom_remote_mem_register function."); + GELOGE(FAILED, "[Check][Param] Symbol HcomRegRemoteAccessMem can't find in %s", canonical_path.c_str()); return FAILED; } HcclResult hccl_ret = hcom_remote_mem_register(reg_addrs.get(), table_len); if (hccl_ret != HCCL_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Call hcom_remote_mem_register failed, ret:%d,", - hccl_ret); - GELOGE(HCCL_E_INTERNAL, "Rdma mem register failed, ret: 0x%X", hccl_ret); + REPORT_CALL_ERROR("E19999", "Call hcom_remote_mem_register failed, ret:%d,", hccl_ret); + GELOGE(HCCL_E_INTERNAL, "[Call][HcomRemoteMemRegister] Rdma mem register failed, ret:0x%X", hccl_ret); return HCCL_E_INTERNAL; } return SUCCESS; @@ -88,14 +86,14 @@ Status MallocSharedMemory(const TensorInfo &tensor_info, uint64_t &dev_addr, uin uint32_t type_size = 0; bool result = TypeUtils::GetDataTypeLength(tensor_info.data_type, type_size); if (!result) { - GELOGE(GRAPH_FAILED, "GetDataTypeLength failed, data_type=(%s).", + GELOGE(GRAPH_FAILED, "[Get][DataTypeLength] failed, data_type=(%s).", TypeUtils::DataTypeToSerialString(tensor_info.data_type).c_str()); return GRAPH_FAILED; } memory_size = type_size; for (auto dim : tensor_info.dims) { if (dim <= 0) { - GELOGE(GRAPH_FAILED, "Tensor dims should be positive"); + GELOGE(GRAPH_FAILED, "[Check][Param] Tensor dims should be positive"); return GRAPH_FAILED; } memory_size *= dim; @@ -103,7 +101,7 @@ Status MallocSharedMemory(const TensorInfo &tensor_info, uint64_t &dev_addr, uin SharedMemInfo mem_info(tensor_info.var_name, memory_size); Status ret = HostMemManager::Instance().MallocSharedMemory(mem_info); if (ret != SUCCESS) { - GELOGE(GRAPH_FAILED, "MallocSharedMemory failed op name [%s]", tensor_info.var_name.c_str()); + GELOGE(GRAPH_FAILED, "[Malloc][SharedMemory] failed, op name [%s]", tensor_info.var_name.c_str()); return GRAPH_FAILED; } dev_addr = reinterpret_cast(reinterpret_cast(mem_info.device_address)); diff --git a/ge/graph/manager/model_manager/event_manager.cc b/ge/graph/manager/model_manager/event_manager.cc index 69a946f9..339e9894 100644 --- a/ge/graph/manager/model_manager/event_manager.cc +++ b/ge/graph/manager/model_manager/event_manager.cc @@ -45,7 +45,7 @@ Status EventManager::Init(size_t event_num) { void EventManager::Release() noexcept { for (size_t i = 0; i < this->event_list_.size(); ++i) { rtError_t rt_ret = rtEventDestroy(this->event_list_[i]); - RETURN_IF_COND_NOT_MET(rt_ret == RT_ERROR_NONE, "Destroy event failed, idx is %zu, ret is 0x%x.", i, rt_ret); + RETURN_IF_COND_NOT_MET(rt_ret == RT_ERROR_NONE, "[Destroy][Event] failed, idx is %zu, ret is 0x%x.", i, rt_ret); } this->event_list_.clear(); diff --git a/ge/graph/manager/rdma_pool_allocator.cc b/ge/graph/manager/rdma_pool_allocator.cc index 58829adb..4297be95 100644 --- a/ge/graph/manager/rdma_pool_allocator.cc +++ b/ge/graph/manager/rdma_pool_allocator.cc @@ -82,8 +82,8 @@ Status RdmaPoolAllocator::InitMemory(size_t mem_size) { auto device_id = GetContext().DeviceId(); GELOGD("Init Rdma Memory with size [%zu] for devid:[%u]", mem_size, device_id); if (rdma_base_addr_ != nullptr) { - REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is nullptr, check invalid"); - GELOGE(GE_MULTI_INIT, "Rdma pool has been malloced"); + REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is not nullptr, devid:%u, check invalid", device_id); + GELOGE(GE_MULTI_INIT, "[Check][Param] Rdma pool has been malloced, devid:%u", device_id); return GE_MULTI_INIT; } const std::string purpose = "Memory for rdma pool."; @@ -95,15 +95,15 @@ Status RdmaPoolAllocator::InitMemory(size_t mem_size) { rdma_base_addr_ = memory_allocator_->MallocMemory(purpose, mem_size, device_id); if (rdma_base_addr_ == nullptr) { - GELOGE(GE_GRAPH_MALLOC_FAILED, "Rdma pool memory malloc failed"); + GELOGE(GE_GRAPH_MALLOC_FAILED, "[Malloc][Memory] failed, size:%zu, device_id:%u", mem_size, device_id); return GE_GRAPH_MALLOC_FAILED; } rdma_mem_size_ = mem_size; // Init with a base block. auto *base_block = new (std::nothrow) Block(device_id, mem_size, rdma_base_addr_); if (base_block == nullptr) { - REPORT_CALL_ERROR("E19999", "New Block failed, device_id:%u", device_id); - GELOGE(GE_GRAPH_MALLOC_FAILED, "Block malloc failed"); + REPORT_CALL_ERROR("E19999", "New Block failed, size:%zu, device_id:%u", mem_size, device_id); + GELOGE(GE_GRAPH_MALLOC_FAILED, "[New][Block] failed, size:%zu, device_id:%u", mem_size, device_id); return GE_GRAPH_MALLOC_FAILED; } block_bin_.insert(base_block); @@ -123,7 +123,7 @@ uint8_t *RdmaPoolAllocator::Malloc(size_t size, uint32_t device_id) { if (block->ptr == nullptr) { REPORT_INNER_ERROR("E19999", "Rdmapool memory address is nullptr, device_id:%u, check invalid", device_id); - GELOGE(INTERNAL_ERROR, "Rdmapool memory address is nullptr."); + GELOGE(INTERNAL_ERROR, "[Check][Param] Rdmapool memory address is nullptr, device_id:%u", device_id); return nullptr; } allocated_blocks_.emplace(block->ptr, block); @@ -155,9 +155,8 @@ uint8_t *RdmaPoolAllocator::Malloc(size_t size, uint32_t device_id) { Status RdmaPoolAllocator::Free(uint8_t *memory_addr, uint32_t device_id) { GELOGI("Free rdma memory, device id = %u", device_id); if (memory_addr == nullptr) { - REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, device_id:%u, check invalid", - device_id); - GELOGE(GE_GRAPH_FREE_FAILED, "Invalid memory pointer"); + REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, device_id:%u, check invalid", device_id); + GELOGE(GE_GRAPH_FREE_FAILED, "[Check][Param] Invalid memory pointer, device id:%u", device_id); return GE_GRAPH_FREE_FAILED; } @@ -166,7 +165,7 @@ Status RdmaPoolAllocator::Free(uint8_t *memory_addr, uint32_t device_id) { if (it == allocated_blocks_.end()) { REPORT_INNER_ERROR("E19999", "Param memory_addr is not allocated before, device_id:%u, " "check invalid", device_id); - GELOGE(PARAM_INVALID, "Invalid memory pointer"); + GELOGE(PARAM_INVALID, "[Check][Param] Invalid memory pointer, device id:%u", device_id); return PARAM_INVALID; } @@ -209,7 +208,7 @@ void RdmaPoolAllocator::MergeBlocks(Block *dst, Block *src) { Status RdmaPoolAllocator::GetBaseAddr(uint64_t &base_addr, uint64_t &mem_size) { if (rdma_base_addr_ == nullptr) { REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is nullptr, check invalid"); - GELOGE(INTERNAL_ERROR, "Rdma base addr is nullptr."); + GELOGE(INTERNAL_ERROR, "[Check][Param] Rdma base addr is nullptr."); return INTERNAL_ERROR; } base_addr = static_cast(reinterpret_cast(rdma_base_addr_)); diff --git a/ge/graph/manager/trans_var_data_utils.cc b/ge/graph/manager/trans_var_data_utils.cc index 9c1290fa..621eba79 100644 --- a/ge/graph/manager/trans_var_data_utils.cc +++ b/ge/graph/manager/trans_var_data_utils.cc @@ -37,7 +37,8 @@ class RtContextSwitchGuard { if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, device_id:%u, ret:0x%X,", device_id, ret); - GELOGE(RT_FAILED, "Failed to get current context from rt, error-code %d", ret); + GELOGE(RT_FAILED, "[Call][RtCtxGetCurrent] Failed to get current context, device_id:%u, ret:0x%X", + device_id, ret); return; } @@ -45,15 +46,14 @@ class RtContextSwitchGuard { if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxCreate failed, device_id:%u, ret:0x%X,", device_id, ret); - GELOGE(RT_FAILED, "Failed to create new context for device %u, error-code %d", device_id, ret); + GELOGE(RT_FAILED, "[Call][RtCtxCreate] Failed to create new context for device:%u, ret:%d", device_id, ret); return; } ret = rtCtxSetCurrent(current_); if (ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, device_id:%u, ret:0x%X,", - device_id, ret); - GELOGE(RT_FAILED, "Failed to switch context to normal, context %p, device %u", current_, device_id); + REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, device_id:%u, ret:0x%X", device_id, ret); + GELOGE(RT_FAILED, "[Call][RtCtxSetCurrent] failed, device_id:%u, ret:0x%X", device_id, ret); return; } GELOGD("Create and switch rt context %p type %d for device %u, backup last %p.", current_, mode, device_id, last_); @@ -80,7 +80,7 @@ int64_t CalcVarSizeInBytes(const GeTensorDesc &desc) { if (var_size <= 0) { REPORT_INNER_ERROR("E19999", "Data type:%s in desc, it's size:%ld < 0, check invalid", TypeUtils::DataTypeToSerialString(desc.GetDataType()).c_str(), var_size); - GELOGE(PARAM_INVALID, "Failed to calc var data size from data type %s", + GELOGE(PARAM_INVALID, "[Calc][VarDataSize] by data type %s failed.", TypeUtils::DataTypeToSerialString(desc.GetDataType()).c_str()); return -1; } @@ -99,7 +99,8 @@ Status CopyVarToDevice(const NodePtr &var, const formats::TransResult &trans_res if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, op:%s(%s), size:%lu, ret:0x%X,", var->GetName().c_str(), var->GetType().c_str(), trans_result.length, ret); - GELOGE(RT_FAILED, "Failed to copy memory to device, size %zu", trans_result.length); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, op:%s(%s), size:%lu, ret:0x%X,", var->GetName().c_str(), + var->GetType().c_str(), trans_result.length, ret); return RT_FAILED; } return SUCCESS; @@ -111,21 +112,17 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt GE_CHECK_NOTNULL(var); auto ret = VarManager::Instance(session_id)->GetVarAddr(var->GetName(), input_desc, &var_logic); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, - "Failed to copy var %s from device, can not find it" - " from var manager %u", - var->GetName().c_str(), ret); + GELOGE(INTERNAL_ERROR, "[Get][VarAddr] failed, node:%s, session_id:%lu, ret:%d", + var->GetName().c_str(), session_id, ret); return INTERNAL_ERROR; } uint8_t *var_addr = VarManager::Instance(session_id)->GetVarMemoryAddr(var_logic, RT_MEMORY_HBM); if (var_addr == nullptr) { - REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, op:%s(%s), session_id:%lu,", + REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, op:%s(%s), session_id:%lu", RT_MEMORY_HBM, var->GetName().c_str(), var->GetType().c_str(), session_id); - GELOGE(INTERNAL_ERROR, - "Failed to copy var %s from device, cant not get " - "var addr from logic addr %p", - var->GetName().c_str(), var_logic); + GELOGE(INTERNAL_ERROR, "[Get][VarMemoryAddr] failed, mem_type:%d, op:%s(%s), session_id:%lu", + RT_MEMORY_HBM, var->GetName().c_str(), var->GetType().c_str(), session_id); return INTERNAL_ERROR; } @@ -136,9 +133,10 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt std::unique_ptr var_host(new(std::nothrow) uint8_t[var_size_bytes]); if (var_host == nullptr) { - REPORT_CALL_ERROR("E19999", "New host memory failed, size:%ld, op:%s(%s), session_id:%lu,", + REPORT_CALL_ERROR("E19999", "New host memory failed, size:%ld, op:%s(%s), session_id:%lu", var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id); - GELOGE(OUT_OF_MEMORY, "Failed to malloc rt-host memory, size %ld", var_size_bytes); + GELOGE(OUT_OF_MEMORY, "[New][Memory] for rt-host failed, size:%ld, op:%s(%s), session_id:%lu", + var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id); return OUT_OF_MEMORY; } @@ -147,10 +145,8 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%ld, op:%s(%s), session_id:%lu, ret:0x%X", var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id, ret); - GELOGE(RT_FAILED, - "Failed to copy var memory from device, var %s, size %ld," - " rt-error-code %u", - var->GetName().c_str(), var_size_bytes, ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%ld, op:%s(%s), session_id:%lu, ret:0x%X", + var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id, ret); return RT_FAILED; } @@ -197,9 +193,7 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats formats::ShapeToString(src_shape).c_str(), formats::ShapeToString(dst_shape).c_str(), TypeUtils::DataTypeToSerialString(data_type).c_str(), ret); - GELOGE(INTERNAL_ERROR, - "Failed to trans format from %s to %s, shape %s to %s, " - "data type %s error code %u", + GELOGE(INTERNAL_ERROR, "[Trans][Format] from %s to %s, shape %s to %s failed, data type %s error code %u", TypeUtils::FormatToSerialString(src_format).c_str(), TypeUtils::FormatToSerialString(dst_format).c_str(), formats::ShapeToString(src_shape).c_str(), formats::ShapeToString(dst_shape).c_str(), TypeUtils::DataTypeToSerialString(data_type).c_str(), ret); @@ -221,7 +215,7 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats TypeUtils::DataTypeToSerialString(src_data_type).c_str(), TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), formats::ShapeToString(input_shape).c_str(), src_data_size, ret); - GELOGE(INTERNAL_ERROR, "Failed to trans data type from %s to %s, input shape %s, data size %ld, error code %u", + GELOGE(INTERNAL_ERROR, "[Trans][DataType] from %s to %s failed, input shape %s, data size %ld, error code %u", TypeUtils::DataTypeToSerialString(src_data_type).c_str(), TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), formats::ShapeToString(input_shape).c_str(), src_data_size, ret); @@ -230,7 +224,7 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats } else { REPORT_INNER_ERROR("E19999", "Trans var data failed, the trans type %s does not supported, check invalid", trans_info.node_type.c_str()); - GELOGE(UNSUPPORTED, "Failed to trans var data, the trans type %s does not supported", + GELOGE(UNSUPPORTED, "[Trans][VarData] failed, the trans type %s does not supported", trans_info.node_type.c_str()); return UNSUPPORTED; } @@ -255,10 +249,8 @@ Status ReAssignVarAddr(uint64_t session_id, uint8_t *var_logic = nullptr; Status ret = VarManager::Instance(session_id)->GetVarAddr(var_name, tensor_desc, &var_logic); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, - "Failed to get var %s device addr, can not find it" - " from var manager %u", - var_name.c_str(), ret); + GELOGE(INTERNAL_ERROR, "[Get][VarAddr] failed, var name:%s, session_id:%lu, ret:%u", + var_name.c_str(), session_id, ret); return INTERNAL_ERROR; } @@ -266,7 +258,8 @@ Status ReAssignVarAddr(uint64_t session_id, if (var_addr == nullptr) { REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, var_name:%s, session_id:%lu,", RT_MEMORY_HBM, var_name.c_str(), session_id); - GELOGE(INTERNAL_ERROR, "Failed to convert var %s logic addr to real addr", var_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Get][VarMemoryAddr] failed, mem_type:%d, var_name:%s, session_id:%lu", + RT_MEMORY_HBM, var_name.c_str(), session_id); return INTERNAL_ERROR; } *var_device = var_addr; @@ -293,9 +286,8 @@ Status TransVarData(const NodePtr &var, const VarTransRoad &trans_road, uint64_t // Sync var data from device std::unique_ptr var_data; if (trans_road.empty()) { - REPORT_INNER_ERROR("E19999", "Param trans_road is empty, session_id:%lu, check invalid", - session_id); - GELOGE(INTERNAL_ERROR, "Failed to get trans_road, trans_road is empty."); + REPORT_INNER_ERROR("E19999", "Param trans_road is empty, session_id:%lu, check invalid", session_id); + GELOGE(INTERNAL_ERROR, "[Check][Param] trans_road is empty, session_id:%lu", session_id); return INTERNAL_ERROR; } const GeTensorDesc &input_desc = trans_road.begin()->input; @@ -307,7 +299,7 @@ Status TransVarData(const NodePtr &var, const VarTransRoad &trans_road, uint64_t formats::TransResult trans_result{}; ret = TransVarOnHost(var_data.get(), trans_road, trans_result); if (ret != SUCCESS) { - GELOGE(ret, "Failed to trans var data on host, error code %u", ret); + GELOGE(ret, "[Call][TransVarOnHost] failed, session_id:%lu, ret:%u", session_id, ret); return ret; } @@ -319,14 +311,15 @@ Status TransVarData(const NodePtr &var, const VarTransRoad &trans_road, uint64_t /// TensorDesc needs to be removed. This change is large and needs to be performed step by step. ret = ReAssignVarAddr(session_id, var->GetName(), trans_road.rbegin()->output, &var_device); if (ret != SUCCESS) { - GELOGE(ret, "Failed to re-assign memory on device, size %zu", trans_result.length); + GELOGE(ret, "[Call][ReAssignVarAddr] failed, session id:%lu, op:%s, ret:%u", + session_id, var->GetName().c_str(), ret); return ret; } // sync new data to device ret = CopyVarToDevice(var, trans_result, var_device); if (ret != SUCCESS) { - GELOGE(ret, "Failed to send var data to device"); + GELOGE(ret, "[Call][CopyVarToDevice] failed, var:%s, ret:%u", var->GetName().c_str(), ret); return ret; } @@ -350,7 +343,10 @@ Status TransTensor(uint8_t *var_data, const NodePtr &var_src, const NodePtr &var TypeUtils::DataTypeToSerialString(src_data_datatype).c_str(), TypeUtils::DataTypeToSerialString(dst_data_datatype).c_str(), src_data_shape_size, ret); - GELOGE(INTERNAL_ERROR, "trans var data on host failed"); + GELOGE(INTERNAL_ERROR, "[Trans][DataType] from %s to %s failed, data size %ld, ret:%u", + TypeUtils::DataTypeToSerialString(src_data_datatype).c_str(), + TypeUtils::DataTypeToSerialString(dst_data_datatype).c_str(), + src_data_shape_size, ret); return ret; }); return SUCCESS; @@ -366,9 +362,11 @@ Status CopyTensorFromSrcVarNode(const NodePtr &var_src, /// need copy value from var_fp32 to var_fp16. /// [opdesc of var_src and var_dst are checked before passed in, no need to check if they are nullptr] GE_IF_BOOL_EXEC(var_src == nullptr || var_dst == nullptr, - REPORT_INNER_ERROR("E19999", "Param var_src or var_dst is empty, session_id:%lu, device_id:%u, " + REPORT_INNER_ERROR("E19999", "Param var_src or var_dst is nullptr, session_id:%lu, device_id:%u, " "check invalid", session_id, device_id); - GELOGE(FAILED, "node var is nullptr"); return FAILED); + GELOGE(FAILED, "[Check][Param] Param var_src or var_dst is nullptr, session_id:%lu, device_id:%u", + session_id, device_id); + return FAILED); // src_node output_desc (fp32) GeTensorDesc output_desc = var_src->GetOpDesc()->GetOutputDesc(0); auto src_data_type = output_desc.GetDataType(); @@ -390,31 +388,45 @@ Status CopyTensorFromSrcVarNode(const NodePtr &var_src, RtContextSwitchGuard switch_context(RT_CTX_NORMAL_MODE, device_id); // copy from src_node auto ret = CopyVarFromDevice(session_id, var_src, var_src_data, output_desc); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(FAILED, "Copy Var From Device failed"); return ret); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(FAILED, "[Call][CopyVarFromDevice] failed, session id:%lu, var_src:%s", + session_id, var_src->GetName().c_str()); + return ret); // trans dtype formats::TransResult trans_result{}; ret = TransTensor(var_src_data.get(), var_src, var_dst, trans_result); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(INTERNAL_ERROR, "trans var data on host failed"); return ret); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(INTERNAL_ERROR, "[Trans][Tensor] failed, var_src:%s, var_dst:%s", + var_src->GetName().c_str(), var_dst->GetName().c_str()); + return ret); // reset src value. void *var_device = nullptr; ret = ReAssignVarAddr(session_id, var_dst->GetName(), dst_tensor_desc, &var_device); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(INTERNAL_ERROR, "assign mem failed"); return ret); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(INTERNAL_ERROR, "[Call][ReAssignVarAddr] failed, session id:%lu, var_dst:%s", + session_id, var_dst->GetName().c_str()); + return ret); // copy to device ret = CopyVarToDevice(var_dst, trans_result, var_device); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Failed to send var data to device"); return ret); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(ret, "[Call][CopyVarToDevice] failed, var_dst:%s, ret:%u", + var_dst->GetName().c_str(), ret); + return ret); return SUCCESS; } } // namespace Status TransVarDataUtils::SyncVarData2BroadCast(const string &var_name, const ge::GeTensorDesc &src_tensor_desc, uint8_t *dst_addr, int64_t dst_addr_size, uint64_t session_id) { - GE_CHK_BOOL_RET_STATUS(dst_addr != nullptr, FAILED, "dst addr is null. "); + GE_CHK_BOOL_RET_STATUS(dst_addr != nullptr, FAILED, "[Check][Param] dst addr is nullptr."); uint8_t *src_host_addr = nullptr; int64_t src_addr_size = 0; GE_MAKE_GUARD_RTMEM(src_host_addr); GE_CHK_STATUS_RET(SyncTensorToHost(var_name, src_tensor_desc, &src_host_addr, src_addr_size, session_id)); GELOGI("src_addr_size: %ld, dst_addr_size: %ld", src_addr_size, dst_addr_size); - GE_CHK_BOOL_RET_STATUS(src_addr_size == dst_addr_size, FAILED, "var data size is not equal broadcast "); + GE_CHK_BOOL_RET_STATUS(src_addr_size == dst_addr_size, FAILED, + "[Check][Param] src_addr_size:%ld not equal to dst_addr_size:%ld", + src_addr_size, dst_addr_size); GE_CHK_RT_RET(rtMemcpy(dst_addr, dst_addr_size, src_host_addr, src_addr_size, RT_MEMCPY_HOST_TO_DEVICE)); return SUCCESS; @@ -422,7 +434,7 @@ Status TransVarDataUtils::SyncVarData2BroadCast(const string &var_name, const ge Status TransVarDataUtils::SyncBroadCastData2Var(uint8_t *src_addr, int64_t src_addr_size, const string &var_name, const ge::GeTensorDesc &dst_tensor_desc, uint64_t session_id) { - GE_CHK_BOOL_RET_STATUS(src_addr != nullptr, FAILED, "src addr is null. "); + GE_CHK_BOOL_RET_STATUS(src_addr != nullptr, FAILED, "[Check][Param] src addr is nullptr. "); uint8_t *host_addr = nullptr; GE_MAKE_GUARD_RTMEM(host_addr); GE_CHK_RT_RET(rtMallocHost(reinterpret_cast(&host_addr), src_addr_size)); @@ -436,7 +448,7 @@ Status TransVarDataUtils::SyncBroadCastData2Var(uint8_t *src_addr, int64_t src_a Status TransVarDataUtils::SyncTensorToHost(const string &var_name, const ge::GeTensorDesc &src_tensor_desc, uint8_t **host_addr, int64_t &src_tensor_size, uint64_t session_id) { - GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(src_tensor_desc, src_tensor_size), "get size from TensorDesc failed"); + GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(src_tensor_desc, src_tensor_size), "[Get][Size] from TensorDesc failed"); uint8_t *src_addr = nullptr; GE_CHK_STATUS_RET(VarManager::Instance(session_id)->GetVarAddr(var_name, src_tensor_desc, &src_addr)); @@ -493,7 +505,8 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, session_id:%lu, graph_id:%u, ret:0x%X,", session_id, graph_id, rt_ret); - GELOGE(RT_FAILED, "Failed to set context, error_code is: 0x%X.", rt_ret); + GELOGE(RT_FAILED, "[Call][RtCtxSetCurrent] failed, session_id:%lu, graph_id:%u, ret:0x%X,", + session_id, graph_id, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } uint32_t allocated_graph_id = 0; @@ -501,8 +514,8 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Get allocated GraphId failed, session_id:%lu, graph_id:%u, ret:0x%X,", session_id, graph_id, ret); - GELOGE(INTERNAL_ERROR, "var has not been allocated, node:%s, graph_id:%u.", node->GetName().c_str(), - graph_id); + GELOGE(INTERNAL_ERROR, "[Get][AllocatedGraphId] failed, node:%s, graph_id:%u.", + node->GetName().c_str(), graph_id); return INTERNAL_ERROR; } uint32_t changed_graph_id = 0; @@ -518,7 +531,8 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, } ret = TransVarData(node, *trans_road, session_id); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, "TransVarData failed, node:%s, graph_id:%u.", node->GetName().c_str(), graph_id); + GELOGE(INTERNAL_ERROR, "[Trans][VarData] failed, node:%s, graph_id:%u, session_id:%lu.", + node->GetName().c_str(), graph_id, session_id); return INTERNAL_ERROR; } VarManager::Instance(session_id)->RemoveChangedGraphId(node->GetName()); @@ -527,7 +541,7 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, }, node, session_id, context, graph_id, ErrorManager::GetInstance().GetErrorManagerContext()); if (!f.valid()) { - GELOGE(FAILED, "Future is invalid"); + GELOGE(FAILED, "[Check][Param] Future is invalid, session id:%lu, graph id:%u", session_id, graph_id); return FAILED; } vector_future.push_back(std::move(f)); @@ -537,7 +551,7 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, for (size_t i = 0; i < vector_future.size(); ++i) { ret_status = vector_future[i].get(); if (ret_status != SUCCESS) { - GELOGE(ret_status, "TransAllVarData:: trans %zu vardata failed", i); + GELOGE(ret_status, "[Check][Param] trans %zu vardata failed", i); return ret_status; } } @@ -550,7 +564,8 @@ Status TransVarDataUtils::CopyVarData(const ComputeGraphPtr &compute_graph, uint if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, session_id:%lu, device_id:%u, check invalid", session_id, device_id); - GELOGE(FAILED, "compute_graph is nullptr"); + GELOGE(FAILED, "[Check][Param] compute_graph is nullptr, session_id:%lu, device_id:%u", + session_id, device_id); return FAILED; } @@ -568,7 +583,10 @@ Status TransVarDataUtils::CopyVarData(const ComputeGraphPtr &compute_graph, uint GELOGI("current_var_node__: [%s] copy_from_var_node__: [%s].", node->GetName().c_str(), src_node->GetName().c_str()); auto ret = CopyTensorFromSrcVarNode(src_node, node, session_id, device_id); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(FAILED, "copy tensor failed!"); return FAILED); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(FAILED, "[Copy][Tensor] failed, src_node:%s, node:%s, session_id:%lu, device_id:%u", + src_node->GetName().c_str(), node->GetName().c_str(), session_id, device_id); + return FAILED); // only copy once (void) ge::AttrUtils::SetBool(node->GetOpDesc(), "_copy_value", true); // no need to check value } diff --git a/ge/graph/manager/util/debug.cc b/ge/graph/manager/util/debug.cc index 1dd97bc1..d20280fb 100644 --- a/ge/graph/manager/util/debug.cc +++ b/ge/graph/manager/util/debug.cc @@ -63,17 +63,15 @@ Status Debug::DumpDevMem(const char *file, const void *addr, int64_t size) { uint8_t *host_addr = nullptr; rtError_t ret = rtMallocHost(reinterpret_cast(&host_addr), size); if (ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, size:%zu, ret: 0x%X", - size, ret); - GELOGE(FAILED, "Call rt api rtMallocHost failed, ret: 0x%X", ret); + REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, size:%zu, ret:0x%X", size, ret); + GELOGE(FAILED, "[Call][RtMallocHost] failed, size:%zu, ret:0x%X", size, ret); return FAILED; } GE_MAKE_GUARD_RTMEM(host_addr); ret = rtMemcpy(host_addr, size, addr, size, RT_MEMCPY_DEVICE_TO_HOST); if (ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret: 0x%X", - size, ret); - GELOGE(FAILED, "Call rt api rtMemcpy failed, ret: 0x%X", ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", size, ret); + GELOGE(FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", size, ret); return FAILED; } diff --git a/ge/graph/manager/util/hcom_util.cc b/ge/graph/manager/util/hcom_util.cc index a30321f9..1843127f 100644 --- a/ge/graph/manager/util/hcom_util.cc +++ b/ge/graph/manager/util/hcom_util.cc @@ -28,7 +28,8 @@ Status HcomOmeUtil::GetHcclDataType(const ge::ConstOpDescPtr &op_desc, std::vector &kernel_hccl_infos) { GE_CHECK_NOTNULL(op_desc); if (CheckKernelHcclInfo(op_desc, kernel_hccl_infos) != SUCCESS) { - GELOGE(PARAM_INVALID, "HcomOmeUtil:: the number of GETaskKernelHcclInfo is invalid."); + GELOGE(PARAM_INVALID, "[Check][KernelHcclInfo] failed, op:%s(%s).", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } GELOGI("GetHcclDataType start, node[%s], opType[%s].", op_desc->GetName().c_str(), op_desc->GetType().c_str()); @@ -40,10 +41,10 @@ Status HcomOmeUtil::GetHcclDataType(const ge::ConstOpDescPtr &op_desc, if (op_desc->GetType() == HCOMRECEIVE) { bool ret = ge::AttrUtils::GetDataType(op_desc, HCOM_ATTR_DATA_TYPE, src_data_type); if (ret == false) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - HCOM_ATTR_DATA_TYPE.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", HCOM_ATTR_DATA_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "op:HcomReceive, op desc no attr: dtype."); + GELOGE(PARAM_INVALID, "[Get][Attr] %s in op:%s(%s) fail", HCOM_ATTR_DATA_TYPE.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } } else { @@ -55,13 +56,11 @@ Status HcomOmeUtil::GetHcclDataType(const ge::ConstOpDescPtr &op_desc, auto iter = kConstOpHcclDataType.find(static_cast(src_data_type)); if (iter == kConstOpHcclDataType.end()) { REPORT_INNER_ERROR("E19999", "Attr:%s in op:%s(%s), value data_type:%s, not support in kConstOpHcclDataType now, " - "check invalid", HCOM_ATTR_DATA_TYPE.c_str(), - op_desc->GetName().c_str(), op_desc->GetType().c_str(), - ge::TypeUtils::DataTypeToSerialString(src_data_type).c_str()); - GELOGE(PARAM_INVALID, - "HcomOmeUtil:: Node: %s Optype: %s HcomDataType cann't support! Current Davinci Data Type : %s", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), - ge::TypeUtils::DataTypeToSerialString(src_data_type).c_str()); + "check invalid", HCOM_ATTR_DATA_TYPE.c_str(), op_desc->GetName().c_str(), + op_desc->GetType().c_str(), ge::TypeUtils::DataTypeToSerialString(src_data_type).c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s in op:%s(%s), value data_type:%s, " + "not support in kConstOpHcclDataType now", HCOM_ATTR_DATA_TYPE.c_str(), op_desc->GetName().c_str(), + op_desc->GetType().c_str(), ge::TypeUtils::DataTypeToSerialString(src_data_type).c_str()); return PARAM_INVALID; } @@ -73,7 +72,7 @@ Status HcomOmeUtil::GetHcclDataType(const ge::ConstOpDescPtr &op_desc, Status HcomOmeUtil::GetHcclTypeSize(HcclDataType data_type, int32_t &size) { auto iter = kConstOpHcclDataTypeSize.find(data_type); GE_CHK_BOOL_EXEC(iter != kConstOpHcclDataTypeSize.end(), return PARAM_INVALID, - "HcomOmeUtil::HcomDataTypeSize , No DataTypeSize!"); + "[Check][Param] param data_type:%d not find", data_type); size = iter->second; return SUCCESS; @@ -83,21 +82,22 @@ Status HcomOmeUtil::GetHcomCount(const ge::ConstOpDescPtr &op_desc, HcclDataType int &count) { GE_CHECK_NOTNULL(op_desc); if (!IsHCOMOp(op_desc->GetType())) { - REPORT_INNER_ERROR("E19999", "Op:%s(%s) is not hcom op, check invalid", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: operator is not Hcom operator."); + REPORT_INNER_ERROR("E19999", "Op:%s(%s) is not hcom op, check invalid", op_desc->GetName().c_str(), + op_desc->GetType().c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Op:%s(%s) is not hcom op", op_desc->GetName().c_str(), + op_desc->GetType().c_str()); return PARAM_INVALID; } int64_t total_size = 0; int64_t align_size = 512; int32_t size = 0; - GE_CHK_STATUS_RET(HcomOmeUtil::GetHcclTypeSize(data_type, size), "GetHcomCount: GetHcclTypeSize fail!"); + GE_CHK_STATUS_RET(HcomOmeUtil::GetHcclTypeSize(data_type, size), "[Get][HcclTypeSize] fail, datatype:%d", data_type); if (op_desc->GetType() == HCOMRECEIVE) { for (size_t i = 0; i < op_desc->GetOutputsSize(); ++i) { int64_t output_size = 0; GE_CHECK_NOTNULL(op_desc->GetOutputDescPtr(i)); GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(*op_desc->GetOutputDescPtr(i), output_size), - "Get size from TensorDesc failed, op: %s, output index: %zu.", op_desc->GetName().c_str(), i); + "[Get][Size] from TensorDesc failed, op:%s, output index:%zu.", op_desc->GetName().c_str(), i); output_size = (output_size + align_size - 1) / align_size * align_size; total_size += output_size; } @@ -107,42 +107,48 @@ Status HcomOmeUtil::GetHcomCount(const ge::ConstOpDescPtr &op_desc, HcclDataType int64_t block_size = 0; GE_CHECK_NOTNULL(op_desc->GetInputDescPtr(i)); GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(*op_desc->GetInputDescPtr(i), input_size), - "get size from TensorDesc failed, op : %s, input index : %zu", op_desc->GetName().c_str(), i); + "[Get][Size] from TensorDesc failed, op:%s, input index:%zu", op_desc->GetName().c_str(), i); // dynamic shape hccl op get size from output tensor desc if (op_desc->HasAttr(ATTR_NAME_IS_UNKNOWN_SHAPE)) { GE_CHECK_NOTNULL(op_desc->GetOutputDescPtr(i)); GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(*op_desc->GetOutputDescPtr(i), input_size), - "get size from TensorDesc failed, op : %s, input index : %zu", op_desc->GetName().c_str(), i); + "[Get][Size] from TensorDesc failed, op:%s, input index:%zu", op_desc->GetName().c_str(), i); } GE_IF_BOOL_EXEC( op_desc->GetType() == HCOMREDUCESCATTER, int32_t rank_size = 0; GE_CHK_BOOL_RET_STATUS(ge::AttrUtils::GetInt(op_desc, HCOM_ATTR_RANK_SIZE, rank_size), PARAM_INVALID, - "get HCOM_ATTR_RANK_SIZE failed"); - GE_CHK_BOOL_RET_STATUS(rank_size != 0, PARAM_INVALID, "rank size is zero"); - int64_t shape_size = op_desc->GetInputDescPtr(i)->GetShape().GetShapeSize(); GE_CHK_STATUS_RET( - ge::CheckInt64Uint32MulOverflow(shape_size, size), "Product of shape size and size beyond INT64_MAX"); + "[Get][Attr] %s in op:%s(%s) failed", HCOM_ATTR_RANK_SIZE.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GE_CHK_BOOL_RET_STATUS(rank_size != 0, PARAM_INVALID, "[Check][Param] rank size is zero"); + int64_t shape_size = op_desc->GetInputDescPtr(i)->GetShape().GetShapeSize(); + GE_CHK_STATUS_RET(ge::CheckInt64Uint32MulOverflow(shape_size, size), + "[Check][Param] Product of shape size:%ld and size:%d beyond INT64_MAX, op:%s(%s)", + shape_size, size, op_desc->GetName().c_str(), op_desc->GetType().c_str()); block_size = (shape_size * size) / rank_size; - GE_CHK_STATUS_RET(ge::CheckInt64AddOverflow(total_size, block_size), "Total size is beyond the INT64_MAX"); + GE_CHK_STATUS_RET(ge::CheckInt64AddOverflow(total_size, block_size), + "[Check][Param] Total size:%ld is beyond the INT64_MAX, op:%s(%s)", + total_size, op_desc->GetName().c_str(), op_desc->GetType().c_str()); total_size = total_size + block_size; continue;); int64_t shape_size = op_desc->GetInputDescPtr(i)->GetShape().GetShapeSize(); GELOGD("hcom util node %s inputsize %ld, shapesize %ld, datasize %d.", op_desc->GetName().c_str(), input_size, shape_size, size); GE_CHK_STATUS_RET(ge::CheckInt64Int32MulOverflow(shape_size, size), - "Product of shape size and size beyond INT64_MAX"); + "[Check][Param] Product of shape size:%ld and size:%d beyond INT64_MAX", shape_size, size); GE_IF_BOOL_EXEC(is_allgather, block_size = shape_size * size;); GE_IF_BOOL_EXEC(!is_allgather, block_size = (input_size + align_size - 1) / align_size * align_size;); - GE_CHK_STATUS_RET(ge::CheckInt64AddOverflow(total_size, block_size), "Total size is beyond the INT64_MAX"); + GE_CHK_STATUS_RET(ge::CheckInt64AddOverflow(total_size, block_size), + "[Check][Param] Total size:%ld is beyond the INT64_MAX", total_size); total_size = total_size + block_size; } } - GE_CHK_BOOL_RET_STATUS(size != 0, PARAM_INVALID, "Size is zero"); + GE_CHK_BOOL_RET_STATUS(size != 0, PARAM_INVALID, "[Check][Param] Size is zero"); count = static_cast(total_size / size); - GE_CHK_BOOL_EXEC(total_size % size == 0, return PARAM_INVALID, "total_size:%ld is not divisiable by size:%d.", - total_size, size); + GE_CHK_BOOL_EXEC(total_size % size == 0, return PARAM_INVALID, + "[Check][Param] total_size:%ld is not divisiable by size:%d.", total_size, size); return SUCCESS; } @@ -153,32 +159,34 @@ Status HcomOmeUtil::GetHorovodCount(const ge::ConstOpDescPtr &op_desc, if (!IsHorovodOp(op_desc->GetType())) { REPORT_INNER_ERROR("E19999", "Op:%s(%s) is not horovod op, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: operator is not Horovod operator."); + GELOGE(PARAM_INVALID, "[Call][IsHorovodOp] failed, Op:%s(%s) is not horovod op", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } int64_t align_size = 512; int32_t size = 0; for (size_t i = 0; i < op_desc->GetInputsSize(); i++) { GE_CHK_STATUS_RET(HcomOmeUtil::GetHcclTypeSize(static_cast(kernel_hccl_infos[i].dataType), size), - "GetHorovodCount: GetHcclTypeSize fail!"); + "[Call][GetHcclTypeSize] fail, op:%s(%s)", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); int64_t input_size = 0; int64_t block_size = 0; GE_CHECK_NOTNULL(op_desc->GetInputDescPtr(i)); GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(*op_desc->GetInputDescPtr(i), input_size), - "get size from TensorDesc failed, op : %s, input index : %zu", op_desc->GetName().c_str(), i); + "[Get][Size] from TensorDesc failed, op:%s, input index:%zu", op_desc->GetName().c_str(), i); int64_t shape_size = op_desc->GetInputDescPtr(i)->GetShape().GetShapeSize(); GE_CHK_STATUS_RET(ge::CheckInt64Int32MulOverflow(shape_size, size), - "Product of shape size and size beyond INT64_MAX"); + "[Check][Param] Product of shape size:%ld and size:%d beyond INT64_MAX", shape_size, size); if (kernel_hccl_infos[0].hccl_type == HVDCALLBACKALLGATHER) { block_size = shape_size * size; } else { block_size = (input_size + align_size - 1) / align_size * align_size; } - GE_CHK_BOOL_RET_STATUS(size != 0, PARAM_INVALID, "Size is zero"); - GE_CHK_BOOL_EXEC(block_size % size == 0, return PARAM_INVALID, "block_size:%ld is not divisiable by size:%d.", - block_size, size); + GE_CHK_BOOL_RET_STATUS(size != 0, PARAM_INVALID, "[Check][Param] Size is zero"); + GE_CHK_BOOL_EXEC(block_size % size == 0, return PARAM_INVALID, + "[Check][Param] block_size:%ld is not divisiable by size:%d.", block_size, size); kernel_hccl_infos[i].count = static_cast(block_size / size); } @@ -191,7 +199,8 @@ Status HcomOmeUtil::GetHcclCount(const ge::ConstOpDescPtr &op_desc, Status ret; ret = CheckKernelHcclInfo(op_desc, kernel_hccl_infos); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "HcomOmeUtil:: the number of GETaskKernelHcclInfo is invalid."); + GELOGE(PARAM_INVALID, "[Check][KernelHcclInfo] failed, the number of GETaskKernelHcclInfo is invalid, op:%s(%s).", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } GELOGI("GetHcclCount start, node[%s], opType[%s].", op_desc->GetName().c_str(), op_desc->GetType().c_str()); @@ -200,7 +209,7 @@ Status HcomOmeUtil::GetHcclCount(const ge::ConstOpDescPtr &op_desc, ret = GetHcomCount(op_desc, static_cast(kernel_hccl_infos[0].dataType), kernel_hccl_infos[0].hccl_type == HCOMALLGATHER, count); if (ret != SUCCESS) { - GELOGE(ret, "HcomOmeUtil:: Node: %s Optype: %s get the Hcom operator hccl count fail.", + GELOGE(ret, "[Call][GetHcomCount] Node:%s Optype:%s get the Hcom operator hccl count fail.", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } @@ -210,7 +219,7 @@ Status HcomOmeUtil::GetHcclCount(const ge::ConstOpDescPtr &op_desc, if (IsHorovodOp(op_desc->GetType())) { ret = GetHorovodCount(op_desc, kernel_hccl_infos); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "HcomOmeUtil:: Node: %s Optype: %s get the Horovod hccl operator count fail.", + GELOGE(PARAM_INVALID, "[Call][GetHorovodCount] Node:%s Optype:%s get the Horovod hccl operator count fail.", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } @@ -225,11 +234,10 @@ Status HcomOmeUtil::GetHcclOperationType(const ge::ConstOpDescPtr &op_desc, Hccl if (IsHCOMOp(op_desc->GetType())) { std::string hcom_op_type; GE_CHK_BOOL_EXEC(ge::AttrUtils::GetStr(op_desc, HCOM_ATTR_REDUCE_TYPE, hcom_op_type), - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - HCOM_ATTR_REDUCE_TYPE.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", HCOM_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID, - "HcomOmeUtil:: Node: %s Optype: %s Get HCOM_ATTR_REDUCE_TYPE fail, not support!", + "[Get][Attr] %s in op:%s(%s) fail", HCOM_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); if (hcom_op_type == "min") { @@ -244,7 +252,9 @@ Status HcomOmeUtil::GetHcclOperationType(const ge::ConstOpDescPtr &op_desc, Hccl REPORT_INNER_ERROR("E19999", "Attr:%s in Op:%s(%s), hcom_op_type value:%s is not support now, " "check invalid", HCOM_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), hcom_op_type.c_str()); - GELOGE(PARAM_INVALID, "HcomOmeUtil::Get HCOM_ATTR_REDUCE_TYPE fail, [%s] not support!", hcom_op_type.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s in Op:%s(%s), hcom_op_type value:%s is not support now", + HCOM_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), + op_desc->GetType().c_str(), hcom_op_type.c_str()); return PARAM_INVALID; } } @@ -256,7 +266,7 @@ Status HcomOmeUtil::GetHcclOperationType(const ge::ConstOpDescPtr &op_desc, Hccl ATTR_HOROVOD_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID, - "HcomOmeUtil:: Node: %s Optype: %s Get ATTR_HOROVOD_ATTR_REDUCE_TYPE fail, not support!", + "[Get][Attr] %s in op:%s(%s) fail", ATTR_HOROVOD_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); auto iter = kHorovodRedOpToHcclRedOp.find(static_cast(horovod_op_type)); @@ -264,8 +274,8 @@ Status HcomOmeUtil::GetHcclOperationType(const ge::ConstOpDescPtr &op_desc, Hccl REPORT_INNER_ERROR("E19999", "Attr:%s in Op:%s(%s), horovod_op_type value:%ld is not support now, " "check invalid", ATTR_HOROVOD_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), horovod_op_type); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: Node: %s Optype: %s HcomOpType cann't support! Current HcomOpType : %ld", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), horovod_op_type); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s in Op:%s(%s), horovod_op_type value:%ld is not support now", + ATTR_HOROVOD_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), horovod_op_type); return PARAM_INVALID; } op_type = iter->second; @@ -281,7 +291,7 @@ Status HcomOmeUtil::GetHcclRootId(const ge::ConstOpDescPtr &op_desc, int64_t &ro HCOM_ATTR_ROOT_RANK.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID, - "HcomOmeUtil::Node %s Optype: %s Get HCOM_ATTR_ROOT_INDEX fail, not support!", + "[Get][Attr] %s in op:%s(%s) fail", HCOM_ATTR_ROOT_RANK.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return SUCCESS; @@ -296,7 +306,7 @@ Status HcomOmeUtil::GetAllRootId(const ge::ConstOpDescPtr &op_desc, int64_t root_id = 0; Status dmrt = GetHcclRootId(op_desc, root_id); if (dmrt != SUCCESS) { - GELOGE(FAILED, "davinci_model: GetHcomRootId fail! domi error: %u", dmrt); + GELOGE(FAILED, "[Get][HcclRootId] fail! domi error: %u", dmrt); return FAILED; } @@ -324,7 +334,8 @@ Status HcomOmeUtil::CheckKernelHcclInfo(const ge::ConstOpDescPtr &op_desc, REPORT_INNER_ERROR("E19999", "Op:%s(%s) is not hcom op or param kernel_hccl_infos.size:%zu != 1, " "check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), kernel_hccl_infos.size()); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: in Hcom scenario, the number of GETaskKernelHcclInfo is invalid."); + GELOGE(PARAM_INVALID, "[Check][Param] Op:%s(%s) is not hcom op or param kernel_hccl_infos.size:%zu != 1", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), kernel_hccl_infos.size()); return PARAM_INVALID; } @@ -337,7 +348,9 @@ Status HcomOmeUtil::CheckKernelHcclInfo(const ge::ConstOpDescPtr &op_desc, "in op:%s(%s), check invalid", kernel_hccl_infos.size(), op_desc->GetInputsSize(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: in Horovod scenario, the number of GETaskKernelHcclInfo is invalid."); + GELOGE(PARAM_INVALID, "Param kernel_hccl_infos.size:%zu is empty or not equal to " + "input_desc size:%zu in op:%s(%s)", kernel_hccl_infos.size(), op_desc->GetInputsSize(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } } @@ -360,7 +373,7 @@ Status HcomOmeUtil::GetHorovodInputs(const ge::ConstOpDescPtr &op_desc, } if (CheckKernelHcclInfo(op_desc, kernel_hccl_infos) != SUCCESS) { - GELOGE(PARAM_INVALID, "HcomOmeUtil:: Node: %s Optype: %s the number of GETaskKernelHcclInfo is invalid.", + GELOGE(PARAM_INVALID, "[Check][KernelHcclInfo] Node:%s Optype:%s the number of GETaskKernelHcclInfo is invalid.", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } diff --git a/ge/graph/manager/util/variable_accelerate_ctrl.cc b/ge/graph/manager/util/variable_accelerate_ctrl.cc index 22f9169c..12ec5a49 100644 --- a/ge/graph/manager/util/variable_accelerate_ctrl.cc +++ b/ge/graph/manager/util/variable_accelerate_ctrl.cc @@ -54,7 +54,7 @@ void VarAccelerateCtrl::SetVarChanged(const std::string &var_name) { void VarAccelerateCtrl::AddGraph(uint32_t graph_id, const ComputeGraphPtr &compute_graph) { std::lock_guard lock(mutex_); if (compute_graph == nullptr) { - GELOGE(PARAM_INVALID, "Failed to add graph %u, the compute graph is null", graph_id); + GELOGE(PARAM_INVALID, "[Check][Param] Failed to add graph %u, the compute graph is null", graph_id); return; } auto &var_names = graph_ids_to_var_names_[graph_id]; From 65837e1718a2b64da952937c3935e275d4f42f94 Mon Sep 17 00:00:00 2001 From: liudingyan Date: Wed, 19 May 2021 09:28:37 +0800 Subject: [PATCH 11/48] modify desc of err param dynamic_image_size --- ge/ir_build/option_utils.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ge/ir_build/option_utils.cc b/ge/ir_build/option_utils.cc index c23da519..9943834e 100755 --- a/ge/ir_build/option_utils.cc +++ b/ge/ir_build/option_utils.cc @@ -253,8 +253,7 @@ bool CheckDynamicImagesizeInputShapeValid(map> shape_map for (auto str : split_set) { split_dim = StringUtils::Split(str, ','); if (split_dim.size() != static_cast(kDynamicImageSizeNum)) { - ErrorManager::GetInstance().ATCReportErrMessage("E10020", {"DynamicImageSizeNum"}, - {std::to_string(kDynamicImageSizeNum)}); + ErrorManager::GetInstance().ATCReportErrMessage("E10020"); GELOGE(ge::PARAM_INVALID, "[Check][DynamicImagesizeInputShape] invalid value:%s number of dimensions of each group must be %ld.", dynamic_image_size.c_str(), kDynamicImageSizeNum); From 3074cd4ee5252fb7724f46166df12716d0b42756 Mon Sep 17 00:00:00 2001 From: lichun Date: Wed, 19 May 2021 10:24:11 +0800 Subject: [PATCH 12/48] fwkacllib support inference when ge lib is not inited --- .../aicore/aicore_node_executor.cc | 3 -- .../aicore/aicore_task_compiler.cc | 13 +++++ .../aicore/aicore_task_compiler.h | 2 + tests/ut/ge/CMakeLists.txt | 1 + .../aicore/aicore_task_compiler_unittest.cc | 48 +++++++++++++++++++ 5 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 tests/ut/ge/hybrid/node_executor/aicore/aicore_task_compiler_unittest.cc diff --git a/ge/hybrid/node_executor/aicore/aicore_node_executor.cc b/ge/hybrid/node_executor/aicore/aicore_node_executor.cc index 29ae831c..7ebb9e39 100755 --- a/ge/hybrid/node_executor/aicore/aicore_node_executor.cc +++ b/ge/hybrid/node_executor/aicore/aicore_node_executor.cc @@ -41,9 +41,6 @@ AiCoreNodeTask::AiCoreNodeTask(std::vector> &&task Status AiCoreNodeExecutor::Initialize() { compiler_ = TaskCompilerFactory::GetInstance().GetTaskCompiler(); - if (compiler_ != nullptr) { - GE_CHK_STATUS_RET(compiler_->Initialize(), "[Init][TaskCompiler] failed."); - } return SUCCESS; } diff --git a/ge/hybrid/node_executor/aicore/aicore_task_compiler.cc b/ge/hybrid/node_executor/aicore/aicore_task_compiler.cc index 742b3ca2..0cdea5d5 100755 --- a/ge/hybrid/node_executor/aicore/aicore_task_compiler.cc +++ b/ge/hybrid/node_executor/aicore/aicore_task_compiler.cc @@ -31,6 +31,11 @@ REGISTER_TASK_COMPILER(AiCoreTaskCompiler); std::mutex AiCoreTaskCompiler::mu_; Status AiCoreTaskCompiler::Initialize() { + std::lock_guard lk(mu_); + if (is_initialized_) { + return SUCCESS; + } + auto ge_lib = GELib::GetInstance(); GE_CHECK_NOTNULL(ge_lib); if (!ge_lib->InitFlag()) { @@ -41,6 +46,7 @@ Status AiCoreTaskCompiler::Initialize() { auto &kernel_manager = ge_lib->OpsKernelManagerObj(); aic_kernel_store_ = kernel_manager.GetOpsKernelInfoStore("AIcoreEngine"); GE_CHECK_NOTNULL(aic_kernel_store_); + is_initialized_ = true; return SUCCESS; } @@ -57,6 +63,13 @@ Status AiCoreTaskCompiler::DoCompileOp(const NodePtr &node) const { } Status AiCoreTaskCompiler::CompileOp(const NodePtr &node, std::vector &tasks) { + Status ret = Initialize(); + if (ret != SUCCESS) { + GELOGE(FAILED, "[Check][State][%s] Offline inference not support online compile.", node->GetName().c_str()); + REPORT_INNER_ERROR("E19999", "[%s] Offline inference not support online compile.", node->GetName().c_str()); + return ret; + } + GE_CHECK_NOTNULL(node); GELOGI("AiCoreTaskCompiler(%s) CompileOp Start.", node->GetName().c_str()); diff --git a/ge/hybrid/node_executor/aicore/aicore_task_compiler.h b/ge/hybrid/node_executor/aicore/aicore_task_compiler.h index b6dfd82b..cc733e99 100755 --- a/ge/hybrid/node_executor/aicore/aicore_task_compiler.h +++ b/ge/hybrid/node_executor/aicore/aicore_task_compiler.h @@ -18,6 +18,7 @@ #define GE_HYBRID_KERNEL_AICORE_TASK_COMPILER_H_ #include +#include #include "opskernel_manager/ops_kernel_manager.h" #include "aicore_node_executor.h" @@ -34,6 +35,7 @@ class AiCoreTaskCompiler : public TaskCompiler { Status DoCompileOp(const NodePtr &node) const; Status DoGenerateTask(const Node &node, std::vector &tasks); OpsKernelInfoStorePtr aic_kernel_store_; + std::atomic_bool is_initialized_{false}; static std::mutex mu_; }; } // namespace hybrid diff --git a/tests/ut/ge/CMakeLists.txt b/tests/ut/ge/CMakeLists.txt index 96da50d1..3f272023 100755 --- a/tests/ut/ge/CMakeLists.txt +++ b/tests/ut/ge/CMakeLists.txt @@ -829,6 +829,7 @@ set(HYBRID_TEST_FILES "hybrid/model/hybrid_model_builder_unittest.cc" "hybrid/node_executor/rts/rts_node_task_unittest.cc" "hybrid/executor/hybrid_model_async_executor_unittest.cc" + "hybrid/node_executor/aicore/aicore_task_compiler_unittest.cc" ) set(OTHERS_TEST_FILES diff --git a/tests/ut/ge/hybrid/node_executor/aicore/aicore_task_compiler_unittest.cc b/tests/ut/ge/hybrid/node_executor/aicore/aicore_task_compiler_unittest.cc new file mode 100644 index 00000000..3371cd5c --- /dev/null +++ b/tests/ut/ge/hybrid/node_executor/aicore/aicore_task_compiler_unittest.cc @@ -0,0 +1,48 @@ +/** + * Copyright 2021-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#define private public +#define protected public +#include "init/gelib.h" +#include "hybrid/node_executor/aicore/aicore_task_compiler.h" +#undef private +#undef protected + +using namespace std; +using namespace testing; + +namespace ge { +using namespace hybrid; + +class UtestAiCoreTaskCompiler : public testing::Test { + protected: + void SetUp() {} + void TearDown() {} +}; + +TEST_F(UtestAiCoreTaskCompiler, test_aicore_task_compiler_init) { + ge::hybrid::AiCoreTaskCompiler aicore_task_compiler; + NodePtr node = MakeShared(); + std::vector tasks{}; + EXPECT_EQ(aicore_task_compiler.Initialize(), ge::PARAM_INVALID); // cause: ge lib is nullptr + EXPECT_EQ(aicore_task_compiler.CompileOp(node, tasks), ge::PARAM_INVALID); // cause: aicore task compiler init failed. +} +} // namespace ge + From 2083a8e059de8ddfbf6d19bdd6ed398f2463c723 Mon Sep 17 00:00:00 2001 From: lichun Date: Wed, 19 May 2021 11:30:16 +0800 Subject: [PATCH 13/48] fwkacllib support inference when ge lib is not inited --- ge/hybrid/node_executor/aicore/aicore_task_compiler.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ge/hybrid/node_executor/aicore/aicore_task_compiler.h b/ge/hybrid/node_executor/aicore/aicore_task_compiler.h index cc733e99..4cb4dc58 100755 --- a/ge/hybrid/node_executor/aicore/aicore_task_compiler.h +++ b/ge/hybrid/node_executor/aicore/aicore_task_compiler.h @@ -18,7 +18,6 @@ #define GE_HYBRID_KERNEL_AICORE_TASK_COMPILER_H_ #include -#include #include "opskernel_manager/ops_kernel_manager.h" #include "aicore_node_executor.h" @@ -35,7 +34,7 @@ class AiCoreTaskCompiler : public TaskCompiler { Status DoCompileOp(const NodePtr &node) const; Status DoGenerateTask(const Node &node, std::vector &tasks); OpsKernelInfoStorePtr aic_kernel_store_; - std::atomic_bool is_initialized_{false}; + bool is_initialized_ = false; static std::mutex mu_; }; } // namespace hybrid From b90d8105ecfac0e0c8dfb8220aad68b31bbc2c2f Mon Sep 17 00:00:00 2001 From: chenyemeng Date: Wed, 19 May 2021 15:00:48 +0800 Subject: [PATCH 14/48] modify tensor_desc idx when insert identity --- ge/graph/passes/subgraph_pass.cc | 4 +- tests/ut/ge/CMakeLists.txt | 3 +- .../ge/graph/passes/subgraph_pass_unittest.cc | 129 ++++++++++++++++++ 3 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 tests/ut/ge/graph/passes/subgraph_pass_unittest.cc diff --git a/ge/graph/passes/subgraph_pass.cc b/ge/graph/passes/subgraph_pass.cc index b931eea8..401dee54 100755 --- a/ge/graph/passes/subgraph_pass.cc +++ b/ge/graph/passes/subgraph_pass.cc @@ -464,8 +464,8 @@ Status SubgraphPass::InsertMemcpyNode(const ComputeGraphPtr &graph, const OutDat GE_CHECK_NOTNULL(out_anchor); NodePtr in_node = out_anchor->GetOwnerNode(); OpDescBuilder op_desc_builder(name, IDENTITY); - OpDescPtr op_desc = op_desc_builder.AddInput("x", in_node->GetOpDesc()->GetOutputDesc(0)) - .AddOutput("y", in_node->GetOpDesc()->GetOutputDesc(0)) + OpDescPtr op_desc = op_desc_builder.AddInput("x", in_node->GetOpDesc()->GetOutputDesc(out_anchor->GetIdx())) + .AddOutput("y", in_node->GetOpDesc()->GetOutputDesc(out_anchor->GetIdx())) .Build(); (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); (void)AttrUtils::SetBool(op_desc, ATTR_NAME_CANNOT_BE_DELETED, true); diff --git a/tests/ut/ge/CMakeLists.txt b/tests/ut/ge/CMakeLists.txt index 895c33df..d24b440d 100755 --- a/tests/ut/ge/CMakeLists.txt +++ b/tests/ut/ge/CMakeLists.txt @@ -665,7 +665,7 @@ set(PASS_TEST_FILES "graph/passes/permute_pass_unittest.cc" "graph/passes/print_op_pass_unittest.cc" "graph/passes/shape_operate_op_remove_pass_unittest.cc" - "graph/passes/variable_op_pass_unittest.cc" + "graph/passes/variable_op_pass_unittest.cc" "graph/passes/base_pass_unittest.cc" "graph/passes/addn_pass_unittest.cc" "graph/passes/save_pass_unittest.cc" @@ -674,6 +674,7 @@ set(PASS_TEST_FILES "graph/passes/cond_branch_v1_unittest.cc" "graph/passes/loop_branch_v1_unittest.cc" "graph/passes/switch_dead_branch_elimination_unittest.cc" + "graph/passes/subgraph_pass_unittest.cc" "graph/passes/assert_pass_unittest.cc" "graph/passes/dropout_pass_unittest.cc" "graph/passes/unused_const_pass_unittest.cc" diff --git a/tests/ut/ge/graph/passes/subgraph_pass_unittest.cc b/tests/ut/ge/graph/passes/subgraph_pass_unittest.cc new file mode 100644 index 00000000..f11882e1 --- /dev/null +++ b/tests/ut/ge/graph/passes/subgraph_pass_unittest.cc @@ -0,0 +1,129 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "graph/passes/subgraph_pass.h" +#include "inc/pass_manager.h" + +namespace ge { +namespace { +class UtestGraphPassesSubgraphPass : public testing::Test { + protected: + void SetUp() {} + void TearDown() {} +}; + +OpDescPtr CreateOpDesc(const std::string name, const std::string type, uint32_t input_num, uint32_t output_num) { + OpDescPtr op_desc = std::shared_ptr(new (std::nothrow) OpDesc(name, type)); + if (op_desc == nullptr) { + return nullptr; + } + for (uint32_t i = 0; i < input_num; i++) { + op_desc->AddInputDesc(GeTensorDesc()); + } + for (uint32_t i = 0; i < output_num; i++) { + op_desc->AddOutputDesc(GeTensorDesc()); + } + return op_desc; +} + +bool CheckMemcpyExist(const ComputeGraphPtr &graph) { + for (const auto &node : graph->GetDirectNode()) { + if (node->GetType() == IDENTITY) { + return true; + } + } + return false; +} + +uint32_t CheckMemcpyNum(const ComputeGraphPtr &graph) { + uint32_t num = 0; + for (const auto &node : graph->GetDirectNode()) { + if (node->GetType() == IDENTITY) { + num++; + } + } + return num; +} +} // namespace + +/// +/// ****** root_graph ****** * ****** subgraph branch1 ***** * ****** subgraph branch2 ***** +/// * * +/// Case * Const * Data +/// / \ * | * | +/// data_0 data_1 * NetOutput * NetOutput +/// * * +/// ****** root_graph ****** * ****** subgraph branch1 ***** * ****** subgraph branch2 ***** +/// +TEST(UtestGraphPassesSubgraphPass, add_memcpy_success) { + ComputeGraphPtr graph = std::make_shared("add_memcpy_success"); + NodePtr func_node = graph->AddNode(CreateOpDesc("Case", CASE, 2, 1)); + NodePtr data_node_0 = graph->AddNode(CreateOpDesc("data_0", DATA, 1, 1)); + NodePtr data_node_1 = graph->AddNode(CreateOpDesc("data_1", DATA, 1, 1)); + EXPECT_EQ(GraphUtils::AddEdge(data_node_0->GetOutDataAnchor(0), func_node->GetInDataAnchor(0)), GRAPH_SUCCESS); + EXPECT_EQ(GraphUtils::AddEdge(data_node_1->GetOutDataAnchor(0), func_node->GetInDataAnchor(1)), GRAPH_SUCCESS); + + std::string subgraph_name_1 = "instance_branch_1"; + ComputeGraphPtr subgraph_1 = std::make_shared(subgraph_name_1); + subgraph_1->SetParentNode(func_node); + subgraph_1->SetParentGraph(graph); + size_t index = func_node->GetOpDesc()->GetSubgraphInstanceNames().size(); + EXPECT_EQ(index, 0); + func_node->GetOpDesc()->AddSubgraphName("branch1"); + EXPECT_EQ(func_node->GetOpDesc()->GetSubgraphInstanceNames().size(), 1); + func_node->GetOpDesc()->SetSubgraphInstanceName(index, subgraph_name_1); + EXPECT_EQ(func_node->GetOpDesc()->GetSubgraphInstanceNames().size(), 1); + + std::string subgraph_name_2 = "instance_branch_2"; + ComputeGraphPtr subgraph_2 = std::make_shared(subgraph_name_2); + subgraph_2->SetParentNode(func_node); + subgraph_2->SetParentGraph(graph); + index = func_node->GetOpDesc()->GetSubgraphInstanceNames().size(); + EXPECT_EQ(index, 1); + func_node->GetOpDesc()->AddSubgraphName("branch2"); + EXPECT_EQ(func_node->GetOpDesc()->GetSubgraphInstanceNames().size(), 2); + func_node->GetOpDesc()->SetSubgraphInstanceName(index, subgraph_name_2); + EXPECT_EQ(func_node->GetOpDesc()->GetSubgraphInstanceNames().size(), 2); + + { + // Const->NetOutput in subgraph + NodePtr const_node = subgraph_1->AddNode(CreateOpDesc("const", CONSTANTOP, 0, 1)); + NodePtr output_node = subgraph_1->AddNode(CreateOpDesc(NODE_NAME_NET_OUTPUT, NETOUTPUT, 1, 1)); + EXPECT_EQ(GraphUtils::AddEdge(const_node->GetOutDataAnchor(0), output_node->GetInDataAnchor(0)), SUCCESS); + } + + { + // Data->NetOutput in subgraph but not while body + NodePtr data_node = subgraph_2->AddNode(CreateOpDesc("sata", DATA, 1, 1)); + NodePtr output_node = subgraph_2->AddNode(CreateOpDesc(NODE_NAME_NET_OUTPUT, NETOUTPUT, 1, 1)); + EXPECT_EQ(GraphUtils::AddEdge(data_node->GetOutDataAnchor(0), output_node->GetInDataAnchor(0)), SUCCESS); + EXPECT_TRUE(AttrUtils::SetInt(data_node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, 1)); + } + + PassManager pass_manager; + pass_manager.AddPass("SubgraphPass", new (std::nothrow) SubgraphPass); + EXPECT_EQ(pass_manager.Run(graph), SUCCESS); + EXPECT_FALSE(CheckMemcpyExist(graph)); + EXPECT_EQ(pass_manager.Run(subgraph_1), SUCCESS); + EXPECT_EQ(CheckMemcpyNum(subgraph_1), 1); + EXPECT_EQ(pass_manager.Run(subgraph_2), SUCCESS); + EXPECT_EQ(CheckMemcpyNum(subgraph_2), 1); +} +} // namespace ge From 899d143a4779e489ee74b3bff47acd77934a908e Mon Sep 17 00:00:00 2001 From: y00500818 Date: Thu, 20 May 2021 19:12:30 +0800 Subject: [PATCH 15/48] bugfix for infer format for single op --- ge/generator/ge_generator.cc | 40 +++++++++++++++++-- inc/framework/generator/ge_generator.h | 1 + .../ut/ge/generator/ge_generator_unittest.cc | 22 ++++++++++ tests/ut/ge/graph/ops_stub.h | 11 +++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/ge/generator/ge_generator.cc b/ge/generator/ge_generator.cc index fa6d8fa8..8a94aa9b 100644 --- a/ge/generator/ge_generator.cc +++ b/ge/generator/ge_generator.cc @@ -31,6 +31,7 @@ #include "graph/ge_context.h" #include "graph/manager/graph_manager.h" #include "graph/manager/util/rt_context_util.h" +#include "graph/operator_factory_impl.h" #include "graph/opsproto_manager.h" #include "graph/utils/graph_utils.h" #include "graph/utils/type_utils.h" @@ -803,6 +804,41 @@ Status GeGenerator::CheckForSingleOp(OpDescPtr &op_desc, const vector return SUCCESS; } +Status GeGenerator::InferFormatForSingleOp(OpDescPtr &op_desc) { + GE_CHECK_NOTNULL(op_desc); + if (OperatorFactoryImpl::GetInferFormatFunc(op_desc->GetType()) != nullptr) { + auto node_op = ge::OperatorFactoryImpl::CreateOperator("node_op", op_desc->GetType()); + if (node_op.IsEmpty()) { + GELOGW("get op from OperatorFactory fail. op type: %s", op_desc->GetType().c_str()); + } else { + GELOGD("get op from OperatorFactory success. op type: %s", op_desc->GetType().c_str()); + auto temp_op_desc = ge::OpDescUtils::GetOpDescFromOperator(node_op); + if (temp_op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "GetOpDescFromOperator failed, as return nullptr, type:%s", + op_desc->GetType().c_str()); + GELOGE(FAILED, "[Get][OpDesc] temp op desc is null, type:%s", op_desc->GetType().c_str()); + return FAILED; + } + if (!op_desc->UpdateInputName(temp_op_desc->GetAllInputName())) { + GELOGW("InferFormatForSingleOp UpdateInputName failed"); + } + if (!op_desc->UpdateOutputName(temp_op_desc->GetAllOutputName())) { + GELOGW("InferFormatForSingleOp UpdateOutputName failed"); + } + } + node_op.BreakConnect(); + } + auto op = OpDescUtils::CreateOperatorFromOpDesc(op_desc); + auto ret = op_desc->CallInferFormatFunc(op); + if (ret != GRAPH_SUCCESS) { + REPORT_INNER_ERROR("E19999", "call InferFormatFunc for single op:%s fail", + op_desc->GetName().c_str()); + GELOGE(FAILED, "[Call][InferFormatFunc] for single op:%s fail.", op_desc->GetName().c_str()); + return FAILED; + } + return SUCCESS; +} + Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector &inputs, const vector &outputs, const string &model_file_name, OpEngineType engine_type, ModelBufferData &model_buff, bool is_offline, int32_t compile_flag) { @@ -843,9 +879,7 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector &in Graph graph; GE_CHK_STATUS(BuildSingleOpGraph(op_desc, inputs, outputs, name, graph), "[Build][Graph] for single op:%s fail.", op_desc->GetName().c_str()); - auto op = OpDescUtils::CreateOperatorFromOpDesc(op_desc); - GE_CHK_STATUS_RET(op_desc->CallInferFormatFunc(op), - "[Call][InferFormatFunc] for single op:%s fail.", op_desc->GetName().c_str()); + GE_CHK_STATUS_RET_NOLOG(InferFormatForSingleOp(op_desc)); // 2. check engine type when compile online if (model_file_name == kFileNameSuffix) { diff --git a/inc/framework/generator/ge_generator.h b/inc/framework/generator/ge_generator.h index 24f969dd..ee51d29d 100644 --- a/inc/framework/generator/ge_generator.h +++ b/inc/framework/generator/ge_generator.h @@ -106,6 +106,7 @@ class GE_FUNC_VISIBILITY GeGenerator { bool CheckNoAicore(const ComputeGraphPtr &graph); void RemoveConst(const vector &inputs, vector &outputs); Status CheckForSingleOp(OpDescPtr &op_desc, const vector &inputs, const vector &outputs); + Status InferFormatForSingleOp(OpDescPtr &op_desc); using GeRootModelPtr = std::shared_ptr; Status SetModelNameForDump(const GeRootModelPtr &ge_root_model); diff --git a/tests/ut/ge/generator/ge_generator_unittest.cc b/tests/ut/ge/generator/ge_generator_unittest.cc index fb256c7c..1bb4430f 100644 --- a/tests/ut/ge/generator/ge_generator_unittest.cc +++ b/tests/ut/ge/generator/ge_generator_unittest.cc @@ -23,6 +23,7 @@ #include "graph/attr_value.h" #include "graph/debug/ge_attr_define.h" #include "graph/utils/graph_utils.h" +#include "graph/operator_factory_impl.h" #include "../graph/passes/graph_builder_utils.h" #include "../graph/manager/graph_manager.h" #include "all_ops.h" @@ -79,6 +80,27 @@ TEST_F(UtestGeGenerator, test_build_single_op_offline) { EXPECT_EQ(generator.BuildSingleOpModel(op_desc, inputs, outputs, "offline_"), GE_GENERATOR_GRAPH_MANAGER_BUILD_GRAPH_FAILED); } */ +graphStatus TestFunc(Operator &op) { return 0; } +graphStatus TestFunc1(Operator &op) { return 1; } +TEST_F(UtestGeGenerator, test_infer_format_for_single_op) { + OperatorFactoryImpl::RegisterInferFormatFunc("Add", TestFunc); + shared_ptr op_desc = make_shared("add", "add"); + GeGenerator generator; + EXPECT_EQ(generator.InferFormatForSingleOp(op_desc), SUCCESS); + shared_ptr op_desc1 = make_shared("Add", "Add"); + EXPECT_EQ(generator.InferFormatForSingleOp(op_desc1), SUCCESS); + OperatorFactoryImpl::RegisterInferFormatFunc("MatMulV2", TestFunc1); + shared_ptr op_desc2 = make_shared("MatMulV2", "MatMulV2"); + GeTensorDesc tensor_desc; + EXPECT_EQ(op_desc2->AddInputDesc(tensor_desc), GRAPH_SUCCESS); + EXPECT_EQ(op_desc2->AddInputDesc(tensor_desc), GRAPH_SUCCESS); + EXPECT_EQ(op_desc2->AddInputDesc(tensor_desc), GRAPH_SUCCESS); + EXPECT_EQ(op_desc2->AddInputDesc(tensor_desc), GRAPH_SUCCESS); + EXPECT_EQ(op_desc2->AddInputDesc(tensor_desc), GRAPH_SUCCESS); + EXPECT_EQ(op_desc2->AddOutputDesc(tensor_desc), GRAPH_SUCCESS); + EXPECT_EQ(op_desc2->AddOutputDesc(tensor_desc), GRAPH_SUCCESS); + EXPECT_EQ(generator.InferFormatForSingleOp(op_desc2), FAILED); +} TEST_F(UtestGeGenerator, test_build_single_op_online) { GeTensorDesc tensor_desc; diff --git a/tests/ut/ge/graph/ops_stub.h b/tests/ut/ge/graph/ops_stub.h index 2a71d80a..c122befa 100644 --- a/tests/ut/ge/graph/ops_stub.h +++ b/tests/ut/ge/graph/ops_stub.h @@ -144,6 +144,17 @@ REG_OP(Data) DT_UINT64, DT_BOOL, DT_DOUBLE})) .OP_END_FACTORY_REG(GuaranteeConst) + REG_OP(MatMulV2) + .INPUT(x1, TensorType({DT_FLOAT, DT_FLOAT16, DT_INT32, DT_INT8})) + .INPUT(x2, TensorType({DT_FLOAT, DT_FLOAT16, DT_INT32, DT_INT8})) + .OPTIONAL_INPUT(bias, TensorType({DT_FLOAT, DT_FLOAT16, DT_INT32})) + .OUTPUT(y, TensorType({DT_FLOAT, DT_FLOAT16, DT_INT32})) + .OPTIONAL_INPUT(offset_w, TensorType({DT_INT8})) + .ATTR(transpose_x1, Bool, false) + .ATTR(transpose_x2, Bool, false) + .ATTR(offset_x, Int, 0) + .OP_END_FACTORY_REG(MatMulV2) + IMPLEMT_INFERFUNC(GuaranteeConst, GuaranteeConstInfer) { TensorDesc tensorDesc = op.GetInputDesc("x"); (void)op.UpdateOutputDesc("y", tensorDesc); From 019b097cfe001dece605fd84400e28a2aadfaffa Mon Sep 17 00:00:00 2001 From: zhengyuanhua Date: Thu, 20 May 2021 19:34:27 +0800 Subject: [PATCH 16/48] add checkout field exist when profiling json parse --- ge/common/profiling/profiling_manager.cc | 23 ++++++++---- .../ge_profiling_manager_unittest.cc | 36 +++++++++++++++++-- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index f7015525..f1c3c87b 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -184,7 +184,10 @@ ge::Status ProfilingManager::ParseOptions(const std::string &options) { if (options.find(kTrainingTrace) == std::string::npos) { return ge::SUCCESS; } - const std::string training_trace = prof_options[kTrainingTrace]; + std::string training_trace; + if (prof_options.contains(kTrainingTrace)) { + training_trace = prof_options[kTrainingTrace]; + } if (training_trace.empty()) { GELOGI("Training trace will not take effect."); return ge::SUCCESS; @@ -196,8 +199,12 @@ ge::Status ProfilingManager::ParseOptions(const std::string &options) { REPORT_INNER_ERROR("E19999", "Training trace param:%s is invalid.", training_trace.c_str()); return ge::PARAM_INVALID; } - fp_point_ = prof_options[kFpPoint]; - bp_point_ = prof_options[kBpPoint]; + if (prof_options.contains(kFpPoint)) { + fp_point_ = prof_options[kFpPoint]; + } + if (prof_options.contains(kBpPoint)) { + bp_point_ = prof_options[kBpPoint]; + } if (!fp_point_.empty() && !bp_point_.empty()) { GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); } @@ -1014,10 +1021,12 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpP if (is_profiling_valid) { try { Json prof_options = Json::parse(profiling_options); - - fp_point_ = prof_options[kFpPoint]; - bp_point_ = prof_options[kBpPoint]; - + if (prof_options.contains(kFpPoint)) { + fp_point_ = prof_options[kFpPoint]; + } + if (prof_options.contains(kBpPoint)) { + bp_point_ = prof_options[kBpPoint]; + } fp_point = fp_point_; bp_point = bp_point_; if (!fp_point_.empty() && !bp_point_.empty()) { diff --git a/tests/ut/ge/profiling/ge_profiling_manager_unittest.cc b/tests/ut/ge/profiling/ge_profiling_manager_unittest.cc index 2da80b32..9c615317 100644 --- a/tests/ut/ge/profiling/ge_profiling_manager_unittest.cc +++ b/tests/ut/ge/profiling/ge_profiling_manager_unittest.cc @@ -24,6 +24,7 @@ #define protected public #define private public #include "common/profiling/profiling_manager.h" +#include "graph/ge_local_context.h" #undef protected #undef private @@ -64,11 +65,12 @@ TEST_F(UtestGeProfilinganager, ParseOptions) { options.profiling_mode = "1"; options.profiling_options = R"({"result_path":"/data/profiling","training_trace":"on","task_trace":"on","aicpu_trace":"on","fp_point":"Data_0","bp_point":"addn","ai_core_metrics":"ResourceConflictRatio"})"; - struct MsprofGeOptions prof_conf = {{ 0 }}; - Status ret = ProfilingManager::Instance().ParseOptions(options.profiling_options); EXPECT_EQ(ret, ge::SUCCESS); + EXPECT_EQ(ProfilingManager::Instance().is_training_trace_, true); + EXPECT_EQ(ProfilingManager::Instance().fp_point_, "Data_0"); + EXPECT_EQ(ProfilingManager::Instance().bp_point_, "addn"); } TEST_F(UtestGeProfilinganager, plungin_init_) { @@ -83,4 +85,34 @@ TEST_F(UtestGeProfilinganager, report_data_) { std::string data = "ge is better than tensorflow."; std::string tag_name = "fmk"; ProfilingManager::Instance().ReportData(0, data, tag_name); +} + +TEST_F(UtestGeProfilinganager, get_fp_bp_point_) { + map options_map = { + {OPTION_EXEC_PROFILING_OPTIONS, + R"({"result_path":"/data/profiling","training_trace":"on","task_trace":"on","aicpu_trace":"on","fp_point":"Data_0","bp_point":"addn","ai_core_metrics":"ResourceConflictRatio"})"}}; + GEThreadLocalContext &context = GetThreadLocalContext(); + context.SetGraphOption(options_map); + + std::string fp_point; + std::string bp_point; + ProfilingManager::Instance().GetFpBpPoint(fp_point, bp_point); + EXPECT_EQ(fp_point, "Data_0"); + EXPECT_EQ(bp_point, "addn"); +} + +TEST_F(UtestGeProfilinganager, get_fp_bp_point_empty) { + // fp bp empty + map options_map = { + { OPTION_EXEC_PROFILING_OPTIONS, + R"({"result_path":"/data/profiling","training_trace":"on","task_trace":"on","aicpu_trace":"on","ai_core_metrics":"ResourceConflictRatio"})"}}; + GEThreadLocalContext &context = GetThreadLocalContext(); + context.SetGraphOption(options_map); + std::string fp_point = "fp"; + std::string bp_point = "bp"; + ProfilingManager::Instance().bp_point_ = ""; + ProfilingManager::Instance().fp_point_ = ""; + ProfilingManager::Instance().GetFpBpPoint(fp_point, bp_point); + EXPECT_EQ(fp_point, ""); + EXPECT_EQ(bp_point, ""); } \ No newline at end of file From 25069f6dab26f10c7f86917ad219eeae300226d9 Mon Sep 17 00:00:00 2001 From: zhengyuanhua Date: Sat, 22 May 2021 16:53:20 +0800 Subject: [PATCH 17/48] pass --- ge/graph/passes/memcpy_addr_async_pass.cc | 14 +++--- tests/ut/ge/CMakeLists.txt | 1 + .../passes/memcpy_addr_async_unittest.cc | 47 +++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 tests/ut/ge/graph/passes/memcpy_addr_async_unittest.cc diff --git a/ge/graph/passes/memcpy_addr_async_pass.cc b/ge/graph/passes/memcpy_addr_async_pass.cc index aff89f35..e8e4ebd8 100755 --- a/ge/graph/passes/memcpy_addr_async_pass.cc +++ b/ge/graph/passes/memcpy_addr_async_pass.cc @@ -25,15 +25,15 @@ namespace ge { Status MemcpyAddrAsyncPass::Run(ComputeGraphPtr graph) { GE_CHECK_NOTNULL(graph); - for (const auto &node : graph->GetAllNodes()) { - if (node->GetType() == STREAMSWITCH) { - auto sub_graph = node->GetOwnerComputeGraph(); - if (sub_graph != nullptr && !sub_graph->GetGraphUnknownFlag()) { - GE_CHK_STATUS_RET(AddMemcpyAsyncNode(node), "Add memcpyasync node failed in known subgraph."); + if (graph->GetGraphUnknownFlag()) { + for (const auto &node : graph->GetAllNodes()) { + if (node->GetType() == STREAMSWITCH) { + auto sub_graph = node->GetOwnerComputeGraph(); + if (sub_graph != nullptr && !sub_graph->GetGraphUnknownFlag()) { + GE_CHK_STATUS_RET(AddMemcpyAsyncNode(node), "Add memcpyasync node failed in known subgraph."); + } } } - } - if (graph->GetGraphUnknownFlag()) { GELOGD("Graph[%s] is unknown graph, skip.", graph->GetName().c_str()); return SUCCESS; } diff --git a/tests/ut/ge/CMakeLists.txt b/tests/ut/ge/CMakeLists.txt index afe2f2b9..e957d119 100755 --- a/tests/ut/ge/CMakeLists.txt +++ b/tests/ut/ge/CMakeLists.txt @@ -715,6 +715,7 @@ set(PASS_TEST_FILES "graph/passes/mark_node_unknown_shape_pass_unittest.cc" "graph/passes/reshape_recovery_pass_unittest.cc" "graph/passes/cast_remove_pass_unittest.cc" + "graph/passes/memcpy_addr_async_unittest.cc" ) set(KERNEL_TEST_FILES diff --git a/tests/ut/ge/graph/passes/memcpy_addr_async_unittest.cc b/tests/ut/ge/graph/passes/memcpy_addr_async_unittest.cc new file mode 100644 index 00000000..e5bc450e --- /dev/null +++ b/tests/ut/ge/graph/passes/memcpy_addr_async_unittest.cc @@ -0,0 +1,47 @@ +/** + * Copyright 2019-2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#define private public +#include "graph/passes/memcpy_addr_async_pass.h" +#include "common/ge_inner_error_codes.h" +#include "inc/pass_manager.h" +#undef private + +namespace ge { +class UtestMemcpyAddrAsyncPass : public testing::Test { + protected: + void SetUp() {} + void TearDown() {} +}; + +TEST_F(UtestMemcpyAddrAsyncPass, run) { + ge::ComputeGraphPtr graph = std::make_shared("default"); + ge::OpDescPtr op = std::make_shared(); + op->SetType(STREAMSWITCH); + op->SetName("stream_switch"); + op->AddOutputDesc(ge::GeTensorDesc()); + ge::NodePtr node = graph->AddNode(op); + graph->SetGraphUnknownFlag(true); + MemcpyAddrAsyncPass pass; + Status ret = pass.Run(graph); + EXPECT_EQ(ret, SUCCESS); +} +} // namespace ge From c16ae97d6e8b2c02913df1995dd5a50c154ae855 Mon Sep 17 00:00:00 2001 From: zhou_chao1993 Date: Tue, 25 May 2021 09:48:27 +0800 Subject: [PATCH 18/48] opdebug bug --- ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc index 7c26d23d..231655d1 100644 --- a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc +++ b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc @@ -270,7 +270,7 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin } void KernelExTaskInfo::InitDumpTask(void *addr, const OpDescPtr &op_desc) { - if (davinci_model_->OpNeedDump(op_desc->GetName()) || davinci_model_->GetOpDugReg()) { + if (davinci_model_->OpNeedDump(op_desc->GetName())) { GELOGD("Op %s need dump in kernel ex task info", op_desc->GetName().c_str()); dump_flag_ = RT_KERNEL_DUMPFLAG; dump_args_ = addr; From db7eb5f3568c7a22fa9b8d06c466d3e20478c5c5 Mon Sep 17 00:00:00 2001 From: chenyemeng Date: Tue, 11 May 2021 21:20:17 +0800 Subject: [PATCH 19/48] rw lock for GetOrCreateNodeState --- ge/graph/build/stream_allocator.cc | 14 +- ge/graph/build/stream_allocator.h | 5 +- ge/graph/manager/graph_caching_allocator.cc | 1 + ge/graph/manager/host_mem_allocator.cc | 5 +- ge/hybrid/executor/hybrid_execution_context.h | 1 + .../hybrid_model_pipeline_executor.cc | 34 +++- .../executor/hybrid_model_pipeline_executor.h | 1 + ge/hybrid/executor/node_done_manager.cc | 5 + ge/hybrid/executor/node_done_manager.h | 2 + ge/hybrid/executor/subgraph_context.cc | 62 +++++- ge/hybrid/executor/subgraph_context.h | 7 +- ge/hybrid/executor/subgraph_executor.cc | 16 +- ge/hybrid/model/hybrid_model_builder.cc | 18 +- .../node_executor/hccl/hccl_node_executor.cc | 15 +- tests/depends/mmpa/src/mmpa_stub.cc | 84 ++++++++ tests/depends/runtime/src/runtime_stub.cc | 4 +- tests/ut/ge/CMakeLists.txt | 3 + .../graph/build/stream_allocator_unittest.cc | 190 ++++++++++++++++++ .../manager/host_mem_allocator_unittest.cc | 40 ++++ ...hybrid_model_pipeline_executor_unittest.cc | 68 +++++++ .../executor/subgraph_executor_unittest.cc | 17 +- .../model/hybrid_model_builder_unittest.cc | 31 ++- 22 files changed, 589 insertions(+), 34 deletions(-) create mode 100644 tests/ut/ge/graph/build/stream_allocator_unittest.cc create mode 100644 tests/ut/ge/graph/manager/host_mem_allocator_unittest.cc create mode 100644 tests/ut/ge/hybrid/executor/hybrid_model_pipeline_executor_unittest.cc diff --git a/ge/graph/build/stream_allocator.cc b/ge/graph/build/stream_allocator.cc index 0e1a1aba..a5a1112e 100644 --- a/ge/graph/build/stream_allocator.cc +++ b/ge/graph/build/stream_allocator.cc @@ -905,6 +905,7 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { added_stream_num_vec[stream_id]++; new_stream_id_vec[stream_id] = last_stream_id; split_streams[stream_id].emplace(last_stream_id); + split_ori_stream_map_[last_stream_id] = stream_id; node_split_stream_map_[cur_node] = last_stream_id; // Add the send/recv event to the first and last nodes of the split stream. @@ -1104,7 +1105,7 @@ Status StreamAllocator::UpdateActiveStreamsForActiveNode(const vector(new_split_stream)); active_streams.assign(new_active_streams.begin(), new_active_streams.end()); if (!AttrUtils::SetListInt(active_op, ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) { @@ -1148,13 +1150,21 @@ Status StreamAllocator::UpdateActiveStreamsForSubgraphs() const { } bool StreamAllocator::IsActivated(int64_t stream_id) const { + const auto &iter = split_ori_stream_map_.find(stream_id); + if (iter == split_ori_stream_map_.end()) { + REPORT_INNER_ERROR("E19999", "Find original stream_id failed, split_stream_id=%ld", stream_id); + GELOGE(INTERNAL_ERROR, "[CheckActivated][Check] Find original stream_id failed, split_stream_id=%ld", stream_id); + return false; + } + int64_t ori_stream_id = iter->second; for (const auto &node : whole_graph_->GetNodes(whole_graph_->GetGraphUnknownFlag())) { auto op_desc = node->GetOpDesc(); vector active_streams; if (op_desc == nullptr || !AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) { continue; } - if (std::find(active_streams.begin(), active_streams.end(), stream_id) != active_streams.end()) { + if (std::find(active_streams.begin(), active_streams.end(), stream_id) != active_streams.end() || + std::find(active_streams.begin(), active_streams.end(), ori_stream_id) != active_streams.end()) { return true; } } diff --git a/ge/graph/build/stream_allocator.h b/ge/graph/build/stream_allocator.h index 44dcd673..34b2ec3a 100644 --- a/ge/graph/build/stream_allocator.h +++ b/ge/graph/build/stream_allocator.h @@ -66,7 +66,7 @@ class StreamAllocator { Status UpdateActiveStreamsForSwitchNode(NodePtr &switch_node); Status InsertActiveNodesAfterSwitch(NodePtr &switch_nodes, std::vector &switch_active_nodes); Status UpdateActiveStreamsForActiveNode(const std::vector> &split_streams, NodePtr &node); - Status UpdateActiveStreamsForSubgraphs() const; + Status UpdateActiveStreamsForSubgraphs(); bool IsActivated(int64_t stream_id) const; Status SetActiveStreamsForLoop(); Status CheckStreamActived() const; @@ -114,6 +114,7 @@ class StreamAllocator { std::map> specific_activated_streams_nodes_map_; std::map node_split_stream_map_; + std::map split_ori_stream_map_; std::map subgraph_first_active_node_map_; // send events corresponding to the node @@ -123,4 +124,4 @@ class StreamAllocator { std::map> node_to_recv_events_; }; } // namespace ge -#endif // GE_GRAPH_BUILD_STREAM_ALLOCATOR_H_ \ No newline at end of file +#endif // GE_GRAPH_BUILD_STREAM_ALLOCATOR_H_ diff --git a/ge/graph/manager/graph_caching_allocator.cc b/ge/graph/manager/graph_caching_allocator.cc index 75aa5c01..8326038d 100644 --- a/ge/graph/manager/graph_caching_allocator.cc +++ b/ge/graph/manager/graph_caching_allocator.cc @@ -137,6 +137,7 @@ uint8_t *CachingAllocator::Malloc(size_t size, uint8_t *org_ptr, uint32_t device uint8_t *ptr = nullptr; Block *block = FindFreeBlock(size, org_ptr, device_id); if (block == nullptr) { + std::lock_guard lock(mutex_); if (ge::SUCCESS == TryExtendCache(size, device_id)) { block = FindFreeBlock(size, org_ptr, device_id); if (block != nullptr) { diff --git a/ge/graph/manager/host_mem_allocator.cc b/ge/graph/manager/host_mem_allocator.cc index 98f9a313..baf81a60 100644 --- a/ge/graph/manager/host_mem_allocator.cc +++ b/ge/graph/manager/host_mem_allocator.cc @@ -21,7 +21,10 @@ namespace ge { const void *HostMemAllocator::Malloc(const std::shared_ptr &aligned_ptr, size_t size) { if (aligned_ptr == nullptr) { - GELOGW("Insert a null aligned_ptr"); + GELOGW("Insert a null aligned_ptr, size=%zu", size); + if (size == 0) { + allocated_blocks_[nullptr] = { size, nullptr }; + } return nullptr; } GELOGD("allocate existed host memory succ, size=%zu", size); diff --git a/ge/hybrid/executor/hybrid_execution_context.h b/ge/hybrid/executor/hybrid_execution_context.h index f2628409..489d6d99 100644 --- a/ge/hybrid/executor/hybrid_execution_context.h +++ b/ge/hybrid/executor/hybrid_execution_context.h @@ -62,6 +62,7 @@ struct GraphExecutionContext { const HybridModel *model = nullptr; const GEThreadLocalContext *ge_context = nullptr; rtStream_t stream = nullptr; + rtStream_t hccl_stream = nullptr; rtContext_t rt_context = nullptr; rtContext_t rt_gen_context = nullptr; std::unique_ptr callback_manager = nullptr; diff --git a/ge/hybrid/executor/hybrid_model_pipeline_executor.cc b/ge/hybrid/executor/hybrid_model_pipeline_executor.cc index ba24d78d..c0bd5c7d 100644 --- a/ge/hybrid/executor/hybrid_model_pipeline_executor.cc +++ b/ge/hybrid/executor/hybrid_model_pipeline_executor.cc @@ -18,14 +18,26 @@ const char *const kEnvProfilingLevel = "HYBRID_PROFILING_LEVEL"; StageExecutor::StageExecutor(int id, HybridModel *model, PipeExecutionConfig *config) : id_(id), model_(model), pipe_config_(config) {} -StageExecutor::~StageExecutor() { GELOGD("~StageExecutor(), id = %d", id_); } +StageExecutor::~StageExecutor() { + GELOGD("~StageExecutor(), id = %d", id_); + if (stream_ != nullptr) { + GE_CHK_RT(rtStreamDestroy(stream_)); + stream_ = nullptr; + } + if (hccl_stream_ != nullptr) { + GE_CHK_RT(rtStreamDestroy(hccl_stream_)); + hccl_stream_ = nullptr; + } +} Status StageExecutor::Init() { GELOGD("[Executor: %d] Start to init StateExecutor", id_); context_.rt_context = pipe_config_->rt_context; GE_CHK_STATUS_RET_NOLOG(InitExecutionContext()); GE_CHK_RT_RET(rtStreamCreate(&stream_, RT_STREAM_PRIORITY_DEFAULT)); + GE_CHK_RT_RET(rtStreamCreate(&hccl_stream_, RT_STREAM_PRIORITY_DEFAULT)); context_.stream = stream_; + context_.hccl_stream = hccl_stream_; root_graph_executor_.reset(new (std::nothrow) SubgraphExecutor(model_->GetRootGraphItem(), &context_)); GE_CHECK_NOTNULL(root_graph_executor_); @@ -78,11 +90,11 @@ Status StageExecutor::Start(const std::vector &inputs, const std::v if (task_info.event != nullptr) { GELOGD("[%d] Add StreamWaitEvent", id_); GE_CHK_RT_RET(rtStreamWaitEvent(stream_, task_info.event)); - RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] End", task_info.iteration - 1, + RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] EventWait End", task_info.iteration, task_info.stage); } - RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %lld] [Stage = %d] Start", task_info.iteration, + RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] Start", task_info.iteration, task_info.stage); if (task_info.stage == 0) { @@ -102,6 +114,10 @@ Status StageExecutor::Start(const std::vector &inputs, const std::v StageTask next_task; next_task.stage = task_info.stage; next_task.iteration = task_info.iteration + 1; + if ((task_info.iteration + 1) % iteration_count > 0) { + GE_CHK_RT_RET(rtEventCreate(&next_task.event)); + GE_CHK_RT_RET(rtEventRecord(next_task.event, context_.hccl_stream)); + } auto sync_result = Synchronize(); if (sync_result != SUCCESS) { @@ -110,15 +126,22 @@ Status StageExecutor::Start(const std::vector &inputs, const std::v id_, sync_result, task_info.iteration); REPORT_CALL_ERROR("E19999", "[Executor: %d] Failed to sync result:%d. iteration = %ld", id_, sync_result, task_info.iteration); - context_.profiler->Dump(std::cout); + if (context_.profiler != nullptr) { + context_.profiler->Dump(std::cout); + } context_.callback_manager->Destroy(); RuntimeInferenceContext::DestroyContext(std::to_string(context_.context_id)); return sync_result; } + if (task_info.event != nullptr) { + GE_CHK_RT_RET(rtEventDestroy(task_info.event)); + RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] EventDestroy End", task_info.iteration, + task_info.stage); + } RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] End", task_info.iteration, task_info.stage); - // if not end stage + // if end stage if (task_info.stage >= pipe_config_->num_stages - 1) { RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] Schedule End", task_info.iteration); GELOGD("[Executor: %d] End of iteration [%ld]", id_, task_info.iteration); @@ -163,6 +186,7 @@ Status StageExecutor::InitExecutionContext() { context_.callback_manager = std::unique_ptr(new (std::nothrow) CallbackManager()); GE_CHECK_NOTNULL(context_.callback_manager); context_.dump_properties = DumpManager::GetInstance().GetDumpProperties(context_.session_id); + context_.is_eos_ = false; if (IsLogEnable(GE_MODULE_NAME, DLOG_DEBUG)) { context_.trace_enabled = true; } diff --git a/ge/hybrid/executor/hybrid_model_pipeline_executor.h b/ge/hybrid/executor/hybrid_model_pipeline_executor.h index cb08d872..c59e1462 100644 --- a/ge/hybrid/executor/hybrid_model_pipeline_executor.h +++ b/ge/hybrid/executor/hybrid_model_pipeline_executor.h @@ -63,6 +63,7 @@ class StageExecutor { StageExecutor *next_executor_ = nullptr; rtStream_t stream_ = nullptr; + rtStream_t hccl_stream_ = nullptr; }; class HybridModelPipelineExecutor { diff --git a/ge/hybrid/executor/node_done_manager.cc b/ge/hybrid/executor/node_done_manager.cc index d31765c2..0ea04661 100644 --- a/ge/hybrid/executor/node_done_manager.cc +++ b/ge/hybrid/executor/node_done_manager.cc @@ -121,5 +121,10 @@ void NodeDoneManager::Reset(const NodePtr &node) { GELOGD("[%s] Node reset.", node->GetName().c_str()); } } + +void NodeDoneManager::Reset() { + subjects_.clear(); + destroyed_ = false; +} } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/executor/node_done_manager.h b/ge/hybrid/executor/node_done_manager.h index 292d1369..bedbff3d 100644 --- a/ge/hybrid/executor/node_done_manager.h +++ b/ge/hybrid/executor/node_done_manager.h @@ -35,6 +35,8 @@ class NodeDoneManager { void Destroy(); + void Reset(); + private: class Cond { public: diff --git a/ge/hybrid/executor/subgraph_context.cc b/ge/hybrid/executor/subgraph_context.cc index 9a9a97c2..5de0828f 100644 --- a/ge/hybrid/executor/subgraph_context.cc +++ b/ge/hybrid/executor/subgraph_context.cc @@ -15,8 +15,6 @@ */ #include "subgraph_context.h" - -#include "common/debug/log.h" #include "hybrid/executor/hybrid_model_executor.h" namespace ge { @@ -25,6 +23,13 @@ SubgraphContext::SubgraphContext(const GraphItem *graph_item, const GraphExecuti : graph_item_(graph_item), execution_context_(execution_context) { } +SubgraphContext::~SubgraphContext() { + if (mmRWLockDestroy(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "Destroy rw_lock failed"); + GELOGE(INTERNAL_ERROR, "[RWLock][Destroy] Destroy rw_lock failed"); + } +} + Status SubgraphContext::Init() { GE_CHECK_NOTNULL(graph_item_); GELOGD("[%s] Start to init subgraph context. total inputs = %d, total outputs = %d", @@ -33,7 +38,11 @@ Status SubgraphContext::Init() { graph_item_->TotalOutputs()); all_inputs_.resize(static_cast(graph_item_->TotalInputs())); all_outputs_.resize(static_cast(graph_item_->TotalOutputs())); - + if (mmRWLockInit(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "Init rw_lock failed"); + GELOGE(INTERNAL_ERROR, "[RWLock][Init] Init rw_lock failed"); + return INTERNAL_ERROR; + } return SUCCESS; } @@ -42,13 +51,48 @@ void SubgraphContext::ResetContext(const NodePtr &node) { } NodeStatePtr SubgraphContext::GetOrCreateNodeState(const NodeItem *node_item) { - std::lock_guard lk(mu_); + GELOGD("[%s] lock for read", node_item->NodeName().c_str()); + if (mmRWLockRDLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Lock for read failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Lock][Node:%s] Lock for read failed", node_item->NodeName().c_str()); + return nullptr; + } + const auto &iter = node_states_.find(node_item); + if (iter != node_states_.end()) { + auto state = iter->second; + GELOGD("[%s] unlock for read", node_item->NodeName().c_str()); + if (mmRDLockUnLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Unlock for read failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Unlock][Node:%s] Unlock for read failed", node_item->NodeName().c_str()); + return nullptr; + } + return state; + } + GELOGD("[%s] unlock for read", node_item->NodeName().c_str()); + if (mmRDLockUnLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Unlock for read failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Unlock][Node:%s] Unlock for read failed", node_item->NodeName().c_str()); + return nullptr; + } + + GELOGD("[%s] lock for write", node_item->NodeName().c_str()); + if (mmRWLockWRLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Lock for write failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Lock][Node:%s] Lock for write failed", node_item->NodeName().c_str()); + return nullptr; + } auto &node_state = node_states_[node_item]; if (node_state == nullptr) { const auto &guard = node_item->MutexGuard("GetOrCreateNodeState"); - node_state.reset(new(std::nothrow)NodeState(*node_item, this)); + node_state = std::move(std::unique_ptr(new(std::nothrow)NodeState(*node_item, this))); (void)guard; } + GELOGD("[%s] unlock for write", node_item->NodeName().c_str()); + if (mmWRLockUnLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Unlock for write failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Unlock][Node:%s] Unlock for write failed", node_item->NodeName().c_str()); + return nullptr; + } return node_state; } @@ -144,5 +188,13 @@ void SubgraphContext::OnError(Status error) { void SubgraphContext::NodeDone(const NodePtr &node) { node_done_manager_.NodeDone(node); } + +void SubgraphContext::Reset() { + node_done_manager_.Reset(); + if (mmRWLockWRLock(&rw_lock_) == EN_OK) { + node_states_.clear(); + (void)mmWRLockUnLock(&rw_lock_); + } +} } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/executor/subgraph_context.h b/ge/hybrid/executor/subgraph_context.h index ff692ed9..7a99e324 100644 --- a/ge/hybrid/executor/subgraph_context.h +++ b/ge/hybrid/executor/subgraph_context.h @@ -18,7 +18,7 @@ #define GE_HYBRID_EXECUTOR_ITERATION_CONTEXT_H_ #include - +#include "mmpa/mmpa_api.h" #include "hybrid/common/tensor_value.h" #include "hybrid/executor/hybrid_execution_context.h" #include "hybrid/executor/node_state.h" @@ -31,10 +31,11 @@ namespace hybrid { class SubgraphContext { public: explicit SubgraphContext(const GraphItem *graph_item, const GraphExecutionContext *execution_context); - ~SubgraphContext() = default; + ~SubgraphContext(); Status Init(); void ResetContext(const NodePtr &node); + void Reset(); NodeStatePtr GetOrCreateNodeState(const NodeItem *node_item); void OnError(Status error); @@ -52,7 +53,7 @@ class SubgraphContext { friend class TaskContext; const GraphItem *graph_item_; const GraphExecutionContext *execution_context_; - std::mutex mu_; + mmRWLock_t rw_lock_; std::vector all_inputs_; std::vector all_outputs_; NodeDoneManager node_done_manager_; diff --git a/ge/hybrid/executor/subgraph_executor.cc b/ge/hybrid/executor/subgraph_executor.cc index 60895c7e..3536f295 100644 --- a/ge/hybrid/executor/subgraph_executor.cc +++ b/ge/hybrid/executor/subgraph_executor.cc @@ -704,7 +704,21 @@ Status SubgraphExecutor::PartialExecuteAsync(int task_group) { Status SubgraphExecutor::InitForPartialExecution(const vector &inputs, const vector &input_desc) { - return Init(inputs, input_desc); + if (subgraph_context_ == nullptr) { + return Init(inputs, input_desc); + } + subgraph_context_->Reset(); + if (graph_item_->IsDynamic()) { + GE_CHK_STATUS_RET(InitInputsForUnknownShape(inputs, input_desc), + "[%s] Failed to set inputs.", + graph_item_->GetName().c_str()); + } else { + GE_CHK_STATUS_RET(InitInputsForKnownShape(inputs), + "[Invoke][InitInputsForKnownShape][%s] Failed to init subgraph executor for known shape subgraph", + graph_item_->GetName().c_str()); + } + + return SUCCESS; } } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/model/hybrid_model_builder.cc b/ge/hybrid/model/hybrid_model_builder.cc index 86530fe0..91188326 100755 --- a/ge/hybrid/model/hybrid_model_builder.cc +++ b/ge/hybrid/model/hybrid_model_builder.cc @@ -1005,14 +1005,18 @@ Status HybridModelBuilder::InitConstantOps() { // Tensors return by api GetWeights share data with proto, whose addr is not confirmed to be aligned GeTensor aligned_tensor = ge_tensor->Clone(); GELOGD("Init tensor with host constant %s size = %zu", var_name.c_str(), aligned_tensor.MutableData().GetSize()); - if (MemManager::Instance().HostMemInstance(RT_MEMORY_HBM).Malloc(aligned_tensor.GetAlignedPtr(), - aligned_tensor.GetData().size()) == nullptr) { - GELOGE(MEMALLOC_FAILED, "[Malloc][HostMemory] for an existed GeTensor failed, model_name_:%s.", - GetGraphName()); - return MEMALLOC_FAILED; + if (aligned_tensor.GetData().size() > 0) { + if (MemManager::Instance().HostMemInstance(RT_MEMORY_HBM).Malloc(aligned_tensor.GetAlignedPtr(), + aligned_tensor.GetData().size()) == nullptr) { + GELOGE(MEMALLOC_FAILED, "[Malloc][HostMemory] for an existed GeTensor failed, model_name_:%s.", + GetGraphName()); + return MEMALLOC_FAILED; + } + var_tensor.reset(new(std::nothrow)TensorValue(aligned_tensor.MutableData().data(), + aligned_tensor.GetData().size())); + } else { + var_tensor.reset(new(std::nothrow)TensorValue(nullptr, 0)); } - var_tensor.reset(new(std::nothrow)TensorValue(aligned_tensor.MutableData().data(), - aligned_tensor.GetData().size())); } else { GE_CHK_STATUS_RET_NOLOG(VarNodeToTensor(var_node, var_tensor)); GELOGD("Init const op tensor. name = %s, size = %ld", var_name.c_str(), var_tensor->GetSize()); diff --git a/ge/hybrid/node_executor/hccl/hccl_node_executor.cc b/ge/hybrid/node_executor/hccl/hccl_node_executor.cc index 20684194..c46d5080 100644 --- a/ge/hybrid/node_executor/hccl/hccl_node_executor.cc +++ b/ge/hybrid/node_executor/hccl/hccl_node_executor.cc @@ -314,21 +314,26 @@ Status RdmaNodeTask::ExecuteAsync(TaskContext &context, std::function do return SUCCESS; } + rtEvent_t evt = nullptr; + if (context.GetExecutionContext()->hccl_stream != nullptr) { + GE_CHK_RT_RET(rtEventCreateWithFlag(&evt, 0x01)); + GE_CHK_RT_RET(rtStreamWaitEvent(context.GetExecutionContext()->hccl_stream, evt)); + } TaskContext *p_ctx = &context; - auto callback = [p_ctx, done_callback](HcclResult status) { + auto callback = [p_ctx, done_callback, evt](HcclResult status) { if (status != HCCL_SUCCESS) { GELOGE(HCCL_E_INTERNAL, "[Call][HcomExcutorInitialize] failed for node:%s(%s), ret: 0x%X", p_ctx->GetNodeName(), p_ctx->GetNodeItem().NodeType().c_str(), status); p_ctx->SetStatus(FAILED); } done_callback(); + if (evt != nullptr) { + GE_CHK_RT_RET(rtEventRecord(evt, nullptr)); + GE_CHK_RT_RET(rtEventDestroy(evt)); + } GELOGI("rdma callback success."); }; - std::string executor_type = context.GetNodeItem().NodeType(); - if (kRdmaScatterTypes.count(context.GetNodeItem().NodeType()) > 0) { - executor_type = context.GetNodeItem().NodeType() == HCOMREMOTEREFREAD ? HCOMREMOTEREAD : HCOMREMOTEWRITE; - } HcclResult hccl_ret = HcomExecEnqueueRemoteAccess(context.GetNodeItem().NodeType(), addr_infos, callback); if (hccl_ret != HCCL_SUCCESS) { GELOGE(HCCL_E_INTERNAL, "[Call][HcomExecEnqueueRemoteAccess] failed for node:%s(%s), ret: 0x%X", diff --git a/tests/depends/mmpa/src/mmpa_stub.cc b/tests/depends/mmpa/src/mmpa_stub.cc index 62499ca1..a82621ef 100644 --- a/tests/depends/mmpa/src/mmpa_stub.cc +++ b/tests/depends/mmpa/src/mmpa_stub.cc @@ -242,6 +242,90 @@ INT32 mmRealPath(const CHAR *path, CHAR *realPath, INT32 realPathLen) return ret; } +INT32 mmRWLockInit(mmRWLock_t *rwLock) +{ + if (rwLock == NULL) { + return EN_INVALID_PARAM; + } + + INT32 ret = pthread_rwlock_init(rwLock, NULL); + if (ret != MMPA_ZERO) { + return EN_ERROR; + } + + return EN_OK; +} + +INT32 mmRWLockRDLock(mmRWLock_t *rwLock) +{ + if (rwLock == NULL) { + return EN_INVALID_PARAM; + } + + INT32 ret = pthread_rwlock_rdlock(rwLock); + if (ret != MMPA_ZERO) { + return EN_ERROR; + } + + return EN_OK; +} + +INT32 mmRWLockWRLock(mmRWLock_t *rwLock) +{ + if (rwLock == NULL) { + return EN_INVALID_PARAM; + } + + INT32 ret = pthread_rwlock_wrlock(rwLock); + if (ret != MMPA_ZERO) { + return EN_ERROR; + } + + return EN_OK; +} + +INT32 mmRDLockUnLock(mmRWLock_t *rwLock) +{ + if (rwLock == NULL) { + return EN_INVALID_PARAM; + } + + INT32 ret = pthread_rwlock_unlock(rwLock); + if (ret != MMPA_ZERO) { + return EN_ERROR; + } + + return EN_OK; +} + +INT32 mmWRLockUnLock(mmRWLock_t *rwLock) +{ + if (rwLock == NULL) { + return EN_INVALID_PARAM; + } + + INT32 ret = pthread_rwlock_unlock(rwLock); + if (ret != MMPA_ZERO) { + return EN_ERROR; + } + + return EN_OK; +} + +INT32 mmRWLockDestroy(mmRWLock_t *rwLock) +{ + if (rwLock == NULL) { + return EN_INVALID_PARAM; + } + + INT32 ret = pthread_rwlock_destroy(rwLock); + if (ret != MMPA_ZERO) { + return EN_ERROR; + } + + return EN_OK; +} + INT32 mmGetErrorCode() { return 0; diff --git a/tests/depends/runtime/src/runtime_stub.cc b/tests/depends/runtime/src/runtime_stub.cc index 59a98978..2b1af23c 100644 --- a/tests/depends/runtime/src/runtime_stub.cc +++ b/tests/depends/runtime/src/runtime_stub.cc @@ -434,6 +434,8 @@ rtError_t rtGetRtCapability(rtFeatureType_t featureType, int32_t featureInfo, in rtError_t rtGetMaxStreamAndTask(uint32_t streamType, uint32_t *maxStrCount, uint32_t *maxTaskCount) { + *maxStrCount = 1024; + *maxTaskCount = 1024; return RT_ERROR_NONE; } @@ -456,4 +458,4 @@ rtError_t rtDebugUnRegisterForStream(rtStream_t stream) { } #ifdef __cplusplus } -#endif \ No newline at end of file +#endif diff --git a/tests/ut/ge/CMakeLists.txt b/tests/ut/ge/CMakeLists.txt index 3f272023..4db4c0d1 100755 --- a/tests/ut/ge/CMakeLists.txt +++ b/tests/ut/ge/CMakeLists.txt @@ -782,6 +782,7 @@ set(MULTI_PARTS_TEST_FILES "common/ge_auth_file_saver_unittest.cc" "graph/variable_accelerate_ctrl_unittest.cc" "graph/build/logical_stream_allocator_unittest.cc" + "graph/build/stream_allocator_unittest.cc" "graph/build/model_builder_unittest.cc" "graph/build/mem_assigner_unittest.cc" "graph/build/task_generator_unittest.cc" @@ -790,6 +791,7 @@ set(MULTI_PARTS_TEST_FILES "graph/preprocess/graph_preprocess_unittest.cc" "graph/manager/hcom_util_unittest.cc" "graph/manager/graph_caching_allocator_unittest.cc" + "graph/manager/host_mem_allocator_unittest.cc" "graph/manager/session_scope_mem_allocator_unittest.cc" "graph/manager/run_graph_unittest.cc" "graph/partition/dynamic_shape_partition_unittest.cc" @@ -829,6 +831,7 @@ set(HYBRID_TEST_FILES "hybrid/model/hybrid_model_builder_unittest.cc" "hybrid/node_executor/rts/rts_node_task_unittest.cc" "hybrid/executor/hybrid_model_async_executor_unittest.cc" + "hybrid/executor/hybrid_model_pipeline_executor_unittest.cc" "hybrid/node_executor/aicore/aicore_task_compiler_unittest.cc" ) diff --git a/tests/ut/ge/graph/build/stream_allocator_unittest.cc b/tests/ut/ge/graph/build/stream_allocator_unittest.cc new file mode 100644 index 00000000..019e75d1 --- /dev/null +++ b/tests/ut/ge/graph/build/stream_allocator_unittest.cc @@ -0,0 +1,190 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#define protected public +#define private public +#include "graph/build/stream_allocator.h" +#undef protected +#undef private + +#include "graph/debug/ge_attr_define.h" +#include "graph/utils/graph_utils.h" + +namespace ge { +class UtestStreamAllocator : public testing::Test { + protected: + void SetUp() {} + void TearDown() {} + public: + + /// + /// A + /// / \ + /// B C + /// | | + /// D 400 + /// | | + /// | E + /// \ / + /// F + /// + void make_graph_active(const ComputeGraphPtr &graph) { + const auto &a_desc = std::make_shared("A", DATA); + a_desc->AddInputDesc(GeTensorDesc()); + a_desc->AddOutputDesc(GeTensorDesc()); + a_desc->SetStreamId(0); + const auto &a_node = graph->AddNode(a_desc); + + const auto &b_desc = std::make_shared("B", "testa"); + b_desc->AddInputDesc(GeTensorDesc()); + b_desc->AddOutputDesc(GeTensorDesc()); + b_desc->SetStreamId(1); + AttrUtils::SetListStr(b_desc, ATTR_NAME_ACTIVE_LABEL_LIST, {"1"}); + const auto &b_node = graph->AddNode(b_desc); + + const auto &c_desc = std::make_shared("C", "testa"); + c_desc->AddInputDesc(GeTensorDesc()); + c_desc->AddOutputDesc(GeTensorDesc()); + c_desc->SetStreamId(2); + AttrUtils::SetStr(c_desc, ATTR_NAME_STREAM_LABEL, "1"); + const auto &c_node = graph->AddNode(c_desc); + + const auto &d_desc = std::make_shared("D", "testa"); + d_desc->AddInputDesc(GeTensorDesc()); + d_desc->AddOutputDesc(GeTensorDesc()); + d_desc->SetStreamId(1); + const auto &d_node = graph->AddNode(d_desc); + + const auto &e_desc = std::make_shared("E", "testa"); + e_desc->AddInputDesc(GeTensorDesc()); + e_desc->AddOutputDesc(GeTensorDesc()); + e_desc->SetStreamId(2); + const auto &e_node = graph->AddNode(e_desc); + + const auto &f_desc = std::make_shared("F", "testa"); + f_desc->AddInputDesc(GeTensorDesc()); + f_desc->AddInputDesc(GeTensorDesc()); + f_desc->AddOutputDesc(GeTensorDesc()); + f_desc->SetStreamId(2); + const auto &f_node = graph->AddNode(f_desc); + + std::vector node_list(400); + for (int i = 0; i < 400; i++) { + const auto &op_desc = std::make_shared("X", DATA); + op_desc->AddInputDesc(GeTensorDesc()); + op_desc->AddOutputDesc(GeTensorDesc()); + op_desc->SetStreamId(2); + node_list[i] = graph->AddNode(op_desc); + } + + GraphUtils::AddEdge(a_node->GetOutDataAnchor(0), b_node->GetInDataAnchor(0)); + GraphUtils::AddEdge(a_node->GetOutDataAnchor(0), c_node->GetInDataAnchor(0)); + GraphUtils::AddEdge(b_node->GetOutDataAnchor(0), d_node->GetInDataAnchor(0)); + GraphUtils::AddEdge(d_node->GetOutDataAnchor(0), f_node->GetInDataAnchor(0)); + GraphUtils::AddEdge(c_node->GetOutDataAnchor(0), node_list[0]->GetInDataAnchor(0)); + for (uint32_t i = 0; i < 399; i++) { + GraphUtils::AddEdge(node_list[i]->GetOutDataAnchor(0), node_list[i + 1]->GetInDataAnchor(0)); + } + GraphUtils::AddEdge(node_list[399]->GetOutDataAnchor(0), e_node->GetInDataAnchor(0)); + GraphUtils::AddEdge(e_node->GetOutDataAnchor(0), f_node->GetInDataAnchor(1)); + } +}; + +TEST_F(UtestStreamAllocator, test_split_streams_active) { + const auto &graph = std::make_shared("test_split_streams_active_graph"); + EXPECT_NE(graph, nullptr); + make_graph_active(graph); + + StreamAllocator allocator(graph, Graph2SubGraphInfoList()); + allocator.stream_num_ = 3; + EXPECT_EQ(allocator.SetActiveStreamsByLabel(), SUCCESS); + std::vector> split_stream(3); + EXPECT_EQ(allocator.SplitStreams(split_stream), SUCCESS); + EXPECT_EQ(allocator.UpdateActiveStreams(split_stream), SUCCESS); + EXPECT_EQ(allocator.SetActiveStreamsForLoop(), SUCCESS); + EXPECT_EQ(allocator.specific_activated_streams_.count(3), 1); + + const auto &node_b = graph->FindNode("B"); + EXPECT_NE(node_b, nullptr); + std::vector active_stream_list; + EXPECT_TRUE(AttrUtils::GetListInt(node_b->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list)); + EXPECT_EQ(active_stream_list.size(), 2); + const auto &node_e = graph->FindNode("E"); + EXPECT_NE(node_e, nullptr); + EXPECT_EQ(active_stream_list[0], node_e->GetOpDesc()->GetStreamId()); + EXPECT_EQ(active_stream_list[1], 3); +} + +TEST_F(UtestStreamAllocator, test_update_active_streams_for_subgraph) { + const auto &root_graph = std::make_shared("test_update_active_streams_for_subgraph_root_graph"); + EXPECT_NE(root_graph, nullptr); + root_graph->SetGraphUnknownFlag(false); + const auto &sub_graph1 = std::make_shared("test_update_active_streams_for_subgraph_sub_graph1"); + EXPECT_NE(sub_graph1, nullptr); + root_graph->AddSubGraph(sub_graph1); + const auto &sub_graph2 = std::make_shared("test_update_active_streams_for_subgraph_sub_graph2"); + EXPECT_NE(sub_graph2, nullptr); + root_graph->AddSubGraph(sub_graph2); + + const auto &case_desc = std::make_shared("case", CASE); + EXPECT_NE(case_desc, nullptr); + EXPECT_EQ(case_desc->AddInputDesc(GeTensorDesc()), GRAPH_SUCCESS); + EXPECT_EQ(case_desc->AddOutputDesc(GeTensorDesc()), GRAPH_SUCCESS); + case_desc->AddSubgraphName("branch1"); + case_desc->SetSubgraphInstanceName(0, "test_update_active_streams_for_subgraph_sub_graph1"); + case_desc->AddSubgraphName("branch2"); + case_desc->SetSubgraphInstanceName(1, "test_update_active_streams_for_subgraph_sub_graph2"); + const auto &case_node = root_graph->AddNode(case_desc); + EXPECT_NE(case_node, nullptr); + sub_graph1->SetParentNode(case_node); + sub_graph2->SetParentNode(case_node); + + const auto &active_desc1 = std::make_shared("active1", STREAMACTIVE); + EXPECT_NE(active_desc1, nullptr); + EXPECT_TRUE(AttrUtils::SetListInt(active_desc1, ATTR_NAME_ACTIVE_STREAM_LIST, {0})); + const auto &active_node1 = sub_graph1->AddNode(active_desc1); + EXPECT_NE(active_node1, nullptr); + + const auto &active_desc2 = std::make_shared("active2", STREAMACTIVE); + EXPECT_NE(active_desc2, nullptr); + EXPECT_TRUE(AttrUtils::SetListInt(active_desc2, ATTR_NAME_ACTIVE_STREAM_LIST, {1})); + const auto &active_node2 = sub_graph2->AddNode(active_desc2); + EXPECT_NE(active_node2, nullptr); + + StreamAllocator allocator(root_graph, Graph2SubGraphInfoList()); + allocator.node_split_stream_map_[active_node1] = 2; + allocator.node_split_stream_map_[active_node2] = 3; + allocator.split_ori_stream_map_[2] = 0; + allocator.subgraph_first_active_node_map_[sub_graph1] = active_node1; + allocator.subgraph_first_active_node_map_[sub_graph2] = active_node2; + EXPECT_EQ(allocator.UpdateActiveStreamsForSubgraphs(), SUCCESS); + std::vector active_stream_list1; + EXPECT_TRUE(AttrUtils::GetListInt(active_node1->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list1)); + EXPECT_EQ(active_stream_list1.size(), 1); + EXPECT_EQ(active_stream_list1[0], 0); + std::vector active_stream_list2; + EXPECT_TRUE(AttrUtils::GetListInt(active_node2->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list2)); + EXPECT_EQ(active_stream_list2.size(), 2); + EXPECT_EQ(active_stream_list2[0], 1); + EXPECT_EQ(active_stream_list2[1], 3); + EXPECT_EQ(allocator.specific_activated_streams_.size(), 1); + EXPECT_EQ(allocator.specific_activated_streams_.count(3), 1); +} +} diff --git a/tests/ut/ge/graph/manager/host_mem_allocator_unittest.cc b/tests/ut/ge/graph/manager/host_mem_allocator_unittest.cc new file mode 100644 index 00000000..3d8e4890 --- /dev/null +++ b/tests/ut/ge/graph/manager/host_mem_allocator_unittest.cc @@ -0,0 +1,40 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#define protected public +#define private public +#include "graph/manager/host_mem_allocator.h" +#undef protected +#undef private + +namespace ge { +class UtestHostMemManagerTest : public testing::Test { + protected: + void SetUp() {} + void TearDown() {} +}; + +TEST_F(UtestHostMemManagerTest, malloc_zero_size) { + HostMemAllocator allocator(RT_MEMORY_HBM); + EXPECT_EQ(allocator.allocated_blocks_.size(), 0); + EXPECT_EQ(allocator.Malloc(nullptr, 0), nullptr); + EXPECT_EQ(allocator.allocated_blocks_.size(), 1); + EXPECT_EQ(allocator.Malloc(nullptr, 1), nullptr); + EXPECT_EQ(allocator.allocated_blocks_.size(), 1); +} +} // namespace ge diff --git a/tests/ut/ge/hybrid/executor/hybrid_model_pipeline_executor_unittest.cc b/tests/ut/ge/hybrid/executor/hybrid_model_pipeline_executor_unittest.cc new file mode 100644 index 00000000..7dc5a191 --- /dev/null +++ b/tests/ut/ge/hybrid/executor/hybrid_model_pipeline_executor_unittest.cc @@ -0,0 +1,68 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#define private public +#define protected public +#include "hybrid/executor/hybrid_model_pipeline_executor.h" +#include "graph/ge_context.h" + +namespace ge { +using namespace hybrid; + +class UtestStageExecutor : public testing::Test { + protected: + void SetUp() {} + void TearDown() { } +}; + +TEST_F(UtestStageExecutor, run_success) { + ComputeGraphPtr graph = std::make_shared("test"); + GeRootModelPtr ge_root_model = std::make_shared(graph); + HybridModel hybrid_model(ge_root_model); + hybrid_model.root_graph_item_ = std::unique_ptr(new(std::nothrow)GraphItem()); + + PipeExecutionConfig config; + config.device_id = 0; + config.num_executors = 2; + config.num_stages = 1; + config.iteration_end = 2; + rtCtxGetCurrent(&config.rt_context); + StageExecutor executor(0, &hybrid_model, &config); + StageExecutor next_executor(1, &hybrid_model, &config); + executor.SetNext(&next_executor); + EXPECT_EQ(executor.Init(), SUCCESS); + + auto allocator = NpuMemoryAllocator::GetAllocator(config.device_id); + EXPECT_NE(allocator, nullptr); + StageExecutor::StageTask task_info_1; + task_info_1.stage = 0; + task_info_1.iteration = 0; + EXPECT_EQ(rtEventCreate(&task_info_1.event), RT_ERROR_NONE); + EXPECT_EQ(executor.ExecuteAsync(task_info_1), SUCCESS); + EXPECT_EQ(executor.Start({}, {}, 2), SUCCESS); + + StageExecutor::StageTask task_info_2; + task_info_2.stage = 0; + task_info_2.iteration = 1; + EXPECT_EQ(rtEventCreate(&task_info_2.event), RT_ERROR_NONE); + EXPECT_EQ(executor.ExecuteAsync(task_info_2), SUCCESS); + EXPECT_EQ(executor.Start({}, {}, 2), SUCCESS); + executor.Reset(); +} +} // namespace ge diff --git a/tests/ut/ge/hybrid/executor/subgraph_executor_unittest.cc b/tests/ut/ge/hybrid/executor/subgraph_executor_unittest.cc index fbda3776..445382bc 100644 --- a/tests/ut/ge/hybrid/executor/subgraph_executor_unittest.cc +++ b/tests/ut/ge/hybrid/executor/subgraph_executor_unittest.cc @@ -264,4 +264,19 @@ TEST_F(UtestSubgraphExecutor, cond_graph_schedule_tasks) { ASSERT_EQ(state_it_f->second->GetSwitchIndex(), 0); ASSERT_EQ(graph_context.callback_manager->Destroy(), SUCCESS); } -} // namespace ge \ No newline at end of file + +TEST_F(UtestSubgraphExecutor, partial_execution_init) { + ComputeGraphPtr graph = std::make_shared("test"); + ASSERT_NE(graph, nullptr); + GeRootModelPtr ge_root_model = std::make_shared(graph); + ASSERT_NE(ge_root_model, nullptr); + HybridModel hybrid_model(ge_root_model); + hybrid_model.root_graph_item_ = std::unique_ptr(new(std::nothrow)GraphItem()); + hybrid_model.root_graph_item_->is_dynamic_ = false; + GraphExecutionContext graph_context; + SubgraphExecutor executor(hybrid_model.GetRootGraphItem(), &graph_context); + + ASSERT_EQ(executor.Init({}, {}), SUCCESS); + ASSERT_EQ(executor.InitForPartialExecution({}, {}), SUCCESS); +} +} // namespace ge diff --git a/tests/ut/ge/hybrid/model/hybrid_model_builder_unittest.cc b/tests/ut/ge/hybrid/model/hybrid_model_builder_unittest.cc index 9630b193..9c9dab17 100644 --- a/tests/ut/ge/hybrid/model/hybrid_model_builder_unittest.cc +++ b/tests/ut/ge/hybrid/model/hybrid_model_builder_unittest.cc @@ -27,6 +27,7 @@ #include "graph/utils/tensor_utils.h" #include "graph/utils/graph_utils.h" #include "graph/debug/ge_attr_define.h" +#include "graph/ge_local_context.h" using namespace std; using namespace testing; @@ -70,6 +71,15 @@ static NodePtr CreateNode(ComputeGraph &graph, const string &name, const string return graph.AddNode(op_desc); } +static NodePtr CreateConstantNode(const ComputeGraphPtr &graph, const string &name, size_t size) { + OpDescPtr op_desc = std::make_shared(name, CONSTANTOP); + op_desc->AddOutputDesc(GeTensorDesc()); + GeTensorPtr value = std::make_shared(GeTensorDesc(), size); + (void)AttrUtils::SetTensor(op_desc, ATTR_NAME_WEIGHTS, value); + + return graph->AddNode(op_desc); +} + TEST_F(UtestHybridModelBuilder, normal_hybrid_model_build) { /******************************************************************************* * Exit Identify @@ -230,4 +240,23 @@ TEST_F(UtestHybridModelBuilder, stream_switch_n_group) { AttrUtils::SetInt(switch_n->GetOpDesc(), ATTR_NAME_BATCH_NUM, batch_num); ASSERT_EQ(hybrid_model_builder.CreateStreamSwitchNGroup(switch_n, &node_item), SUCCESS); } -} // namespace ge \ No newline at end of file + +TEST_F(UtestHybridModelBuilder, init_constant_op_host_) { + ComputeGraphPtr graph = std::make_shared("test"); + GeRootModelPtr ge_root_model = make_shared(graph); + HybridModel hybrid_model(ge_root_model); + HybridModelBuilder hybrid_model_builder(hybrid_model); + + auto const_1 = CreateConstantNode(graph, "const_1", 0); + hybrid_model_builder.constant_op_nodes_.emplace(const_1->GetName(), const_1); + auto const_2 = CreateConstantNode(graph, "const_2", 10); + hybrid_model_builder.constant_op_nodes_.emplace(const_2->GetName(), const_2); + + std::map options; + options["ge.exec.placement"] = "HOST"; + GetThreadLocalContext().SetGraphOption(options); + + EXPECT_EQ(hybrid_model_builder.InitConstantOps(), SUCCESS); + EXPECT_EQ(hybrid_model_builder.hybrid_model_.variable_tensors_.size(), 2); +} +} // namespace ge From 76b498006afc273561de4aafd2c34ce724e9a26f Mon Sep 17 00:00:00 2001 From: zhou_lili Date: Wed, 26 May 2021 12:35:53 +0800 Subject: [PATCH 20/48] fix dts --- .../task_info/kernel_ex_task_info.cc | 16 +++++++++--- .../task_info/kernel_ex_task_info.h | 3 ++- .../task_info/kernel_task_info.cc | 19 +++++++++----- .../task_info/kernel_task_info.h | 3 ++- .../load/model_manager/task_info/task_info.cc | 2 +- ge/graph/preprocess/multi_batch_options.cc | 4 +-- tests/ut/ge/graph/ge_executor_unittest.cc | 26 ++++++++++++++++--- 7 files changed, 54 insertions(+), 19 deletions(-) diff --git a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc index 231655d1..356919f6 100644 --- a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc +++ b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc @@ -195,7 +195,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin return RT_ERROR_TO_GE_STATUS(rt_ret);) SetIoAddrs(op_desc); - InitDumpTask(input_output_addr, op_desc); + InitDumpFlag(op_desc); + InitDumpArgs(input_output_addr, op_desc); GELOGI("KernelExTaskInfo knonw node Init Success."); return SUCCESS; } @@ -237,7 +238,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, ret:0x%X, size:%lu", rt_ret, addrs_size); return RT_ERROR_TO_GE_STATUS(rt_ret);) - InitDumpTask(input_output_addr_, op_desc); + InitDumpFlag(op_desc); + InitDumpArgs(input_output_addr_, op_desc); } uint64_t input_output_addr = static_cast(reinterpret_cast(input_output_addr_)); @@ -269,10 +271,16 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin return SUCCESS; } -void KernelExTaskInfo::InitDumpTask(void *addr, const OpDescPtr &op_desc) { +void KernelExTaskInfo::InitDumpFlag(const OpDescPtr &op_desc) { if (davinci_model_->OpNeedDump(op_desc->GetName())) { - GELOGD("Op %s need dump in kernel ex task info", op_desc->GetName().c_str()); + GELOGD("Op %s need init dump flag in kernel ex task info", op_desc->GetName().c_str()); dump_flag_ = RT_KERNEL_DUMPFLAG; + } +} + +void KernelExTaskInfo::InitDumpArgs(void *addr, const OpDescPtr &op_desc) { + if (davinci_model_->OpNeedDump(op_desc->GetName())) { + GELOGD("Op %s need dump in kernel ex task info", op_desc->GetName().c_str()); dump_args_ = addr; } if (davinci_model_->GetOpDugReg()) { diff --git a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.h b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.h index bcc17168..1b77b715 100644 --- a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.h +++ b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.h @@ -61,7 +61,8 @@ class KernelExTaskInfo : public TaskInfo { Status CopyTaskInfo(const domi::KernelExDef &kernel_def, const RuntimeParam &rts_param, const OpDescPtr &op_desc); void SetIoAddrs(const OpDescPtr &op_desc); - void InitDumpTask(void *addr, const OpDescPtr &op_desc); + void InitDumpFlag(const OpDescPtr &op_desc); + void InitDumpArgs(void *addr, const OpDescPtr &op_desc); Status InitTaskExtInfo(const std::string &ext_info, const OpDescPtr &op_desc); uint32_t task_id_; diff --git a/ge/graph/load/model_manager/task_info/kernel_task_info.cc b/ge/graph/load/model_manager/task_info/kernel_task_info.cc index c72bfeef..d69d0a8b 100755 --- a/ge/graph/load/model_manager/task_info/kernel_task_info.cc +++ b/ge/graph/load/model_manager/task_info/kernel_task_info.cc @@ -129,6 +129,7 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci ctx_.opIndex2[i] = context.origin_op_index(i); } ctx_.opCount = context.origin_op_index_size(); + InitDumpFlag(); if (kernel_type_ == ccKernelType::TE) { ctx_.opIndex = context.op_index(); uint16_t *args_offset_tmp = reinterpret_cast(const_cast(context.args_offset().data())); @@ -660,7 +661,7 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne if (davinci_model_->IsKnownNode()) { args_ = l2_buffer_on_ ? davinci_model_->GetCurrentHybridArgsAddr(hybrid_args_offset_) : davinci_model_->GetCurrentArgsAddr(args_offset_); - InitDumpTask(offset); + InitDumpArgs(offset); return SUCCESS; } @@ -726,7 +727,7 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne return FAILED; } skt_dump_args_ = static_cast(args_) + offset; - InitDumpTask(offset); + InitDumpArgs(offset); vector virtual_io_addrs; // use virtual address for zero copy key. virtual_io_addrs.insert(virtual_io_addrs.end(), input_data_addrs.begin(), input_data_addrs.end()); @@ -1022,7 +1023,7 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k if (davinci_model_->IsKnownNode()) { args_ = davinci_model_->GetCurrentHybridArgsAddr(hybrid_args_offset_); - InitDumpTask(sizeof(aicpu::AicpuParamHead)); + InitDumpArgs(sizeof(aicpu::AicpuParamHead)); return SUCCESS; } const RuntimeParam &rts_param = davinci_model_->GetRuntimeParam(); @@ -1063,7 +1064,7 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } - InitDumpTask(sizeof(aicpu::AicpuParamHead)); + InitDumpArgs(sizeof(aicpu::AicpuParamHead)); if (kernel_type_ == ccKernelType::CUST_AI_CPU) { dump_flag_ |= RT_KERNEL_CUSTOM_AICPU; @@ -1074,14 +1075,20 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k return SUCCESS; } -void KernelTaskInfo::InitDumpTask(uint32_t offset) { +void KernelTaskInfo::InitDumpFlag() { if (davinci_model_->OpNeedDump(op_desc_->GetName())) { - GELOGD("Op %s need dump in task info", op_desc_->GetName().c_str()); + GELOGD("Op %s init dump flag", op_desc_->GetName().c_str()); if (IsL1FusionOp(op_desc_)) { dump_flag_ = RT_FUSION_KERNEL_DUMPFLAG; } else { dump_flag_ = RT_KERNEL_DUMPFLAG; } + } +} + +void KernelTaskInfo::InitDumpArgs(uint32_t offset) { + if (davinci_model_->OpNeedDump(op_desc_->GetName())) { + GELOGD("Op %s need dump in task info", op_desc_->GetName().c_str()); dump_args_ = static_cast(args_) + offset; } if (davinci_model_->GetOpDugReg()) { diff --git a/ge/graph/load/model_manager/task_info/kernel_task_info.h b/ge/graph/load/model_manager/task_info/kernel_task_info.h index 79347255..d9dd30bb 100644 --- a/ge/graph/load/model_manager/task_info/kernel_task_info.h +++ b/ge/graph/load/model_manager/task_info/kernel_task_info.h @@ -128,7 +128,8 @@ class KernelTaskInfo : public TaskInfo { Status SuperKernelDistribute(); bool IsL1FusionOp(const OpDescPtr &op_desc); void SetIoAddrs(const OpDescPtr &op_desc); - void InitDumpTask(uint32_t offset); + void InitDumpFlag(); + void InitDumpArgs(uint32_t offset); void SetContinuousArgs(uint32_t args_size, DavinciModel *davinci_model); void SetNoncontinuousArgs(uint32_t args_size, DavinciModel *davinci_model); Status CopyNoncontinuousArgs(uint16_t offset); diff --git a/ge/graph/load/model_manager/task_info/task_info.cc b/ge/graph/load/model_manager/task_info/task_info.cc index fad18d37..9f82efad 100755 --- a/ge/graph/load/model_manager/task_info/task_info.cc +++ b/ge/graph/load/model_manager/task_info/task_info.cc @@ -26,7 +26,7 @@ Status TaskInfo::SetStream(uint32_t stream_id, const std::vector &st stream_ = stream_list[stream_id]; } else { REPORT_INNER_ERROR("E19999", "stream_id:%u >= stream_list.size(): %zu, check invalid", - stream_id, stream_list.size()); + stream_id, stream_list.size()); GELOGE(FAILED, "[Check][Param] index:%u >= stream_list.size():%zu.", stream_id, stream_list.size()); return FAILED; } diff --git a/ge/graph/preprocess/multi_batch_options.cc b/ge/graph/preprocess/multi_batch_options.cc index b82d1034..6cb57e6b 100644 --- a/ge/graph/preprocess/multi_batch_options.cc +++ b/ge/graph/preprocess/multi_batch_options.cc @@ -335,9 +335,9 @@ Status DeleteIdentityInsertByAdapter(ComputeGraphPtr &graph) { GE_IF_BOOL_EXEC(peer_in_anchor == nullptr, continue); auto dst_node = peer_in_anchor->GetOwnerNode(); GE_IF_BOOL_EXEC(dst_node == nullptr, continue); - if (dst_node->GetType() == IDENTITY) { + if (dst_node->GetType() == IDENTITY && dst_node->GetAllOutDataAnchors().empty()) { GELOGI("Need to remove %s.", dst_node->GetName().c_str()); - if (ge::GraphUtils::RemoveNodeWithoutRelink(graph, dst_node) != GRAPH_SUCCESS) { + if (GraphUtils::RemoveNodeWithoutRelink(graph, dst_node) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) from graph:%s failed", dst_node->GetName().c_str(), dst_node->GetType().c_str(), graph->GetName().c_str()); GELOGE(FAILED, "Remove Identity node %s failed.", dst_node->GetName().c_str()); diff --git a/tests/ut/ge/graph/ge_executor_unittest.cc b/tests/ut/ge/graph/ge_executor_unittest.cc index 19b8aeab..bbe29007 100644 --- a/tests/ut/ge/graph/ge_executor_unittest.cc +++ b/tests/ut/ge/graph/ge_executor_unittest.cc @@ -157,7 +157,7 @@ TEST_F(UtestGeExecutor, InitFeatureMapAndP2PMem_failed) { EXPECT_EQ(model.InitFeatureMapAndP2PMem(nullptr, 0), PARAM_INVALID); } -TEST_F(UtestGeExecutor, kernel_InitDumpTask) { +TEST_F(UtestGeExecutor, kernel_InitDumpArgs) { DavinciModel model(0, g_label_call_back); model.om_name_ = "testom"; model.name_ = "test"; @@ -173,10 +173,10 @@ TEST_F(UtestGeExecutor, kernel_InitDumpTask) { KernelTaskInfo kernel_task_info; kernel_task_info.davinci_model_ = &model; kernel_task_info.op_desc_ = op_desc; - kernel_task_info.InitDumpTask(0); + kernel_task_info.InitDumpArgs(0); } -TEST_F(UtestGeExecutor, kernel_ex_InitDumpTask) { +TEST_F(UtestGeExecutor, kernel_ex_InitDumpArgs) { DavinciModel model(0, g_label_call_back); model.om_name_ = "testom"; model.name_ = "test"; @@ -191,7 +191,25 @@ TEST_F(UtestGeExecutor, kernel_ex_InitDumpTask) { KernelExTaskInfo kernel_ex_task_info; kernel_ex_task_info.davinci_model_ = &model; - kernel_ex_task_info.InitDumpTask(nullptr, op_desc); + kernel_ex_task_info.InitDumpArgs(nullptr, op_desc); +} + +TEST_F(UtestGeExecutor, kernel_ex_InitDumpFlag) { + DavinciModel model(0, g_label_call_back); + model.om_name_ = "testom"; + model.name_ = "test"; + OpDescPtr op_desc = CreateOpDesc("test", "test"); + + std::map> model_dump_properties_map; + std::set s; + model_dump_properties_map[DUMP_ALL_MODEL] = s; + DumpProperties dp; + dp.model_dump_properties_map_ = model_dump_properties_map; + model.SetDumpProperties(dp); + + KernelExTaskInfo kernel_ex_task_info; + kernel_ex_task_info.davinci_model_ = &model; + kernel_ex_task_info.InitDumpFlag(op_desc); } TEST_F(UtestGeExecutor, execute_graph_with_stream) { From 70ae0883aad6afa839a6ea27d7214399b55fdabf Mon Sep 17 00:00:00 2001 From: wangzhengjun Date: Wed, 26 May 2021 21:46:37 +0800 Subject: [PATCH 21/48] update submodule --- metadef | 2 +- parser | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metadef b/metadef index 7cb171b9..23718da6 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit 7cb171b9c511fec57ccc0ad746ef2126267fe18b +Subproject commit 23718da69af64f8a57051ee64d5515ae1e103c70 diff --git a/parser b/parser index 8d44bebf..9bb03f21 160000 --- a/parser +++ b/parser @@ -1 +1 @@ -Subproject commit 8d44bebfeeb71b793bc7325acc95345090789e19 +Subproject commit 9bb03f21773f028b07d5a912db6f176268962c7d From 894f91c1deb8d497010f2e00926ee4f54c73b1e0 Mon Sep 17 00:00:00 2001 From: wuweikang Date: Wed, 19 May 2021 14:26:35 +0800 Subject: [PATCH 22/48] fix return error through callback twice --- ge/graph/manager/graph_manager.cc | 6 +- .../graph/manager/graph_manager_unittest.cc | 78 +++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index dc1142d0..f9d24ac4 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -3072,7 +3072,6 @@ Status GraphManager::CheckIncreBuildAndPreRun(GraphManager *graph_manager, const "The graph " + std::to_string(graph_node->GetGraphId()) + " need to re-build, you should remove it" " from GE first, then AddGraph again and rebuild it."); - graph_node->Unlock(); return PARAM_INVALID; } // check need incre build. @@ -3177,12 +3176,11 @@ void GraphManager::PreRunThread(GraphManager *graph_manager) { if (ret != SUCCESS) { graph_node->SetRunFlag(false); if (!ge::Analyzer::GetInstance()->IsEnableNetAnalyzeDebug()) { - ReturnError(graph_manager, args.callback, ret, "CheckIncreBuildAndPreRun Failed, thread exit.."); + GELOGE(ret, "CheckIncreBuildAndPreRun Failed, thread exit.."); graph_node->Unlock(); return; } else { - ReturnError(graph_manager, graph_node, args.callback, ret, - "CheckIncreBuildAndPreRun Failed, keep geop continue!"); + GELOGE(ret, "CheckIncreBuildAndPreRun Failed, keep geop continue!"); graph_node->Unlock(); continue; } diff --git a/tests/ut/ge/graph/manager/graph_manager_unittest.cc b/tests/ut/ge/graph/manager/graph_manager_unittest.cc index 96a0fa64..f68b5080 100644 --- a/tests/ut/ge/graph/manager/graph_manager_unittest.cc +++ b/tests/ut/ge/graph/manager/graph_manager_unittest.cc @@ -16,6 +16,7 @@ #include #include +#include #define protected public #define private public #include "graph/manager/graph_manager.h" @@ -115,6 +116,7 @@ #include "common/formats/utils/formats_trans_utils.h" #include "register/custom_pass_helper.h" #include "graph/ops_stub.h" +#include "ge_attr_value.h" using namespace std; using namespace testing; @@ -458,6 +460,82 @@ TEST_F(UtestGraphManagerTest, ParseInputsDimsForData_success) { graph_manager.ParseInputsDimsForData(input_tensors); } +TEST_F(UtestGraphManagerTest, test_prerunthread_failed_1) { + GraphId graph_id = 1; + GraphManager graph_manager; + graph_manager.thread_run_flag_ = true; + ComputeGraphPtr compute_graph = MakeShared("test_graph"); + GeRootModelPtr ge_root_model = MakeShared(compute_graph); + GraphManager::PreRunArgs args; + error_message::Context error_ctx{1, "1st_stage", "2nd_stage", "log_header"}; + Status st = 0; + args.callback = [&st](Status st_return, std::vector &) { st = st_return; }; + args.graph_id = graph_id; + args.session_id = 1; + args.error_context = error_ctx; + args.context = GetThreadLocalContext(); + // create graph + Graph graph = GraphUtils::CreateGraphFromComputeGraph(compute_graph); + std::shared_ptr graph_ptr = MakeShared(graph); + GraphNodePtr graph_node = MakeShared(graph_id); + graph_node->SetGraph(graph_ptr); + + graph_manager.options_.local_fmk_op_flag = false; + // need build while buildflag is true, var format changed + graph_node->SetBuildFlag(true); + graph_manager.var_acc_ctrl_.graph_ids_need_rebuild_.insert(graph_id); + + graph_manager.graph_map_.insert({graph_id, graph_node}); + graph_manager.graph_count_.insert({graph_id, 1}); + graph_node->SetRunFlag(false); + // function return. + graph_manager.prerun_args_q_.Push(args); + auto t1 = std::thread(GraphManager::PreRunThread, &graph_manager); + if (t1.joinable()) { + t1.join(); + } + EXPECT_EQ(st, ge::PARAM_INVALID); +} + +TEST_F(UtestGraphManagerTest, test_prerunthread_failed_2) { + GraphId graph_id = 1; + GraphManager graph_manager; + graph_manager.thread_run_flag_ = true; + ComputeGraphPtr compute_graph = MakeShared("test_graph"); + GeRootModelPtr ge_root_model = MakeShared(compute_graph); + GraphManager::PreRunArgs args; + error_message::Context error_ctx{1, "1st_stage", "2nd_stage", "log_header"}; + Status st; + args.callback = [&st, &graph_manager](Status st_return, std::vector &) { st = st_return; + graph_manager.thread_run_flag_ = false;}; + args.graph_id = graph_id; + args.session_id = 1; + args.error_context = error_ctx; + args.context = GetThreadLocalContext(); + // create graph + Graph graph = GraphUtils::CreateGraphFromComputeGraph(compute_graph); + std::shared_ptr graph_ptr = MakeShared(graph); + GraphNodePtr graph_node = MakeShared(graph_id); + graph_node->SetGraph(graph_ptr); + + graph_manager.options_.local_fmk_op_flag = false; + // need build while buildflag is true, var format changed + graph_node->SetBuildFlag(true); + graph_manager.var_acc_ctrl_.graph_ids_need_rebuild_.insert(graph_id); + + graph_manager.graph_map_.insert({graph_id, graph_node}); + graph_manager.graph_count_.insert({graph_id, 1}); + graph_node->SetRunFlag(false); + // function continue + int ret = setenv("ENABLE_NETWORK_ANALYSIS_DEBUG", "1", 1); + EXPECT_EQ(ret, 0); + graph_manager.prerun_args_q_.Push(args); + auto t1 = std::thread(GraphManager::PreRunThread, &graph_manager); + if (t1.joinable()) { + t1.join(); + } + EXPECT_EQ(st, ge::PARAM_INVALID); +} // TEST_F(UtestGraphManagerTest, ParseInputsDimsForGetNexNosinkAndData_success) { // GraphManager graph_manager; From 46ee33364ccf11cef8f976d1f1c251c9c4e255c3 Mon Sep 17 00:00:00 2001 From: "gengchao4@huawei.com" Date: Thu, 27 May 2021 10:56:34 +0800 Subject: [PATCH 23/48] add inner offset for var addr update --- ge/graph/build/memory/graph_mem_assigner.cc | 8 ++++---- ge/graph/load/model_manager/model_utils.cc | 4 ++-- .../ut/ge/graph/build/mem_assigner_unittest.cc | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/ge/graph/build/memory/graph_mem_assigner.cc b/ge/graph/build/memory/graph_mem_assigner.cc index 166162aa..4423c2c5 100755 --- a/ge/graph/build/memory/graph_mem_assigner.cc +++ b/ge/graph/build/memory/graph_mem_assigner.cc @@ -1514,7 +1514,7 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector< output_list = last_peer_out_op_desc->GetOutputOffset(); auto out_index = static_cast(peer_out_anchor->GetIdx()); if (output_list.size() > static_cast(out_index)) { - int64_t peer_out_inner_offset; + int64_t peer_out_inner_offset = 0; if (ge::AttrUtils::GetInt(last_peer_out_op_desc->MutableOutputDesc(out_index), ATTR_NAME_INNER_OFFSET, peer_out_inner_offset)) { (void)ge::AttrUtils::SetInt(tmp_op_desc->MutableInputDesc(anchor->GetIdx()), ATTR_NAME_INNER_OFFSET, @@ -1534,7 +1534,7 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector< GE_ERRORLOG_AND_ERRORMSG(ge::FAILED, error.c_str()); return ge::FAILED; } - int64_t inner_offset; + int64_t inner_offset = 0; (void)ge::AttrUtils::GetInt(tmp_op_desc->MutableInputDesc(anchor->GetIdx()), ATTR_NAME_INNER_OFFSET, inner_offset); GELOGD("Node[%s] input[%d] has origin offset[%ld] origin_inner_offset[%ld]", tmp_op_desc->GetName().c_str(), @@ -1575,7 +1575,7 @@ ge::Status GraphMemoryAssigner::UpdateRefOpOutputOffset(const NodePtr &node, con const int ref_in, const int64_t input_offset) const { auto opdesc = node->GetOpDesc(); GE_CHECK_NOTNULL(opdesc); - int64_t inner_offset; + int64_t inner_offset = 0; bool has_inner_offset = ge::AttrUtils::GetInt(opdesc->MutableInputDesc(ref_in), ATTR_NAME_INNER_OFFSET, inner_offset); for (const auto &out2in : out2ins) { auto out_i = out2in.first; @@ -1591,7 +1591,7 @@ ge::Status GraphMemoryAssigner::UpdateRefOpOutputOffset(const NodePtr &node, con origin_output_list[out_i] = input_offset; opdesc->SetOutputOffset(origin_output_list); if (has_inner_offset) { - (void)ge::AttrUtils::SetInt(opdesc->MutableOutputDesc(out_i), ATTR_NAME_INNER_OFFSET,inner_offset); + (void)ge::AttrUtils::SetInt(opdesc->MutableOutputDesc(out_i), ATTR_NAME_INNER_OFFSET, inner_offset); } GELOGI("Node[%s] output[%d] is updated from reuse input index[%d] to offset[%ld], inner_offset[%ld]", opdesc->GetName().c_str(), out_i, ref_in, input_offset, inner_offset); diff --git a/ge/graph/load/model_manager/model_utils.cc b/ge/graph/load/model_manager/model_utils.cc index f1748b35..a35a4669 100755 --- a/ge/graph/load/model_manager/model_utils.cc +++ b/ge/graph/load/model_manager/model_utils.cc @@ -342,7 +342,7 @@ vector ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co int64_t input_offset = v_input_offset[non_const_index]; non_const_index++; - int64_t inner_offset; + int64_t inner_offset = 0; (void)ge::AttrUtils::GetInt(op_desc->MutableInputDesc(i), ATTR_NAME_INNER_OFFSET, inner_offset); GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(input_offset - inner_offset), uint8_t *variable_addr = nullptr; @@ -454,7 +454,7 @@ vector ModelUtils::GetOutputDataAddrs(const RuntimeParam &model_param, C GELOGD("%s is an optional output, the address don't need to be saved.", tensor_desc->GetName().c_str()); continue; } - int64_t inner_offset; + int64_t inner_offset = 0; (void)ge::AttrUtils::GetInt(op_desc->MutableOutputDesc(i), ATTR_NAME_INNER_OFFSET, inner_offset); GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(v_output_offset[i] - inner_offset), uint8_t *variable_addr = nullptr; diff --git a/tests/ut/ge/graph/build/mem_assigner_unittest.cc b/tests/ut/ge/graph/build/mem_assigner_unittest.cc index 785af2ef..7b6b8479 100644 --- a/tests/ut/ge/graph/build/mem_assigner_unittest.cc +++ b/tests/ut/ge/graph/build/mem_assigner_unittest.cc @@ -503,6 +503,24 @@ TEST_F(UtestMemoryAssignerTest, graph_memory_assign_set_input_offset) { EXPECT_EQ(memoryAssigner.CheckOffset(), GRAPH_SUCCESS); } +TEST_F(UtestMemoryAssignerTest, graph_memory_assign_check_inner_offset) { + ge::ComputeGraphPtr graph = MakeRefNodeGraph(); + auto assign = graph->FindNode("assgin"); + auto op_desc = assign->GetOpDesc(); + int64_t inner_offset = 0; + EXPECT_EQ(ge::AttrUtils::GetInt(op_desc->MutableInputDesc(0), ATTR_NAME_INNER_OFFSET, inner_offset), false); + EXPECT_EQ(ge::AttrUtils::GetInt(op_desc->MutableInputDesc(1), ATTR_NAME_INNER_OFFSET, inner_offset), false); + GraphMemoryAssigner memoryAssigner(graph); + MemoryOffset memory_offset(RT_MEMORY_HBM, 0); + memoryAssigner.memory_offset_.emplace(RT_MEMORY_HBM, memory_offset); + EXPECT_EQ(memoryAssigner.SetInputOffset(), GRAPH_SUCCESS); + EXPECT_EQ(ge::AttrUtils::GetInt(op_desc->MutableInputDesc(0), ATTR_NAME_INNER_OFFSET, inner_offset), true); + EXPECT_EQ(inner_offset, 100); + EXPECT_EQ(ge::AttrUtils::GetInt(op_desc->MutableOutputDesc(0), ATTR_NAME_INNER_OFFSET, inner_offset), true); + EXPECT_EQ(inner_offset, 100); + EXPECT_EQ(ge::AttrUtils::GetInt(op_desc->MutableInputDesc(1), ATTR_NAME_INNER_OFFSET, inner_offset), false); +} + TEST_F(UtestMemoryAssignerTest, graph_memory_assign_update_ref_op_offset_reverse) { ge::ut::GraphBuilder builder("graph"); auto data_input = builder.AddNode("data", "Data", 1, 1); From c457cce7f5c9144ea50147b8b3a90864b91ea916 Mon Sep 17 00:00:00 2001 From: zhaoxinxin Date: Wed, 19 May 2021 11:14:22 +0800 Subject: [PATCH 24/48] modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.cc modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.h modified: ge/graph/passes/hccl_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.h modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.h modified: ge/graph/passes/hccl_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.h modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.h modified: ge/graph/passes/hccl_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.h modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.h modified: ge/graph/passes/hccl_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.h modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc modified: ge/graph/optimize/mem_rw_conflict_optimize.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.cc modified: ge/graph/passes/hccl_continuous_memcpy_pass.h modified: ge/graph/passes/hccl_memcpy_pass.cc modified: ge/graph/passes/hccl_memcpy_pass.h modified: tests/ut/ge/CMakeLists.txt new file: tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file: tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file: tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc --- ge/graph/optimize/mem_rw_conflict_optimize.cc | 155 +++++++++--------- .../passes/hccl_continuous_memcpy_pass.cc | 91 +++------- ge/graph/passes/hccl_continuous_memcpy_pass.h | 2 +- ge/graph/passes/hccl_memcpy_pass.cc | 108 ++++-------- ge/graph/passes/hccl_memcpy_pass.h | 2 +- tests/ut/ge/CMakeLists.txt | 7 +- .../mem_rw_conflict_optimize_unittest.cc | 150 +++++++++++++++++ .../passes/hccl_continuous_pass_unittest.cc | 79 +++++++++ .../graph/passes/hccl_memcpy_pass_unittest.cc | 80 +++++++++ 9 files changed, 445 insertions(+), 229 deletions(-) create mode 100644 tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc create mode 100644 tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc create mode 100644 tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc diff --git a/ge/graph/optimize/mem_rw_conflict_optimize.cc b/ge/graph/optimize/mem_rw_conflict_optimize.cc index 077ed110..7c1fc4ab 100644 --- a/ge/graph/optimize/mem_rw_conflict_optimize.cc +++ b/ge/graph/optimize/mem_rw_conflict_optimize.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -22,6 +22,7 @@ #include "graph/optimize/graph_optimize.h" #include "graph/utils/graph_utils.h" #include "graph/utils/node_utils.h" +#include "graph/utils/op_desc_utils.h" namespace { using namespace ge; @@ -32,12 +33,14 @@ const int kCaseReadOnly = 0; const int kCaseScopeWriteable = 2; const int kCaseWriteable = 3; const int kCaseInvalidRWType = 5; +// attr _input_mutable = true means node will modify its input in runtime +const char *const kModifyInput = "_input_mutable"; // rw type of input. enum class InputRWType { kReadOnly, // Normal op input only read kWriteable, // Op like Assign/ApplyMomentum - kScopeWriteable, // Op like hcom_allreduce, it will modify input ,but not expect take effect on pre ouput + kScopeWriteable, // Op like hcom_allreduce/while, it will modify input ,but not expect take effect on pre ouput kInvalidRWType }; // rw type of output @@ -154,7 +157,7 @@ bool IsSubgraphOutputNode(const NodePtr &node) { return true; } -NodePtr CreateIdentityAfterSrcNode(const Node &src_node, int out_anchor_idx) { +NodePtr AddIdentityToGraph(const Node &src_node, int out_anchor_idx) { if (src_node.GetOpDesc() == nullptr) { return nullptr; } @@ -162,30 +165,19 @@ NodePtr CreateIdentityAfterSrcNode(const Node &src_node, int out_anchor_idx) { auto next_num = identity_num.fetch_add(1); // 1. create new identity op desc string identity_name = src_node.GetName() + "_" + IDENTITY + std::to_string(next_num); - auto identity_opdesc = MakeShared(identity_name, IDENTITY); - if (identity_opdesc == nullptr) { - GELOGE(OUT_OF_MEMORY, "Failed to insert identity node, name %s", identity_name.c_str()); - return nullptr; - } + OpDescBuilder op_desc_builder(identity_name, IDENTITY); auto data_desc = src_node.GetOpDesc()->GetOutputDesc(out_anchor_idx); - // 2. add input_desc & output_desc for new identity - Status ret = identity_opdesc->AddInputDesc("x", data_desc); - if (ret != SUCCESS) { - GELOGE(ret, "Add Input desc failed for new identity %s.", identity_name.c_str()); - return nullptr; - } - ret = identity_opdesc->AddOutputDesc("y", data_desc); - if (ret != SUCCESS) { - GELOGE(ret, "Add Output desc failed for new Identity %s.", identity_name.c_str()); - return nullptr; - } + auto identity_op_desc = op_desc_builder.AddInput("x", data_desc) + .AddOutput("y", data_desc) + .Build(); + GELOGI("Insert new Identity node %s.", identity_name.c_str()); auto graph = src_node.GetOwnerComputeGraph(); if (graph == nullptr) { GELOGE(GRAPH_PARAM_INVALID, "Node %s owner compute graph is null.", src_node.GetName().c_str()); return nullptr; } - return graph->AddNode(identity_opdesc); + return graph->AddNode(identity_op_desc); } OutputRWType GetOutputRWTypeByIndex(const Node &node, uint32_t index) { @@ -274,8 +266,6 @@ InputRWType GetInputRWTypeByIndex(const Node &node, uint32_t index) { // single node without sub graph return GetSingleNodeInputRWTypeByIndex(node, index); } else { - // node with sub graph - std::set node_rw_type_set; auto data_node_vec = NodeUtils::GetSubgraphDataNodesByIndex(node, index); // get all input data node in subgraph std::set anchor_rw_type_set; @@ -345,12 +335,24 @@ Status MarkRWTypeForSubgraph(const ComputeGraphPtr &sub_graph) { auto parent_node = sub_graph->GetParentNode(); if (pre_output_rw_type == OutputRWType::kWriteable && parent_node->GetType() != PARTITIONEDCALL) { // insert identity - auto identity_node = CreateIdentityAfterSrcNode(*pre_node, pre_out_anchor->GetIdx()); + auto identity_node = AddIdentityToGraph(*pre_node, pre_out_anchor->GetIdx()); GE_CHECK_NOTNULL(identity_node); - auto ret = GraphUtils::InsertNodeBetweenDataAnchors(pre_out_anchor, in_data_anchor, identity_node); - if (ret != SUCCESS) { - GELOGE(ret, "Fail to insert identity"); - return ret; + if (GraphUtils::InsertNodeAfter(pre_out_anchor, {in_data_anchor}, identity_node) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Insert Identity node %s(%s) between %s(%s) -> %s(%s) failed.", + identity_node->GetName().c_str(), + identity_node->GetType().c_str(), + pre_node->GetName().c_str(), + pre_node->GetType().c_str(), + node->GetName().c_str(), + node->GetType().c_str()); + GELOGE(FAILED, "Insert Identity node %s(%s) between %s(%s) -> %s(%s) failed.", + identity_node->GetName().c_str(), + identity_node->GetType().c_str(), + pre_node->GetName().c_str(), + pre_node->GetType().c_str(), + node->GetName().c_str(), + node->GetType().c_str()); + return FAILED; } GELOGI("InsertNode %s between %s and %s successfully.", identity_node->GetName().c_str(), pre_node->GetName().c_str(), node->GetName().c_str()); @@ -505,34 +507,24 @@ Status SplitIdentityAlongAnchor(const OutDataAnchorPtr &out_data_anchor, const I auto peer_in_data_node = peer_in_data_anchor->GetOwnerNode(); GE_CHECK_NOTNULL(peer_in_data_node); auto input_rw_type = GetInputRWTypeByIndex(*peer_in_data_node, peer_in_data_anchor->GetIdx()); - auto ret = out_data_anchor->Unlink(peer_in_data_anchor); auto old_identity = out_data_anchor->GetOwnerNode(); - if (ret != SUCCESS) { - GELOGE(ret, "Failed to unlink from %s %dth out to %s.", old_identity->GetName().c_str(), out_data_anchor->GetIdx(), - peer_in_data_anchor->GetOwnerNode()->GetName().c_str()); - return ret; - } if (input_rw_type == InputRWType::kScopeWriteable || input_rw_type == InputRWType::kWriteable) { - auto new_identity = CreateIdentityAfterSrcNode(*pre_node, pre_out_data_anchor->GetIdx()); + auto new_identity = AddIdentityToGraph(*pre_node, pre_out_data_anchor->GetIdx()); GE_CHECK_NOTNULL(new_identity); - if (GraphUtils::AddEdge(pre_out_data_anchor, new_identity->GetInDataAnchor(kIdentityAnchorIndex)) != SUCCESS - || GraphUtils::AddEdge(new_identity->GetOutDataAnchor(kIdentityAnchorIndex), peer_in_data_anchor) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to insert Identity between node %s and %s", - pre_out_data_anchor->GetOwnerNode()->GetName().c_str(), - peer_in_data_anchor->GetOwnerNode()->GetName().c_str()); - return INTERNAL_ERROR; - } - - // 2. copy in-control-edge from dst to Identity - if (GraphUtils::CopyInCtrlEdges(peer_in_data_node, new_identity) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to copy in_control edges from node %s to %s", peer_in_data_node->GetName().c_str(), - new_identity->GetName().c_str()); - return INTERNAL_ERROR; + auto ret = GraphUtils::InsertNodeBefore(peer_in_data_anchor, new_identity, kIdentityAnchorIndex, + kIdentityAnchorIndex); + if (ret != SUCCESS) { + GELOGE(ret, "Failed to insert Identity %s before %s %dth input.", + new_identity->GetName().c_str(), + peer_in_data_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_data_anchor->GetIdx()); + return ret; } GELOGI("Node %s intput rw type is %s. Insert Identity between %s and %s.", peer_in_data_node->GetName().c_str(), InputRWTypeToSerialString(input_rw_type).c_str(), pre_out_data_anchor->GetOwnerNode()->GetName().c_str(), peer_in_data_anchor->GetOwnerNode()->GetName().c_str()); } else { + (void) out_data_anchor->Unlink(peer_in_data_anchor); // copy control edge to pre and peer node if (GraphUtils::CopyInCtrlEdges(old_identity, peer_in_data_node) != SUCCESS || GraphUtils::CopyOutCtrlEdges(old_identity, pre_node) != SUCCESS) { @@ -613,16 +605,14 @@ Status InsertIdentityAsNeeded(const NodePtr &node) { GELOGD("No need insert Identity."); continue; case INSERT_IDENTITY: - auto identity_node = CreateIdentityAfterSrcNode(*node, out_data_anchor->GetIdx()); - if (identity_node == nullptr) { - GELOGE(FAILED, "Create identity node failed."); - return FAILED; - } - auto ret = GraphUtils::InsertNodeBetweenDataAnchors(out_data_anchor, peer_in_data_anchor, identity_node); - if (ret != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to insert reshape between node %s and %s", node->GetName().c_str(), - peer_in_node->GetName().c_str()); - return INTERNAL_ERROR; + auto identity_node = AddIdentityToGraph(*node, out_data_anchor->GetIdx()); + GE_CHECK_NOTNULL(identity_node); + auto ret = GraphUtils::InsertNodeBefore(peer_in_data_anchor, identity_node, kIdentityAnchorIndex, + kIdentityAnchorIndex); + if (ret != SUCCESS) { + GELOGE(ret, "Fail to insert %s before %s %dth input.", identity_node->GetName().c_str(), + peer_in_data_anchor->GetOwnerNode()->GetName().c_str(), peer_in_data_anchor->GetIdx()); + return ret; } GELOGI("Insert Identity between %s and %s to handle memory conflict.", node->GetName().c_str(), peer_in_node->GetName().c_str()); @@ -633,28 +623,35 @@ Status InsertIdentityAsNeeded(const NodePtr &node) { return SUCCESS; } Status HandleAllreduceDuplicateInput(ComputeGraphPtr &compute_graph) { - for (const auto &node : compute_graph->GetDirectNode()) { - if (node->GetType() == HCOMALLREDUCE) { - std::set pre_out_anchor_set; - for (const auto &in_data_anchor : node->GetAllInDataAnchors()) { - auto pre_out_anchor = in_data_anchor->GetPeerOutAnchor(); - GE_CHECK_NOTNULL(pre_out_anchor); - if (pre_out_anchor_set.find(pre_out_anchor) == pre_out_anchor_set.end()) { - pre_out_anchor_set.emplace(pre_out_anchor); - continue; - } - // need insert identity - auto pre_node = pre_out_anchor->GetOwnerNode(); - auto identity_node = CreateIdentityAfterSrcNode(*pre_node, pre_out_anchor->GetIdx()); - GE_CHECK_NOTNULL(identity_node); - auto ret = GraphUtils::InsertNodeBetweenDataAnchors(pre_out_anchor, in_data_anchor, identity_node); - GE_CHK_STATUS_RET(ret, "Fail to insert identity."); - GELOGI("InsertNode %s between %s and %s successfully.", identity_node->GetName().c_str(), - pre_node->GetName().c_str(), node->GetName().c_str()); - } - } - } - return SUCCESS; + for (const auto &node : compute_graph->GetDirectNode()) { + bool mutable_input_flag = false; + (void)AttrUtils::GetBool(node->GetOpDesc(), kModifyInput, mutable_input_flag); + if (!mutable_input_flag) { + continue; + } + std::set pre_out_anchor_set; + for (const auto &in_data_anchor : node->GetAllInDataAnchors()) { + auto pre_out_anchor = in_data_anchor->GetPeerOutAnchor(); + GE_CHECK_NOTNULL(pre_out_anchor); + if (pre_out_anchor_set.insert(pre_out_anchor).second) { + continue; + } + // need insert identity + auto pre_node = pre_out_anchor->GetOwnerNode(); + auto identity_node = AddIdentityToGraph(*pre_node, pre_out_anchor->GetIdx()); + GE_CHECK_NOTNULL(identity_node); + auto ret = + GraphUtils::InsertNodeBefore(in_data_anchor, identity_node, kIdentityAnchorIndex, kIdentityAnchorIndex); + if (ret != SUCCESS) { + GELOGE(ret, "Failed to insert node %s before %s %dth input.", identity_node->GetName().c_str(), + node->GetName().c_str(), in_data_anchor->GetIdx()); + return ret; + } + GELOGI("InsertNode %s between %s and %s successfully.", identity_node->GetName().c_str(), + pre_node->GetName().c_str(), node->GetName().c_str()); + } + } + return SUCCESS; } } // namespace diff --git a/ge/graph/passes/hccl_continuous_memcpy_pass.cc b/ge/graph/passes/hccl_continuous_memcpy_pass.cc index 790661bc..61066d63 100644 --- a/ge/graph/passes/hccl_continuous_memcpy_pass.cc +++ b/ge/graph/passes/hccl_continuous_memcpy_pass.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,11 +24,12 @@ #include "common/ge/ge_util.h" #include "framework/common/types.h" #include "graph/utils/graph_utils.h" +#include "graph/utils/op_desc_utils.h" namespace { -const int kAnchorNum = 0; const int32_t kAnchorAssignRefIndex = 0; const int32_t kAnchorAssignValueIndex = 1; +const int32_t kAnchorIdentityIndex = 0; } // namespace namespace ge { Status HcclContinuousMemcpyPass::Run(ge::ComputeGraphPtr graph) { @@ -161,41 +162,23 @@ NodePtr HcclContinuousMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &grap std::string node_name = pre_node->GetName() + "_" + IDENTITY; node_name = CheckDuplicateName(node_name); - OpDescPtr op_desc = MakeShared(node_name.c_str(), IDENTITY); - if (op_desc == nullptr) { - REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(INTERNAL_ERROR, "Create Identity op: MakeShared op_desc fail."); - return nullptr; - } - GELOGI("Create Identity op:%s.", op_desc->GetName().c_str()); - - graphStatus ret = op_desc->AddInputDesc("x", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); - if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op: add input desc fail."); - return nullptr; - } - - ret = op_desc->AddOutputDesc("y", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); - if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op: add output desc fail."); + OpDescBuilder op_desc_builder(node_name, IDENTITY); + auto data_desc = pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx()); + auto identity_op_desc = op_desc_builder.AddInput("x", data_desc).AddOutput("y", data_desc).Build(); + if (identity_op_desc == nullptr) { return nullptr; } // because history reason ,this pass can not do work after constant fold so mark it - (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); + (void)AttrUtils::SetBool(identity_op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); - NodePtr memcpy_node = graph->AddNode(op_desc); - if (memcpy_node == nullptr) { + NodePtr identity_node = graph->AddNode(identity_op_desc); + if (identity_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); + identity_node->GetName().c_str(), identity_node->GetType().c_str(), graph->GetName().c_str()); GELOGE(INTERNAL_ERROR, "Insert Identity node fail."); return nullptr; } - - return memcpy_node; + return identity_node; } /// @@ -256,50 +239,24 @@ Status HcclContinuousMemcpyPass::ModifyEdgeConnection(const ComputeGraphPtr &gra Status HcclContinuousMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr &graph, const OutDataAnchorPtr &src_out_anchor, const InDataAnchorPtr &hccl_in_anchor) { - GELOGI("Between op %s and op %s need insert memcpy async op.", src_out_anchor->GetOwnerNode()->GetName().c_str(), + GELOGI("Between op %s and op %s need insert identity op.", src_out_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - NodePtr memcpy_node = CreateIdentityNode(graph, src_out_anchor); - GE_CHECK_NOTNULL(memcpy_node); + NodePtr identity_node = CreateIdentityNode(graph, src_out_anchor); + GE_CHECK_NOTNULL(identity_node); - Status ret1 = src_out_anchor->Unlink(hccl_in_anchor); - if (ret1 != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed", - src_out_anchor->GetOwnerNode()->GetName().c_str(), - src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetType().c_str(), - hccl_in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "The op %s Unlink anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - return FAILED; - } - auto out_data_anchor_0 = memcpy_node->GetOutDataAnchor(kAnchorNum); - GE_CHECK_NOTNULL(out_data_anchor_0); - ret1 = out_data_anchor_0->LinkTo(hccl_in_anchor); - if (ret1 != SUCCESS) { + auto ret = GraphUtils::InsertNodeBefore(hccl_in_anchor, identity_node, kAnchorIdentityIndex, kAnchorIdentityIndex); + if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d link to op:%s(%s) in index:%d failed", - out_data_anchor_0->GetOwnerNode()->GetName().c_str(), - out_data_anchor_0->GetOwnerNode()->GetType().c_str(), out_data_anchor_0->GetIdx(), + "Op:Fail to insert %s(%s) before %s(%s) on index:%d input anchor.", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetType().c_str(), hccl_in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", memcpy_node->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - return FAILED; - } - - Status ret = src_out_anchor->LinkTo(memcpy_node->GetInDataAnchor(kAnchorNum)); - if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed", - src_out_anchor->GetOwnerNode()->GetName().c_str(), - src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), - memcpy_node->GetName().c_str(), memcpy_node->GetType().c_str(), - kAnchorNum); - GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), - memcpy_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "Fail to insert %s(%s) before %s(%s) on index:%d input anchor.", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), + hccl_in_anchor->GetOwnerNode()->GetName().c_str(), + hccl_in_anchor->GetOwnerNode()->GetType().c_str(), + hccl_in_anchor->GetIdx()); return FAILED; } return SUCCESS; diff --git a/ge/graph/passes/hccl_continuous_memcpy_pass.h b/ge/graph/passes/hccl_continuous_memcpy_pass.h index 538e89e9..5fbb6fd0 100644 --- a/ge/graph/passes/hccl_continuous_memcpy_pass.h +++ b/ge/graph/passes/hccl_continuous_memcpy_pass.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ge/graph/passes/hccl_memcpy_pass.cc b/ge/graph/passes/hccl_memcpy_pass.cc index 2d2f8220..dd251ea6 100755 --- a/ge/graph/passes/hccl_memcpy_pass.cc +++ b/ge/graph/passes/hccl_memcpy_pass.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,13 +24,15 @@ #include "common/ge/ge_util.h" #include "framework/common/types.h" #include "graph/utils/graph_utils.h" +#include "graph/utils/op_desc_utils.h" namespace { const int32_t kAnchorSize = 1; -const int kAnchorNum = 0; const int32_t kAnchorAssignRefIndex = 0; const int32_t kAnchorAssignValueIndex = 1; -const char *const kInputMutable = "_input_mutable"; +const int32_t kAnchorIdentityIndex = 0; +// attr _input_mutable = true means hccl node will modify its input in runtime +const char *const kModifyInput = "_input_mutable"; } // namespace namespace ge { Status HcclMemcpyPass::Run(ge::ComputeGraphPtr graph) { @@ -58,24 +60,13 @@ Status HcclMemcpyPass::Run(ge::ComputeGraphPtr graph) { // need to inset memcpy node between. // also works on situation that input is variable or const. Status HcclMemcpyPass::MutableInputProcess(const ComputeGraphPtr &graph, const NodePtr node) { - auto op_desc = node->GetOpDesc(); - bool node_input_mutable = false; - if (!AttrUtils::HasAttr(op_desc, kInputMutable)) { - return SUCCESS; - } - - if (!AttrUtils::GetBool(op_desc, kInputMutable, node_input_mutable)) { - REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", kInputMutable, - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "node:%s get attr:_input_mutable failed.", node->GetName().c_str()); - return FAILED; - } + (void)AttrUtils::GetBool(node->GetOpDesc(), kModifyInput, node_input_mutable); if (!node_input_mutable) { return SUCCESS; } - GELOGI("input mutable hcom op is:%s.", op_desc->GetName().c_str()); + GELOGI("input mutable hcom op is:%s.", node->GetName().c_str()); for (auto &hccl_in_anchor : node->GetAllInDataAnchors()) { if (hccl_in_anchor == nullptr) { continue; @@ -127,41 +118,23 @@ NodePtr HcclMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &graph, const O std::string node_name = pre_node->GetName() + "_" + IDENTITY; node_name = CheckDuplicateName(node_name); - OpDescPtr op_desc = MakeShared(node_name.c_str(), IDENTITY); - if (op_desc == nullptr) { - REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(INTERNAL_ERROR, "Create Identity op: MakeShared op_desc fail."); - return nullptr; - } - GELOGI("Create Identity op:%s.", op_desc->GetName().c_str()); - - graphStatus ret = op_desc->AddInputDesc("x", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); - if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, name:x", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op: add input desc fail."); - return nullptr; - } - - ret = op_desc->AddOutputDesc("y", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); - if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, name:y", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op: add output desc fail."); + OpDescBuilder op_desc_builder(node_name, IDENTITY); + auto data_desc = pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx()); + auto identity_op_desc = op_desc_builder.AddInput("x", data_desc).AddOutput("y", data_desc).Build(); + if (identity_op_desc == nullptr) { return nullptr; } // because history reason ,this pass can not do work after constant fold so mark it - (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); + (void)AttrUtils::SetBool(identity_op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); - NodePtr memcpy_node = graph->AddNode(op_desc); - if (memcpy_node == nullptr) { + NodePtr identity_node = graph->AddNode(identity_op_desc); + if (identity_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); + identity_node->GetName().c_str(), identity_node->GetType().c_str(), graph->GetName().c_str()); GELOGE(INTERNAL_ERROR, "Insert Identity node fail."); return nullptr; } - - return memcpy_node; + return identity_node; } /// @@ -220,49 +193,24 @@ Status HcclMemcpyPass::ModifyEdgeConnection(const ComputeGraphPtr &graph, const /// Status HcclMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr &graph, const OutDataAnchorPtr &src_out_anchor, const InDataAnchorPtr &hccl_in_anchor) { - GELOGI("Between op %s and op %s need insert memcpy async op.", src_out_anchor->GetOwnerNode()->GetName().c_str(), + GELOGI("Between op %s and op %s need insert identity op.", src_out_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - NodePtr memcpy_node = CreateIdentityNode(graph, src_out_anchor); - GE_CHECK_NOTNULL(memcpy_node); + NodePtr identity_node = CreateIdentityNode(graph, src_out_anchor); + GE_CHECK_NOTNULL(identity_node); - Status ret1 = src_out_anchor->Unlink(hccl_in_anchor); - if (ret1 != SUCCESS) { + auto ret = GraphUtils::InsertNodeBefore(hccl_in_anchor, identity_node, kAnchorIdentityIndex, kAnchorIdentityIndex); + if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed", - src_out_anchor->GetOwnerNode()->GetName().c_str(), - src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetType().c_str(), hccl_in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "The op %s Unlink anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - return FAILED; - } - auto out_data_anchor_0 = memcpy_node->GetOutDataAnchor(kAnchorNum); - GE_CHECK_NOTNULL(out_data_anchor_0); - ret1 = out_data_anchor_0->LinkTo(hccl_in_anchor); - if (ret1 != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d link to op:%s(%s) in index:%d failed", - out_data_anchor_0->GetOwnerNode()->GetName().c_str(), - out_data_anchor_0->GetOwnerNode()->GetType().c_str(), out_data_anchor_0->GetIdx(), + "Op:Fail to insert %s(%s) before %s(%s) on index:%d input anchor.", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetType().c_str(), hccl_in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", memcpy_node->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - return FAILED; - } - - Status ret = src_out_anchor->LinkTo(memcpy_node->GetInDataAnchor(kAnchorNum)); - if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed", - src_out_anchor->GetOwnerNode()->GetName().c_str(), - src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), - memcpy_node->GetName().c_str(), memcpy_node->GetType().c_str(), - kAnchorNum); - GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), - memcpy_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "Fail to insert %s(%s) before %s(%s) on index:%d input anchor.", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), + hccl_in_anchor->GetOwnerNode()->GetName().c_str(), + hccl_in_anchor->GetOwnerNode()->GetType().c_str(), + hccl_in_anchor->GetIdx()); return FAILED; } return SUCCESS; diff --git a/ge/graph/passes/hccl_memcpy_pass.h b/ge/graph/passes/hccl_memcpy_pass.h index 7ab63c59..b75b27d1 100755 --- a/ge/graph/passes/hccl_memcpy_pass.h +++ b/ge/graph/passes/hccl_memcpy_pass.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/tests/ut/ge/CMakeLists.txt b/tests/ut/ge/CMakeLists.txt index 16f3672b..40a94dd2 100755 --- a/tests/ut/ge/CMakeLists.txt +++ b/tests/ut/ge/CMakeLists.txt @@ -372,6 +372,7 @@ set(COMMON_FORMAT_SRC_FILES set(GRAPH_OPTIMIZE_COMMON_SRC_FILES "${GE_CODE_DIR}/ge/graph/optimize/graph_optimize.cc" "${GE_CODE_DIR}/ge/graph/optimize/summary_optimize.cc" + "${GE_CODE_DIR}/ge/graph/optimize/mem_rw_conflict_optimize.cc" ) @@ -715,7 +716,10 @@ set(PASS_TEST_FILES "graph/passes/mark_node_unknown_shape_pass_unittest.cc" "graph/passes/reshape_recovery_pass_unittest.cc" "graph/passes/cast_remove_pass_unittest.cc" - "graph/passes/memcpy_addr_async_unittest.cc" + "graph/passes/memcpy_addr_async_unittest.cc" + "graph/passes/hccl_continuous_pass_unittest.cc" + "graph/passes/hccl_memcpy_pass_unittest.cc" + ) set(KERNEL_TEST_FILES @@ -798,6 +802,7 @@ set(MULTI_PARTS_TEST_FILES "graph/manager/run_graph_unittest.cc" "graph/partition/dynamic_shape_partition_unittest.cc" "graph/manager/graph_manager_unittest.cc" + "graph/optimize/mem_rw_conflict_optimize_unittest.cc" "session/omg_omg_unittest.cc" "session/ge_api_unittest.cc" "session/inner_session_unittest.cc" diff --git a/tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc b/tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc new file mode 100644 index 00000000..22b0b2c0 --- /dev/null +++ b/tests/ut/ge/graph/optimize/mem_rw_conflict_optimize_unittest.cc @@ -0,0 +1,150 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#define protected public +#define private public +#include "graph/optimize/graph_optimize.h" +#undef protected +#undef private +#include "../passes/graph_builder_utils.h" +#include "graph/debug/ge_attr_define.h" + +namespace ge { +class UTest_Graph_Mem_RW_Conflict_Optimize : public testing::Test { + protected: + void SetUp() {} + void TearDown() {} +}; +namespace { +/* + * Data -cast - netoutput + */ +ComputeGraphPtr BuildGraph_Readonly_Subgraph(const string subraph_name){ + auto sub_builder = ut::GraphBuilder(subraph_name); + auto data1 = sub_builder.AddNode("data1", DATA, 0,1); + auto cast = sub_builder.AddNode("cast", CAST, 1,1); + auto netoutput = sub_builder.AddNode("netoutput",NETOUTPUT, 1,1); + AttrUtils::SetInt(data1->GetOpDesc(),ATTR_NAME_PARENT_NODE_INDEX, 1); + AttrUtils::SetInt(netoutput->GetOpDesc(),ATTR_NAME_PARENT_NODE_INDEX,0); + + sub_builder.AddDataEdge(data1,0,cast,0); + sub_builder.AddDataEdge(cast,0,netoutput,0); + return sub_builder.GetGraph(); +} +/* + * const - allreduce + * \ if + * insert identity + */ +ComputeGraphPtr BuildGraph_Readonly_ScopeWrite() { + auto builder = ut::GraphBuilder("test"); + auto const1 = builder.AddNode("const1", CONSTANT, 0, 1); + auto ctrl_const = builder.AddNode("ctrl_const", CONSTANT, 0, 1); + auto allreduce = builder.AddNode("allreduce", HCOMALLREDUCE, 1, 1); + auto if_node = builder.AddNode("if", IF, 1,0); + + builder.AddDataEdge(const1, 0, allreduce, 0); + builder.AddDataEdge(const1, 0, if_node, 0); + builder.AddControlEdge(ctrl_const, allreduce); + + auto root_graph = builder.GetGraph(); + string subgraph_name = "then_branch"; + ComputeGraphPtr then_branch_graph = BuildGraph_Readonly_Subgraph(subgraph_name); + then_branch_graph->SetParentNode(if_node); + then_branch_graph->SetParentGraph(root_graph); + if_node->GetOpDesc()->AddSubgraphName(subgraph_name); + if_node->GetOpDesc()->SetSubgraphInstanceName(0,subgraph_name); + root_graph->AddSubgraph(subgraph_name, then_branch_graph); + return root_graph; +} +/* const1---allreduce const1--identity - allreduce + * / / + * var-identity--cast1 ==> var-----cast1 + * \ \ + * if if + */ +ComputeGraphPtr BuildGraph_Identiyt_Split(){ + auto builder = ut::GraphBuilder("g1"); + auto var = builder.AddNode("var", VARIABLE, 0, 1); + auto identity = builder.AddNode("identity", IDENTITY, 1, 1); + auto const1 = builder.AddNode("const1", CONSTANT, 0, 1); + auto allreduce = builder.AddNode("allreduce", HCOMALLREDUCE, 1, 1); + auto cast1 = builder.AddNode("cast1", CAST, 1, 1); + auto if_node = builder.AddNode("if", IF, 1,0); + + builder.AddDataEdge(var, 0 , identity, 0); + builder.AddDataEdge(identity, 0 , allreduce, 0); + builder.AddDataEdge(identity, 0 , cast1, 0); + builder.AddDataEdge(identity, 0 , if_node, 0); + builder.AddControlEdge(const1, allreduce); + + auto root_graph = builder.GetGraph(); + string subgraph_name = "then_branch"; + ComputeGraphPtr then_branch_graph = BuildGraph_Readonly_Subgraph(subgraph_name); + then_branch_graph->SetParentNode(if_node); + then_branch_graph->SetParentGraph(root_graph); + if_node->GetOpDesc()->AddSubgraphName(subgraph_name); + if_node->GetOpDesc()->SetSubgraphInstanceName(0,subgraph_name); + root_graph->AddSubgraph(subgraph_name, then_branch_graph); + return root_graph; +} +/* + * mul == allreduce + * need insert identity + */ +ComputeGraphPtr BuildGraph_mul_1To2_ScopeWrite() { + auto builder = ut::GraphBuilder("test"); + auto mul = builder.AddNode("mul", MUL, 2,1); + auto allreduce = builder.AddNode("allreduce", HCOMALLREDUCE, 2,0); + AttrUtils::SetBool(allreduce->GetOpDesc(), "_input_mutable", true); + builder.AddDataEdge(mul,0,allreduce,0); + builder.AddDataEdge(mul,0,allreduce,1); + return builder.GetGraph(); +} +} // namespace +// const -> allreduce +// const -> Identity -> allreduce +TEST(UtestGraphPassesHcclMemcpyPass, testReadonlyScopeWriteConflict) { + ComputeGraphPtr graph = BuildGraph_Readonly_ScopeWrite(); + GraphOptimize graph_optimizer; + auto ret = graph_optimizer.HandleMemoryRWConflict(graph); + EXPECT_EQ(ret, SUCCESS); + auto allreduce = graph->FindNode("allreduce"); + EXPECT_EQ(allreduce->GetInDataNodes().at(0)->GetType(), IDENTITY); +} +TEST(UtestGraphPassesHcclMemcpyPass, testIdentiytSplit) { + ComputeGraphPtr graph = BuildGraph_Identiyt_Split(); + GraphOptimize graph_optimizer; + auto ret = graph_optimizer.HandleMemoryRWConflict(graph); + EXPECT_EQ(ret, SUCCESS); + auto allreduce = graph->FindNode("allreduce"); + auto allreduce_in_node = allreduce->GetInDataNodes().at(0); + EXPECT_EQ(allreduce_in_node->GetType(), IDENTITY); + EXPECT_EQ(allreduce_in_node->GetInControlNodes().at(0)->GetType(), CONSTANT); +} +TEST(UtestGraphPassesHcclMemcpyPass, testMul_1To2_ScopeWrite) { + ComputeGraphPtr graph = BuildGraph_mul_1To2_ScopeWrite(); + EXPECT_EQ(graph->GetDirectNodesSize(), 2); + GraphOptimize graph_optimizer; + auto ret = graph_optimizer.HandleMemoryRWConflict(graph); + EXPECT_EQ(ret, SUCCESS); + EXPECT_EQ(graph->GetDirectNodesSize(), 3); +} +} // namespace ge diff --git a/tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc b/tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc new file mode 100644 index 00000000..fb18162b --- /dev/null +++ b/tests/ut/ge/graph/passes/hccl_continuous_pass_unittest.cc @@ -0,0 +1,79 @@ +/** + * Copyright 2019-2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "common/ge_inner_error_codes.h" +#define protected public +#define private public +#include "graph/passes/hccl_continuous_memcpy_pass.h" +#undef protected +#undef private +#include "graph_builder_utils.h" + +namespace ge { +class UtestGraphPassesHcclContinuousMemcpyPass : public testing::Test { + protected: + void SetUp() {} + void TearDown() {} +}; +namespace { + +/* + * var var + * | \ | \ + * | assign | assign + * | // =======> | // + * allreduce identity + * | | + * netoutput allreduce + * | + * netoutput + */ +ComputeGraphPtr BuildGraph_Allreduce_Read_Var_After_Assign(){ + auto builder = ut::GraphBuilder("test"); + auto var = builder.AddNode("var", VARIABLE, 0, 1); + auto assign = builder.AddNode("assign", ASSIGN, 1, 1); + auto allreduce = builder.AddNode("allreduce", HCOMALLREDUCE, 1, 1); + auto netoutput1 = builder.AddNode("netoutput", NETOUTPUT, 1, 0); + + builder.AddDataEdge(var, 0, assign, 0); + builder.AddDataEdge(var,0,allreduce,0); + builder.AddControlEdge(assign, allreduce); + return builder.GetGraph(); +} +} // namespace + +// const -> allreduce +// const -> Identity -> allreduce +TEST(UtestGraphPassesHcclContinuousMemcpyPass, testInsertIdentityBeforeHccl) { + ComputeGraphPtr graph = BuildGraph_Allreduce_Read_Var_After_Assign(); + auto src_node = graph->FindNode("var"); + auto dst_node = graph->FindNode("allreduce"); + // test InsertIdentityBeforeHccl + HcclContinuousMemcpyPass hccl_continuous_memcpy_pass; + hccl_continuous_memcpy_pass.InsertIdentityBeforeHccl(graph, src_node->GetOutDataAnchor(0), dst_node->GetInDataAnchor(0)); + + // check + dst_node = graph->FindNode("allreduce"); + auto in_node_before_dst_node = dst_node->GetInDataAnchor(0)->GetPeerOutAnchor()->GetOwnerNode(); + EXPECT_EQ(in_node_before_dst_node->GetType(), IDENTITY); + EXPECT_EQ(in_node_before_dst_node->GetInControlNodes().size(), 1); + EXPECT_EQ(in_node_before_dst_node->GetInControlNodes().at(0)->GetName(), "assign"); +} +} // namespace ge diff --git a/tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc b/tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc new file mode 100644 index 00000000..35edeb47 --- /dev/null +++ b/tests/ut/ge/graph/passes/hccl_memcpy_pass_unittest.cc @@ -0,0 +1,80 @@ +/** + * Copyright 2019-2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "common/ge_inner_error_codes.h" +#define protected public +#define private public +#include "graph/passes/hccl_memcpy_pass.h" +#undef protected +#undef private +#include "graph_builder_utils.h" + +namespace ge { +class UtestGraphPassesHcclMemcpyPass : public testing::Test { + protected: + void SetUp() {} + void TearDown() {} +}; +namespace { + +/* + * var var + * | \ | \ + * | assign | assign + * | // =======> | // + * allreduce identity + * | | + * netoutput allreduce + * | + * netoutput + */ +ComputeGraphPtr BuildGraph_Allreduce_Read_Var_After_Assign(){ + auto builder = ut::GraphBuilder("test"); + auto var = builder.AddNode("var", VARIABLE, 0, 1); + auto assign = builder.AddNode("assign", ASSIGN, 1, 1); + auto allreduce = builder.AddNode("allreduce", HCOMALLREDUCE, 1, 1); + auto netoutput1 = builder.AddNode("netoutput", NETOUTPUT, 1, 0); + + builder.AddDataEdge(var, 0, assign, 0); + builder.AddDataEdge(var,0,allreduce,0); + builder.AddControlEdge(assign, allreduce); + return builder.GetGraph(); +} +} // namespace + +// const -> allreduce +// const -> Identity -> allreduce +TEST(UtestGraphPassesHcclMemcpyPass, testInsertIdentityBeforeHccl) { + ComputeGraphPtr graph = BuildGraph_Allreduce_Read_Var_After_Assign(); + auto src_node = graph->FindNode("var"); + auto dst_node = graph->FindNode("allreduce"); + // test InsertIdentityBeforeHccl + HcclMemcpyPass hccl_memcpy_pass; + hccl_memcpy_pass.InsertIdentityBeforeHccl(graph, src_node->GetOutDataAnchor(0), + dst_node->GetInDataAnchor(0)); + + // check + dst_node = graph->FindNode("allreduce"); + auto in_node_before_dst_node = dst_node->GetInDataAnchor(0)->GetPeerOutAnchor()->GetOwnerNode(); + EXPECT_EQ(in_node_before_dst_node->GetType(), IDENTITY); + EXPECT_EQ(in_node_before_dst_node->GetInControlNodes().size(), 1); + EXPECT_EQ(in_node_before_dst_node->GetInControlNodes().at(0)->GetName(), "assign"); +} +} // namespace ge From f5dc59ad7d34a5e25a9619db74acf4cfb80f4555 Mon Sep 17 00:00:00 2001 From: wei-gangqiang Date: Mon, 24 May 2021 16:39:01 +0800 Subject: [PATCH 25/48] add tool-chains for graphengine create container name by different code path remove no use console for update add ge config command for server addr config add readme for ge config add readme for ge config --- .gitignore | 9 + build.sh | 0 scripts/build/ge_build.sh | 25 + scripts/check_clang_format.sh | 76 +- scripts/clean/ge_clean.sh | 90 + scripts/config/ge_config.sh | 115 + scripts/cov/ge_cov.sh | 136 + scripts/docs/Doxyfile_all | 2639 +++++++++++++++++ scripts/docs/Doxyfile_brief | 2638 ++++++++++++++++ scripts/docs/ge_docs.sh | 87 + scripts/env/Dockerfile | 42 + scripts/env/ge_env.sh | 146 + .../ge_format.sh} | 48 +- scripts/ge.sh | 77 + scripts/readme.md | 331 +++ scripts/test/ge_test.sh | 80 + scripts/update/deps_config.sh | 47 + scripts/update/ge_update.sh | 136 + 18 files changed, 6661 insertions(+), 61 deletions(-) mode change 100644 => 100755 build.sh create mode 100755 scripts/build/ge_build.sh mode change 100644 => 100755 scripts/check_clang_format.sh create mode 100755 scripts/clean/ge_clean.sh create mode 100755 scripts/config/ge_config.sh create mode 100755 scripts/cov/ge_cov.sh create mode 100755 scripts/docs/Doxyfile_all create mode 100755 scripts/docs/Doxyfile_brief create mode 100755 scripts/docs/ge_docs.sh create mode 100755 scripts/env/Dockerfile create mode 100755 scripts/env/ge_env.sh rename scripts/{format_source_code.sh => format/ge_format.sh} (67%) mode change 100644 => 100755 create mode 100755 scripts/ge.sh create mode 100755 scripts/readme.md create mode 100755 scripts/test/ge_test.sh create mode 100755 scripts/update/deps_config.sh create mode 100755 scripts/update/ge_update.sh diff --git a/.gitignore b/.gitignore index 891c0f87..d4acd86b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,17 @@ /output /prebuilts /cov +/deps +.autotools +.project +.cproject +.settings/ +/tests/frm/ *.ir *.out +*.DS_Store +.DS_Store +server_config.sh # Dynamic libraries # *.so diff --git a/build.sh b/build.sh old mode 100644 new mode 100755 diff --git a/scripts/build/ge_build.sh b/scripts/build/ge_build.sh new file mode 100755 index 00000000..c230ec09 --- /dev/null +++ b/scripts/build/ge_build.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e +export PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../} +function main(){ + ${PROJECT_HOME}/build.sh "$@" +} + +main "$@" + +set +e diff --git a/scripts/check_clang_format.sh b/scripts/check_clang_format.sh old mode 100644 new mode 100755 index 2425345b..29434fd4 --- a/scripts/check_clang_format.sh +++ b/scripts/check_clang_format.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2020 Huawei Technologies Co., Ltd +# Copyright 2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - set -e CLANG_FORMAT=$(which clang-format) || (echo "Please install 'clang-format' tool first"; exit 1) @@ -25,10 +24,10 @@ if [[ "${version}" -lt "8" ]]; then fi CURRENT_PATH=$(pwd) -SCRIPTS_PATH=$(dirname "$0") +PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../} echo "CURRENT_PATH=$CURRENT_PATH" -echo "SCRIPTS_PATH=$SCRIPTS_PATH" +echo "PROJECT_HOME=$PROJECT_HOME" # print usage message function usage() @@ -81,45 +80,46 @@ function checkopts() checkopts "$@" # switch to project root path, which contains clang-format config file '.clang-format' -cd "${SCRIPTS_PATH}/.." || exit 1 +pushd "${CURRENT_PATH}" + CHECK_LIST_FILE='__checked_files_list__' + cd "${PROJECT_HOME}" || exit 1 -CHECK_LIST_FILE='__checked_files_list__' + if [ "X${mode}" == "Xall" ]; then + find src -type f -name "*" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true + find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${CHECK_LIST_FILE}" || true + elif [ "X${mode}" == "Xchanged" ]; then + # --diff-filter=ACMRTUXB will ignore deleted files in commit + git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true + else # "X${mode}" == "Xlastcommit" + git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true + fi -if [ "X${mode}" == "Xall" ]; then - find src -type f -name "*" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true - find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${CHECK_LIST_FILE}" || true -elif [ "X${mode}" == "Xchanged" ]; then - # --diff-filter=ACMRTUXB will ignore deleted files in commit - git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true -else # "X${mode}" == "Xlastcommit" - git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true -fi + CHECK_RESULT_FILE=__code_format_check_result__ + echo "0" > "$CHECK_RESULT_FILE" -CHECK_RESULT_FILE=__code_format_check_result__ -echo "0" > "$CHECK_RESULT_FILE" + # check format of files modified in the lastest commit + while read line; do + BASE_NAME=$(basename "${line}") + TEMP_FILE="__TEMP__${BASE_NAME}" + cp "${line}" "${TEMP_FILE}" + ${CLANG_FORMAT} -i "${TEMP_FILE}" + set +e + diff "${TEMP_FILE}" "${line}" + ret=$? + set -e + rm "${TEMP_FILE}" + if [[ "${ret}" -ne 0 ]]; then + echo "File ${line} is not formated, please format it." + echo "1" > "${CHECK_RESULT_FILE}" + break + fi + done < "${CHECK_LIST_FILE}" -# check format of files modified in the lastest commit -while read line; do - BASE_NAME=$(basename "${line}") - TEMP_FILE="__TEMP__${BASE_NAME}" - cp "${line}" "${TEMP_FILE}" - ${CLANG_FORMAT} -i "${TEMP_FILE}" - set +e - diff "${TEMP_FILE}" "${line}" - ret=$? - set -e - rm "${TEMP_FILE}" - if [[ "${ret}" -ne 0 ]]; then - echo "File ${line} is not formated, please format it." - echo "1" > "${CHECK_RESULT_FILE}" - break - fi -done < "${CHECK_LIST_FILE}" + result=$(cat "${CHECK_RESULT_FILE}") + rm "${CHECK_RESULT_FILE}" + rm "${CHECK_LIST_FILE}" +popd -result=$(cat "${CHECK_RESULT_FILE}") -rm "${CHECK_RESULT_FILE}" -rm "${CHECK_LIST_FILE}" -cd "${CURRENT_PATH}" || exit 1 if [[ "X${result}" == "X0" ]]; then echo "Check PASS: specified files are well formated!" fi diff --git a/scripts/clean/ge_clean.sh b/scripts/clean/ge_clean.sh new file mode 100755 index 00000000..90fb939e --- /dev/null +++ b/scripts/clean/ge_clean.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../} + +function help(){ + cat <<-EOF +Usage: ge clean [OPTIONS] + +Options: + -b, --build Clean build dir + -d, --docs Clean generate docs + -i, --install Clean dependenices + -a, --all Clean all + -h, --help +EOF + +} + +function clean_relative_dir(){ + rm -rf "${PROJECT_HOME}/${1:-output}" +} + +function parse_args(){ + parsed_args=$(getopt -a -o bdiah --long build,docs,install,all,help -- "$@") || { + help + exit 1 + } + + if [ $# -lt 1 ]; then + clean_relative_dir "build" + clean_relative_dir "output" + exit 1 + fi + + eval set -- "$parsed_args" + while true; do + case "$1" in + -b | --build) + clean_relative_dir "build" + clean_relative_dir "output" + ;; + -d | --docs) + clean_relative_dir "docs/doxygen" + ;; + -i | --install) + clean_relative_dir "deps" + ;; + -a | --all) + clean_relative_dir "deps" + clean_relative_dir "build" + clean_relative_dir "output" + clean_relative_dir "docs/doxygen" + ;; + -h | --help) + help + ;; + --) + shift; break; + ;; + *) + help; exit 1 + ;; + esac + shift + done +} + +function main(){ + parse_args "$@" +} + +main "$@" + +set +e \ No newline at end of file diff --git a/scripts/config/ge_config.sh b/scripts/config/ge_config.sh new file mode 100755 index 00000000..a5aba2a8 --- /dev/null +++ b/scripts/config/ge_config.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../} +PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd) + +function help(){ + cat <<-EOF +Usage: ge config [OPTIONS] + +update server config for ge, you need input all config info (ip, user, password) + +Options: + -i, --ip Config ip config + -u, --user Config user name + -p, --password Config password + -h, --help + +Example: ge config -i=121.36.**.** -u=asc**, -p=Asc***\#@\!\$ (Need add escape character \ before special charactor $、#、!) + +EOF + +} + +function write_config_file(){ + local IP=$1 + local USER=$2 + local PASSWORD=$3 + if [[ -z "$IP" ]] || [[ -z "$USER" ]] || [[ -z "$USER" ]]; then + echo "You need input all info （ip, user,password）obout server config !!!" + help + exit 1 + fi + + local PASSWORD=${PASSWORD//!/\\!} + local PASSWORD=${PASSWORD//#/\\#} + local PASSWORD=${PASSWORD/\$/\\\$} + local SERVER_CONFIG_FILE=${PROJECT_HOME}/scripts/config/server_config.sh + [ -n "${SERVER_CONFIG_FILE}" ] && rm -rf "${SERVER_CONFIG_FILE}" + + cat>${SERVER_CONFIG_FILE}<<-EOF +SERVER_PATH=http://${IP}/package/etrans +DEP_USER=${USER} +DEP_PASSWORD=${PASSWORD} + +EOF + +} + + + +function parse_args(){ + parsed_args=$(getopt -a -o i::u::p::h --long ip::,user::,password::,help -- "$@") || { + help + exit 1 + } + + if [ $# -lt 1 ]; then + help + exit 1 + fi + local IP= + local USER= + local PASSWORD= + + eval set -- "$parsed_args" + while true; do + case "$1" in + -i | --ip) + IP=$2 + ;; + -u | --user) + USER=$2 + ;; + -p | --password) + PASSWORD=$2 + ;; + -h | --help) + help; exit; + ;; + --) + shift; break; + ;; + *) + help; exit 1 + ;; + esac + shift 2 + done + + write_config_file $IP $USER $PASSWORD +} + +function main(){ + parse_args "$@" +} + +main "$@" + +set +e \ No newline at end of file diff --git a/scripts/cov/ge_cov.sh b/scripts/cov/ge_cov.sh new file mode 100755 index 00000000..27c5b5f7 --- /dev/null +++ b/scripts/cov/ge_cov.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + + +function help(){ + cat <<-EOF +Usage: ge cov [OPTIONS] + +Options: + -a, --all Full coverage + -i, --increment Increment coverage + -d, --directory Coverage of directory + -h, --help +EOF + +} + +PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../} +PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd) + +ALL_COV_GEN_PATH=${PROJECT_HOME}/cov/all +DIFF_FILE_PATH=${PROJECT_HOME}/cov/diff +DIFF_FILE_NAME=${DIFF_FILE_PATH}/inc_change_diff.txt + +function process_diff_format(){ + sed -i "s/--- a/--- \/code\/Turing\/graphEngine/g" ${DIFF_FILE_NAME} + sed -i "s/+++ b/+++ \/code\/Turing\/graphEngine/g" ${DIFF_FILE_NAME} +} + + +function add_cov_generate(){ + addlcov --diff ${ALL_COV_GEN_PATH}/coverage.info ${DIFF_FILE_NAME} -o ${PROJECT_HOME}/cov/diff/inc_coverage.info +} + +function gen_add_cov_html(){ + genhtml --prefix ${PROJECT_HOME} -o ${PROJECT_HOME}/cov/diff/html ${PROJECT_HOME}/cov/diff/inc_coverage.info --legend -t CHG --no-branch-coverage --no-function-coverage +} + +function increment_cov_for_directory(){ + [ -n "${DIFF_FILE_PATH}" ] && rm -rf "${DIFF_FILE_PATH}" + mkdir -p ${DIFF_FILE_PATH} + git diff HEAD -- $1 >>${DIFF_FILE_NAME} + process_diff_format + add_cov_generate + gen_add_cov_html +} + +function run_all_coverage(){ + [ -n "${ALL_COV_GEN_PATH}" ] && rm -rf ${ALL_COV_GEN_PATH} + mkdir -p ${ALL_COV_GEN_PATH} + pushd "${PWD}" >/dev/null + cd ${PROJECT_HOME} + lcov -c -d build/tests/ut/ge -d build/tests/ut/common/graph/ -o ${ALL_COV_GEN_PATH}/tmp.info + lcov -r ${ALL_COV_GEN_PATH}/tmp.info '*/output/*' '*/build/opensrc/*' '*/build/proto/*' '*/third_party/*' '*/tests/*' '/usr/local/*' '/usr/include/*' '*/metadef/*' '*/parser/*' -o ${ALL_COV_GEN_PATH}/coverage.info + cd ${ALL_COV_GEN_PATH} + genhtml coverage.info + popd >/dev/null +} + +function do_coverage_run(){ + local cov_mode=$1 + local directory_dir=$2 + + run_all_coverage + + if [ "$cov_mode" = "all" ]; then + exit 1 + elif [ -n "$directory_dir" ]; then + increment_cov_for_directory $directory_dir + else + increment_cov_for_directory "ge" + fi +} + +function parse_args(){ + parsed_args=$(getopt -a -o aid::h --long all,increment,directory::,help -- "$@") || { + help + exit 1 + } + + if [ $# -lt 1 ]; then + run_all_coverage + exit 1 + fi + + local cov_mode="increment" + local directory_dir= + eval set -- "$parsed_args" + while true; do + case "$1" in + -a | --all) + cov_mode="all" + ;; + -i | --increment) + ;; + -d | --directory) + directory_dir=$2 + shift + ;; + -h | --help) + help; exit 1; + ;; + --) + shift; break; + ;; + *) + help; exit 1; + ;; + esac + shift + done + do_coverage_run $cov_mode $directory_dir +} + +function main(){ + parse_args "$@" +} + +main "$@" + +set +e \ No newline at end of file diff --git a/scripts/docs/Doxyfile_all b/scripts/docs/Doxyfile_all new file mode 100755 index 00000000..100f4e62 --- /dev/null +++ b/scripts/docs/Doxyfile_all @@ -0,0 +1,2639 @@ +# Doxyfile 1.9.1 + +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the configuration +# file that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# https://www.gnu.org/software/libiconv/ for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "graph engine" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = /code/Turing/graphEngine/docs/doxygen + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = YES + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all generated output in the proper direction. +# Possible values are: None, LTR, RTL and Context. +# The default value is: None. + +OUTPUT_TEXT_DIRECTION = None + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = YES + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line +# such as +# /*************** +# as being the beginning of a Javadoc-style comment "banner". If set to NO, the +# Javadoc-style will behave just like regular comments and it will not be +# interpreted by doxygen. +# The default value is: NO. + +JAVADOC_BANNER = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# By default Python docstrings are displayed as preformatted text and doxygen's +# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the +# doxygen's special commands can be used and the contents of the docstring +# documentation blocks is shown as doxygen documentation. +# The default value is: YES. + +PYTHON_DOCSTRING = YES + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines (in the resulting output). You can put ^^ in the value part of an +# alias to insert a newline as if a physical newline was in the original file. +# When you need a literal { or } or , in the value part of an alias you have to +# escape them by means of a backslash (\), this can lead to conflicts with the +# commands \{ and \} for these it is advised to use the version @{ and @} or use +# a double escape (\\{ and \\}) + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice +# sources only. Doxygen will then generate output that is more tailored for that +# language. For instance, namespaces will be presented as modules, types will be +# separated into more groups, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_SLICE = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, +# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, +# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser +# tries to guess whether the code is fixed or free formatted code, this is the +# default for Fortran type files). For instance to make doxygen treat .inc files +# as Fortran files (default is PHP), and .f files as C (default is Fortran), +# use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See https://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 5. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 5 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = YES + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use +# during processing. When set to 0 doxygen will based this on the number of +# cores available in the system. You can set it explicitly to a value larger +# than 0 to get more control over the balance between CPU load and processing +# speed. At this moment only the input processing can be done using multiple +# threads. Since this is still an experimental feature the default is set to 1, +# which efficively disables parallel processing. Please report any issues you +# encounter. Generating dot graphs in parallel is controlled by the +# DOT_NUM_THREADS setting. +# Minimum value: 0, maximum value: 32, default value: 1. + +NUM_PROC_THREADS = 1 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual +# methods of a class will be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIV_VIRTUAL = NO + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# declarations. If set to NO, these declarations will be included in the +# documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# With the correct setting of option CASE_SENSE_NAMES doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and MacOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. +# The default value is: system dependent. + +CASE_SENSE_NAMES = NO + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = YES + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = YES + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. If +# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the doxygen process doxygen will return with a non-zero status. +# Possible values are: NO, YES and FAIL_ON_WARNINGS. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT =/code/Turing/graphEngine/ge \ + /code/Turing/graphEngine/inc \ + /code/Turing/graphEngine/parser \ + /code/Turing/graphEngine/metadef \ + /code/Turing/graphEngine/third_party/inc + + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), +# *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, *.vhdl, +# *.ucf, *.qsf and *.ice. + +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.pyw \ + *.f90 \ + *.f95 \ + *.f03 \ + *.f08 \ + *.f18 \ + *.f \ + *.for \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf \ + *.ice + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = * + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# +# +# where is the value of the INPUT_FILTER tag, and is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# entity all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see https://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# https://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = NO + +# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML +# documentation will contain a main index with vertical navigation menus that +# are dynamically created via JavaScript. If disabled, the navigation index will +# consists of multiple levels of tabs that are statically embedded in every HTML +# page. Disable this option to support browsers that do not have JavaScript, +# like the Qt help browser. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_MENUS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy +# genXcode/_index.html for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: +# https://www.microsoft.com/en-us/download/details.aspx?id=21138) on Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the main .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to +# run qhelpgenerator on the generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg +# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see +# https://inkscape.org) to generate formulas as SVG images instead of PNGs for +# the HTML output. These images will generally look nicer at scaled resolutions. +# Possible values are: png (the default) and svg (looks nicer but requires the +# pdf2svg or inkscape tool). +# The default value is: png. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FORMULA_FORMAT = png + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANSPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands +# to create new LaTeX commands to be used in formulas as building blocks. See +# the section "Including formulas" for details. + +FORMULA_MACROFILE = + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# https://www.mathjax.org) which uses client side JavaScript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from https://www.mathjax.org before deployment. +# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = https://cdn.jsdelivr.net/npm/mathjax@2 + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use + S +# (what the is depends on the OS and browser, but it is typically +# , /