From fdf860087bbbb7d4e0741e3d6c11d2aa42135036 Mon Sep 17 00:00:00 2001 From: liudingyan Date: Thu, 6 May 2021 16:14:26 +0800 Subject: [PATCH] modify geloge info --- ge/graph/build/graph_builder.cc | 116 +-- ge/graph/build/label_allocator.cc | 12 +- ge/graph/build/logical_stream_allocator.cc | 22 +- ge/graph/build/memory/block_mem_assigner.cc | 32 +- ge/graph/build/memory/graph_mem_assigner.cc | 13 +- ge/graph/build/memory/hybrid_mem_assigner.cc | 8 +- ge/graph/build/memory/memory_assigner.cc | 17 +- ge/graph/build/memory/var_mem_assign_util.cc | 40 +- ge/graph/build/model_builder.cc | 156 ++-- ge/graph/build/run_context.cc | 40 +- ge/graph/build/stream_allocator.cc | 146 ++-- ge/graph/build/stream_graph_optimizer.cc | 12 +- ge/graph/build/task_generator.cc | 135 ++-- ge/graph/common/bcast.cc | 4 +- ge/graph/common/omg_util.cc | 32 +- ge/graph/execute/graph_execute.cc | 125 ++-- ge/graph/label/case_label_maker.cc | 22 +- ge/graph/label/if_label_maker.cc | 20 +- ge/graph/label/label_maker.cc | 38 +- .../label/partitioned_call_label_maker.cc | 6 +- ge/graph/label/while_label_maker.cc | 20 +- .../load/model_manager/cpu_queue_schedule.cc | 152 ++-- ge/graph/load/model_manager/data_dumper.cc | 143 ++-- ge/graph/load/model_manager/data_inputer.cc | 2 +- ge/graph/load/model_manager/davinci_model.cc | 678 ++++++++++-------- .../task_info/end_graph_task_info.cc | 13 +- .../task_info/event_record_task_info.cc | 9 +- .../task_info/event_wait_task_info.cc | 14 +- .../task_info/fusion_start_task_info.cc | 7 +- .../task_info/fusion_stop_task_info.cc | 4 +- .../model_manager/task_info/hccl_task_info.cc | 51 +- .../task_info/kernel_ex_task_info.cc | 114 ++- .../task_info/kernel_task_info.cc | 344 +++++---- .../task_info/label_goto_ex_task_info.cc | 19 +- .../task_info/label_set_task_info.cc | 15 +- .../label_switch_by_index_task_info.cc | 39 +- .../task_info/memcpy_addr_async_task_info.cc | 16 +- .../task_info/memcpy_async_task_info.cc | 14 +- .../task_info/model_exit_task_info.cc | 9 +- .../task_info/profiler_trace_task_info.cc | 8 +- .../task_info/stream_active_task_info.cc | 22 +- .../task_info/stream_switch_task_info.cc | 45 +- .../task_info/stream_switchn_task_info.cc | 48 +- .../task_info/super_kernel/super_kernel.cc | 12 +- .../super_kernel/super_kernel_factory.cc | 20 +- .../load/model_manager/task_info/task_info.cc | 2 +- 46 files changed, 1459 insertions(+), 1357 deletions(-) diff --git a/ge/graph/build/graph_builder.cc b/ge/graph/build/graph_builder.cc index bcd80b0c..8b172e63 100644 --- a/ge/graph/build/graph_builder.cc +++ b/ge/graph/build/graph_builder.cc @@ -79,7 +79,8 @@ Status HandleSubgraphDataNode(NodePtr &src_node, OutDataAnchorPtr &src_out_ancho if (!AttrUtils::GetInt(src_node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, index)) { REPORT_INNER_ERROR("E19999", "get attr:%s failed from node:%s", ATTR_NAME_PARENT_NODE_INDEX.c_str(), src_node->GetName().c_str()); - GELOGE(FAILED, "Get attr ATTR_NAME_PARENT_NODE_INDEX failed, node:%s.", src_node->GetName().c_str()); + GELOGE(FAILED, "[Get][Attr] %s failed, node:%s.", ATTR_NAME_PARENT_NODE_INDEX.c_str(), + src_node->GetName().c_str()); return FAILED; } const NodePtr &parent_node = src_node->GetOwnerComputeGraph()->GetParentNode(); @@ -113,7 +114,8 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) { if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { REPORT_INNER_ERROR("E19999", "check gelib instance null, graph:%s", graph->GetName().c_str()); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GraphBuilder: GE is not initialized"); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Check][GELib] GraphBuilder: GE is not initialized, graph:%s", + graph->GetName().c_str()); return GE_CLI_GE_NOT_INITIALIZED; } @@ -127,7 +129,7 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) { if (kernel_lib_name.empty()) { REPORT_INNER_ERROR("E19999", "op kernel lib is empty in node:%s(%s)", node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Get node:%s(%s) kernel lib failed.", node_ptr->GetName().c_str(), + GELOGE(INTERNAL_ERROR, "[Get][KernelLibName] of node:%s(%s) failed.", node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); return INTERNAL_ERROR; } @@ -137,7 +139,7 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) { if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Set node:%s(%s) inputDesc size failed", node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); - GELOGE(ret, "Set node inputDesc size failed, node name is %s", node_ptr->GetName().c_str()); + GELOGE(ret, "[Set][InputSize] to node:%s failed.", node_ptr->GetName().c_str()); return ret; } @@ -145,7 +147,7 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) { if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Call Calculate op:%s(%s) running param failed", node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); - GELOGE(ret, "Calculate op running param failed, node name is %s", node_ptr->GetName().c_str()); + GELOGE(ret, "[Call][Calculate] op running param failed, node name is %s", node_ptr->GetName().c_str()); return ret; } GE_CHK_STATUS_RET(AddOutputMemTypeForNode(node_ptr)); @@ -202,7 +204,7 @@ Status GraphBuilder::UpdateParentNodeOutputSize(const ge::ComputeGraphPtr &graph Status GraphBuilder::Build(ComputeGraphPtr &comp_graph, GeRootModelPtr &ge_root_model_ptr, uint64_t session_id) { if (comp_graph == nullptr) { REPORT_INNER_ERROR("E19999", "check compute_graph nullptr, session_id:%lu", session_id); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "Graph build comp_graph is null."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] comp_graph is null, session_id:%lu", session_id); return GE_GRAPH_PARAM_NULLPTR; } ge_root_model_ptr = MakeShared(comp_graph); @@ -216,12 +218,13 @@ Status GraphBuilder::Build(ComputeGraphPtr &comp_graph, GeRootModelPtr &ge_root_ if (is_dynamic_shape || comp_graph->GetGraphUnknownFlag()) { GE_CHK_STATUS_RET( BuildForDynamicShapeGraph(comp_graph, ge_root_model_ptr, ge_model_ptr, session_id), - "Build for dynamic shape graph failed."); + "[Build][DynamicShapeGraph] failed, graph:%s, session id:%lu.", comp_graph->GetName().c_str(), session_id); return SUCCESS; } GE_CHK_STATUS_RET(BuildForKnownShapeGraph(comp_graph, ge_model_ptr, session_id), - "Build for known shape graph failed."); + "[Build][KnownShapeGraph] failed, graph:%s, session id:%lu.", + comp_graph->GetName().c_str(), session_id); ge_root_model_ptr->SetSubgraphInstanceNameToModel(comp_graph->GetName(), ge_model_ptr); return SUCCESS; } @@ -229,28 +232,29 @@ Status GraphBuilder::Build(ComputeGraphPtr &comp_graph, GeRootModelPtr &ge_root_ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph, GeModelPtr &ge_model_ptr, uint64_t session_id) { if (ge::GetContext().GetHostExecFlag()) { - GE_CHK_STATUS_RET(BuildForHostCpuGraph(comp_graph, ge_model_ptr, session_id), "Build for host-cpu graph failed."); + GE_CHK_STATUS_RET(BuildForHostCpuGraph(comp_graph, ge_model_ptr, session_id), + "[Build][HostCpuGraph] failed, graph:%s, session id:%lu.", + comp_graph->GetName().c_str(), session_id); return SUCCESS; } ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kPreBuild); GELOGI("Begin to build known shape graph[%s].", comp_graph->GetName().c_str()); Status ret = SecondPartition(comp_graph); - GE_CHK_STATUS_RET(ret, "Graph[%s] second partition Failed.", comp_graph->GetName().c_str()); + GE_CHK_STATUS_RET(ret, "[Call][SecondPartition] for Graph[%s] failed.", comp_graph->GetName().c_str()); auto subgraph_map = graph_partitioner_.GetSubGraphMap(); GE_TIMESTAMP_START(BuildSubgraph); ge::ModelBuilder builder(session_id, comp_graph, subgraph_map, stream_max_parallel_num_, hcom_parallel_, build_mode_); GE_DUMP(comp_graph, "BeforePreBuildModel"); GE_TIMESTAMP_START(PreBuildModel); - GE_CHK_STATUS_RET(builder.PreBuildModel(), "Graph[%s] builder PreBuildModel() return fail.", + GE_CHK_STATUS_RET(builder.PreBuildModel(), "[PreBuild][Model] failed, Graph[%s].", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(PreBuildModel, "GraphBuilder::PreBuildModel"); GE_DUMP(comp_graph, "AfterPreBuildModel"); GE_TIMESTAMP_START(CalcOpParam); - GE_CHK_STATUS_RET(CalcOpParam(comp_graph), "Graph[%s] builder CalcOpParam() return fail.", - comp_graph->GetName().c_str()); + GE_CHK_STATUS_RET(CalcOpParam(comp_graph), "[Calc][OpParam] fail, Graph[%s].", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(CalcOpParam, "GraphBuilder::CalcOpParam"); GE_DUMP(comp_graph, "AfterCalcOpParam"); @@ -259,7 +263,7 @@ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph, return MEMALLOC_FAILED; } GE_TIMESTAMP_START(BuildModelForGetTask); - GE_CHK_STATUS_RET(builder.BuildModelForGetTask(*model_ptr), "Graph[%s] builder BuildModelForGetTask() return fail.", + GE_CHK_STATUS_RET(builder.BuildModelForGetTask(*model_ptr), "[Build][Model] ForGetTask fail, Graph[%s].", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(BuildModelForGetTask, "GraphBuilder::BuildModelForGetTask"); GE_DUMP(comp_graph, "AfterBuildModel"); @@ -270,7 +274,7 @@ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph, GE_TIMESTAMP_END(GetTaskInfo, "GraphBuilder::GetTaskInfo"); GE_DUMP(comp_graph, "AfterGetTask"); if (ret != SUCCESS) { - GELOGE(ret, "Graph[%s] builder GetTaskInfo() return fail.", comp_graph->GetName().c_str()); + GELOGE(ret, "[Get][TaskInfo] fail, Graph[%s].", comp_graph->GetName().c_str()); return ret; } @@ -280,7 +284,7 @@ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph, return MEMALLOC_FAILED; } GE_CHK_STATUS_RET(builder.SaveDataToModel(*model_ptr, *ge_model_ptr), - "Graph[%s] builder SaveDataToModel() return fail.", comp_graph->GetName().c_str()); + "[Save][Data] ToModel fail, Graph[%s].", comp_graph->GetName().c_str()); GELOGD("Success to build graph[%s] model.", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(BuildSubgraph, "GraphBuilder::Build"); return SUCCESS; @@ -315,7 +319,7 @@ Status GraphBuilder::SetConstantInputOffset(ComputeGraphPtr &comp_graph) { if (weights.empty()) { REPORT_INNER_ERROR("E19999", "check weights size of node %s(%s) is empty", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "weights size of node %s is empty", node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] weights size of node %s is empty", node->GetName().c_str()); return FAILED; } GeTensorPtr weight = weights[0]; @@ -342,23 +346,21 @@ Status GraphBuilder::BuildForUnknownShapeGraph(ComputeGraphPtr &comp_graph, GeMo ge::ModelBuilder builder(session_id, comp_graph, subgraph_map, stream_max_parallel_num_, hcom_parallel_, build_mode_); GE_DUMP(comp_graph, "BeforePreBuildModel"); GE_TIMESTAMP_START(PreBuildModel); - GE_CHK_STATUS_RET(builder.PreBuildModel(), "Graph[%s] builder PreBuildModel() return fail.", - comp_graph->GetName().c_str()); + GE_CHK_STATUS_RET(builder.PreBuildModel(), "[PreBuild][Model] fail, Graph[%s].", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(PreBuildModel, "GraphBuilder::PreBuildModel"); GE_DUMP(comp_graph, "AfterPreBuildModel"); GE_TIMESTAMP_START(CalcOpParam); - GE_CHK_STATUS_RET(CalcOpParam(comp_graph), "Graph[%s] builder CalcOpParam() return fail.", - comp_graph->GetName().c_str()); + GE_CHK_STATUS_RET(CalcOpParam(comp_graph), "[Calc][OpParam] fail, Graph[%s].", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(CalcOpParam, "GraphBuilder::CalcOpParam"); GE_DUMP(comp_graph, "AfterCalcOpParam"); GE_TIMESTAMP_START(SetConstantInputOffset); GE_CHK_STATUS_RET(SetConstantInputOffset(comp_graph), - "Graph[%s] failed to set constant input offset.", comp_graph->GetName().c_str()); + "[Set][Offset] Graph[%s] failed to set constant input offset.", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(SetConstantInputOffset, "GraphBuilder::SetConstantInputOffset"); GE_TIMESTAMP_START(MergeWeights); - GE_CHK_STATUS_RET(builder.MergeWeights(), "Graph[%s] failed to merge weights.", comp_graph->GetName().c_str()); + GE_CHK_STATUS_RET(builder.MergeWeights(), "[Merge][Weights] failed for Graph[%s].", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(MergeWeights, "GraphBuilder::MergeWeights"); ModelPtr model_ptr = MakeShared(); @@ -367,7 +369,7 @@ Status GraphBuilder::BuildForUnknownShapeGraph(ComputeGraphPtr &comp_graph, GeMo } GE_TIMESTAMP_START(BuildModelForGetDynShapeTask); GE_CHK_STATUS_RET(builder.BuildModelForGetDynShapeTask(*model_ptr), - "Graph[%s] builder BuildModelForGetDynShapeTask() return fail.", comp_graph->GetName().c_str()); + "[Build][Model] ForGetDynShapeTask fail, Graph[%s].", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(BuildModelForGetDynShapeTask, "GraphBuilder::BuildModelForGetDynShapeTask"); ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kTaskGenerate); GE_TIMESTAMP_START(GetTaskInfo); @@ -378,7 +380,7 @@ Status GraphBuilder::BuildForUnknownShapeGraph(ComputeGraphPtr &comp_graph, GeMo GraphUtils::DumpGEGraph(comp_graph, "AfterGetTask"); GraphUtils::DumpGEGraphToOnnx(*comp_graph, "AfterGetTask"); if (ret != SUCCESS) { - GELOGE(ret, "Graph[%s] builder GetTaskInfo() return fail.", comp_graph->GetName().c_str()); + GELOGE(ret, "[Get][TaskInfo] fail, Graph[%s].", comp_graph->GetName().c_str()); return ret; } ge_model_ptr = MakeShared(); @@ -386,7 +388,7 @@ Status GraphBuilder::BuildForUnknownShapeGraph(ComputeGraphPtr &comp_graph, GeMo return MEMALLOC_FAILED; } GE_CHK_STATUS_RET(builder.SaveDataToModel(*model_ptr, *ge_model_ptr), - "Graph[%s] builder SaveDataToModel() return fail.", comp_graph->GetName().c_str()); + "[Save][Data] ToModel fail, Graph[%s].", comp_graph->GetName().c_str()); GELOGD("Success to build graph[%s] model.", comp_graph->GetName().c_str()); return SUCCESS; } @@ -433,8 +435,9 @@ Status GraphBuilder::MarkFpBpProfilingTaskAttr(ComputeGraphPtr &com_graph) { GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep), REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id " "for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Multiply result is out of range."); - return FAILED); + GELOGE(FAILED, "[Check][Param] Multiply result is out of range, node:%s(%s)", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); + return FAILED); int64_t log_id = i * kProfilingArStep + kProfilingArStartLogid; (void)ge::AttrUtils::SetInt(op_desc, ATTR_NAME_INSERT_PROFILILNG_TASK_LOG_ID, log_id); continue; @@ -458,14 +461,14 @@ Status GraphBuilder::BuildForDynamicShapeGraph(ComputeGraphPtr &comp_graph, GE_CHECK_NOTNULL(op_desc); op_desc->SetStreamId(kInvalidStream); if (node->GetType() == DATA) { - GE_CHK_STATUS_RET(CalcDynShapeRootGraphDataSize(op_desc), "Calc dynamic shape root graph data[%s] size failed.", + GE_CHK_STATUS_RET(CalcDynShapeRootGraphDataSize(op_desc), "[Calc][DynShapeRootGraphDataSize] failed, op:%s.", op_desc->GetName().c_str()); } } // Set fp bp profiling task attr for graph if (MarkFpBpProfilingTaskAttr(comp_graph) != SUCCESS) { - GELOGE(FAILED, "Set fp bp profiling task attr for graph."); + GELOGE(FAILED, "[Mark][TaskAttr]Set fp bp profiling task attr for graph:%s failed.", comp_graph->GetName().c_str()); return FAILED; } @@ -482,18 +485,20 @@ Status GraphBuilder::BuildForDynamicShapeGraph(ComputeGraphPtr &comp_graph, if (sub_graph->GetGraphUnknownFlag()) { // unknown shape build flow GE_CHK_STATUS_RET(BuildForUnknownShapeGraph(sub_graph, ge_model_ptr, session_id), - "Build for unknown shape graph failed."); + "[Build][Graph] as unknown shape failed, session id:%lu.", session_id); } else { // reset functional subgraph parent graph as known subgraph for (const auto &node : sub_graph->GetDirectNode()) { for (const auto &sub_graph_name : node->GetOpDesc()->GetSubgraphInstanceNames()) { auto sub_sub_graph = comp_graph->GetSubgraph(sub_graph_name); - GE_CHK_STATUS_RET(sub_graph->AddSubgraph(sub_sub_graph), "Failed add subgraph to known graph."); + GE_CHK_STATUS_RET(sub_graph->AddSubgraph(sub_sub_graph), + "[Add][SubGraph] %s to known graph:%s failed.", sub_sub_graph->GetName().c_str(), + sub_graph->GetName().c_str()); } } // known shape build flow GE_CHK_STATUS_RET(BuildForKnownShapeGraph(sub_graph, ge_model_ptr, session_id), - "Build for known shape graph failed."); + "[Build][Graph] for known shape failed, session id:%lu.", session_id); } ge_root_model_ptr->SetSubgraphInstanceNameToModel(sub_graph->GetName(), ge_model_ptr); } @@ -510,19 +515,20 @@ Status GraphBuilder::GetTaskInfo(const ge::ModelBuilder &builder, const ModelPtr int64_t memory_size = 0; if (!AttrUtils::GetInt(model_ptr, ATTR_MODEL_MEMORY_SIZE, memory_size)) { REPORT_INNER_ERROR("E19999", "Get Attr:%s fail in model", ATTR_MODEL_MEMORY_SIZE.c_str()); - GELOGE(INTERNAL_ERROR, "Get memory size fail."); + GELOGE(INTERNAL_ERROR, "[Get][Attr] memory size fail, graph:%s, session id:%lu.", comp_graph->GetName().c_str(), + session_id); return INTERNAL_ERROR; } int64_t p2p_memory_size = 0; if (!AttrUtils::GetInt(model_ptr, ATTR_MODEL_P2P_MEMORY_SIZE, p2p_memory_size)) { REPORT_INNER_ERROR("E19999", "Get Attr:%s fail in model", ATTR_MODEL_P2P_MEMORY_SIZE.c_str()); - GELOGE(INTERNAL_ERROR, "Get p2p memory size fail."); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s fail in model", ATTR_MODEL_P2P_MEMORY_SIZE.c_str()); return INTERNAL_ERROR; } int64_t weight_size = 0; if (!AttrUtils::GetInt(model_ptr, ATTR_MODEL_WEIGHT_SIZE, weight_size)) { REPORT_INNER_ERROR("E19999", "Get Attr:%s fail in model", ATTR_MODEL_WEIGHT_SIZE.c_str()); - GELOGE(INTERNAL_ERROR, "Get weight memory size fail."); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s fail in model", ATTR_MODEL_WEIGHT_SIZE.c_str()); return INTERNAL_ERROR; } @@ -548,20 +554,20 @@ Status GraphBuilder::GetTaskInfo(const ge::ModelBuilder &builder, const ModelPtr Status ret = run_context.InitMemInfo(get_mem_base, memory_size, mem_type_to_data_mem_base, mem_type_to_data_mem_size, get_weight_mem_base, weight_size); if (ret != SUCCESS) { - GELOGE(ret, "task_generator init mem info fail."); + GELOGE(ret, "[Init][MemInfo] fail, ret:%d.", ret); return ret; } auto weight_buffer = builder.GetWeightBuffer(); ret = run_context.CreateRunContext(*model_ptr, comp_graph, weight_buffer, session_id); if (ret != SUCCESS) { - GELOGE(ret, "runContext create run context fail."); + GELOGE(ret, "[Create][RunContext] fail, ret:%d, graph:%s.", ret, comp_graph->GetName().c_str()); return ret; } StreamGraphOptimizer stream_optimizer; ret = stream_optimizer.OptimizeStreamedSubGraph(comp_graph, subgraph_map, run_context.GetRunContext()); if (ret != SUCCESS) { - GELOGE(ret, "Optimize streamed subGraph fail."); + GELOGE(ret, "[Optimize][StreamedSubGraph] fail, graph:%s.", comp_graph->GetName().c_str()); return ret; } GE_DUMP(comp_graph, "AfterOptimizeStreamedSubGraph"); @@ -578,13 +584,13 @@ Status GraphBuilder::SetInputSize(const ge::NodePtr &node_ptr) { if (node_ptr->GetType() == DATA) { bool is_unknown_shape = false; GE_CHK_STATUS_RET(ge::NodeUtils::GetNodeUnknownShapeStatus(*node_ptr, is_unknown_shape), - "Get data node[%s] shape status failed!", node_ptr->GetName().c_str()); + "[Get][Status] of data node[%s] shape failed!", node_ptr->GetName().c_str()); if (is_unknown_shape) { GELOGD("data node: %s is unknown shape, do not set input size!", node_ptr->GetName().c_str()); return SUCCESS; } if (UpdateDataInputSize(node_ptr) != SUCCESS) { - GELOGE(FAILED, "Update data input size failed."); + GELOGE(FAILED, "[Update][Data] input size failed, node:%s.", node_ptr->GetName().c_str()); return FAILED; } } @@ -632,7 +638,7 @@ Status GraphBuilder::UpdateDataInputSize(const ge::NodePtr &node_ptr) { const auto &op_desc = node_ptr->GetOpDesc(); if (op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "check op_desc is nullptr"); - GELOGE(FAILED, "Op desc is nullptr."); + GELOGE(FAILED, "[Check][Param] Op desc is nullptr."); return FAILED; } // data op only has one output anchor @@ -651,7 +657,7 @@ Status GraphBuilder::UpdateDataInputSize(const ge::NodePtr &node_ptr) { if (graph_status != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Get tensor size in bytes failed for op:%s(%s) index:0", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Get tensor size in bytes failed."); + GELOGE(FAILED, "[Get][TensorSize] in bytes failed, op:%s.", op_desc->GetName().c_str()); return FAILED; } // data op only has one input anchor @@ -660,7 +666,7 @@ Status GraphBuilder::UpdateDataInputSize(const ge::NodePtr &node_ptr) { if (op_desc->UpdateInputDesc(0, input_desc) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update input desc size failed for op:%s(%s) index:0", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Update input desc size failed."); + GELOGE(FAILED, "[Update][InputDesc] failed, op:%s.", op_desc->GetName().c_str()); return FAILED; } } @@ -690,7 +696,7 @@ Status GraphBuilder::CalcDynShapeRootGraphDataSize(const ge::OpDescPtr &op_desc) if (graph_status != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Get tensor size in bytes failed for op:%s(%s) index:0 ", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Get tensor size in bytes failed."); + GELOGE(FAILED, "[Get][TensorSize] in bytes failed, op:%s.", op_desc->GetName().c_str()); return FAILED; } @@ -699,7 +705,7 @@ Status GraphBuilder::CalcDynShapeRootGraphDataSize(const ge::OpDescPtr &op_desc) if (op_desc->UpdateOutputDesc(0, output_desc) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update output desc size failed for op:%s(%s) index:0 ", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Update dynamic shape graph data output desc size failed."); + GELOGE(FAILED, "[Update][OutputDesc] for dynamic shape graph data failed, op:%s.", op_desc->GetName().c_str()); return FAILED; } } @@ -710,15 +716,13 @@ Status GraphBuilder::SecondPartition(ge::ComputeGraphPtr &comp_graph) { GE_TIMESTAMP_START(GraphPartition2); auto ret = graph_partitioner_.Partition(comp_graph, GraphPartitioner::kSecondPartitioning); if (ret != SUCCESS) { - GELOGE(ret, "Graph partition Failed"); + GELOGE(ret, "[Call][Partition] for Graph Failed"); return ret; } - GE_CHK_STATUS_RET(ret, "Graph partition Failed."); const auto &graph_2_subgraphlist = graph_partitioner_.GetSubGraphMap(); if (graph_2_subgraphlist.find(comp_graph) == graph_2_subgraphlist.end()) { - REPORT_INNER_ERROR("E19999", "find subgraphlis in graph:%s failed", - comp_graph->GetName().c_str()); - GELOGE(FAILED, "Find subgraph failed."); + REPORT_INNER_ERROR("E19999", "find subgraphlis in graph:%s failed", comp_graph->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Find subgraph graph:%s failed.", comp_graph->GetName().c_str()); return FAILED; } GE_TIMESTAMP_END(GraphPartition2, "GraphPartitioner::Partition2"); @@ -749,18 +753,18 @@ Status GraphBuilder::AddOutputMemTypeForNode(const NodePtr &node) { REPORT_INNER_ERROR("E19999", "Set Attr:%s for node:%s(%s) out_index:%u failed", ATTR_OUTPUT_MEMORY_TYPE.c_str(), src_desc->GetName().c_str(), src_desc->GetType().c_str(), src_out_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "Set out_memory_type attr for [%s:%d] failed.", src_desc->GetName().c_str(), + GELOGE(INTERNAL_ERROR, "[Set][Attr] out_memory_type for [%s:%d] failed.", src_desc->GetName().c_str(), src_out_anchor->GetIdx()); return INTERNAL_ERROR; } switch (TransferNodeType(src_node)) { case kSubgraphNode: - GE_CHK_STATUS_RET(HandleSubgraphNode(src_node, src_out_anchor), "Handle subgraph node %s failed", - src_node->GetName().c_str()); + GE_CHK_STATUS_RET(HandleSubgraphNode(src_node, src_out_anchor), + "[Handle][Node] %s in subgraph failed", src_node->GetName().c_str()); break; case kSubgraphData: - GE_CHK_STATUS_RET(HandleSubgraphDataNode(src_node, src_out_anchor), "Handle Data node %s in subgraph failed", - src_node->GetName().c_str()); + GE_CHK_STATUS_RET(HandleSubgraphDataNode(src_node, src_out_anchor), + "[Handle][DataNode] %s in subgraph failed", src_node->GetName().c_str()); break; case kOthers: default: diff --git a/ge/graph/build/label_allocator.cc b/ge/graph/build/label_allocator.cc index b6ef8dc9..32bdd0a3 100644 --- a/ge/graph/build/label_allocator.cc +++ b/ge/graph/build/label_allocator.cc @@ -29,7 +29,7 @@ LabelAllocator::LabelAllocator(const ComputeGraphPtr &graph) : compute_graph_(gr Status LabelAllocator::AssignFunctionalLabels() { if (compute_graph_ == nullptr) { REPORT_INNER_ERROR("E19999", "check param compute_graph nullptr"); - GELOGE(INTERNAL_ERROR, "ComputeGraph not set, Assign labels failed."); + GELOGE(INTERNAL_ERROR, "[Check][Param] ComputeGraph not set, Assign labels failed."); return INTERNAL_ERROR; } @@ -49,14 +49,14 @@ Status LabelAllocator::AssignFunctionalLabels() { if (maker == nullptr) { REPORT_CALL_ERROR("E19999", "Check Node:%s(%s) label maker not registed", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Node: %s label maker not registed.", node->GetType().c_str()); + GELOGE(INTERNAL_ERROR, "[Create][LabelMaker] Node: %s label maker not registed.", node->GetType().c_str()); return INTERNAL_ERROR; } if (maker->Run(label_index) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Node:%s(%s) run label maker failed", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Node: %s run label maker failed.", node->GetType().c_str()); + GELOGE(INTERNAL_ERROR, "[Call][Run] Node: %s run label maker failed.", node->GetType().c_str()); return INTERNAL_ERROR; } } @@ -69,7 +69,7 @@ Status LabelAllocator::AssignFunctionalLabels() { bool LabelAllocator::CollectFunctionalNode(ComputeGraphPtr &graph, std::set &functional_nodes) { if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "check param compute_graph nullptr"); - GELOGE(INTERNAL_ERROR, "Sub ComputeGraph is null."); + GELOGE(INTERNAL_ERROR, "[Check][Param] Sub ComputeGraph is null."); return false; } @@ -82,7 +82,7 @@ bool LabelAllocator::CollectFunctionalNode(ComputeGraphPtr &graph, std::setGetName().c_str(), func_node->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Parent functional node not set: %s.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Get][Node] Parent functional node not set: %s.", graph->GetName().c_str()); return false; } @@ -90,7 +90,7 @@ bool LabelAllocator::CollectFunctionalNode(ComputeGraphPtr &graph, std::setGetName().c_str(), func_node->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "ComputeGraph owner not set: %s.", func_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Get][Graph] ComputeGraph owner not set: %s.", func_node->GetName().c_str()); return false; } diff --git a/ge/graph/build/logical_stream_allocator.cc b/ge/graph/build/logical_stream_allocator.cc index 88b4a97f..c74cdf7a 100644 --- a/ge/graph/build/logical_stream_allocator.cc +++ b/ge/graph/build/logical_stream_allocator.cc @@ -322,7 +322,7 @@ Status SingleStreamPass::Run(ComputeGraphPtr graph, const vector &s if (!stream_label.empty()) { REPORT_INNER_ERROR("E19999", "Stream labels are not supported in SingleStream mode " "(subgraph: %s, stream label: %s)", subgraph->name.c_str(), stream_label.c_str()); - GELOGE(INTERNAL_ERROR, "Stream labels are not supported (subgraph: %s, stream label: %s).", + GELOGE(INTERNAL_ERROR, "[Get][Label] Stream labels are not supported (subgraph: %s, stream label: %s).", subgraph->name.c_str(), stream_label.c_str()); return INTERNAL_ERROR; } @@ -341,8 +341,8 @@ Status NodeStreamUpdatePass::Run(ComputeGraphPtr graph, const vectorname.c_str(), engine_name.c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph %s has not yet been assigned a stream (engine: %s).", subgraph->name.c_str(), - engine_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Subgraph %s has not yet been assigned a stream (engine: %s).", + subgraph->name.c_str(), engine_name.c_str()); return INTERNAL_ERROR; } else { GELOGI("[Assign][StreamId] %ld for Subgraph %s (engine: %s).", subgraph->stream_id, subgraph->name.c_str(), @@ -402,7 +402,7 @@ Status UpdateForParallelGroupPass::Run(ComputeGraphPtr graph, const vectorGetName().c_str()); + GELOGE(FAILED, "[Get][Attr] ATTR_NAME_PARALLEL_GROUP of node %s failed.", op_desc->GetName().c_str()); REPORT_INNER_ERROR("E19999", "Get node %s ATTR_NAME_PARALLEL_GROUP failed.", op_desc->GetName().c_str()); return FAILED; } @@ -606,7 +606,7 @@ Status LogicalStreamAllocator::Assign(const ComputeGraphPtr &root_graph, const G Status status = DoAssign(root_graph, subgraph_map, engine_confs); if (status != SUCCESS) { - GELOGE(status, "Assign streams failed."); + GELOGE(status, "[Assign][Streams] failed, graph:%s.", root_graph->GetName().c_str()); return status; } @@ -614,7 +614,7 @@ Status LogicalStreamAllocator::Assign(const ComputeGraphPtr &root_graph, const G for (const ComputeGraphPtr &subgraph : subgraphs) { Status status = DoAssign(subgraph, subgraph_map, engine_confs); if (status != SUCCESS) { - GELOGE(status, "Assign streams failed."); + GELOGE(status, "[Assign][Streams] failed, graph:%s.", subgraph->GetName().c_str()); return status; } } @@ -642,7 +642,7 @@ Status LogicalStreamAllocator::DoAssign(const ComputeGraphPtr &graph, const Grap if (iter == subgraph_map.end()) { REPORT_INNER_ERROR("E19999", "Graph %s not found in subgraph_map when do logical stream assign ", graph->GetName().c_str()); - GELOGE(FAILED, "Graph %s not found.", graph->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Graph %s not found.", graph->GetName().c_str()); return FAILED; } @@ -652,7 +652,7 @@ Status LogicalStreamAllocator::DoAssign(const ComputeGraphPtr &graph, const Grap Status status = ConvertSubgraphs(subgraph_info_list, engine_confs, subgraphs); GE_TIMESTAMP_END(ConvertSubgraphs, "GraphBuilder::AssignStreamConvertSubgraphs"); if (status != SUCCESS) { - GELOGE(status, "Create subgraphs failed."); + GELOGE(status, "[Convert][SubGraphs] failed."); return status; } @@ -683,8 +683,8 @@ Status LogicalStreamAllocator::ConvertSubgraphs(const vector &s if ((engine_conf_iter == engine_confs.end()) || (engine_conf_iter->second == nullptr)) { REPORT_INNER_ERROR("E19999", "Engine conf of subgraph %s not found (engine name: %s)", subgraph_name.c_str(), engine_name.c_str()); - GELOGE(INTERNAL_ERROR, "Engine conf of subgraph %s not found (engine name: %s).", subgraph_name.c_str(), - engine_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Engine conf of subgraph %s not found (engine name: %s).", + subgraph_name.c_str(), engine_name.c_str()); return INTERNAL_ERROR; } @@ -731,7 +731,7 @@ Status LogicalStreamAllocator::RunPasses(const ComputeGraphPtr &graph, const vec GELOGD("[Show][Status]Stream pass %s return NOT_CHANGED.", pass->GetName().c_str()); } else { REPORT_CALL_ERROR("E19999", "Stream pass %s run failed.", pass->GetName().c_str()); - GELOGE(status, "Stream pass %s failed.", pass->GetName().c_str()); + GELOGE(status, "[Call][Run] Stream pass %s failed.", pass->GetName().c_str()); return status; } } diff --git a/ge/graph/build/memory/block_mem_assigner.cc b/ge/graph/build/memory/block_mem_assigner.cc index 6f255b59..2756c6fe 100755 --- a/ge/graph/build/memory/block_mem_assigner.cc +++ b/ge/graph/build/memory/block_mem_assigner.cc @@ -798,7 +798,8 @@ bool IsContinuousInputNodeMaxLife(const NodePtr &n, uint32_t out_index) { } auto peer_in_node_desc = peer_in_anchor->GetOwnerNode()->GetOpDesc(); GE_IF_BOOL_EXEC(peer_in_node_desc == nullptr, - GELOGE(FAILED, "Node[%s] output[%u] peer in node desc is null.", n->GetName().c_str(), out_index); + GELOGE(FAILED, "[Get][OpDesc] Node[%s] output[%u] peer in node desc is null.", + n->GetName().c_str(), out_index); return false;); if(peer_in_node_desc->GetId() > max_node_life_time) { @@ -1074,7 +1075,8 @@ void BlockMemAssigner::UpdateOpTensorMemType(std::list node_index_i bool BlockMemAssigner::IsContinuousOutput(const NodePtr &n) { if (n == nullptr) { - GELOGE(FAILED, "Node is null."); + REPORT_INNER_ERROR("E19999", "param n is nullptr, check invalid."); + GELOGE(FAILED, "[Check][Param] Node is null."); return false; } @@ -1082,7 +1084,8 @@ bool BlockMemAssigner::IsContinuousOutput(const NodePtr &n) { bool is_output_continuous = false; auto node_desc = n->GetOpDesc(); if (node_desc == nullptr) { - GELOGE(FAILED, "Node[%s] nodedesc is null.", n->GetName().c_str()); + REPORT_INNER_ERROR("E19999", "param node:%s opdesc is nullptr, check invalid.", n->GetName().c_str()); + GELOGE(FAILED, "[Get][OpDesc] Node[%s] nodedesc is null.", n->GetName().c_str()); return false; } @@ -1450,7 +1453,9 @@ MemoryBlock *BlockMemAssigner::ApplyOutMemory(const NodePtr &n, uint32_t index, auto op_desc = owner_node->GetOpDesc(); GE_IF_BOOL_EXEC(op_desc == nullptr, continue); Params *instance = Params::Instance(); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(instance == nullptr, return nullptr, "Params instance is nullptr."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(instance == nullptr, + REPORT_INNER_ERROR("E19999", "Params instance is nullptr."); + return nullptr, "[Get][Instance] Params instance is nullptr."); if (!((instance->GetTarget() == TARGET_TYPE_TINY) && (op_desc->GetType() == NETOUTPUT))) { out_count++; } @@ -1462,7 +1467,9 @@ MemoryBlock *BlockMemAssigner::ApplyOutMemory(const NodePtr &n, uint32_t index, bool IsOutputBlock(const ge::InDataAnchorPtr &in_data_anchor) { auto peer_out_anchor = in_data_anchor->GetPeerOutAnchor(); - GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, GELOGE(FAILED, "Peer out anchor is nullptr."); return false); + GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, + REPORT_INNER_ERROR("E19999", "Peer out anchor is nullptr."); + GELOGE(FAILED, "[Check][Param] Peer out anchor is nullptr."); return false); auto src = peer_out_anchor->GetOwnerNode(); int32_t index = peer_out_anchor->GetIdx(); auto iter = GetLocalOmgContext().out_nodes_map.find(src->GetName()); @@ -1511,8 +1518,11 @@ bool IsKnownSubgraphData(const NodePtr &node) { void BlockMemAssigner::ReleaseMemory(MemoryBlock *to_release, vector &reusable_memory, bool same_stream) { - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(to_release == nullptr, return, "Input parameter to_release is null."); - GE_CHK_TRUE_EXEC_INFO(to_release->ref_count_ <= 0, return, "Release memory"); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(to_release == nullptr, + return, "[Check][Param] Input parameter to_release is null."); + GE_CHK_TRUE_EXEC_INFO(to_release->ref_count_ <= 0, + return, "[Check][Param] to_release->ref_count_ must greater than 0"); + GE_CHK_TRUE_EXEC_INFO(!to_release->reuse_mem_, return, "[Check][Param] doesn't reuse memory"); --to_release->ref_count_; GE_CHK_TRUE_EXEC_INFO(!to_release->reuse_mem_, return, "doesn't reuse memory"); if (!same_stream) { @@ -1613,13 +1623,13 @@ void CheckAndGetOpReuseEnv(const string &env, vector &env_vec, bool &op_ string env_str; env_str = string(env); if (env_str.size() > kReuseMaxCharNum) { - GELOGE(FAILED, "The OP_NO_REUSE_MEM has more than %d characters.", kReuseMaxCharNum); + GELOGE(FAILED, "[Check][Param] The OP_NO_REUSE_MEM has more than %d characters.", kReuseMaxCharNum); return; } SplitStringByComma(env_str, env_vec); if (env_vec.size() > kReuseMaxOpNum) { - GELOGE(FAILED, "The OP_NO_REUSE_MEM has more than %d nodes.", kReuseMaxOpNum); + GELOGE(FAILED, "[Check][Param] The OP_NO_REUSE_MEM has more than %d nodes.", kReuseMaxOpNum); return; } @@ -1860,7 +1870,9 @@ void BlockMemAssigner::CheckWorkspaceReuse(const vector &workspace_reuse_f void BlockMemAssigner::GetNodeWorkSpaceSize(const NodePtr &node, vector &workspace_memory, int64_t &total_size) { - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(node->GetOpDesc() == nullptr, return, "Op desc is null."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(node->GetOpDesc() == nullptr, + REPORT_INNER_ERROR("E19999", "param node opdesc is nullptr, check invalid."); + return, "[Check][Param] Op desc is null."); vector workspace_byte_nums = node->GetOpDesc()->GetWorkspaceBytes(); GELOGD("node[%s] size:%zu", node->GetOpDesc()->GetName().c_str(), workspace_byte_nums.size()); diff --git a/ge/graph/build/memory/graph_mem_assigner.cc b/ge/graph/build/memory/graph_mem_assigner.cc index 2f339bde..a45fb239 100755 --- a/ge/graph/build/memory/graph_mem_assigner.cc +++ b/ge/graph/build/memory/graph_mem_assigner.cc @@ -233,9 +233,12 @@ Status GraphMemoryAssigner::ReAssignMemory(bool is_loop_graph, mapGetName().c_str()); + GE_CHK_STATUS_RET(ReAssignAtomicMemory(is_loop_graph), + "[ReAssign][AtomicMemory] Failed! graph:%s", compute_graph_->GetName().c_str()); + GE_CHK_STATUS_RET(AssignBufferPoolMemory(), + "[Assign][BufferPoolMemory] Failed! graph:%s", compute_graph_->GetName().c_str()); size_t total_mem_offset = 0; for (auto pair : memory_offset_) { @@ -1009,7 +1012,9 @@ Status GraphMemoryAssigner::AssignReferenceMemory() { node->GetName().c_str()); auto out_op_desc = node->GetOpDesc(); - GE_IF_BOOL_EXEC(out_op_desc == nullptr, GELOGE(ge::FAILED, "out_op_desc is null."); return ge::FAILED); + GE_IF_BOOL_EXEC(out_op_desc == nullptr, + REPORT_INNER_ERROR("E19999", "out_op_desc is null."); + GELOGE(ge::FAILED, "[Check][Param] out_op_desc is null."); return ge::FAILED); vector output_list = out_op_desc->GetOutputOffset(); if (out_op_desc->GetOutputsSize() > output_list.size()) { diff --git a/ge/graph/build/memory/hybrid_mem_assigner.cc b/ge/graph/build/memory/hybrid_mem_assigner.cc index 4ea52d9d..eff821bf 100755 --- a/ge/graph/build/memory/hybrid_mem_assigner.cc +++ b/ge/graph/build/memory/hybrid_mem_assigner.cc @@ -29,7 +29,7 @@ Status HybridMemAssigner::AssignMemory(std::unique_ptr &block_ vector ranges; GE_CHECK_NOTNULL(block_assigner); if (block_assigner->GetMemoryRanges(ranges) != SUCCESS) { - GELOGE(FAILED, "GetMemoryRanges Fail!"); + GELOGE(FAILED, "[Get][MemoryRanges] Fail!"); return FAILED; } GE_IF_BOOL_EXEC(ranges.empty(), return SUCCESS); @@ -43,7 +43,7 @@ Status HybridMemAssigner::AssignMemory(std::unique_ptr &block_ Status HybridMemAssigner::Assign() { if (GraphUtils::GetRefMapping(compute_graph_, symbol_to_anchors_, anchor_to_symbol_) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Get ref-mapping for graph %s failed", compute_graph_->GetName().c_str()); - GELOGE(FAILED, "Get ref-mapping for graph %s failed.", compute_graph_->GetName().c_str()); + GELOGE(FAILED, "[Get][RefMapping] for graph %s failed.", compute_graph_->GetName().c_str()); return FAILED; } @@ -58,8 +58,8 @@ Status HybridMemAssigner::Assign() { size_t bin_mem_size = 0; size_t max_mem_size = 0; - GE_CHK_STATUS_RET(AssignMemory(binary_assigner, bin_mem_size), "BinaryBlock Method AssignMemory Fail!"); - GE_CHK_STATUS_RET(AssignMemory(max_assigner, max_mem_size), "MaxBlock Method AssignMemory Fail!"); + GE_CHK_STATUS_RET(AssignMemory(binary_assigner, bin_mem_size), "[Assign][Memory] Fail!"); + GE_CHK_STATUS_RET(AssignMemory(max_assigner, max_mem_size), "[Assign][Memory] Fail!"); std::unique_ptr priority_assigner; diff --git a/ge/graph/build/memory/memory_assigner.cc b/ge/graph/build/memory/memory_assigner.cc index d074fce0..34b97c60 100755 --- a/ge/graph/build/memory/memory_assigner.cc +++ b/ge/graph/build/memory/memory_assigner.cc @@ -24,45 +24,46 @@ Status MemoryAssigner::AssignMemory(bool is_loop_graph, map &me GraphMemoryAssigner graph_mem_assigner(compute_graph_); if (graph_mem_assigner.AssignMemory() != ge::SUCCESS) { - GELOGE(ge::FAILED, "Memory assigner failed"); + GELOGE(ge::FAILED, "[Assign][Memory] failed, graph:%s", compute_graph_->GetName().c_str()); return ge::FAILED; } // Reassign memory for special nodes if (graph_mem_assigner.ReAssignMemory(is_loop_graph, mem_offset) != ge::SUCCESS) { - GELOGE(ge::FAILED, "Memory assigner failed"); + GELOGE(ge::FAILED, "[ReAssign][Memory] failed, graph:%s", compute_graph_->GetName().c_str()); return ge::FAILED; } // Assign memory (block and offset) for zero copy nodes if (graph_mem_assigner.AssignZeroCopyMemory(mem_offset, zero_copy_mem_size) != ge::SUCCESS) { - GELOGE(ge::FAILED, "Zero copy memory assigner failed"); + GELOGE(ge::FAILED, "[Assign][ZeroCopyMemory] failed, graph:%s", compute_graph_->GetName().c_str()); return ge::FAILED; } if (graph_mem_assigner.AssignMemory2HasRefAttrNode() != ge::SUCCESS) { - GELOGE(ge::FAILED, "Assign memory to node which has ref attr failed!"); + GELOGE(ge::FAILED, "[Assign][Memory] to node which has ref attr failed! graph:%s", + compute_graph_->GetName().c_str()); return ge::FAILED; } // Assign memory for reference if (graph_mem_assigner.AssignReferenceMemory() != ge::SUCCESS) { - GELOGE(ge::FAILED, "Assign reference memory failed!"); + GELOGE(ge::FAILED, "[Assign][ReferenceMemory] failed! graph:%s", compute_graph_->GetName().c_str()); return ge::FAILED; } // Must do variable attr assign after all the memory assigned if (graph_mem_assigner.AssignVarAttr2Nodes() != SUCCESS) { - GELOGE(FAILED, "Variable Memory assigner failed"); + GELOGE(FAILED, "[Variable][Memory] assigner failed, graph:%s", compute_graph_->GetName().c_str()); return FAILED; } if (graph_mem_assigner.SetInputOffset() != ge::SUCCESS) { - GELOGE(ge::FAILED, "SetInputOffset Fail!"); + GELOGE(ge::FAILED, "[Set][InputOffset] Fail! graph:%s", compute_graph_->GetName().c_str()); return ge::FAILED; } if (graph_mem_assigner.CheckOffset() != SUCCESS) { - GELOGE(FAILED, "CheckOffset Fail!"); + GELOGE(FAILED, "[Check][Offset] Fail! graph:%s", compute_graph_->GetName().c_str()); return FAILED; } diff --git a/ge/graph/build/memory/var_mem_assign_util.cc b/ge/graph/build/memory/var_mem_assign_util.cc index 4cbde2af..b8138a30 100755 --- a/ge/graph/build/memory/var_mem_assign_util.cc +++ b/ge/graph/build/memory/var_mem_assign_util.cc @@ -53,9 +53,8 @@ Status VarMemAssignUtil::AssignStaticMemory2Node(ge::ComputeGraphPtr &compute_gr GE_IF_BOOL_EXEC(ge::AttrUtils::GetStr(n->GetOpDesc(), REF_VAR_SRC_VAR_NAME, ref_var_src_var_name), continue); string node_name = n->GetName(); GE_IF_BOOL_EXEC(n->GetOpDesc()->GetAllOutputsDesc().empty(), - REPORT_INNER_ERROR("E19999", "check node:%s has no OutputDesc", - n->GetName().c_str()); - GELOGE(FAILED, "node:%s has no OutputDesc.", n->GetName().c_str()); + REPORT_INNER_ERROR("E19999", "check node:%s has no OutputDesc", n->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] node:%s has no OutputDesc.", n->GetName().c_str()); return FAILED); ge::ConstGeTensorDescPtr tensor_desc = n->GetOpDesc()->GetOutputDescPtr(0); GE_CHECK_NOTNULL(tensor_desc); @@ -118,9 +117,8 @@ Status VarMemAssignUtil::SetOutVariableAttr(const ge::NodePtr &node, const ge::N GE_CHECK_NOTNULL(node->GetOpDesc()); output_list = node->GetOpDesc()->GetOutputOffset(); if (output_list.empty()) { - REPORT_INNER_ERROR("E19999", "check node:%s output_offset_list is empty", - node->GetName().c_str()); - GELOGE(PARAM_INVALID, "Output_list is empty"); + REPORT_INNER_ERROR("E19999", "check node:%s output_offset_list is empty", node->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] node:%s Output_list is empty", node->GetName().c_str()); return PARAM_INVALID; } GE_CHECK_NOTNULL(var_node->GetOpDesc()); @@ -133,7 +131,8 @@ Status VarMemAssignUtil::SetOutVariableAttr(const ge::NodePtr &node, const ge::N if (index >= out_list_size) { REPORT_INNER_ERROR("E19999", "param index:%d >= output_list.size() %d in node %s, check invalid", index, out_list_size, node->GetName().c_str()); - GELOGE(FAILED, "index %d >= output_list.size() %d", index, out_list_size); + GELOGE(FAILED, "[Check][Param] index %d >= output_list.size() %d in node %s", index, out_list_size, + node->GetName().c_str()); return FAILED; } @@ -169,7 +168,8 @@ Status VarMemAssignUtil::DealBroadCastNode(uint32_t graph_id, const ge::NodePtr broad_cast_info.broadcast_name = node->GetName(); auto op_desc = node->GetOpDesc(); - GE_CHK_BOOL_RET_STATUS(op_desc != nullptr, FAILED, "Get broadcast op %s desc is nullptr", node->GetName().c_str()); + GE_CHK_BOOL_RET_STATUS(op_desc != nullptr, FAILED, + "[Check][Param] Get broadcast op %s desc is nullptr", node->GetName().c_str()); GE_IF_BOOL_EXEC(broad_cast_info.idx < 0, GELOGI("Broadcast input index must be positive, actual %d", broad_cast_info.idx); @@ -180,8 +180,8 @@ Status VarMemAssignUtil::DealBroadCastNode(uint32_t graph_id, const ge::NodePtr if (input_tensor_desc_ptr_vistor.size() <= broad_cast_index) { REPORT_INNER_ERROR("E19999", "Get broadcast op %s input tensor desc size [%zu] < idx [%d]", node->GetName().c_str(), input_tensor_desc_ptr_vistor.size(), broad_cast_info.idx); - GELOGE(FAILED, "Get broadcast op %s input tensor desc size [%zu] < idx [%d]", node->GetName().c_str(), - input_tensor_desc_ptr_vistor.size(), broad_cast_info.idx); + GELOGE(FAILED, "[Check][Param] Get broadcast op %s input tensor desc size [%zu] < idx [%d]", + node->GetName().c_str(), input_tensor_desc_ptr_vistor.size(), broad_cast_info.idx); return FAILED; } const ge::GeTensorDescPtr input_tensor_desc = @@ -192,8 +192,8 @@ Status VarMemAssignUtil::DealBroadCastNode(uint32_t graph_id, const ge::NodePtr vector output_list = op_desc->GetOutputOffset(); GE_CHK_BOOL_RET_STATUS(output_list.size() > broad_cast_index, FAILED, - "Get broadcast op %s output_list size [%zu] < idx [%d]", node->GetName().c_str(), - output_list.size(), broad_cast_info.idx); + "[Check][Param] Get broadcast op %s output_list size [%zu] < idx [%d]", + node->GetName().c_str(), output_list.size(), broad_cast_info.idx); broad_cast_info.input_offset = output_list[broad_cast_info.idx]; broad_cast_info.output_offset = output_list[broad_cast_info.idx]; @@ -201,16 +201,16 @@ Status VarMemAssignUtil::DealBroadCastNode(uint32_t graph_id, const ge::NodePtr auto output_tensor_desc_ptr_vistor = op_desc->GetAllOutputsDescPtr(); GE_CHK_BOOL_RET_STATUS(output_tensor_desc_ptr_vistor.size() > broad_cast_index, FAILED, - "Get broadcast op %s output tensor desc size [%zu] < idx [%d]", node->GetName().c_str(), - output_tensor_desc_ptr_vistor.size(), broad_cast_info.idx); + "[Check][Param] Get broadcast op %s output tensor desc size [%zu] < idx [%d]", + node->GetName().c_str(), output_tensor_desc_ptr_vistor.size(), broad_cast_info.idx); const ge::GeTensorDescPtr output_tensor_desc = output_tensor_desc_ptr_vistor.at(static_cast(broad_cast_info.idx)); int64_t output_size = 0; - GE_CHK_STATUS(TensorUtils::GetSize(*output_tensor_desc, output_size), "get input size failed."); + GE_CHK_STATUS(TensorUtils::GetSize(*output_tensor_desc, output_size), "[Check][Param] get output size failed."); broad_cast_info.output_size = output_size; GE_CHK_BOOL_RET_STATUS(broad_cast_info.output_size == broad_cast_info.input_size, FAILED, - "Broadcast op input size[%lu] is not equal output size[%lu]", broad_cast_info.input_size, - broad_cast_info.output_size); + "[Check][Param] Broadcast op input size[%lu] is not equal output size[%lu]", + broad_cast_info.input_size, broad_cast_info.output_size); GE_CHK_STATUS_RET(VarManager::Instance(session_id)->SaveBroadCastInfo(graph_id, broad_cast_info)); return SUCCESS; @@ -298,7 +298,9 @@ Status VarMemAssignUtil::SetOutTransNodeToAssign(const ge::NodePtr &node, const vector output_list = node->GetOpDesc()->GetOutputOffset(); auto out_list_size = output_list.size(); GE_CHECK_SIZE(out_list_size); - GE_CHK_BOOL_RET_STATUS(index < out_list_size, FAILED, "index %zu >= output_list.size() %zu", index, out_list_size); + GE_CHK_BOOL_RET_STATUS(index < out_list_size, FAILED, + "[Check][Param] index %zu >= output_list.size() %zu, node:%s", + index, out_list_size, node->GetName().c_str()); // final_trans_node outputOffset[0] to assign_node outputOffset[0] GELOGI("final_trans_node outputOffset[0] is: %ld", final_trans_output_list[0]); @@ -372,7 +374,7 @@ Status VarMemAssignUtil::AssignData2VarRef(const ge::NodePtr &has_ref_attr_node, GE_CHECK_SIZE(ref_attr_node_output_list.size()); GE_CHK_BOOL_RET_STATUS(out_index < ref_attr_node_output_list.size(), FAILED, - "out_index %u >= ref_attr_node_output_list.size() %zu", out_index, + "[Check][Param] out_index %u >= ref_attr_node_output_list.size() %zu", out_index, ref_attr_node_output_list.size()); ref_attr_node_output_list[out_index] = static_cast(reinterpret_cast(dev_ptr)); diff --git a/ge/graph/build/model_builder.cc b/ge/graph/build/model_builder.cc index 5fdbfb31..0e625990 100755 --- a/ge/graph/build/model_builder.cc +++ b/ge/graph/build/model_builder.cc @@ -118,14 +118,16 @@ Status ModelBuilder::CalcOutputSize(const ge::NodePtr &n) { if (graph_status != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Get tensor size in bytes failed for op:%s(%s) index:%u", node_op_desc->GetName().c_str(), node_op_desc->GetType().c_str(), index); - GELOGE(graph_status, "GetTensorMemorySizeInBytes failed!"); + GELOGE(graph_status, "[Get][TensorMemorySize] In Bytes failed for op:%s(%s) index:%u", + node_op_desc->GetName().c_str(), node_op_desc->GetType().c_str(), index); return FAILED; } TensorUtils::SetSize(desc_temp, size_temp); if (node_op_desc->UpdateOutputDesc(index, desc_temp) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Update Output desc size failed for op:%s(%s) index:%u", node_op_desc->GetName().c_str(), node_op_desc->GetType().c_str(), index); - GELOGE(FAILED, "UpdateOutputDesc failed."); + GELOGE(FAILED, "[Update][OutputDesc] failed for op:%s(%s) index:%u", + node_op_desc->GetName().c_str(), node_op_desc->GetType().c_str(), index); return FAILED; } @@ -212,14 +214,14 @@ Status ModelBuilder::AdjustConstWeightSize(const ge::NodePtr &node, size_t &mem_ if (weights.empty()) { REPORT_INNER_ERROR("E19999", "Check weights size of node %s(%s) is empty", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "weights size of node %s is empty", node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] weights size of node %s is empty", node->GetName().c_str()); return FAILED; } GeTensorPtr weight = weights[0]; if (weight == nullptr) { REPORT_INNER_ERROR("E19999", "Check weight of node %s(%s) is nullptr", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "weights[0] is null."); + GELOGE(FAILED, "[Check][Param] weights[0] is null, node:%s.", node->GetName().c_str()); return FAILED; } GeTensorDesc &tensor_desc = weight->MutableTensorDesc(); @@ -271,15 +273,16 @@ Status ModelBuilder::SetInputOutputDesc() { bool is_unknow = false; (void)NodeUtils::GetNodeUnknownShapeStatus(*n, is_unknow); if ((IsGeLocalOp(n->GetOpDesc())) && (!is_unknow)) { - GE_CHK_STATUS_RET(CalcOutputSize(n), "Calculate output size failed"); + GE_CHK_STATUS_RET(CalcOutputSize(n), "[Calc][OutputSize] failed, node:%s", n->GetName().c_str()); } ret = AdjustConstWeightSize(n, weight_offset_); - GE_CHK_STATUS_RET(ret, "AdjustConstWeightSize failed"); + GE_CHK_STATUS_RET(ret, "[Adjust][ConstWeightSize] failed, node:%s", n->GetName().c_str()); GE_IF_BOOL_EXEC(((weight_offset_ > 0) && (weight_offset_ % MEM_ALIGN_SIZE != 0)), weight_offset_ = (weight_offset_ + MEM_ALIGN_SIZE - 1) / MEM_ALIGN_SIZE * MEM_ALIGN_SIZE); } - GE_CHK_STATUS_RET(compute_graph_->TopologicalSorting(), "TopologicalSorting failed"); + GE_CHK_STATUS_RET(compute_graph_->TopologicalSorting(), "[Call][TopologicalSorting] failed, graph:%s", + compute_graph_->GetName().c_str()); return SUCCESS; } @@ -363,7 +366,8 @@ Status ModelBuilder::AdjustInputTensorFlag() { REPORT_CALL_ERROR("E19999", "Update Input desc size failed for op:%s(%s) index:%u", owner_node_op_desc->GetName().c_str(), owner_node_op_desc->GetType().c_str(), in_anchors->GetIdx()); - GELOGE(FAILED, "UpdateOutputDesc failed."); + GELOGE(FAILED, "[Update][InputDesc] failed for op:%s(%s) index:%u", + owner_node_op_desc->GetName().c_str(), owner_node_op_desc->GetType().c_str(), in_anchors->GetIdx()); return FAILED; } } @@ -391,61 +395,52 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) { max_mem_offset_ = mem_type_to_mem_offset_[RT_MEMORY_HBM]; GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_MEMORY_SIZE, max_mem_offset_), - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_MODEL_MEMORY_SIZE.c_str()); - GELOGE(FAILED, "SetInt of ATTR_MODEL_MEMORY_SIZE failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_MEMORY_SIZE.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_MODEL_MEMORY_SIZE.c_str()); return FAILED); if (mem_type_to_mem_offset_.find(RT_MEMORY_P2P_DDR) != mem_type_to_mem_offset_.end()) { p2p_mem_offset_ = mem_type_to_mem_offset_[RT_MEMORY_P2P_DDR]; } GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_P2P_MEMORY_SIZE, p2p_mem_offset_), - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_MODEL_P2P_MEMORY_SIZE.c_str()); - GELOGE(FAILED, "SetInt of ATTR_MODEL_P2P_MEMORY_SIZE failed."); - return FAILED); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_P2P_MEMORY_SIZE.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_MODEL_P2P_MEMORY_SIZE.c_str()); + return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_WEIGHT_SIZE, weight_offset_), - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_MODEL_WEIGHT_SIZE.c_str()); - GELOGE(FAILED, "SetInt of ATTR_MODEL_WEIGHT_SIZE failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_WEIGHT_SIZE.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_MODEL_WEIGHT_SIZE.c_str()); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_STREAM_NUM, stream_num_), - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_MODEL_STREAM_NUM.c_str()); - GELOGE(FAILED, "SetInt of ATTR_MODEL_STREAM_NUM failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_STREAM_NUM.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_MODEL_STREAM_NUM.c_str()); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_EVENT_NUM, event_num_), - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_MODEL_EVENT_NUM.c_str()); - GELOGE(FAILED, "SetInt of ATTR_MODEL_EVENT_NUM failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_EVENT_NUM.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_MODEL_EVENT_NUM.c_str()); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(&model, ATTR_MODEL_HUGE_STREAM_LIST, huge_streams_), - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_MODEL_HUGE_STREAM_LIST.c_str()); - GELOGE(FAILED, "SetInt of ATTR_MODEL_HUGE_STREAM_LIST failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_HUGE_STREAM_LIST.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_MODEL_HUGE_STREAM_LIST.c_str()); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_LABEL_NUM, label_num_), - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_MODEL_LABEL_NUM.c_str()); - GELOGE(FAILED, "SetInt of ATTR_MODEL_LABEL_NUM failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_LABEL_NUM.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_MODEL_LABEL_NUM.c_str()); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_ZERO_COPY_MEMORY_SIZE, zero_copy_mem_size_), REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_ZERO_COPY_MEMORY_SIZE.c_str()); - GELOGE(FAILED, "SetInt of ATTR_MODEL_ZERO_COPY_MEMORY_SIZE failed."); + GELOGE(FAILED, "[Set][Attr] %s in model failed.", ATTR_MODEL_ZERO_COPY_MEMORY_SIZE.c_str()); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(&model, ATTR_MODEL_OUT_NODES_NAME, GetLocalOmgContext().net_out_nodes), - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_MODEL_OUT_NODES_NAME.c_str()); - GELOGE(FAILED, "SetListStr of ATTR_MODEL_OUT_NODES_NAME failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_OUT_NODES_NAME.c_str()); + GELOGE(FAILED, "[Set][Str] %s in model failed.", ATTR_MODEL_OUT_NODES_NAME.c_str()); return FAILED); GELOGI("For model, max_mem_offset_: %zu, p2p_mem_size: %zu, zero_copy_mem_size_: %zu", max_mem_offset_, p2p_mem_offset_, zero_copy_mem_size_); string fp_ceiling_mode; if (ge::GetContext().GetOption("ge.fpCeilingMode", fp_ceiling_mode) == SUCCESS) { if (!ge::AttrUtils::SetStr(&model, ATTR_FP_CEILING_MODE, fp_ceiling_mode)) { - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_FP_CEILING_MODE.c_str()); - GELOGE(FAILED, "Failed to set attr ATTR_FP_CEILING_MODE"); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_FP_CEILING_MODE.c_str()); + GELOGE(FAILED, "[Set][Str] %s in model failed", ATTR_FP_CEILING_MODE.c_str()); return FAILED; } GELOGI("Set attr ATTR_FP_CEILING_MODE to model, value is %s.", fp_ceiling_mode.c_str()); @@ -459,31 +454,27 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) { int64_t core_type = (ge_core_type == kVectorCore) ? 1 : 0; GELOGI("core_type: %ld", core_type); if (!ge::AttrUtils::SetInt(&model, ATTR_MODEL_CORE_TYPE, core_type)) { - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_MODEL_CORE_TYPE.c_str()); - GELOGE(FAILED, "SetInt of ATTR_CORE_TYPE failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_MODEL_CORE_TYPE.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_MODEL_CORE_TYPE.c_str()); } InitL1FusionOption(); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(&model, ATTR_NAME_SWITCH_FOR_L1_FUSION, is_l1_fusion_enable_), - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_NAME_SWITCH_FOR_L1_FUSION.c_str()); - GELOGE(FAILED, "SetBool of ATTR_NAME_SWITCH_FOR_L1_FUSION failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_NAME_SWITCH_FOR_L1_FUSION.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed.", ATTR_NAME_SWITCH_FOR_L1_FUSION.c_str()); return FAILED); const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(session_id_); bool is_op_debug = dump_properties.IsOpDebugOpen(); if (is_op_debug) { if (!ge::AttrUtils::SetBool(&model, ATTR_OP_DEBUG_FLAG, is_op_debug)) { - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_OP_DEBUG_FLAG.c_str()); - GELOGE(FAILED, "SetBool of ATTR_OP_DEBUG_FLAG failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_OP_DEBUG_FLAG.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_OP_DEBUG_FLAG.c_str()); return FAILED; } uint32_t op_debug_mode = dump_properties.GetOpDebugMode(); GELOGI("Get op debug mode:%d", op_debug_mode); if (!ge::AttrUtils::SetInt(&model, ATTR_OP_DEBUG_MODE, op_debug_mode)) { - REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", - ATTR_OP_DEBUG_MODE.c_str()); - GELOGE(FAILED, "SetBool of ATTR_OP_DEBUG_MODE failed."); + REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed", ATTR_OP_DEBUG_MODE.c_str()); + GELOGE(FAILED, "[Set][Attr] %s in model failed", ATTR_OP_DEBUG_MODE.c_str()); return FAILED; } } @@ -556,7 +547,7 @@ Status ModelBuilder::MergeWeights() { if (weight == nullptr) { REPORT_INNER_ERROR("E19999", "Can't get const weight in op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Can't get const op weight, name: %s", node->GetName().c_str()); + GELOGE(FAILED, "[Call][MutableTensor] Can't get const op weight, name:%s", node->GetName().c_str()); return FAILED; } @@ -581,14 +572,15 @@ Status ModelBuilder::MergeWeights() { GE_IF_BOOL_EXEC(base_addr == nullptr, REPORT_INNER_ERROR("E19999", "Check weight in op:%s(%s) is nullptr", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Base addr is nullptr."); + GELOGE(FAILED, "[Check][Param] weight in op:%s(%s) is nullptr", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED); if (weight_offset_ - offset < weight_data.size()) { REPORT_INNER_ERROR("E19999", "left weight size not enough for op:%s(%s) left_size:%zu, weight_size:%zu", op_desc->GetName().c_str(), op_desc->GetType().c_str(), weight_offset_ - offset, weight_data.size()); - GELOGE(FAILED, "left weight size not enough. left_size:%lu, weight_size:%lu", - weight_offset_ - offset, weight_data.size()); + GELOGE(FAILED, "[Check][Param] left weight size not enough for op:%s(%s). left_size:%lu, weight_size:%lu", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), weight_offset_ - offset, weight_data.size()); return FAILED; } uintptr_t dst_ptr = reinterpret_cast(base_addr) + offset; @@ -615,7 +607,7 @@ Status ModelBuilder::MergeWeights() { REPORT_CALL_ERROR("E19999", "mem copy failed. errret:%u, " "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu,", err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN); - GELOGE(FAILED, "mem copy failed. errret:%u, " + GELOGE(FAILED, "[Update][Data] mem copy failed. errret:%u, " "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu", err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN); return FAILED; @@ -714,9 +706,10 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) { } GE_IF_BOOL_EXEC(tbe_kernel == nullptr, continue); if (tbe_name_set.count(tbe_kernel->GetName()) > 0) { - REPORT_INNER_ERROR("E19999", "tbe_kernel name %s can't be the same, judge for op:%s(%s),", + REPORT_INNER_ERROR("E19999", "tbe_kernel name %s can't be the same, judge for op:%s(%s)", tbe_kernel->GetName().c_str(), n->GetName().c_str(), n->GetType().c_str()); - GELOGE(FAILED, "tbe_kernel name %s can't be the same", tbe_kernel->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] tbe_kernel name %s can't be the same, judge for op:%s(%s)", + tbe_kernel->GetName().c_str(), n->GetName().c_str(), n->GetType().c_str()); return FAILED; } tbe_name_set.insert(tbe_kernel->GetName()); @@ -734,9 +727,10 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) { node_op_desc->TryGetExtAttr(ge::OP_EXTATTR_CUSTAICPU_KERNEL, CustAICPUKernelPtr()); GE_IF_BOOL_EXEC(cust_aicpu_kernel == nullptr, continue); if (aicpu_name_set.count(cust_aicpu_kernel->GetName()) > 0) { - REPORT_INNER_ERROR("E19999", "aicpu_kernel name %s can't be the same, judge for op:%s(%s),", + REPORT_INNER_ERROR("E19999", "aicpu_kernel name %s can't be the same, judge for op:%s(%s)", cust_aicpu_kernel->GetName().c_str(), n->GetName().c_str(), n->GetType().c_str()); - GELOGE(FAILED, "aicpu_kernel name %s can't be the same", cust_aicpu_kernel->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] aicpu_kernel name %s can't be the same, judge for op:%s(%s)", + cust_aicpu_kernel->GetName().c_str(), n->GetName().c_str(), n->GetType().c_str()); return FAILED; } aicpu_name_set.insert(cust_aicpu_kernel->GetName()); @@ -745,11 +739,11 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) { } if (!tbe_kernel_store_.Build()) { - GELOGE(FAILED, "TBE Kernels store build failed!"); + GELOGE(FAILED, "[Call][Build] TBE Kernels store build failed!"); return FAILED; } if (!cust_aicpu_kernel_store_.Build()) { - GELOGE(FAILED, "custom AICPU kernels store build failed!"); + GELOGE(FAILED, "[Call][Build] custom AICPU kernels store build failed!"); return FAILED; } ge_model.SetTBEKernelStore(tbe_kernel_store_); @@ -759,14 +753,14 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) { GeAttrValue::BYTES task_def_bytes; if (!AttrUtils::GetZeroCopyBytes(model, MODEL_ATTR_TASKS, task_def_bytes)) { REPORT_CALL_ERROR("E19999", "Get attr:%s in model failed", MODEL_ATTR_TASKS.c_str()); - GELOGE(INTERNAL_ERROR, "Get zero copy bytes fail."); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s in model failed", MODEL_ATTR_TASKS.c_str()); return INTERNAL_ERROR; } int byte_size = static_cast(task_def_bytes.GetSize()); std::shared_ptr task = ge::MakeShared(); GE_CHECK_NOTNULL(task); GE_CHK_BOOL_EXEC(ReadProtoFromArray(task_def_bytes.GetData(), byte_size, task.get()), return INTERNAL_ERROR, - "ReadProtoFromArray failed."); + "[Read][Proto] From Array failed."); ge_model.SetModelTaskDef(task); // Add graph @@ -795,11 +789,12 @@ void ModelBuilder::SetModelVersion(ge::Model &model) { Status ModelBuilder::PreBuildModel() { if ((compute_graph_ == nullptr) || !(compute_graph_->IsValid())) { REPORT_INNER_ERROR("E19999", "Check compute_graph no valid"); - GELOGE(FAILED, "Graph_ is not valid."); + GELOGE(FAILED, "[Check][Param] Graph_ is not valid."); return FAILED; } - GE_CHK_STATUS_RET(SetInputOutputDesc(), "SetInputOutputDesc Failed!"); + GE_CHK_STATUS_RET(SetInputOutputDesc(), + "[Set][InputOutputDesc] Failed! graph:%s", compute_graph_->GetName().c_str()); AddNodeInputProperty(); @@ -807,14 +802,15 @@ Status ModelBuilder::PreBuildModel() { } Status ModelBuilder::BuildModelForGetTask(ge::Model &model) { - GE_CHK_STATUS_RET(AdjustInputTensorFlag(), "AdjustInputTensorFlag failed!"); + GE_CHK_STATUS_RET(AdjustInputTensorFlag(), "[Adjust][InputTensorFlag] failed! graph:%s", + compute_graph_->GetName().c_str()); ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kStreamAlloc); // Assign logical streams. StreamAllocator stream_allocator(compute_graph_, subgraphs_); GE_TIMESTAMP_START(AssignLogicalStreams); GE_CHK_STATUS_RET(stream_allocator.AssignLogicalStreams(stream_max_parallel_num_, hcom_parallel_), - "Assign logical streams failed."); + "[Assign][LogicalStreams] failed. graph:%s", compute_graph_->GetName().c_str()); GE_TIMESTAMP_END(AssignLogicalStreams, "GraphBuilder::AssignLogicalStreams"); ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kMemoryAlloc); @@ -825,34 +821,36 @@ Status ModelBuilder::BuildModelForGetTask(ge::Model &model) { GE_TIMESTAMP_START(AssignMemory); MemoryAssigner mem_assigner(compute_graph_); GE_CHK_STATUS_RET(mem_assigner.AssignMemory(is_loop_graph_, mem_type_to_mem_offset_, zero_copy_mem_size_), - "Assign Memory Failed!"); + "[Assign][Memory] Failed! graph:%s", compute_graph_->GetName().c_str()); GE_TIMESTAMP_END(AssignMemory, "GraphBuilder::AssignMemory"); ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther); GE_TIMESTAMP_START(SetInputOutputOffset); SetInputOutputOffsetPass input_output_offset; - GE_CHK_STATUS_RET(input_output_offset.Run(compute_graph_), "Set input output offset failed."); + GE_CHK_STATUS_RET(input_output_offset.Run(compute_graph_), + "[Set][InputOutputOffset] failed. graph:%s", compute_graph_->GetName().c_str()); GE_TIMESTAMP_END(SetInputOutputOffset, "SetInputOutputOffsetPass::Run"); // Compile single op in graph build stage GE_TIMESTAMP_START(CompileSingleOp); - GE_CHK_STATUS_RET(CompileSingleOp(), "ATC builder CompileSingleOp() return fail."); + GE_CHK_STATUS_RET(CompileSingleOp(), "[Compile][SingleOp] fail. graph:%s", compute_graph_->GetName().c_str()); GE_TIMESTAMP_EVENT_END(CompileSingleOp, "GraphBuilder::CompileSingleOp"); ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kStreamAlloc); // Refresh real streams and insert event nodes. GE_TIMESTAMP_START(RefreshRealStream); - GE_CHK_STATUS_RET(stream_allocator.RefreshRealStream(stream_num_, event_num_), "RefreshRealStream failed."); + GE_CHK_STATUS_RET(stream_allocator.RefreshRealStream(stream_num_, event_num_), + "[Refresh][RealStream] failed. graph:%s", compute_graph_->GetName().c_str()); huge_streams_ = stream_allocator.GetHugeStreams(); GE_TIMESTAMP_END(RefreshRealStream, "GraphBuilder::RefreshRealStream"); ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther); GE_TIMESTAMP_START(MergeWeights); - GE_CHK_STATUS_RET(MergeWeights(), "MergeWeights Failed!"); + GE_CHK_STATUS_RET(MergeWeights(), "[Merge][Weights] Failed! graph:%s", compute_graph_->GetName().c_str()); GE_TIMESTAMP_END(MergeWeights, "GraphBuilder::MergeWeights"); GE_TIMESTAMP_START(BuildModelDef); - GE_CHK_STATUS_RET(BuildModelDef(model), "BuildModelDef failed!"); + GE_CHK_STATUS_RET(BuildModelDef(model), "[Build][ModelDef] failed! graph:%s", compute_graph_->GetName().c_str()); GE_TIMESTAMP_END(BuildModelDef, "GraphBuilder::BuildModelDef"); SetModelVersion(model); @@ -862,7 +860,7 @@ Status ModelBuilder::BuildModelForGetTask(ge::Model &model) { Status ModelBuilder::BuildModelForGetDynShapeTask(ge::Model &model_def) { GE_TIMESTAMP_START(BuildModelDef); - GE_CHK_STATUS_RET(BuildModelDef(model_def), "BuildModelDef failed!"); + GE_CHK_STATUS_RET(BuildModelDef(model_def), "[Build][ModelDef] failed!"); GE_TIMESTAMP_END(BuildModelDef, "GraphBuilder::BuildModelDef"); SetModelVersion(model_def); return SUCCESS; @@ -875,7 +873,7 @@ Status ModelBuilder::CompileSingleOp() { std::shared_ptr instance = ge::GELib::GetInstance(); if ((instance == nullptr) || !instance->InitFlag()) { REPORT_INNER_ERROR("E19999", "Check GELib instance not init before"); - GELOGE(ge::GE_CLI_GE_NOT_INITIALIZED, "CompileSingleOp failed."); + GELOGE(ge::GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] CompileSingleOp failed."); return ge::GE_CLI_GE_NOT_INITIALIZED; } @@ -898,7 +896,7 @@ Status ModelBuilder::CompileSingleOp() { if (kernel_lib_name.empty()) { REPORT_INNER_ERROR("E19999", "Check kernel lib name empty of op:%s(%s)", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(ge::INTERNAL_ERROR, "Get node:%s(%s) kernel lib failed.", node->GetName().c_str(), + GELOGE(ge::INTERNAL_ERROR, "[Get][Name] of node:%s(%s) kernel lib failed.", node->GetName().c_str(), node->GetType().c_str()); return ge::INTERNAL_ERROR; } @@ -910,7 +908,7 @@ Status ModelBuilder::CompileSingleOp() { } else { REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s,", node->GetName().c_str(), node->GetType().c_str(), kernel_lib_name.c_str()); - GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "Get op %s ops kernel info store failed", node->GetName().c_str()); + GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "[Get][OpsKernelInfoStore] for op %s failed", node->GetName().c_str()); return ge::GE_GRAPH_PARAM_NULLPTR; } } @@ -927,7 +925,7 @@ Status ModelBuilder::CompileSingleOp() { if (ret != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Batch compile op failed, kernel lib name, node size:%zu,", node_vector.size()); - GELOGE(ret, "Compile op failed, kernel lib name is %s", kernel_lib_name.c_str()); + GELOGE(ret, "[Compile][Op] failed, kernel lib name is %s", kernel_lib_name.c_str()); return ret; } } @@ -975,10 +973,10 @@ void ModelBuilder::SetModelCheckAicpuAttr(ge::Model &model, std::setGetName().c_str(), aicpu_op_types.size(), aicpu_optype_list.size(), aicpu_tf_op_types.size(), aicpu_tf_optype_list.size()); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(&model, "needCheckCpu", aicpu_optype_list), return, - "Set attr needCheckCpu fail."); + "[Set][Attr] needCheckCpu fail."); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(&model, "needCheckTf", aicpu_tf_optype_list), return, - "Set attr needCheckTf fail."); + "[Set][Attr] needCheckTf fail."); return; } } // namespace ge diff --git a/ge/graph/build/run_context.cc b/ge/graph/build/run_context.cc index eca8b31b..05e40b63 100644 --- a/ge/graph/build/run_context.cc +++ b/ge/graph/build/run_context.cc @@ -29,13 +29,15 @@ Status RunContextUtil::InitMemInfo(uint8_t *data_mem_base, uint64_t data_mem_siz uint64_t weight_mem_size) { if ((data_mem_size > 0) && (data_mem_base == nullptr)) { REPORT_INNER_ERROR("E19999", "InitMemInfo param data_mem_base is null but data_mem_size = %lu", data_mem_size); - GELOGE(PARAM_INVALID, "InitMemInfo param data_mem_base is null but data_mem_size = %lu.", data_mem_size); + GELOGE(PARAM_INVALID, "[Check][Param] InitMemInfo param data_mem_base is null but data_mem_size = %lu.", + data_mem_size); return PARAM_INVALID; } if ((weight_mem_size > 0) && (weight_mem_base == nullptr)) { REPORT_INNER_ERROR("E19999", "InitMemInfo param weight_mem_base is null but weight_mem_size = %lu", weight_mem_size); - GELOGE(PARAM_INVALID, "InitMemInfo param weight_mem_base is null but weight_mem_size = %lu.", weight_mem_size); + GELOGE(PARAM_INVALID, "[Check][Param] InitMemInfo param weight_mem_base is null but weight_mem_size = %lu.", + weight_mem_size); return PARAM_INVALID; } if (mem_type_to_data_mem_base.empty() || mem_type_to_data_mem_size.empty() || @@ -44,9 +46,8 @@ Status RunContextUtil::InitMemInfo(uint8_t *data_mem_base, uint64_t data_mem_siz "is not equal to the size of mem_type_to_data_mem_size[%zu].", mem_type_to_data_mem_base.size(), mem_type_to_data_mem_size.size()); GELOGE(PARAM_INVALID, - "InitMemInfo param mem_type_to_data_mem_base size[%zu] is not equal to the size of " - "mem_type_to_data_mem_size[%zu].", - mem_type_to_data_mem_base.size(), mem_type_to_data_mem_size.size()); + "[Check][Param] InitMemInfo param mem_type_to_data_mem_base size[%zu] is not equal to the size of " + "mem_type_to_data_mem_size[%zu].", mem_type_to_data_mem_base.size(), mem_type_to_data_mem_size.size()); return PARAM_INVALID; } data_mem_base_ = data_mem_base; @@ -63,7 +64,7 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even rtError_t rt_ret = rtModelCreate(&rt_model_, 0); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "call rtModelCreate failed, ret:%d,", static_cast(rt_ret)); - GELOGE(RT_FAILED, "rtModelCreate failed. rt_ret = %d", static_cast(rt_ret)); + GELOGE(RT_FAILED, "[Call][RtModelCreate] failed. rt_ret = %d", static_cast(rt_ret)); return RT_FAILED; } @@ -74,7 +75,7 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "call rtStreamCreate failed, ret:%d, index:%u,", static_cast(rt_ret), i); - GELOGE(RT_FAILED, "rtStreamCreate failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); + GELOGE(RT_FAILED, "[Call][RtStreamCreate] failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); return RT_FAILED; } stream_list_.emplace_back(stream); @@ -83,7 +84,7 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "call rtModelBindStream failed, ret:%d, index:%u,", static_cast(rt_ret), i); - GELOGE(RT_FAILED, "Bind stream and model failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); + GELOGE(RT_FAILED, "[Bind][StreamAndModel] failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); return RT_FAILED; } } @@ -97,7 +98,7 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "call rtEventCreate failed, ret:%d, index:%u,", static_cast(rt_ret), i); - GELOGE(RT_FAILED, "rtEventCreate failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); + GELOGE(RT_FAILED, "[Call][RtEventCreate] failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); return RT_FAILED; } event_list_.emplace_back(event); @@ -110,7 +111,7 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "call rtLabelCreateV2 failed, ret:%d, index:%u,", static_cast(rt_ret), i); - GELOGE(RT_FAILED, "rtLabelCreate failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); + GELOGE(RT_FAILED, "[Call][RtLabelCreate] failed. rt_ret = %d, index = %u", static_cast(rt_ret), i); return RT_FAILED; } label_list_.emplace_back(label); @@ -162,40 +163,43 @@ Status RunContextUtil::CreateRunContext(Model &model, const ComputeGraphPtr &gra // check params if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Check param graph nullptr, session_id:%lu,", session_id); - GELOGE(PARAM_INVALID, "CreateRunContext param graph is null. session_id=%lu", session_id); + GELOGE(PARAM_INVALID, "[Check][Param] CreateRunContext param graph is null. session_id=%lu", session_id); return PARAM_INVALID; } uint32_t stream_num = 0; if (!AttrUtils::GetInt(&model, ATTR_MODEL_STREAM_NUM, stream_num)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s failed for model, session_id:%lu,", + REPORT_INNER_ERROR("E19999", "Get Attr:%s failed from model, session_id:%lu,", ATTR_MODEL_STREAM_NUM.c_str(), session_id); - GELOGE(INTERNAL_ERROR, "Get stream_num attr from model_def failed. session_id=%lu", session_id); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s failed from model. session_id=%lu", + ATTR_MODEL_STREAM_NUM.c_str(), session_id); return INTERNAL_ERROR; } GELOGD("Stream_num = %u", stream_num); uint32_t event_num = 0; if (!AttrUtils::GetInt(&model, ATTR_MODEL_EVENT_NUM, event_num)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s failed for model, session_id:%lu,", + REPORT_INNER_ERROR("E19999", "Get Attr:%s failed from model, session_id:%lu,", ATTR_MODEL_EVENT_NUM.c_str(), session_id); - GELOGE(INTERNAL_ERROR, "Get event_num attr from model failed. session_id=%lu", session_id); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s failed from model, session_id:%lu,", + ATTR_MODEL_EVENT_NUM.c_str(), session_id); return INTERNAL_ERROR; } GELOGD("Event_num = %u", event_num); uint32_t label_num = 0; if (!AttrUtils::GetInt(&model, ATTR_MODEL_LABEL_NUM, label_num)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s failed for model, session_id:%lu,", + REPORT_INNER_ERROR("E19999", "Get Attr:%s failed from model, session_id:%lu,", ATTR_MODEL_LABEL_NUM.c_str(), session_id); - GELOGE(INTERNAL_ERROR, "Get label_num attr from model failed. session_id=%lu", session_id); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s failed from model, session_id:%lu,", + ATTR_MODEL_LABEL_NUM.c_str(), session_id); return INTERNAL_ERROR; } GELOGD("Label_num = %u", label_num); Status ret = CreateRtModelResources(stream_num, event_num, label_num); if (ret != SUCCESS) { - GELOGE(ret, "CreateRtModelResources failed. session_id=%lu", session_id); + GELOGE(ret, "[Create][RtModelResources] failed. session_id=%lu", session_id); DestroyRtModelResources(); return ret; } diff --git a/ge/graph/build/stream_allocator.cc b/ge/graph/build/stream_allocator.cc index 8218588f..0e1a1aba 100644 --- a/ge/graph/build/stream_allocator.cc +++ b/ge/graph/build/stream_allocator.cc @@ -176,8 +176,8 @@ Status StreamAllocator::AssignLogicalStreams(const std::map &m auto gelib = GELib::GetInstance(); if (gelib == nullptr) { - REPORT_INNER_ERROR("E19999", "Check GELib instance nullptr"); - GELOGE(FAILED, "Get GELib instance failed."); + REPORT_INNER_ERROR("E19999", "Check GELib instance nullptr, graph:%s", whole_graph_->GetName().c_str()); + GELOGE(FAILED, "[Get][Instance] of GELib failed. graph:%s", whole_graph_->GetName().c_str()); return FAILED; } @@ -188,7 +188,7 @@ Status StreamAllocator::AssignLogicalStreams(const std::map &m Status status = logical_allocator.Assign(whole_graph_, subgraphs_, stream_num_); if (status != SUCCESS) { - GELOGE(status, "Assign logical streams failed."); + GELOGE(status, "[Assign][LogicalStreams] failed. graph:%s", whole_graph_->GetName().c_str()); return status; } GE_DUMP(whole_graph_, "AfterAssignedLogicalStreams"); @@ -203,62 +203,62 @@ Status StreamAllocator::RefreshRealStream(int64_t &stream_num, int64_t &event_nu Status status = AssignSingleStream(); if (status != SUCCESS) { - GELOGE(status, "AssignSingleStream failed!"); + GELOGE(status, "[Assign][SingleStream] failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = SetActiveStreamsByLabel(); if (status != SUCCESS) { - GELOGE(status, "SetActiveStreamsByLabel failed!"); + GELOGE(status, "[Set][ActiveStreams] By Label failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = SetActiveStreamsForSubgraphs(); if (status != SUCCESS) { - GELOGE(status, "SetActiveStreamsForSubgraphs failed."); + GELOGE(status, "[Set][ActiveStreams] For Subgraphs failed. graph:%s", whole_graph_->GetName().c_str()); return status; } status = InsertSyncEvents(); if (status != SUCCESS) { - GELOGE(status, "InsertSyncEventId failed!"); + GELOGE(status, "[Insert][SyncEventId] failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = OptimizeSyncEvents(); if (status != SUCCESS) { - GELOGE(status, "OptimizeSyncEventId failed!"); + GELOGE(status, "[Optimize][SyncEventId] failed! graph:%s", whole_graph_->GetName().c_str()); return status; } vector> split_streams(stream_num_); status = SplitStreams(split_streams); if (status != SUCCESS) { - GELOGE(status, "SplitStreams failed!"); + GELOGE(status, "[Split][Streams] failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = UpdateActiveStreams(split_streams); if (status != SUCCESS) { - GELOGE(status, "UpdateActiveStreams failed!"); + GELOGE(status, "[Update][ActiveStreams] failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = RefreshContinuousEvents(); if (status != SUCCESS) { - GELOGE(status, "RefreshContinuousEvents failed!"); + GELOGE(status, "[Refresh][ContinuousEvents] failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = RefreshEventsWithReuse(); if (status != SUCCESS) { - GELOGE(status, "[Refresh][Events]RefreshEventsWithReuse failed!"); + GELOGE(status, "[Refresh][Events] With Reuse failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = InsertSyncEventNodes(); if (status != SUCCESS) { - GELOGE(status, "InsertSyncEventNode failed!"); + GELOGE(status, "[Insert][SyncEventNode] failed! graph:%s", whole_graph_->GetName().c_str()); return status; } @@ -293,7 +293,7 @@ Status StreamAllocator::AssignSingleStream() { if (stream_num_ > 1) { REPORT_INNER_ERROR("E19999", "The number of ts streams is %ld, only one is supported", stream_num_); - GELOGE(FAILED, "The number of ts streams is %ld, only one is supported.", stream_num_); + GELOGE(FAILED, "[Check][Param] The number of ts streams is %ld, only one is supported.", stream_num_); return FAILED; } @@ -311,7 +311,7 @@ Status StreamAllocator::AssignSingleStream() { uint32_t max_normal_task_count = 0; Status status = GetMaxStreamAndTask(false, max_normal_stream_count, max_normal_task_count); if (status != SUCCESS) { - GELOGE(status, "Get max task count of normal stream failed."); + GELOGE(status, "[Get][MaxCount] of normal stream and task failed. graph:%s", whole_graph_->GetName().c_str()); return status; } @@ -369,7 +369,8 @@ Status StreamAllocator::SetActiveStreamsByLabel() { REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "SetListInt failed."); + GELOGE(FAILED, "[Set][Attr] %s for op:%s(%s) failed", + ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), node->GetName().c_str(), node->GetType().c_str()); return FAILED); } @@ -422,7 +423,7 @@ Status StreamAllocator::SetActiveStreamsForSubgraphs() { REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), first_active_node->GetName().c_str(), first_active_node->GetType().c_str()); - GELOGE(FAILED, "Set active streams for node %s failed.", first_active_node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] active streams for node %s failed.", first_active_node->GetName().c_str()); return FAILED; } } @@ -438,7 +439,7 @@ Status StreamAllocator::InsertSyncEvents() { NodePtr next_node = peer_in_anchor->GetOwnerNode(); Status status = InsertOneEventInTwoNodes(cur_node, next_node); if (status != SUCCESS) { - GELOGE(status, "InsertOneEventInTwoNodes failed!"); + GELOGE(status, "[Insert][OneEvent] In Two Nodes failed! cur node:%s", cur_node->GetName().c_str()); return status; } } @@ -451,7 +452,7 @@ Status StreamAllocator::InsertSyncEvents() { NodePtr next_node = peer_in_anchor->GetOwnerNode(); Status status = InsertOneEventInTwoNodes(cur_node, next_node); if (status != SUCCESS) { - GELOGE(status, "InsertOneEventInTwoNodes failed!"); + GELOGE(status, "[Insert][OneEvent] In Two Nodes failed! cur node:%s", cur_node->GetName().c_str()); return status; } } @@ -460,7 +461,8 @@ Status StreamAllocator::InsertSyncEvents() { Status status = InsertEventsForSubgraph(); if (status != SUCCESS) { - GELOGE(status, "InsertEventsBetweenSubAndParentGraphNodes failed!"); + GELOGE(status, "[Insert][Events] Between Sub And Parent GraphNodes failed! graph:%s", + whole_graph_->GetName().c_str()); return status; } @@ -493,7 +495,8 @@ Status StreamAllocator::InsertOneEventInTwoNodes(const NodePtr &cur_node, const if (next_stream_id == kInvalidStream) { REPORT_INNER_ERROR("E19999", "Stream id of next_node %s(%s) should not be %ld", next_node->GetName().c_str(), next_node->GetType().c_str(), kInvalidStream); - GELOGE(FAILED, "Stream id of next_node %s should not be %ld", next_node->GetName().c_str(), kInvalidStream); + GELOGE(FAILED, "[Check][Param] Stream id of next_node %s should not be %ld", + next_node->GetName().c_str(), kInvalidStream); return FAILED; } @@ -542,7 +545,7 @@ Status StreamAllocator::InsertEventsForSubgraph() { for (const auto &next_node : parent_node->GetOutAllNodes()) { Status status = InsertOneEventInTwoNodes(node, next_node); if (status != SUCCESS) { - GELOGE(status, "InsertOneEventInTwoNodes failed!"); + GELOGE(status, "[Insert][OneEvent] In Two Nodes failed! node:%s", node->GetName().c_str()); return status; } } @@ -566,19 +569,19 @@ Status StreamAllocator::OptimizeSyncEvents() { Status status = OptimizeBySendEvents(stream_nodes); if (status != SUCCESS) { - GELOGE(status, "OptimizeBySendEvents failed!"); + GELOGE(status, "[Optimize][StreamNodes] By Send Events failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = OptimizeByRecvEvents(stream_nodes); if (status != SUCCESS) { - GELOGE(status, "OptimizeByRecvEvents failed!"); + GELOGE(status, "[Optimize][StreamNodes] By Recv Events failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = OptimizeByStreamActivate(); if (status != SUCCESS) { - GELOGE(status, "OptimizeByStreamActivate failed!"); + GELOGE(status, "[Call][OptimizeByStreamActivate] failed! graph:%s", whole_graph_->GetName().c_str()); return status; } for (auto pair : node_to_send_events_) { @@ -708,11 +711,11 @@ Status StreamAllocator::OptimizeByStreamActivate() { bool StreamAllocator::IsRecvNodeActivatedBySendNode(const NodePtr &send_node_ptr, const NodePtr &recv_node_ptr) const { GE_CHECK_NOTNULL_EXEC(send_node_ptr->GetOpDesc(), REPORT_INNER_ERROR("E19999", "Check param send_node_ptr nullptr"); - GELOGE(FAILED, "op desc is nullptr"); + GELOGE(FAILED, "[Check][Param] op desc is nullptr"); return false); GE_CHECK_NOTNULL_EXEC(recv_node_ptr->GetOpDesc(), REPORT_INNER_ERROR("E19999", "Check param recv_node_ptr nullptr"); - GELOGE(FAILED, "op desc is nullptr"); + GELOGE(FAILED, "[Check][Param] op desc is nullptr"); return false); auto cur_stream_id = send_node_ptr->GetOpDesc()->GetStreamId(); if (AttrUtils::HasAttr(recv_node_ptr->GetOpDesc(), ATTR_NAME_STREAM_LABEL)) { @@ -826,7 +829,7 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { uint32_t max_stream_count = 0; uint32_t max_task_count = 0; GE_CHK_STATUS_RET(GetMaxStreamAndTask(false, max_stream_count, max_task_count), - "Get max stream and task count failed."); + "[Get][MaxCount] of stream and task failed."); for (const auto &cur_node : whole_graph_->GetNodes(whole_graph_->GetGraphUnknownFlag())) { GE_CHECK_NOTNULL(cur_node); @@ -839,7 +842,7 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { if (stream_id > last_stream_id) { REPORT_INNER_ERROR("E19999", "streamid(%ld) > last_stream_id(%ld), check invalid", stream_id, last_stream_id); - GELOGE(FAILED, "SplitStreams:streamid(%ld) > last_stream_id(%ld)", stream_id, last_stream_id); + GELOGE(FAILED, "[Check][Param] SplitStreams:streamid(%ld) > last_stream_id(%ld)", stream_id, last_stream_id); return FAILED; } bool is_stream_first_node = (stream_node_num_vec[stream_id] == 0); @@ -854,7 +857,7 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { if (stream_continuous_2_node_num_map[continuous_stream_label] > max_node_num_one_stream) { REPORT_INNER_ERROR("E19999", "Check node[%s] stream_id[%ld] continuous stream label[%s] unsatisfied", op_desc->GetName().c_str(), stream_id, continuous_stream_label.c_str()); - GELOGE(FAILED, "SplitStreams:node[%s] stream_id[%ld] continuous stream label[%s] unsatisfied ", + GELOGE(FAILED, "[Check][Param] SplitStreams:node[%s] stream_id[%ld] continuous stream label[%s] unsatisfied ", op_desc->GetName().c_str(), stream_id, continuous_stream_label.c_str()); return FAILED; } @@ -877,7 +880,8 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { if (HasContinuousStreamLabel(op_desc, cur_continuous_stream_label)) { // get stored nodes auto nodes = stream_continuous_2_nodes_map[cur_continuous_stream_label]; - GE_RETURN_WITH_LOG_IF_FALSE(!nodes.empty(), "split stream with continuous stream label %s failed", + GE_RETURN_WITH_LOG_IF_FALSE(!nodes.empty(), + "[Check][Param] split stream with continuous stream label %s failed", cur_continuous_stream_label.c_str()); for (const auto &node : nodes) { auto stored_op_desc = node->GetOpDesc(); @@ -893,7 +897,7 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { auto iter = std::find(stream_2_nodes_map[stream_id].begin(), stream_2_nodes_map[stream_id].end(), not_cur); GE_RETURN_WITH_LOG_IF_FALSE( (iter != stream_2_nodes_map[stream_id].end()) && (iter != stream_2_nodes_map[stream_id].begin()), - "split stream with continuous stream label %s failed", cur_continuous_stream_label.c_str()); + "[Check][Param] split stream with continuous stream label %s failed", cur_continuous_stream_label.c_str()); iter--; pre_node = *iter; } @@ -905,7 +909,9 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { // Add the send/recv event to the first and last nodes of the split stream. if (pre_node != nullptr) { - GE_CHK_STATUS_RET(AddEventId(pre_node, not_cur, cur_node, not_use_cur), "AddEventId failed."); + GE_CHK_STATUS_RET(AddEventId(pre_node, not_cur, cur_node, not_use_cur), + "[Add][EventId] failed, pre node:%s, not cur node:%s, cur node:%s.", + pre_node->GetName().c_str(), not_cur->GetName().c_str(), cur_node->GetName().c_str()); } } @@ -943,12 +949,12 @@ Status StreamAllocator::UpdateActiveStreams(const vector> &split_st for (auto &node : whole_graph_->GetNodes(whole_graph_->GetGraphUnknownFlag())) { if ((node->GetType() == STREAMSWITCH) || (node->GetType() == STREAMSWITCHN)) { if (UpdateActiveStreamsForSwitchNode(node) != SUCCESS) { - GELOGE(FAILED, "Update active streams for switch node: %s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Update][ActiveStreams] for switch node: %s failed.", node->GetName().c_str()); return FAILED; } } else { if (UpdateActiveStreamsForActiveNode(split_streams, node) != SUCCESS) { - GELOGE(FAILED, "Update active streams for active node: %s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Update][ActiveStreams] for active node: %s failed.", node->GetName().c_str()); return FAILED; } } @@ -956,13 +962,13 @@ Status StreamAllocator::UpdateActiveStreams(const vector> &split_st Status status = UpdateActiveStreamsForSubgraphs(); if (status != SUCCESS) { - GELOGE(status, "Update active streams for subgraphs failed!"); + GELOGE(status, "[Update][ActiveStreams] for subgraphs failed! graph:%s", whole_graph_->GetName().c_str()); return status; } status = SetActiveStreamsForLoop(); if (status != SUCCESS) { - GELOGE(status, "SetActiveStreamsForLoop failed!"); + GELOGE(status, "[Set][ActiveStreams] For Loop failed! graph:%s", whole_graph_->GetName().c_str()); return status; } @@ -990,7 +996,7 @@ void StreamAllocator::UpdateLabelStreams(const vector> &split_strea Status StreamAllocator::UpdateActiveStreamsForSwitchNode(NodePtr &switch_node) { vector active_nodes; if (InsertActiveNodesAfterSwitch(switch_node, active_nodes) != SUCCESS) { - GELOGE(FAILED, "Insert active nodes after node %s failed.", switch_node->GetName().c_str()); + GELOGE(FAILED, "[Insert][ActiveNodes] after node %s failed.", switch_node->GetName().c_str()); return FAILED; } if (active_nodes.empty()) { @@ -1010,7 +1016,8 @@ Status StreamAllocator::UpdateActiveStreamsForSwitchNode(NodePtr &switch_node) { if (!AttrUtils::SetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, stream_ids)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "SetListInt failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } @@ -1024,21 +1031,21 @@ Status StreamAllocator::InsertActiveNodesAfterSwitch(NodePtr &switch_node, vecto vector ori_active_label_list; if (!AttrUtils::GetListStr(switch_desc, ATTR_NAME_ACTIVE_LABEL_LIST, ori_active_label_list) || ori_active_label_list.empty()) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_LABEL_LIST.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s fail from op:%s(%s)", ATTR_NAME_ACTIVE_LABEL_LIST.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Get active label list of switch %s failed.", switch_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Get][Attr] active label list of switch %s failed.", switch_node->GetName().c_str()); return INTERNAL_ERROR; } vector active_label_list; vector added_active_nodes; if (AddActiveNodes(switch_node, ori_active_label_list, active_label_list, added_active_nodes) != SUCCESS) { - GELOGE(FAILED, "Add active nodes after node %s failed.", switch_node->GetName().c_str()); + GELOGE(FAILED, "[Add][ActiveNodes] after node %s failed.", switch_node->GetName().c_str()); return FAILED; } if (SetActiveLabelList(switch_node, active_label_list) != SUCCESS) { - GELOGE(FAILED, "set active label list failed"); + GELOGE(FAILED, "[Set][ActiveLabelList] failed, node:%s", switch_node->GetName().c_str()); return FAILED; } @@ -1051,7 +1058,8 @@ Status StreamAllocator::InsertActiveNodesAfterSwitch(NodePtr &switch_node, vecto if (switch_node->GetOutControlAnchor()->LinkTo(active_node->GetInControlAnchor()) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Link from %s to %s failed", switch_node->GetName().c_str(), active_node->GetName().c_str()); - GELOGE(FAILED, "Link %s to %s failed.", switch_node->GetName().c_str(), active_node->GetName().c_str()); + GELOGE(FAILED, "[Link][Nodes] from %s to %s failed.", + switch_node->GetName().c_str(), active_node->GetName().c_str()); return FAILED; } active_nodes.emplace_back(active_node); @@ -1068,7 +1076,8 @@ Status StreamAllocator::UpdateActiveStreamsForActiveNode(const vector(logical_stream) >= split_streams.size()) { REPORT_INNER_ERROR("E19999", "Check logical stream:%u is out of range:%zu", logical_stream, split_streams.size()); - GELOGE(FAILED, "logical stream is out of range."); + GELOGE(FAILED, "[Check][Param] logical stream:%u is out of range(0, %zu).", + logical_stream, split_streams.size()); return FAILED; } const set &new_split_streams = split_streams[logical_stream]; @@ -1088,7 +1097,7 @@ Status StreamAllocator::UpdateActiveStreamsForActiveNode(const vectorGetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, new_active_streams)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Set active streams for node %s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] active streams for node %s failed.", node->GetName().c_str()); return FAILED; } } @@ -1130,7 +1139,7 @@ Status StreamAllocator::UpdateActiveStreamsForSubgraphs() const { if (!AttrUtils::SetListInt(active_op, ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), active_op->GetName().c_str(), active_op->GetType().c_str()); - GELOGE(FAILED, "Set active streams for node %s failed.", active_node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] active streams for node %s failed.", active_node->GetName().c_str()); return FAILED; } } @@ -1200,7 +1209,7 @@ Status StreamAllocator::SetActiveStreamsForLoop() { if (pre_switch_node == nullptr) { REPORT_INNER_ERROR("E19999", "Find switch node before loop active node %s fail", node->GetName().c_str()); - GELOGE(FAILED, "find switch node before loop active node %s failed", node->GetName().c_str()); + GELOGE(FAILED, "[Find][SwitchNode] before loop active node %s failed", node->GetName().c_str()); return FAILED; } @@ -1210,7 +1219,8 @@ Status StreamAllocator::SetActiveStreamsForLoop() { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "SetListInt failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED); for (const auto &stream_id : loop_active_streams) { GELOGI("Active stream %u for node: %s.", stream_id, node->GetName().c_str()); @@ -1258,7 +1268,7 @@ Status StreamAllocator::CheckStreamActived() const { if (iter != active_streams.end()) { REPORT_INNER_ERROR("E19999", "Node:%s(%s) cannot active its own stream %u, check invalid ", node->GetName().c_str(), node->GetType().c_str(), stream_id); - GELOGE(FAILED, "Node %s cannot active its own stream %u.", node->GetName().c_str(), stream_id); + GELOGE(FAILED, "[Check][Param] Node %s cannot active its own stream %u.", node->GetName().c_str(), stream_id); return FAILED; } } @@ -1376,7 +1386,7 @@ Status StreamAllocator::RefreshContinuousEvents() { auto find_it = old_to_new_events.find(send_events[i]); if (find_it == old_to_new_events.end()) { REPORT_INNER_ERROR("E19999", "Check invalid send event %u", send_events[i]); - GELOGE(FAILED, "RefreshContinuousEvents: invalid send event %u", send_events[i]); + GELOGE(FAILED, "[Check][Param] RefreshContinuousEvents: invalid send event %u", send_events[i]); return FAILED; } send_events[i] = find_it->second; @@ -1390,7 +1400,7 @@ Status StreamAllocator::RefreshContinuousEvents() { auto find_it = old_to_new_events.find(recv_events[i]); if (find_it == old_to_new_events.end()) { REPORT_INNER_ERROR("E19999", "Check invalid recv event %u", recv_events[i]); - GELOGE(FAILED, "RefreshContinuousEvents: invalid recv event %u", recv_events[i]); + GELOGE(FAILED, "[Check][Param] RefreshContinuousEvents: invalid recv event %u", recv_events[i]); return FAILED; } recv_events[i] = find_it->second; @@ -1430,7 +1440,8 @@ Status StreamAllocator::InsertSyncEventNodes() { REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed, event_id:%u,", RECV_ATTR_EVENT_ID.c_str(), node->GetName().c_str(), node->GetType().c_str(), event_id); - GELOGE(FAILED, "SetInt failed."); + GELOGE(FAILED, "[Set][Attr] %s for op:%s(%s) failed, event_id:%u,", + RECV_ATTR_EVENT_ID.c_str(), node->GetName().c_str(), node->GetType().c_str(), event_id); return FAILED); (void)AttrUtils::SetListStr(op_desc_ptr, ATTR_NAME_DATA_DUMP_ORIGIN_OP_NAMES, std::move(std::vector())); @@ -1441,7 +1452,7 @@ Status StreamAllocator::InsertSyncEventNodes() { if (status != SUCCESS) { REPORT_INNER_ERROR("E19999", "Add edge from node %s to node %s failed", recv_node->GetName().c_str(), node->GetName().c_str()); - GELOGE(status, "Add edge for node %s and node %s failed.", recv_node->GetName().c_str(), + GELOGE(status, "[Add][Edge] for node %s and node %s failed.", recv_node->GetName().c_str(), node->GetName().c_str()); return status; } @@ -1478,7 +1489,7 @@ Status StreamAllocator::InsertSyncEventNodes() { if (status != SUCCESS) { REPORT_INNER_ERROR("E19999", "Add edge from node %s to node %s failed", node->GetName().c_str(), send_node->GetName().c_str()); - GELOGE(status, "Add edge for node %s and node %s failed.", node->GetName().c_str(), + GELOGE(status, "[Add][Edge] for node %s and node %s failed.", node->GetName().c_str(), send_node->GetName().c_str()); return status; } @@ -1491,7 +1502,8 @@ Status StreamAllocator::InsertSyncEventNodes() { if (status != SUCCESS) { REPORT_CALL_ERROR("E19999", "Insert Graph Events fail, graph:%s,", whole_graph_->GetName().c_str()); - GELOGE(status, "Graph ReorderEventNodes failed"); + GELOGE(status, "[Insert][GraphEvents] Graph ReorderEventNodes failed, graph:%s,", + whole_graph_->GetName().c_str()); return status; } @@ -1544,7 +1556,8 @@ Status StreamAllocator::GetMaxStreamAndTask(bool huge_stream, uint32_t &max_stre if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "call rtGetMaxStreamAndTask fail, ret:%d, stream_type:%u,", static_cast(ret), stream_type); - GELOGE(FAILED, "Get max stream and task count by rts failed."); + GELOGE(FAILED, "[Call][RtGetMaxStreamAndTask] Get max stream and task count by rts failed, " + "ret:%d, stream_type:%u,", static_cast(ret), stream_type); return FAILED; } GELOGD("Allowed max stream count: %u, max task count per stream: %u.", max_stream_count, max_task_count); @@ -1687,7 +1700,7 @@ Status StreamAllocator::AddActiveNodes(NodePtr &switch_node, const vectorGetOutControlAnchor()->Unlink(node->GetInControlAnchor()) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Unlink %s to %s failed", switch_node->GetName().c_str(), node->GetName().c_str()); - GELOGE(FAILED, "Unlink %s to %s failed.", switch_node->GetName().c_str(), node->GetName().c_str()); + GELOGE(FAILED, "[Unlink][Nodes] %s to %s failed.", switch_node->GetName().c_str(), node->GetName().c_str()); return FAILED; } GE_CHECK_NOTNULL(active_node->GetOutControlAnchor()); if (active_node->GetOutControlAnchor()->LinkTo(node->GetInControlAnchor()) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Link %s to %s failed", active_node->GetName().c_str(), node->GetName().c_str()); - GELOGE(FAILED, "Link %s to %s failed.", active_node->GetName().c_str(), node->GetName().c_str()); + GELOGE(FAILED, "[Link][Nodes] %s to %s failed.", active_node->GetName().c_str(), node->GetName().c_str()); return FAILED; } } if (SetSwitchBranchNodeLabel(active_node, name) != SUCCESS) { - GELOGE(FAILED, "Set switch branch node label failed."); + GELOGE(FAILED, "[Set][SwitchBranchNodeLabel] failed, node:%s.", active_node->GetName().c_str()); return FAILED; } if (SetStreamLabel(active_node, name) != SUCCESS) { - GELOGE(FAILED, "Set stream label failed."); + GELOGE(FAILED, "[Set][StreamLabel] failed, node:%s.", active_node->GetName().c_str()); return FAILED; } if (SetActiveLabelList(active_node, {active_label}) != SUCCESS) { - GELOGE(FAILED, "Set active label list failed."); + GELOGE(FAILED, "[Set][ActiveLabelList] failed, node:%s.", active_node->GetName().c_str()); return FAILED; } if (SetActiveStreamList(active_node, active_label) != SUCCESS) { - GELOGE(FAILED, "Set active stream list failed."); + GELOGE(FAILED, "[Set][ActiveStreamList] failed, node:%s.", active_node->GetName().c_str()); return FAILED; } @@ -1753,7 +1766,7 @@ Status StreamAllocator::AddActiveNodes(NodePtr &switch_node, const vector &streams = labeled_streams_[active_label]; @@ -1761,7 +1774,8 @@ Status StreamAllocator::SetActiveStreamList(NodePtr &active_node, const string & if (!AttrUtils::SetListInt(active_node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(FAILED, "SetListInt of %s failed.", ATTR_NAME_ACTIVE_STREAM_LIST.c_str()); + GELOGE(FAILED, "[Set][Attr] %s failed for op:%s(%s).", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); return FAILED; } return SUCCESS; diff --git a/ge/graph/build/stream_graph_optimizer.cc b/ge/graph/build/stream_graph_optimizer.cc index c71c31be..30142c2b 100644 --- a/ge/graph/build/stream_graph_optimizer.cc +++ b/ge/graph/build/stream_graph_optimizer.cc @@ -128,8 +128,8 @@ Status StreamGraphOptimizer::OptimizeStreamedSubGraph(const ComputeGraphPtr &com REPORT_INNER_ERROR("E19999", "Check stream_id:%ld in op:%s(%s) is bigger than " "run_context.graphStreamList.size():%zu", stream_id, op_desc->GetName().c_str(), op_desc->GetType().c_str(), run_context.graphStreamList.size()); - GELOGE(FAILED, "stream_id %ld is bigger than run_context.graphStreamList.size() %zu", stream_id, - run_context.graphStreamList.size()); + GELOGE(FAILED, "[Check][Param] stream_id %ld is bigger than run_context.graphStreamList.size() %zu", + stream_id, run_context.graphStreamList.size()); return FAILED; } run_context.stream = run_context.graphStreamList[stream_id]; @@ -145,11 +145,9 @@ Status StreamGraphOptimizer::OptimizeStreamedSubGraph(const ComputeGraphPtr &com REPORT_CALL_ERROR("E19999", "Call optimize streamed subgraph failed, subgraph: %s, engine_name: %s, graph " "Optimizer num: %zu, ret: %u", subgraph->GetName().c_str(), engine_name.c_str(), graph_optimizers.size(), ret); - GELOGE( - ret, - "[optimizeStreamedSubGraph]: optimize streamed subgraph failed, subgraph: %s, engine_name: %s, graph " - "Optimizer num: %zu, ret: %u", - subgraph->GetName().c_str(), engine_name.c_str(), graph_optimizers.size(), ret); + GELOGE(ret, "[Optimize][StreamGraph] failed, subgraph: %s, engine_name: %s, graph " + "Optimizer num: %zu, ret: %u", + subgraph->GetName().c_str(), engine_name.c_str(), graph_optimizers.size(), ret); return ret; } GELOGD( diff --git a/ge/graph/build/task_generator.cc b/ge/graph/build/task_generator.cc index 633f541c..dabdc5d2 100755 --- a/ge/graph/build/task_generator.cc +++ b/ge/graph/build/task_generator.cc @@ -72,7 +72,7 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t // Check params if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Check param graph is null, session_id:%lu", session_id); - GELOGE(PARAM_INVALID, "GetTaskInfo param graph is null. session_id=%lu", session_id); + GELOGE(PARAM_INVALID, "[Check][Param] GetTaskInfo param graph is null. session_id=%lu", session_id); return PARAM_INVALID; } @@ -83,7 +83,7 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t GE_DUMP(graph, "GenerateTaskAfter"); if (ret != SUCCESS) { - GELOGE(ret, "GenerateTask failed. session_id=%lu", session_id); + GELOGE(ret, "[Generate][Task] failed. session_id=%lu", session_id); return ret; } @@ -98,7 +98,8 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(model, ATTR_MODEL_TASK_INDEX_OP_NAME, op_name), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s", ATTR_MODEL_TASK_INDEX_OP_NAME.c_str(), model.GetName().c_str()); - GELOGE(FAILED, "SetListStr failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for model:%s", + ATTR_MODEL_TASK_INDEX_OP_NAME.c_str(), model.GetName().c_str()); return FAILED); GELOGI("GenerateTask Success, task list:%zu, op map:%zu, logic mem base:%p, logic weight base:%p, logic var base:%p", @@ -113,7 +114,8 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t if (task_def == nullptr) { REPORT_INNER_ERROR("E19999", "Add task_def in ModelTaskDef fail, session_id:%lu, graph:%s, model:%s", session_id, graph->GetName().c_str(), model.GetName().c_str()); - GELOGE(FAILED, "task_def is nullptr."); + GELOGE(FAILED, "[Check][Param] task_def is nullptr, session_id:%lu, graph:%s, model:%s", + session_id, graph->GetName().c_str(), model.GetName().c_str()); return FAILED; } *task_def = task_def_temp; @@ -121,7 +123,7 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t ret = AddModelTaskToModel(model_task_def, session_id, model, run_context); if (ret != SUCCESS) { - GELOGE(ret, "AddModelTaskToModel failed. session_id=%lu", session_id); + GELOGE(ret, "[Add][ModelTask] To Model failed. session_id=%lu", session_id); return ret; } @@ -135,28 +137,33 @@ Status TaskGenerator::AddModelTaskToModel(const ModelTaskDef &model_task_def, ui AttrUtils::SetInt(model, MODEL_ATTR_TASK_GEN_BASE_ADDR, reinterpret_cast(run_context.dataMemBase)), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s", MODEL_ATTR_TASK_GEN_BASE_ADDR.c_str(), model.GetName().c_str()); - GELOGE(FAILED, "SetInt MODEL_ATTR_TASK_GEN_BASE_ADDR failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for model:%s", + MODEL_ATTR_TASK_GEN_BASE_ADDR.c_str(), model.GetName().c_str()); return FAILED); GE_CHK_BOOL_EXEC( AttrUtils::SetInt(model, MODEL_ATTR_TASK_GEN_WEIGHT_ADDR, reinterpret_cast(run_context.weightMemBase)), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s", MODEL_ATTR_TASK_GEN_WEIGHT_ADDR.c_str(), model.GetName().c_str()); - GELOGE(FAILED, "SetInt MODEL_ATTR_TASK_GEN_WEIGHT_ADDR failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for model:%s", + MODEL_ATTR_TASK_GEN_WEIGHT_ADDR.c_str(), model.GetName().c_str()); return FAILED); GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, ATTR_MODEL_TASK_GEN_VAR_ADDR, reinterpret_cast(var_mem_base_)), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s", ATTR_MODEL_TASK_GEN_VAR_ADDR.c_str(), model.GetName().c_str()); - GELOGE(FAILED, "SetInt ATTR_MODEL_TASK_GEN_VAR_ADDR failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for model:%s", + ATTR_MODEL_TASK_GEN_VAR_ADDR.c_str(), model.GetName().c_str()); return FAILED); GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, ATTR_MODEL_VAR_SIZE, var_mem_size_), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s", ATTR_MODEL_VAR_SIZE.c_str(), model.GetName().c_str()); - GELOGE(FAILED, "SetInt ATTR_MODEL_VAR_SIZE failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for model:%s", + ATTR_MODEL_VAR_SIZE.c_str(), model.GetName().c_str()); return FAILED); GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, MODEL_ATTR_SESSION_ID, session_id), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for mode:%s", MODEL_ATTR_SESSION_ID.c_str(), model.GetName().c_str()); - GELOGE(FAILED, "SetInt MODEL_ATTR_SESSION_ID failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for mode:%s", + MODEL_ATTR_SESSION_ID.c_str(), model.GetName().c_str()); return FAILED); size_t task_size = model_task_def.ByteSizeLong(); @@ -164,15 +171,15 @@ Status TaskGenerator::AddModelTaskToModel(const ModelTaskDef &model_task_def, ui if (!model_task_def.SerializePartialToArray(serial_buff.GetData(), static_cast(task_size))) { REPORT_INNER_ERROR("E19999", "model_task_def's serialize failed, model name = %s, task_size=%zu", model.GetName().c_str(), task_size); - GELOGE(FAILED, "model_task_def's serialize failed, model name = %s, task_size=%zu.", model.GetName().c_str(), - task_size); + GELOGE(FAILED, "[Call][SerializePartialToArray] failed, model name = %s, task_size=%zu.", + model.GetName().c_str(), task_size); return FAILED; } if (!AttrUtils::SetZeroCopyBytes(model, MODEL_ATTR_TASKS, std::move(serial_buff))) { REPORT_INNER_ERROR("E19999", "Set model task to model failed, model name = %s, task_size=%zu", model.GetName().c_str(), task_size); - GELOGE(FAILED, "Set model task to model failed, model name = %s, task_size=%zu.", model.GetName().c_str(), - task_size); + GELOGE(FAILED, "[Call][SetZeroCopyBytes] Set model task to model failed, model name = %s, task_size=%zu.", + model.GetName().c_str(), task_size); return FAILED; } @@ -191,7 +198,8 @@ Status TaskGenerator::UpdateOpIsVarAttr(const OpDescPtr &op_desc, uint64_t sessi GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsInputVar, input_var), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsInputVar, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "SetListBool failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", kIsInputVar, + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED); } @@ -204,7 +212,8 @@ Status TaskGenerator::UpdateOpIsVarAttr(const OpDescPtr &op_desc, uint64_t sessi GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsOutputVar, output_var), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsOutputVar, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "SetListBool failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", kIsOutputVar, + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED); } return SUCCESS; @@ -280,10 +289,11 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra std::shared_ptr ge_lib = GELib::GetInstance(); if ((ge_lib == nullptr) || !ge_lib->InitFlag()) { REPORT_INNER_ERROR("E19999", "Check GELib instance not init before"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GenerateTask failed."); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] GenerateTask failed, because GELib instance not init before."); return GE_CLI_GE_NOT_INITIALIZED; } - GE_CHK_STATUS_RET(MarkNodeAndSetIndex(graph), "MarkNodeAndSetIndex failed."); + GE_CHK_STATUS_RET(MarkNodeAndSetIndex(graph), + "[Call][MarkNodeAndSetIndex] failed, graph:%s.", graph->GetName().c_str()); ProfilingPoint profiling_point; vector all_reduce_nodes; GE_CHK_STATUS_RET(FindProfilingTaskIndex(graph, profiling_point, all_reduce_nodes)); @@ -304,12 +314,13 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra rtStream_t stream = nullptr; bool is_unknown_shape = graph->GetGraphUnknownFlag() || GetContext().GetHostExecFlag(); if (is_unknown_shape) { - GE_CHK_STATUS_RET(SetUnknownShapeStream(run_context, stream), "Set unknown shape stream failed."); + GE_CHK_STATUS_RET(SetUnknownShapeStream(run_context, stream), + "[Set][UnknownShapeStream] failed, graph:%s.", graph->GetName().c_str()); } std::function callback = [&]() { if (is_unknown_shape) { if (DestroyUnknownShapeStream(run_context, stream) != SUCCESS) { - GELOGE(FAILED, "Destory unknown shape stream failed."); + GELOGE(FAILED, "[Destroy][UnknownShapeStream] failed."); } } }; @@ -335,7 +346,7 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra FusionTaskInfo{run_context, graph, node, op_desc, node_index, ge_lib, ops_kernel_manager, task_def_list, op_name_map, profiling_point, all_reduce_nodes}; GE_CHK_STATUS_RET(GenerateTaskForFusionNode(fusion_task_info, fusion_nodes, fusion_nodes_seen), - "Call GenerateTaskForFusionNode node:%s(%s) failed", name.c_str(), type.c_str()); + "[Call][GenerateTaskForFusionNode] node:%s(%s) failed", name.c_str(), type.c_str()); // continue directly if (ge::AttrUtils::GetInt(op_desc, ATTR_NAME_FUSION_GROUP_KEY, group_key)) { GELOGI("Fusion node[name:%s, type:%s] do not need generate task again.", name.c_str(), type.c_str()); @@ -349,13 +360,11 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra if (kernel_info_store == nullptr) { REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s", node->GetName().c_str(), node->GetType().c_str(), op_kernel_lib_name.c_str()); - GELOGE(INTERNAL_ERROR, - "No ops kernel store or ops kernel builder found. node:%s(%s), op_kernel_lib_name=%s.", - name.c_str(), - type.c_str(), op_kernel_lib_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Call][GetOpsKernelInfoStore] No ops kernel store or ops kernel builder found. " + "node:%s(%s), op_kernel_lib_name=%s.", name.c_str(), type.c_str(), op_kernel_lib_name.c_str()); return INTERNAL_ERROR; } - GE_CHK_STATUS_RET(UpdateAnchorStatus(node), "Call UpdateAnchorStatus node:%s(%s) failed", name.c_str(), + GE_CHK_STATUS_RET(UpdateAnchorStatus(node), "[Call][UpdateAnchorStatus] node:%s(%s) failed", name.c_str(), type.c_str()); // Profiling task size_t task_list_size_before = task_def_list.size(); @@ -365,7 +374,8 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra int64_t stream_id = 0; if (!is_unknown_shape) { stream_id = op_desc->GetStreamId(); - GE_CHK_STATUS_RET(SetKnownShapeStream(run_context, stream_id), "node[name:%s(%s), id:%ld] stream id is invalid.", + GE_CHK_STATUS_RET(SetKnownShapeStream(run_context, stream_id), + "[Set][KnownShapeStream] node[name:%s(%s), id:%ld] stream id is invalid.", name.c_str(), type.c_str(), op_id); } GELOGD("Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task.", op_kernel_lib_name.c_str(), @@ -376,8 +386,7 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Call OpsKernelBuilderManager GenerateTask fail for op:%s(%s)", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(ret, "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task failed.", - op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), op_id, stream_id); + GELOGE(ret, "[Generate][Task] fail for op:%s(%s)", node->GetName().c_str(), node->GetType().c_str()); return ret; } // Profiling task @@ -388,9 +397,9 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra REPORT_INNER_ERROR("E19999", "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task " "but task num from %zu to %zu, check invalid", op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), op_id, stream_id, task_list_size_before, task_list_size_after); - GELOGE(FAILED, "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task. but task num from %zu to %zu.", - op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), op_id, stream_id, task_list_size_before, - task_list_size_after); + GELOGE(FAILED, "[Check][Param] Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task. " + "but task num from %zu to %zu.", op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), + op_id, stream_id, task_list_size_before, task_list_size_after); return FAILED; } @@ -455,15 +464,15 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s", op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_kernel_lib_name.c_str()); - GELOGE(INTERNAL_ERROR, - "Fusion: No ops kernel store or ops kernel builder found. fusion_node:%s(%s), op_kernel_lib_name=%s.", + GELOGE(INTERNAL_ERROR, "[Call][GetOpsKernelInfoStore] Fusion: No ops kernel store or ops kernel builder found." + " fusion_node:%s(%s), op_kernel_lib_name=%s.", fusion_node_name.c_str(), fusion_node_type.c_str(), op_kernel_lib_name.c_str()); return INTERNAL_ERROR; } ret = UpdateAnchorStatus(fusion_node); if (ret != SUCCESS) { - GELOGE(ret, "Fusion: Call UpdateAnchorStatus fusion_node:%s(%s) failed", fusion_node_name.c_str(), + GELOGE(ret, "[Update][AnchorStatus] fusion_node:%s(%s) failed", fusion_node_name.c_str(), fusion_node_type.c_str()); return ret; } @@ -474,8 +483,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info REPORT_INNER_ERROR("E19999", "Fusion: fusion_node[name:%s(%s), id:%ld] stream id is invalid, " "stream list size=%zu", fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, run_context.graphStreamList.size()); - GELOGE(INTERNAL_ERROR, "Fusion: fusion_node[name:%s(%s), id:%ld] stream id is invalid, stream list size=%zu", - fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, run_context.graphStreamList.size()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Fusion: fusion_node[name:%s(%s), id:%ld] stream id is invalid, " + "stream list size=%zu", fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, + run_context.graphStreamList.size()); return INTERNAL_ERROR; } // profiling task @@ -488,8 +498,7 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info REPORT_CALL_ERROR("E19999", " Call %s to generate fusion_node:[fusion_node_name:%s(%s), " "id:%ld, stream_id:%ld] task failed", op_kernel_lib_name.c_str(), fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, stream_id); - GELOGE(ret, - "Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), " + GELOGE(ret, "[Generate][Task] Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), " "id:%ld, stream_id:%ld] task failed.", op_kernel_lib_name.c_str(), fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, stream_id); return ret; @@ -503,8 +512,7 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info "id:%ld, stream_id:%ld] task, but task num from %zu to %zu, check invalid", fusion_node_name.c_str(), fusion_node_type.c_str(), op_kernel_lib_name.c_str(), op_id, stream_id, task_list_size_before, task_list_size_after); - GELOGE(FAILED, - "Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), " + GELOGE(FAILED, "[Check][Param] Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), " "id:%ld, stream_id:%ld] task. but task num from %zu to %zu.", op_kernel_lib_name.c_str(), fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, stream_id, task_list_size_before, task_list_size_after); @@ -539,16 +547,18 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) { if (NodeUtils::SetAllAnchorStatus(node) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "SetAllAnchorStatus fail for op:%s(%s)", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "NodeUtils::SetAllAnchorStatus failed."); + GELOGE(INTERNAL_ERROR, "[Set][AllAnchorStatus] failed, op:%s(%s)", + node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } for (auto &anchor : node->GetAllInDataAnchors()) { auto peer_anchor = anchor->GetPeerOutAnchor(); if (peer_anchor == nullptr) { if (AnchorUtils::SetStatus(anchor, ANCHOR_SUSPEND) != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set in peer anchor status fail for op:%s(%s), anchor_index:%d,", + REPORT_CALL_ERROR("E19999", "Set in peer anchor status fail for op:%s(%s), anchor_index:%d", node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed."); + GELOGE(INTERNAL_ERROR, "[Set][Status] failed, op:%s(%s), anchor_index:%d", + node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx()); return INTERNAL_ERROR; } continue; @@ -558,16 +568,18 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) { bool is_const = NodeUtils::GetConstOpType(peer_anchor->GetOwnerNode(), const_type); if (is_const && (const_type == CONSTANT)) { if (AnchorUtils::SetStatus(anchor, ANCHOR_CONST) != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set in anchor CONST status fail for op:%s(%s), anchor_index:%d,", + REPORT_CALL_ERROR("E19999", "Set in anchor CONST status fail for op:%s(%s), anchor_index:%d", node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed."); + GELOGE(INTERNAL_ERROR, "[Set][Status] failed. op:%s(%s), anchor_index:%d.", + node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx()); return INTERNAL_ERROR; } } else { if (AnchorUtils::SetStatus(anchor, ANCHOR_DATA) != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set in anchor DATA status fail for op:%s(%s), anchor_index:%d,", + REPORT_CALL_ERROR("E19999", "Set in anchor DATA status fail for op:%s(%s), anchor_index:%d", node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed."); + GELOGE(INTERNAL_ERROR, "[Set][Status] failed, op:%s(%s), anchor_index:%d.", + node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx()); return INTERNAL_ERROR; } } @@ -580,7 +592,7 @@ Status TaskGenerator::MarkNodeAndSetIndex(ComputeGraphPtr &graph) { auto ge_lib = GELib::GetInstance(); if ((ge_lib == nullptr) || !ge_lib->InitFlag()) { REPORT_INNER_ERROR("E19999", "Check GELib instance not init before"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized or is finalized."); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] GE is not initialized or is finalized."); return GE_CLI_GE_NOT_INITIALIZED; } @@ -588,7 +600,7 @@ Status TaskGenerator::MarkNodeAndSetIndex(ComputeGraphPtr &graph) { if (all_nodes.empty()) { REPORT_INNER_ERROR("E19999", "Check param all_nodes empty in graph:%s", graph->GetName().c_str()); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Graph's node is empty"); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Check][Param] Graph's node is empty, graph:%s", graph->GetName().c_str()); return GE_GRAPH_GRAPH_NODE_NULL; } @@ -612,7 +624,7 @@ Status TaskGenerator::MarkNodeAndSetIndex(ComputeGraphPtr &graph) { for (const auto &stream_ops : all_stream_ops) { Status status = MarkFirstAndLastOps(stream_ops.second, is_single_stream); if (status != SUCCESS) { - GELOGE(status, "Mark first and last nodes failed."); + GELOGE(status, "[Mark][FirstAndLastOps] failed, graph:%s.", graph->GetName().c_str()); return status; } } @@ -644,9 +656,8 @@ Status TaskGenerator::MarkFirstAndLastOps(const vector &ops, bool is_ string op_kernel_lib_name = op_desc->GetOpKernelLibName(); if (op_kernel_lib_name.empty()) { REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), - op_kernel_lib_name.c_str()); - GELOGE(INTERNAL_ERROR, "node:%s(%s) get op kernel lib failed.", op_desc->GetName().c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_kernel_lib_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] node:%s(%s) get op kernel lib failed.", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } @@ -664,12 +675,14 @@ Status TaskGenerator::MarkFirstAndLastOps(const vector &ops, bool is_ GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.first, kIsFirstNode, true), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsFirstNode, op_pair.first->GetName().c_str(), op_pair.first->GetType().c_str()); - GELOGE(FAILED, "SetBool failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", kIsFirstNode, + op_pair.first->GetName().c_str(), op_pair.first->GetType().c_str()); return FAILED); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.second, kIsLastNode, true), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsLastNode, op_pair.second->GetName().c_str(), op_pair.second->GetType().c_str()); - GELOGE(FAILED, "SetBool failed."); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", kIsLastNode, + op_pair.second->GetName().c_str(), op_pair.second->GetType().c_str()); return FAILED); } } @@ -980,7 +993,8 @@ Status TaskGenerator::InsertProfilingArTaskBefore(const OpDescPtr &op_desc, std: GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep), REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id " "for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Multiply result is out of range."); + GELOGE(FAILED, "[Check][Param] Multiply result is out of range. node:%s(%s)", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED); ar_log_id = i * kProfilingArStep + kProfilingArStartLogid; is_insert_all_reduce_task = true; @@ -1074,7 +1088,8 @@ Status TaskGenerator::InsertProfilingArTaskAfter(const OpDescPtr &op_desc, std:: GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep), REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id " "for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Multiply result is out of range."); + GELOGE(FAILED, "[Check][Param] Multiply result is out of range. node:%s(%s)", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED); ar_log_id = i * kProfilingArStep + kProfilingArEndLogid; is_insert_all_reduce_task = true; @@ -1184,7 +1199,7 @@ Status TaskGenerator::SetUnknownShapeStream(RunContext &run_context, rtStream_t rtError_t rt_ret = rtModelBindStream(run_context.model, stream, 0); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtModelBindStream failed, ret:0x%X", rt_ret); - GELOGE(FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(FAILED, "[Call][RtModelBindStream] failed, ret: 0x%X", rt_ret); GE_CHK_RT_RET(rtStreamDestroy(stream)); return FAILED; } @@ -1199,7 +1214,7 @@ Status TaskGenerator::DestroyUnknownShapeStream(RunContext &run_context, rtStrea Status TaskGenerator::SetKnownShapeStream(RunContext &run_context, int64_t stream_id) { if (stream_id < 0 || stream_id >= static_cast(run_context.graphStreamList.size())) { - GELOGE(INTERNAL_ERROR, "Stream id[%ld] is invalid, stream list size=%zu", stream_id, + GELOGE(INTERNAL_ERROR, "[Check][Param] Stream id[%ld] is invalid, stream list size=%zu", stream_id, run_context.graphStreamList.size()); return INTERNAL_ERROR; } diff --git a/ge/graph/common/bcast.cc b/ge/graph/common/bcast.cc index 95a93897..b36b50b2 100644 --- a/ge/graph/common/bcast.cc +++ b/ge/graph/common/bcast.cc @@ -37,7 +37,7 @@ Status BCast::GenerateBcastInfo(const kVecInt &sx, const kVecInt &sy) { Reverse(x); Reverse(y); ExtendTensorDim(x, y); - GE_RETURN_WITH_LOG_IF_ERROR(SetShapeDifferentInfo(x, y), "GenerateBcastInfo failed."); + GE_RETURN_WITH_LOG_IF_ERROR(SetShapeDifferentInfo(x, y), "[Set][ShapeDifferentInfo] GenerateBcastInfo failed."); } ReverseAllIntermediateShapes(); return domi::SUCCESS; @@ -76,7 +76,7 @@ Status BCast::SetShapeDifferentInfo(const kVecInt &x, const kVecInt &y) { REPORT_INNER_ERROR("E19999", "SetShapeDifferentInfo failed. Two tensor shapes are not compatible " "according to the broadcasting rule."); GELOGE(domi::PARAM_INVALID, - "SetShapeDifferentInfo failed. Two tensor shapes are not compatible " + "[Check][Param] SetShapeDifferentInfo failed. Two tensor shapes are not compatible " "according to the broadcasting rule."); return domi::PARAM_INVALID; } diff --git a/ge/graph/common/omg_util.cc b/ge/graph/common/omg_util.cc index 8012af80..1dba8c51 100644 --- a/ge/graph/common/omg_util.cc +++ b/ge/graph/common/omg_util.cc @@ -35,9 +35,10 @@ Status GetOriginalType(const ge::NodePtr &node, string &type) { GE_CHECK_NOTNULL(node->GetOpDesc()); bool ret = ge::AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE, type); if (!ret) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s fail for op:%s(%s)", ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s fail from op:%s(%s)", ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Get FrameWorkOp original type [%s]", type.c_str()); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s fail from op:%s(%s)", ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } GELOGD("Get FrameWorkOp original type [%s]", type.c_str()); @@ -58,7 +59,8 @@ Status SetStreamLabel(const ge::NodePtr &node, const std::string &label) { if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_STREAM_LABEL, label)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_STREAM_LABEL.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Op: %s set ATTR_NAME_STREAM_LABEL failed", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_STREAM_LABEL.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -77,7 +79,8 @@ Status SetCycleEvent(const ge::NodePtr &node) { if (!AttrUtils::SetBool(tmp_desc, ge::ATTR_NAME_STREAM_CYCLE_EVENT_FLAG, true)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_STREAM_CYCLE_EVENT_FLAG.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Op: %s set ATTR_NAME_STREAM_CYCLE_EVENT_FLAG failed", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_STREAM_CYCLE_EVENT_FLAG.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -97,7 +100,8 @@ Status SetActiveLabelList(const ge::NodePtr &node, const std::vectorGetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Op: %s set ATTR_NAME_ACTIVE_LABEL_LIST failed", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_ACTIVE_LABEL_LIST.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -117,7 +121,8 @@ Status SetSwitchBranchNodeLabel(const ge::NodePtr &node, const std::string &bran if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_SWITCH_BRANCH_NODE_LABEL, branch_label)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_SWITCH_BRANCH_NODE_LABEL.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Op: %s set ATTR_NAME_SWITCH_BRANCH_NODE_LABEL failed", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_SWITCH_BRANCH_NODE_LABEL.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -137,7 +142,8 @@ Status SetSwitchTrueBranchFlag(const ge::NodePtr &node, bool value) { if (!AttrUtils::SetBool(tmp_desc, ge::ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG, value)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Op: %s set ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG failed", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -157,7 +163,8 @@ Status SetOriginalNodeName(const ge::NodePtr &node, const std::string &orig_name if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_ORIG_NODE_NAME, orig_name)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ORIG_NODE_NAME.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Op: %s set ATTR_NAME_ORIG_NODE_NAME failed", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_ORIG_NODE_NAME.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -176,7 +183,8 @@ Status SetCyclicDependenceFlag(const ge::NodePtr &node) { if (!AttrUtils::SetBool(tmp_desc, ge::ATTR_NAME_CYCLIC_DEPENDENCE_FLAG, true)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_CYCLIC_DEPENDENCE_FLAG.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Op: %s set ATTR_NAME_CYCLIC_DEPENDENCE_FLAG failed", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_CYCLIC_DEPENDENCE_FLAG.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -197,7 +205,8 @@ Status SetNextIteration(const ge::NodePtr &node, const std::string &next) { if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_NEXT_ITERATION, next)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_NEXT_ITERATION.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Op: %s set ATTR_NAME_NEXT_ITERATION failed", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_NEXT_ITERATION.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -275,7 +284,8 @@ void MarkForceUnknownShape(const NodePtr &node, bool force_unknown) { if (!AttrUtils::SetBool(node->GetOpDesc(), ATTR_NAME_FORCE_UNKNOWN_SHAPE, force_unknown)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_FORCE_UNKNOWN_SHAPE.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Op: %s set %s failed", node->GetName().c_str(), ATTR_NAME_FORCE_UNKNOWN_SHAPE.c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_FORCE_UNKNOWN_SHAPE.c_str(), + node->GetName().c_str(), node->GetType().c_str()); } } } // namespace ge diff --git a/ge/graph/execute/graph_execute.cc b/ge/graph/execute/graph_execute.cc index 5649d483..87dadb41 100755 --- a/ge/graph/execute/graph_execute.cc +++ b/ge/graph/execute/graph_execute.cc @@ -43,7 +43,7 @@ GraphExecutor::~GraphExecutor() { rt_ret = rtFreeHost(buffer_addr); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtFreeHost failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "[GraphManager] subgraph free buffer failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtFreeHost] subgraph free buffer failed, ret: 0x%X", rt_ret); } } } @@ -55,17 +55,17 @@ Status GraphExecutor::SetCondition(std::mutex *mutex, std::condition_variable *c std::shared_ptr listener) { if (mutex == nullptr) { REPORT_INNER_ERROR("E19999", "Check param mutex nullptr"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetCondition] input param mutex is nullptr."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] input param mutex is nullptr."); return GE_GRAPH_PARAM_NULLPTR; } if (cond == nullptr) { REPORT_INNER_ERROR("E19999", "Check param cond nullptr"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetCondition] input param cond is nullptr."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] input param cond is nullptr."); return GE_GRAPH_PARAM_NULLPTR; } if (listener == nullptr) { REPORT_INNER_ERROR("E19999", "Check param listener nullptr"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetCondition] input param listener is nullptr."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] input param listener is nullptr."); return GE_GRAPH_PARAM_NULLPTR; } @@ -82,7 +82,7 @@ Status GraphExecutor::SetCondition(std::mutex *mutex, std::condition_variable *c Status GraphExecutor::SetGraphContext(GraphContextPtr graph_context_ptr) { if (graph_context_ptr == nullptr) { REPORT_INNER_ERROR("E19999", "Check param graph_context_ptr nullptr"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetGraphContext] input param graph_context_ptr is nullptr"); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] input param graph_context_ptr is nullptr"); return GE_GRAPH_PARAM_NULLPTR; } graph_context_ = graph_context_ptr; @@ -94,7 +94,7 @@ Status GraphExecutor::SetDynamicSize(uint32_t model_id, const std::vectorSetDynamicSize(model_id, batch_num, dynamic_type); if (ret != SUCCESS) { - GELOGE(ret, "SetDynamicSize failed"); + GELOGE(ret, "[Set][DynamicSize] failed, model_id:%u", model_id); return ret; } return SUCCESS; @@ -109,7 +109,7 @@ Status GraphExecutor::FreeInOutBuffer() { rt_ret = rtFreeHost(*iter); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtFreeHost failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "[GraphManager] subgraph free buffer failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtFreeHost] subgraph free buffer failed, ret: 0x%X", rt_ret); (void)buffer_addr_.erase(buffer_addr_.begin(), iter); return GE_GRAPH_FREE_FAILED; } @@ -144,7 +144,7 @@ Status GraphExecutor::MallocInOutBuffer(const std::vector &buffer_size buffer_size_.clear(); auto rt_ret = FreeInOutBuffer(); if (rt_ret != SUCCESS) { - GELOGE(RT_FAILED, "[SubGraphInfo] MallocInOutBuffer free buffer failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Free][Buffer] failed, ret: 0x%X", rt_ret); return RT_FAILED; } } @@ -154,9 +154,8 @@ Status GraphExecutor::MallocInOutBuffer(const std::vector &buffer_size void *tmp_buf = nullptr; rt_ret = rtMallocHost(&tmp_buf, buffer_size[i]); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, size:%lu, ret:0x%X", - buffer_size[i], rt_ret); - GELOGE(RT_FAILED, "[GraphManager] subgraph malloc buffer failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, size:%lu, ret:0x%X", buffer_size[i], rt_ret); + GELOGE(RT_FAILED, "[Malloc][Buffer] failed, size:%lu, ret:0x%X", buffer_size[i], rt_ret); return GE_GRAPH_MALLOC_FAILED; } malloc_flag_ = true; @@ -190,7 +189,7 @@ Status GraphExecutor::PrepareInputData(const std::vector &input_tensor Status ret = MallocInOutBuffer(bufferSizeVec, addrVec); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_MALLOC_FAILED, "[GraphExecutor] Malloc mem failed"); + GELOGE(GE_GRAPH_MALLOC_FAILED, "[Malloc][Mem] failed"); return GE_GRAPH_MALLOC_FAILED; } @@ -203,7 +202,8 @@ Status GraphExecutor::PrepareInputData(const std::vector &input_tensor if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, dst_size:%lu, src_size:%zu, ret:0x%X", bufferSizeVec[i], in_tensor->GetData().size(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, dst_size:%lu, src_size:%zu, ret:0x%X", + bufferSizeVec[i], in_tensor->GetData().size(), rt_ret); return RT_FAILED; } } @@ -247,7 +247,7 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vectorResetResult() != SUCCESS) { - REPORT_CALL_ERROR("E19999", "Call graph_run_listener_.ResetResult fail, model_id:%u", - model_id); - GELOGE(GE_GRAPH_EXECUTE_FAILED, "Reset result failed"); + REPORT_CALL_ERROR("E19999", "Call graph_run_listener_.ResetResult fail, model_id:%u", model_id); + GELOGE(GE_GRAPH_EXECUTE_FAILED, "[Reset][Result] failed, model_id:%u", model_id); return GE_GRAPH_EXECUTE_FAILED; } @@ -272,7 +271,7 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vectorGetResultCode(); if (result_code != SUCCESS && result_code != END_OF_SEQUENCE) { - REPORT_CALL_ERROR("E19999", "Graph_run_listener_ run fail, result:%u, model_id:%u", - result_code, model_id); - GELOGE(GE_GRAPH_EXECUTE_FAILED, "[GraphExecutor] execute model failed, ret=%u, modelId=%u.", result_code, - model_id); + REPORT_CALL_ERROR("E19999", "Graph_run_listener_ run fail, result:%u, model_id:%u", result_code, model_id); + GELOGE(GE_GRAPH_EXECUTE_FAILED, "[Execute][Model] failed, ret=%u, modelId=%u.", result_code, model_id); return GE_GRAPH_EXECUTE_FAILED; } } @@ -299,13 +296,13 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vector outBufTmp(new (std::nothrow) uint8_t[outputDataTmp.length]); if (outBufTmp == nullptr) { - REPORT_CALL_ERROR("E19999", "New output buffer fail, length:%lu, model:%u", - outputDataTmp.length, model_id); - GELOGE(FAILED, "Failed to allocate memory."); + REPORT_CALL_ERROR("E19999", "New output buffer fail, length:%lu, model:%u", outputDataTmp.length, model_id); + GELOGE(FAILED, "[Allocate][Memory] failed, length:%lu, model:%u", outputDataTmp.length, model_id); return FAILED; } GE_PRINT_DYNAMIC_MEMORY(new, "the output memory of data on training.", sizeof(uint8_t) * outputDataTmp.length) @@ -314,7 +311,8 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vector shapeDims; @@ -348,7 +346,7 @@ void GraphExecutor::InitModelIdInfo(std::vector &out_model_id_info, Status GraphExecutor::FreeExecuteMemory() { auto ret = FreeInOutBuffer(); if (ret != SUCCESS) { - GELOGE(ret, "[FreeExecuteMemory] FreeInOutBuffer Error!"); + GELOGE(ret, "[Free][InOutBuffer] Error!"); return ret; } @@ -368,13 +366,14 @@ Status GraphExecutor::ExecuteGraph(GraphId graph_id, const GeRootModelPtr &ge_ro if (!init_flag_) { REPORT_INNER_ERROR("E19999", "No SetCondition called before, graph:%u, check invalid", graph_id); - GELOGE(GE_GRAPH_EXECUTE_NOT_INIT, "[GraphExecutor] AI Core Engine without calling SetCondition!"); + GELOGE(GE_GRAPH_EXECUTE_NOT_INIT, "[Check][Param] AI Core Engine without calling SetCondition! graph id:%u", + graph_id); return GE_GRAPH_EXECUTE_NOT_INIT; } GE_CHECK_NOTNULL_EXEC(ge_root_model, return FAILED); Status ret = SyncExecuteModel(ge_root_model->GetModelId(), input_tensor, output_tensor); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_SYNC_MODEL_FAILED, "[GraphExecutor] SyncExecuteModel Error!"); + GELOGE(GE_GRAPH_SYNC_MODEL_FAILED, "[SyncExecute][Model] Error! graph id:%u", graph_id); return GE_GRAPH_SYNC_MODEL_FAILED; } @@ -395,7 +394,7 @@ Status GraphExecutor::ExecuteGraphAsync(GraphId graph_id, const GeRootModelPtr & GE_CHECK_NOTNULL_EXEC(ge_root_model, return FAILED); Status ret = AsyncExecuteModel(ge_root_model, input_tensor, callback); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_SYNC_MODEL_FAILED, "[GraphExecutor] AsyncExecuteModel Error!"); + GELOGE(GE_GRAPH_SYNC_MODEL_FAILED, "[AsyncExecute][Model] Error! graph id:%u", graph_id); return GE_GRAPH_SYNC_MODEL_FAILED; } @@ -426,7 +425,8 @@ Status GraphExecutor::ExecuteGraphWithStream(GraphId graph_id, if (!init_flag_) { REPORT_INNER_ERROR("E19999", "No SetCondition called before, graph id = %u, stream = %p, check invalid.", graph_id, stream); - GELOGE(GE_GRAPH_EXECUTE_NOT_INIT, "[GraphExecutor] AI Core Engine without calling SetCondition!"); + GELOGE(GE_GRAPH_EXECUTE_NOT_INIT, "[Check][Param] AI Core Engine without calling SetCondition! graph id = %u", + graph_id); return GE_GRAPH_EXECUTE_NOT_INIT; } @@ -515,14 +515,14 @@ Status GraphExecutor::SetCallback(uint32_t model_id, const GeRootModelPtr &ge_ro auto model = model_manager->GetHybridModel(model_id); GE_CHECK_NOTNULL(model); if (model->SetRunAsyncListenerCallback(callback) != SUCCESS) { - GELOGE(FAILED, "SetRunAsyncListenerCallback failed."); + GELOGE(FAILED, "[Set][RunAsyncListenerCallback] failed, model_id %u", model_id); return FAILED; } } else { auto model = model_manager->GetModel(model_id); GE_CHECK_NOTNULL(model); if (model->SetRunAsyncListenerCallback(callback) != SUCCESS) { - GELOGE(FAILED, "SetRunAsyncListenerCallback failed."); + GELOGE(FAILED, "[Set][RunAsyncListenerCallback] failed, model_id %u", model_id); return FAILED; } } @@ -541,24 +541,24 @@ Status GraphExecutor::AsyncExecuteModel(const GeRootModelPtr &ge_root_model, con GE_CHECK_NOTNULL(model_manager); GELOGI("RunAsync begin.model_id %u", model_id); if (SetCallback(model_id, ge_root_model, callback) != SUCCESS) { - GELOGE(FAILED, "RunAsync: SetCallBack for model fail"); + GELOGE(FAILED, "[Set][CallBack] for model fail, model_id %u", model_id); return FAILED; } Status ret = model_manager->DataInputTensor(model_id, inputs); if (ret != SUCCESS) { - GELOGE(ret, "RunAsync: DataInput fail"); + GELOGE(ret, "[Call][DataInputTensor] RunAsync: DataInput fail, model_id %u", model_id); return ret; } GELOGI("RunAsync success."); } catch (std::bad_alloc &) { - REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed"); - GELOGE(MEMALLOC_FAILED, "RunAsync failed, bad memory allocation occur !"); + REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed, model_id %u", model_id); + GELOGE(MEMALLOC_FAILED, "[Run][Async] failed, bad memory allocation occur, model_id %u", model_id); return MEMALLOC_FAILED; } catch (...) { - REPORT_INNER_ERROR("E19999", "Some exceptions occur failed"); - GELOGE(FAILED, "RunAsync failed, some exceptions occur !"); + REPORT_INNER_ERROR("E19999", "Some exceptions occur failed, model_id %u", model_id); + GELOGE(FAILED, "[Run][Async] failed, some exceptions occur, model_id %u", model_id); return FAILED; } @@ -571,16 +571,16 @@ Status GraphExecutor::DataInput(const InputData &input_data, OutputData &output_ GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->DataInput(input_data, output_data); if (ret != SUCCESS) { - GELOGE(ret, "DataInput: DataInput failed."); + GELOGE(ret, "[Call][DataInput] failed."); return ret; } } catch (std::bad_alloc &) { REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed"); - GELOGE(MEMALLOC_FAILED, "DataInput failed, bad memory allocation occur !"); + GELOGE(MEMALLOC_FAILED, "[Call][DataInput] failed, bad memory allocation occur !"); return MEMALLOC_FAILED; } catch (...) { REPORT_INNER_ERROR("E19999", "Some exceptions occur failed"); - GELOGE(FAILED, "DataInput failed, some exceptions occur !"); + GELOGE(FAILED, "[Call][DataInput] failed, some exceptions occur !"); return FAILED; } @@ -594,16 +594,16 @@ Status GraphExecutor::GetInputOutputDescInfo(const uint32_t model_id, vectorGetInputOutputDescInfo(model_id, input_desc, output_desc); if (ret != SUCCESS) { - GELOGE(ret, "GetInputOutputDescInfo failed."); + GELOGE(ret, "[Get][InputOutputDescInfo] failed, model_id:%u.", model_id); return ret; } } catch (std::bad_alloc &) { - REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed"); - GELOGE(MEMALLOC_FAILED, "GetInputOutputDescInfo failed, bad memory allocation occur !"); + REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed, model_id:%u.", model_id); + GELOGE(MEMALLOC_FAILED, "[Get][InputOutputDescInfo] failed, bad memory allocation occur, model_id:%u.", model_id); return MEMALLOC_FAILED; } catch (...) { - REPORT_INNER_ERROR("E19999", "Some exceptions occur failed"); - GELOGE(FAILED, "GetInputOutputDescInfo failed, some exceptions occur !"); + REPORT_INNER_ERROR("E19999", "Some exceptions occur failed, model_id:%u.", model_id); + GELOGE(FAILED, "[Get][InputOutputDescInfo] failed, some exceptions occur, model_id:%u.", model_id); return FAILED; } @@ -620,16 +620,16 @@ Status GraphExecutor::GetInputOutputDescInfo(const uint32_t model_id, vectorGetInputOutputDescInfo(model_id, input_desc, output_desc, input_formats, out_formats, new_model_desc); if (ret != SUCCESS) { - GELOGE(ret, "GetInputOutputDescInfo failed."); + GELOGE(ret, "[Get][InputOutputDescInfo] failed, model_id:%u.", model_id); return ret; } } catch (std::bad_alloc &) { - REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed"); - GELOGE(MEMALLOC_FAILED, "GetInputOutputDescInfo failed, bad memory allocation occur !"); + REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed, model_id:%u.", model_id); + GELOGE(MEMALLOC_FAILED, "[Get][InputOutputDescInfo] failed, bad memory allocation occur, model_id:%u.", model_id); return MEMALLOC_FAILED; } catch (...) { - REPORT_INNER_ERROR("E19999", "Some exceptions occur failed"); - GELOGE(FAILED, "GetInputOutputDescInfo failed, some exceptions occur !"); + REPORT_INNER_ERROR("E19999", "Some exceptions occur failed, model_id:%u.", model_id); + GELOGE(FAILED, "[Get][InputOutputDescInfo] failed, some exceptions occur, model_id:%u.", model_id); return FAILED; } @@ -649,7 +649,7 @@ Status GraphExecutor::GetDynamicBatchInfo(uint32_t model_id, std::vectorGetDynamicBatchInfo(model_id, batch_info, dynamic_type); if (ret != SUCCESS) { - GELOGE(ret, "GetDynamicBatchInfo failed."); + GELOGE(ret, "[Get][DynamicBatchInfo] failed, model_id:%u.", model_id); return ret; } return SUCCESS; @@ -667,7 +667,7 @@ Status GraphExecutor::GetCombinedDynamicDims(uint32_t model_id, std::vectorGetCombinedDynamicDims(model_id, batch_info); if (ret != SUCCESS) { - GELOGE(ret, "GetCombinedDynamicDims failed."); + GELOGE(ret, "[Call][GetCombinedDynamicDims] failed, model_id:%u.", model_id); return ret; } return SUCCESS; @@ -686,7 +686,7 @@ ge::Status GraphExecutor::GetUserDesignateShapeOrder(uint32_t model_id, GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->GetUserDesignateShapeOrder(model_id, user_input_shape_order); if (ret != SUCCESS) { - GELOGE(ret, "GetUserDesignateShapeOrder failed."); + GELOGE(ret, "[Get][UserDesignateShapeOrder] failed, model_id:%u.", model_id); return ret; } return SUCCESS; @@ -697,7 +697,7 @@ Status GraphExecutor::GetCurShape(const uint32_t model_id, std::vector GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->GetCurShape(model_id, batch_info, dynamic_type); if (ret != SUCCESS) { - GELOGE(ret, "GetCurShape failed"); + GELOGE(ret, "[Get][CurShape] failed, model_id:%u", model_id); return ret; } return SUCCESS; @@ -721,7 +721,7 @@ Status GraphExecutor::GetModelAttr(uint32_t model_id, std::vector &dynam GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->GetModelAttr(model_id, dynamic_output_shape_info); if (ret != SUCCESS) { - GELOGE(FAILED, "GetModelAttr failed"); + GELOGE(FAILED, "[Get][ModelAttr] failed, model_id:%u", model_id); return ret; } return SUCCESS; @@ -754,7 +754,7 @@ Status GraphExecutor::GetOrigInputInfo(uint32_t model_id, uint32_t index, Origin GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->GetOrigInputInfo(model_id, index, orig_input_info); if (ret != SUCCESS) { - GELOGE(ret, "GetOrigInputInfo failed."); + GELOGE(ret, "[Get][OrigInputInfo] failed, model_id:%u, index:%u.", model_id, index); return ret; } @@ -768,7 +768,7 @@ Status GraphExecutor::GetAllAippInputOutputDims(uint32_t model_id, uint32_t inde GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->GetAllAippInputOutputDims(model_id, index, input_dims, output_dims); if (ret != SUCCESS) { - GELOGE(ret, "GetAllAippInputOutputDims failed."); + GELOGE(ret, "[Get][AllAippInputOutputDims] failed, model_id:%u, index:%u.", model_id, index); return ret; } @@ -781,7 +781,8 @@ Status GraphExecutor::GetOpDescInfo(uint32_t device_id, uint32_t stream_id, uint GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->GetOpDescInfo(device_id, stream_id, task_id, op_desc_info); if (ret != SUCCESS) { - GELOGE(ret, "GetOpDescInfo failed."); + GELOGE(ret, "[Get][OpDescInfo] failed, device_id:%u, stream_id:%u, task_id:%u.", + device_id, stream_id, task_id); return ret; } return SUCCESS; diff --git a/ge/graph/label/case_label_maker.cc b/ge/graph/label/case_label_maker.cc index 8bf5de71..3fdb1783 100644 --- a/ge/graph/label/case_label_maker.cc +++ b/ge/graph/label/case_label_maker.cc @@ -44,8 +44,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { if (graph_names.empty() || graph_names.size() > kMaxCaseBranch) { REPORT_INNER_ERROR("E19999", "Node:%s(%s) subgraph size: %zu, check invalid", case_desc->GetName().c_str(), case_desc->GetType().c_str(), graph_names.size()); - GELOGE(INTERNAL_ERROR, "Node: %s has invalid subgraph, graph size: %zu.", case_desc->GetName().c_str(), - graph_names.size()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Node: %s has invalid subgraph, graph size: %zu.", + case_desc->GetName().c_str(), graph_names.size()); return FAILED; } @@ -71,7 +71,7 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { if (stream_active == nullptr) { REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail", graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][StreamActive] in Subgraph: %s failed.", graph->GetName().c_str()); return FAILED; } @@ -81,7 +81,7 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { if (label == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail", graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Call][AddLabelSetEnter] Subgraph: %s add label set failed.", graph->GetName().c_str()); return FAILED; } switch_labels.emplace_back(curr_label_index); @@ -96,7 +96,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { if (AddLabelGotoLeave(graph, label_goto_name, last_label_index) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelGotoLeave node in graph:%s fail", graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label goto failed.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Call][AddLabelGotoLeave] Subgraph: %s add label goto failed.", + graph->GetName().c_str()); return FAILED; } } else { @@ -105,7 +106,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { if (AddLabelSetLeave(graph, last_label_name, last_label_index) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSetLeave node in graph:%s fail", graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Call][AddLabelSetLeave] Subgraph: %s add label set failed.", + graph->GetName().c_str()); return FAILED; } } @@ -122,7 +124,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { if (switch_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSwitchEnter node in graph:%s fail", first_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label switch failed.", first_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Call][AddLabelSwitchEnter] Subgraph: %s add label switch failed.", + first_graph->GetName().c_str()); return FAILED; } @@ -130,7 +133,7 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { if (GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), first_label->GetInControlAnchor()) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", switch_node->GetName().c_str(), first_label->GetName().c_str(), first_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add ctrl edge to %s failed.", first_label->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][CtrlEdge] to %s failed.", first_label->GetName().c_str()); return FAILED; } @@ -139,7 +142,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) { if (AddLabelSwitchIndex(first_graph, data_name, pred_desc, switch_node, parent_index) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSwitchIndex node in graph:%s fail", first_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add switch input failed.", first_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Call][AddLabelSwitchIndex] Subgraph: %s add switch input failed.", + first_graph->GetName().c_str()); return FAILED; } diff --git a/ge/graph/label/if_label_maker.cc b/ge/graph/label/if_label_maker.cc index cf4cdd39..72b33015 100644 --- a/ge/graph/label/if_label_maker.cc +++ b/ge/graph/label/if_label_maker.cc @@ -47,7 +47,7 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { "then branch graph: %s, else branch graph: %s", if_desc->GetName().c_str(), if_desc->GetType().c_str(), then_branch_name.c_str(), else_branch_name.c_str()); - GELOGE(INTERNAL_ERROR, "Node: %s has invalid subgraph, then branch: %s, else branch: %s.", + GELOGE(INTERNAL_ERROR, "[Check][Param] Node: %s has invalid subgraph, then branch: %s, else branch: %s.", if_desc->GetName().c_str(), then_branch_name.c_str(), else_branch_name.c_str()); return FAILED; } @@ -72,7 +72,7 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { if (then_stream_active == nullptr) { REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail", then_sub_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", then_sub_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][StreamActive] in Subgraph:%s failed.", then_sub_graph->GetName().c_str()); return FAILED; } @@ -80,14 +80,14 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { if (then_enter_label == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail", then_sub_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", then_sub_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelSetEnter] in Subgraph:%s failed.", then_sub_graph->GetName().c_str()); return FAILED; } if (AddLabelGotoLeave(then_sub_graph, then_leave_name, else_leave_index) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelGotoLeave node in graph:%s fail", then_sub_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label goto failed.", then_sub_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelGotoLeave] in Subgraph:%s failed.", then_sub_graph->GetName().c_str()); return FAILED; } @@ -95,20 +95,20 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { if (else_stream_active == nullptr) { REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail", else_stream_active->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", else_sub_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][StreamActive] in Subgraph:%s failed.", else_sub_graph->GetName().c_str()); return FAILED; } if (AddLabelSetEnter(else_sub_graph, else_enter_name, else_enter_index, else_stream_active) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail", else_sub_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", else_sub_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelSetEnter] in Subgraph:%s failed.", else_sub_graph->GetName().c_str()); return FAILED; } if (AddLabelSetLeave(else_sub_graph, else_leave_name, else_leave_index) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSetLeave node in graph:%s fail", else_sub_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", else_sub_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelSetLeave] in Subgraph:%s failed.", else_sub_graph->GetName().c_str()); return FAILED; } @@ -121,7 +121,7 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { if (switch_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSwitchEnter node in graph:%s fail", then_sub_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label switch failed.", then_sub_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelSwitchEnter] in Subgraph:%s failed.", then_sub_graph->GetName().c_str()); return FAILED; } @@ -129,7 +129,7 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { if (GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), then_enter_label->GetInControlAnchor()) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", switch_node->GetName().c_str(), then_enter_label->GetName().c_str(), then_sub_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add ctrl edge to %s failed.", then_enter_label->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][CtrlEdge] to %s failed.", then_enter_label->GetName().c_str()); return FAILED; } @@ -138,7 +138,7 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) { if (AddLabelSwitchIndex(then_sub_graph, data_name, pred_desc, switch_node, parent_index) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSwitchIndex node in graph:%s fail", then_sub_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add switch input failed.", then_sub_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelSwitchIndex] in Subgraph:%s failed.", then_sub_graph->GetName().c_str()); return FAILED; } diff --git a/ge/graph/label/label_maker.cc b/ge/graph/label/label_maker.cc index 156748e8..638cbbae 100644 --- a/ge/graph/label/label_maker.cc +++ b/ge/graph/label/label_maker.cc @@ -58,7 +58,7 @@ void LabelMaker::LinkToGraphHead(const ComputeGraphPtr &graph, const NodePtr &no if (GraphUtils::AddEdge(node->GetOutControlAnchor(), n->GetInControlAnchor()) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", node->GetName().c_str(), n->GetName().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Add ctrl edge from %s to %s failed.", node->GetName().c_str(), n->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][CtrlEdge] from %s to %s failed.", node->GetName().c_str(), n->GetName().c_str()); } } } @@ -82,7 +82,7 @@ void LabelMaker::LinkToGraphTail(const ComputeGraphPtr &graph, const NodePtr &no if (GraphUtils::AddEdge(tail->GetOutControlAnchor(), node->GetInControlAnchor()) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", tail->GetName().c_str(), node->GetName().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Add ctrl edge from %s to %s failed.", tail->GetName().c_str(), node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][CtrlEdge] from %s to %s failed.", tail->GetName().c_str(), node->GetName().c_str()); } return; } @@ -101,7 +101,7 @@ NodePtr LabelMaker::AddStreamActive(const ComputeGraphPtr &graph, const std::str const auto &node_list = graph->GetDirectNode(); if (node_list.empty()) { REPORT_INNER_ERROR("E19999", "Check param graph:%s has no node", graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSet: Graph %s node is empty.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] LabelSet: Graph %s node is empty.", graph->GetName().c_str()); return nullptr; } @@ -137,7 +137,7 @@ NodePtr LabelMaker::AddLabelSetEnter(const ComputeGraphPtr &graph, const std::st const auto &node_list = graph->GetDirectNode(); if (node_list.empty()) { REPORT_INNER_ERROR("E19999", "Check param graph:%s has no node", graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSet: Graph %s node is empty.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] LabelSet: Graph %s node is empty.", graph->GetName().c_str()); return nullptr; } @@ -153,7 +153,7 @@ NodePtr LabelMaker::AddLabelSetEnter(const ComputeGraphPtr &graph, const std::st if (GraphUtils::AddEdge(label_set->GetOutControlAnchor(), stream_active->GetInControlAnchor()) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", label_set->GetName().c_str(), stream_active->GetName().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Add ctrl edge from %s to %s failed.", label_set->GetName().c_str(), + GELOGE(INTERNAL_ERROR, "[Add][CtrlEdge] from %s to %s failed.", label_set->GetName().c_str(), stream_active->GetName().c_str()); return nullptr; } @@ -202,7 +202,7 @@ NodePtr LabelMaker::AddLabelGotoEnter(const ComputeGraphPtr &graph, const std::s auto it = node_list.begin(); if (it == node_list.end()) { REPORT_INNER_ERROR("E19999", "Check param graph:%s has no node", graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelGoto: Graph %s node is empty.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] LabelGoto: Graph %s node is empty.", graph->GetName().c_str()); return nullptr; } @@ -216,7 +216,7 @@ NodePtr LabelMaker::AddLabelGotoEnter(const ComputeGraphPtr &graph, const std::s if (label_goto == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s fail", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelGoto: Add to graph %s failed.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Node] to graph %s failed.", graph->GetName().c_str()); return nullptr; } @@ -265,7 +265,7 @@ NodePtr LabelMaker::AddLabelSwitchEnter(const ComputeGraphPtr &graph, const std: auto it = node_list.begin(); if (it == node_list.end()) { REPORT_INNER_ERROR("E19999", "Check param graph:%s has no node", graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Graph %s node is empty.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] LabelSwitchByIndex: Graph %s node is empty.", graph->GetName().c_str()); return nullptr; } @@ -277,14 +277,14 @@ NodePtr LabelMaker::AddLabelSwitchEnter(const ComputeGraphPtr &graph, const std: if (op_desc->AddInputDesc(desc) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add input desc into node:%s(%s) in graph:%s fail", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add input desc failed."); + GELOGE(INTERNAL_ERROR, "[Add][InputDesc] failed."); return nullptr; } if (!AttrUtils::SetListInt(op_desc, ATTR_NAME_LABEL_SWITCH_LIST, labels)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_LABEL_SWITCH_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add %s failed.", ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s failed.", ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); return nullptr; } @@ -292,7 +292,7 @@ NodePtr LabelMaker::AddLabelSwitchEnter(const ComputeGraphPtr &graph, const std: if (label_switch == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s ahead fail", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add to graph %s failed.", graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Node] to graph %s failed.", graph->GetName().c_str()); return nullptr; } @@ -320,14 +320,15 @@ NodePtr LabelMaker::AddLabelSwitchLeave(const ComputeGraphPtr &graph, const std: if (op_desc->AddInputDesc(desc) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add input desc into node:%s(%s) in graph:%s fail", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add input desc failed."); + GELOGE(INTERNAL_ERROR, "[Add][InputDesc] into node:%s(%s) in graph:%s fail", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr; } if (!AttrUtils::SetListInt(op_desc, ATTR_NAME_LABEL_SWITCH_LIST, labels)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_LABEL_SWITCH_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add %s failed.", ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s failed.", ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); return nullptr; } @@ -360,20 +361,23 @@ NodePtr LabelMaker::AddLabelSwitchIndex(const ComputeGraphPtr &graph, const std: if (op_desc->AddInputDesc(desc) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add input desc into node:%s(%s) in graph:%s fail", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add data input desc failed."); + GELOGE(INTERNAL_ERROR, "[Add][InputDesc] into node:%s(%s) in graph:%s fail", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr; } if (op_desc->AddOutputDesc(desc) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add output desc into node:%s(%s) in graph:%s fail", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add data output desc failed."); + GELOGE(INTERNAL_ERROR, "[Add][OutputDesc] into node:%s(%s) in graph:%s fail", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr; } if (!AttrUtils::SetInt(op_desc, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_PARENT_NODE_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add %s failed.", ATTR_NAME_PARENT_NODE_INDEX.c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_PARENT_NODE_INDEX.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } NodePtr op_data = graph->AddNodeFront(op_desc); @@ -384,7 +388,7 @@ NodePtr LabelMaker::AddLabelSwitchIndex(const ComputeGraphPtr &graph, const std: if (GraphUtils::AddEdge(op_data->GetOutDataAnchor(0), sw_node->GetInDataAnchor(0)) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", op_data->GetName().c_str(), sw_node->GetName().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add input edge to %s failed.", op_data->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][CtrlEdge] to %s failed.", op_data->GetName().c_str()); return nullptr; } diff --git a/ge/graph/label/partitioned_call_label_maker.cc b/ge/graph/label/partitioned_call_label_maker.cc index d9a89ef2..7b4bcbd8 100644 --- a/ge/graph/label/partitioned_call_label_maker.cc +++ b/ge/graph/label/partitioned_call_label_maker.cc @@ -41,7 +41,7 @@ Status PartitionedCallLabelMaker::Run(uint32_t &label_index) { if (sub_graph_name.empty()) { REPORT_INNER_ERROR("E19999", "Node:%s(%s) subgraph_index:%d name is empty, check invalid", call_desc->GetName().c_str(), call_desc->GetType().c_str(), kSubGraphIndex); - GELOGE(INTERNAL_ERROR, "Node: %s has no subgraph name.", sub_graph_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Node:%s has no subgraph name.", sub_graph_name.c_str()); return FAILED; } @@ -50,7 +50,7 @@ Status PartitionedCallLabelMaker::Run(uint32_t &label_index) { REPORT_INNER_ERROR("E19999", "Node:%s(%s) subgraph_name:%s is not exist in parent_graph, check invalid", call_desc->GetName().c_str(), call_desc->GetType().c_str(), sub_graph_name.c_str()); - GELOGE(INTERNAL_ERROR, "Node: %s has no subgraph.", sub_graph_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Get][SubGraph] Node:%s has no subgraph.", sub_graph_name.c_str()); return FAILED; } @@ -59,7 +59,7 @@ Status PartitionedCallLabelMaker::Run(uint32_t &label_index) { if (stream_active == nullptr) { REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail", sub_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active node failed.", sub_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][StreamActive] in Subgraph:%s failed.", sub_graph->GetName().c_str()); return FAILED; } diff --git a/ge/graph/label/while_label_maker.cc b/ge/graph/label/while_label_maker.cc index 22e783e3..cd6b3743 100644 --- a/ge/graph/label/while_label_maker.cc +++ b/ge/graph/label/while_label_maker.cc @@ -47,7 +47,7 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) { REPORT_INNER_ERROR("E19999", "Node:%s(%s) cond subgraph index:%d or body subgraph index:%d name is empty, " "check invalid", while_desc->GetName().c_str(), while_desc->GetType().c_str(), kCondBranchIndex, kBodyBranchIndex); - GELOGE(INTERNAL_ERROR, "Node: %s has invalid subgraph, cond branch: %s, body branch: %s.", + GELOGE(INTERNAL_ERROR, "[Check][Param] Node: %s has invalid subgraph, cond branch: %s, body branch: %s.", while_desc->GetName().c_str(), cond_name.c_str(), body_name.c_str()); return FAILED; } @@ -72,14 +72,14 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) { if (cond_stream_active == nullptr) { REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail", cond_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", cond_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][StreamActive] in Subgraph:%s failed.", cond_graph->GetName().c_str()); return FAILED; } if (AddLabelSetEnter(cond_graph, cond_enter_name, cond_enter_index, cond_stream_active) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail", cond_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", cond_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelSetEnter] in Subgraph:%s failed.", cond_graph->GetName().c_str()); return FAILED; } @@ -87,28 +87,28 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) { if (body_stream_active == nullptr) { REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail", body_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", body_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][StreamActive] in Subgraph:%s failed.", body_graph->GetName().c_str()); return FAILED; } if (AddLabelSetEnter(body_graph, body_enter_name, body_enter_index, body_stream_active) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail", body_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", body_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelSetEnter] in Subgraph:%s failed.", body_graph->GetName().c_str()); return FAILED; } if (AddLabelGotoLeave(body_graph, goto_leave_name, cond_enter_index) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelGotoLeave node in graph:%s fail", body_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label goto failed.", body_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelGotoLeave] in Subgraph:%s failed.", body_graph->GetName().c_str()); return FAILED; } if (AddLabelSetLeave(body_graph, body_leave_name, body_leave_index) == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSetLeave node in graph:%s fail", body_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", body_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelSetLeave] in Subgraph:%s failed.", body_graph->GetName().c_str()); return FAILED; } @@ -126,14 +126,14 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) { if (switch_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add LabelSwitchLeave node in graph:%s fail", cond_graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Subgraph: %s add label switch failed.", cond_graph->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][LabelSwitchLeave] in Subgraph:%s failed.", cond_graph->GetName().c_str()); return FAILED; } // link Data input. const auto &all_in_data = cond_out_node->GetAllInDataAnchors(); if (all_in_data.size() != kCondOutputNum) { - GELOGE(FAILED, "Node: %s Cond sbugraph output size:%zu should equal size:%u.", + GELOGE(FAILED, "[Check][Param] Node: %s Cond sbugraph output size:%zu should equal size:%u.", switch_node->GetName().c_str(), all_in_data.size(), kCondOutputNum); return FAILED; } @@ -144,7 +144,7 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) { REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", in_anchor->GetPeerOutAnchor()->GetOwnerNode()->GetName().c_str(), switch_node->GetName().c_str(), cond_graph->GetName().c_str()); - GELOGE(FAILED, "Node: %s Add pred data input failed.", switch_node->GetName().c_str()); + GELOGE(FAILED, "[Add][PredDataInput] to Node:%s failed.", switch_node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/load/model_manager/cpu_queue_schedule.cc b/ge/graph/load/model_manager/cpu_queue_schedule.cc index c92e2a04..9821aa73 100644 --- a/ge/graph/load/model_manager/cpu_queue_schedule.cc +++ b/ge/graph/load/model_manager/cpu_queue_schedule.cc @@ -51,18 +51,16 @@ CpuTaskInfo::~CpuTaskInfo() { /// Status CpuTaskModelDequeue::Init(uint32_t queue_id, uintptr_t &in_mbuf) { if ((args_ != nullptr) || (args_size_ > 0)) { - REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0," - "check invalid", args_size_); - GELOGE(FAILED, "Task already initialized, size: %u", args_size_); + REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0, check invalid", args_size_); + GELOGE(FAILED, "[Check][Param] Task already initialized, size:%u", args_size_); return FAILED; } args_size_ = sizeof(MbufQueueInfo) + sizeof(uintptr_t); // sizeof(uintptr_t) for save in_mbuf. rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } in_mbuf = reinterpret_cast(args_) + sizeof(MbufQueueInfo); @@ -73,9 +71,8 @@ Status CpuTaskModelDequeue::Init(uint32_t queue_id, uintptr_t &in_mbuf) { queue_info.in_mbuf = in_mbuf; // Placeholder, input mbuf addr will save to this place. status = rtMemcpy(args_, args_size_, &queue_info, sizeof(MbufQueueInfo), RT_MEMCPY_HOST_TO_DEVICE); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } @@ -86,15 +83,14 @@ Status CpuTaskModelDequeue::Distribute() { if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) { REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_:%u is 0 or stream_ is nullptr," "check invalid", args_size_); - GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_); + GELOGE(FAILED, "[Check][Param] Task not initialized, distribute failed, size:%u", args_size_); return FAILED; } rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskModelDequeue, kCoreDim, args_, args_size_, nullptr, stream_); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", - status); - GELOGE(RT_FAILED, "Call rt CpuKernelLaunch ModelDequeue failed, status: 0x%X", status); + REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", status); + GELOGE(RT_FAILED, "[Call][RtCpuKernelLaunch] failed, ret:0x%X", status); return RT_ERROR_TO_GE_STATUS(status); } @@ -111,9 +107,8 @@ Status CpuTaskModelDequeue::Distribute() { /// Status CpuTaskZeroCopy::Init(std::vector &mbuf_list, const map &outside_addrs) { if ((args_ != nullptr) || (args_size_ > 0)) { - REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0," - "check invalid", args_size_); - GELOGE(FAILED, "Task already initialized, size: %u", args_size_); + REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0, check invalid", args_size_); + GELOGE(FAILED, "[Check][Param] Task already initialized, size:%u", args_size_); return FAILED; } @@ -127,7 +122,7 @@ Status CpuTaskZeroCopy::Init(std::vector &mbuf_list, const map dst_addrs; for (const auto &addrs : outside_addrs) { const auto &addrs_mapping_list = addrs.second.GetOutsideAddrs(); - GE_CHK_BOOL_EXEC(!addrs_mapping_list.empty(), return PARAM_INVALID, "not set outside_addrs"); + GE_CHK_BOOL_EXEC(!addrs_mapping_list.empty(), return PARAM_INVALID, "[Check][Param] not set outside_addrs"); std::map> virtual_args_addrs = addrs_mapping_list[0]; for (const auto &virtual_args_addr : virtual_args_addrs) { addr_map_info.addr_num += virtual_args_addr.second.size(); @@ -143,13 +138,21 @@ Status CpuTaskZeroCopy::Init(std::vector &mbuf_list, const map &mbuf_list, const map 0)) { - REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0," - "check invalid", args_size_); - GELOGE(FAILED, "Task already initialized, size: %u", args_size_); + REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0, check invalid", args_size_); + GELOGE(FAILED, "[Check][Param] Task already initialized, size:%u", args_size_); return FAILED; } args_size_ = sizeof(PrepareOutputInfo) + sizeof(uintptr_t); // sizeof(uintptr_t) for save out_mbuf. rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } out_mbuf = reinterpret_cast(args_) + sizeof(PrepareOutputInfo); @@ -240,9 +242,8 @@ Status CpuTaskPrepareOutput::Init(uintptr_t addr, uint32_t size, uintptr_t in_mb prepare.out_mbuf = out_mbuf; // Placeholder, output mbuf addr will save to this place. status = rtMemcpy(args_, args_size_, &prepare, sizeof(PrepareOutputInfo), RT_MEMCPY_HOST_TO_DEVICE); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } @@ -253,15 +254,14 @@ Status CpuTaskPrepareOutput::Distribute() { if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) { REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_:%u is 0 or stream_ is nullptr," "check invalid", args_size_); - GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_); + GELOGE(FAILED, "[Check][Param] Task not initialized, distribute failed, size:%u", args_size_); return FAILED; } rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskPrepareOutput, kCoreDim, args_, args_size_, nullptr, stream_); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", - status); - GELOGE(RT_FAILED, "Call rt CpuKernelLaunch PrepareOutput failed, status: 0x%X", status); + REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", status); + GELOGE(RT_FAILED, "[Call][RtCpuKernelLaunch] failed, ret:0x%X", status); return RT_ERROR_TO_GE_STATUS(status); } @@ -278,9 +278,8 @@ Status CpuTaskPrepareOutput::Distribute() { /// Status CpuTaskModelEnqueue::Init(uint32_t queue_id, uintptr_t out_mbuf) { if ((args_ != nullptr) || (args_size_ > 0)) { - REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0," - "check invalid", args_size_); - GELOGE(FAILED, "Task already initialized, size: %u", args_size_); + REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0, check invalid", args_size_); + GELOGE(FAILED, "[Check][Param] Task already initialized, size:%u", args_size_); return FAILED; } @@ -288,9 +287,8 @@ Status CpuTaskModelEnqueue::Init(uint32_t queue_id, uintptr_t out_mbuf) { args_size_ = sizeof(MbufQueueInfo); rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } GE_PRINT_DYNAMIC_MEMORY(rtMalloc, "args data.", args_size_) @@ -300,9 +298,8 @@ Status CpuTaskModelEnqueue::Init(uint32_t queue_id, uintptr_t out_mbuf) { queue_info.in_mbuf = out_mbuf; status = rtMemcpy(args_, args_size_, &queue_info, args_size_, RT_MEMCPY_HOST_TO_DEVICE); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } @@ -313,15 +310,14 @@ Status CpuTaskModelEnqueue::Distribute() { if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) { REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_ is 0 or stream_ is nullptr, arg_size:%u," "check invalid", args_size_); - GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_); + GELOGE(FAILED, "[Check][Param] Task not initialized, distribute failed, size:%u", args_size_); return FAILED; } rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskModelEnqueue, kCoreDim, args_, args_size_, nullptr, stream_); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", - status); - GELOGE(RT_FAILED, "Call rt CpuKernelLaunch ModelEnqueue failed, status: 0x%X", status); + REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", status); + GELOGE(RT_FAILED, "[Call][RtCpuKernelLaunch] failed, ret:0x%X", status); return RT_ERROR_TO_GE_STATUS(status); } @@ -338,7 +334,7 @@ Status CpuTaskModelEnqueue::Distribute() { Status CpuTaskActiveEntry::Init(rtStream_t stream) { if (stream == nullptr) { REPORT_INNER_ERROR("E19999", "Param stream is nullptr, check invalid"); - GELOGE(FAILED, "Task active stream not valid"); + GELOGE(FAILED, "[Check][Param] Task active stream not valid"); return FAILED; } @@ -348,17 +344,15 @@ Status CpuTaskActiveEntry::Init(rtStream_t stream) { Status CpuTaskActiveEntry::Distribute() { if ((active_stream_ == nullptr) || (stream_ == nullptr)) { - REPORT_INNER_ERROR("E19999", "Param stream is nullptr or active_stream_ is nullptr, " - "check invalid"); - GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_); + REPORT_INNER_ERROR("E19999", "Param stream is nullptr or active_stream_ is nullptr, check invalid"); + GELOGE(FAILED, "[Check][Param] Task not initialized, distribute failed, size:%u", args_size_); return FAILED; } rtError_t ret = rtStreamActive(active_stream_, stream_); if (ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamActive failed, ret:0x%X", - ret); - GELOGE(RT_FAILED, "Call rt StreamActive failed, ret: 0x%X", ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamActive failed, ret:0x%X", ret); + GELOGE(RT_FAILED, "[Call][RtStreamActive] failed, ret:0x%X", ret); return RT_ERROR_TO_GE_STATUS(ret); } @@ -374,27 +368,24 @@ Status CpuTaskActiveEntry::Distribute() { /// Status CpuTaskWaitEndGraph::Init(uint32_t model_id) { if ((args_ != nullptr) || (args_size_ > 0)) { - REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0," - "check invalid", args_size_); - GELOGE(FAILED, "Task already initialized, size: %u", args_size_); + REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0, check invalid", args_size_); + GELOGE(FAILED, "[Check][Param] Task already initialized, size:%u", args_size_); return FAILED; } args_size_ = sizeof(model_id); rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } GE_PRINT_DYNAMIC_MEMORY(rtMalloc, "args data.", args_size_) status = rtMemcpy(args_, args_size_, &model_id, args_size_, RT_MEMCPY_HOST_TO_DEVICE); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } @@ -405,15 +396,14 @@ Status CpuTaskWaitEndGraph::Distribute() { if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) { REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_:%u is 0 or stream_ is nullptr," "check invalid", args_size_); - GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_); + GELOGE(FAILED, "[Check][Param] Task not initialized, distribute failed, size:%u", args_size_); return FAILED; } rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskWaitEndGraph, kCoreDim, args_, args_size_, nullptr, stream_); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", - status); - GELOGE(RT_FAILED, "Call rt CpuKernelLaunch WaitEndGraph failed, status: 0x%X", status); + REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", status); + GELOGE(RT_FAILED, "[Call][RtCpuKernelLaunch] failed, ret:0x%X", status); return RT_ERROR_TO_GE_STATUS(status); } @@ -429,27 +419,24 @@ Status CpuTaskWaitEndGraph::Distribute() { /// Status CpuTaskModelRepeat::Init(uint32_t model_id) { if ((args_ != nullptr) || (args_size_ > 0)) { - REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0," - "check invalid", args_size_); - GELOGE(FAILED, "Task already initialized, size: %u", args_size_); + REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0, check invalid", args_size_); + GELOGE(FAILED, "[Check][Param] Task already initialized, size:%u", args_size_); return FAILED; } args_size_ = sizeof(model_id); rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } GE_PRINT_DYNAMIC_MEMORY(rtMalloc, "args data.", args_size_) status = rtMemcpy(args_, args_size_, &model_id, args_size_, RT_MEMCPY_HOST_TO_DEVICE); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", - args_size_, status); - GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", args_size_, status); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", args_size_, status); return RT_ERROR_TO_GE_STATUS(status); } @@ -460,15 +447,14 @@ Status CpuTaskModelRepeat::Distribute() { if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) { REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_:%u is 0 or stream_ is nullptr," "check invalid", args_size_); - GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_); + GELOGE(FAILED, "[Check][Param] Task not initialized, distribute failed, size:%u", args_size_); return FAILED; } rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskModelRepeat, kCoreDim, args_, args_size_, nullptr, stream_); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", - status); - GELOGE(RT_FAILED, "Call rt CpuKernelLaunch ModelRepeat failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X", status); + GELOGE(RT_FAILED, "[Call][RtCpuKernelLaunch] failed, ret:0x%X", status); return RT_ERROR_TO_GE_STATUS(status); } diff --git a/ge/graph/load/model_manager/data_dumper.cc b/ge/graph/load/model_manager/data_dumper.cc index c7463001..c96b3885 100644 --- a/ge/graph/load/model_manager/data_dumper.cc +++ b/ge/graph/load/model_manager/data_dumper.cc @@ -127,7 +127,7 @@ void DataDumper::ReleaseDevMem(void **ptr) noexcept { if (*ptr != nullptr) { rtError_t rt_ret = rtFree(*ptr); if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtFree failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtFree] failed, ret:0x%X", rt_ret); } *ptr = nullptr; @@ -144,7 +144,7 @@ void DataDumper::SaveDumpInput(const std::shared_ptr &node) { if (node != nullptr) { auto input_op_desc = node->GetOpDesc(); if (input_op_desc == nullptr) { - GELOGE(PARAM_INVALID, "input op desc is null."); + GELOGE(PARAM_INVALID, "[Get][OpDesc] input op desc is null."); return; } @@ -153,7 +153,7 @@ void DataDumper::SaveDumpInput(const std::shared_ptr &node) { ge::NodePtr dst_node = dst_in_data_anchor->GetOwnerNode(); auto op_desc = dst_node->GetOpDesc(); if (op_desc == nullptr) { - GELOGE(PARAM_INVALID, "input op desc is null."); + GELOGE(PARAM_INVALID, "[Get][OpDesc] input op desc is null."); return; } @@ -179,7 +179,7 @@ void DataDumper::SaveOpDebugId(uint32_t task_id, uint32_t stream_id, void *op_de void DataDumper::SaveDumpTask(uint32_t task_id, uint32_t stream_id, const std::shared_ptr &op_desc, uintptr_t args) { if (op_desc == nullptr) { - GELOGE(PARAM_INVALID, "Opdesc is nullptr"); + GELOGE(PARAM_INVALID, "[Check][Param] Opdesc is nullptr"); return; } @@ -190,14 +190,14 @@ void DataDumper::SaveDumpTask(uint32_t task_id, uint32_t stream_id, const std::s InnerInputMapping &inner_input_mapping = iter.first->second; auto &data_op = inner_input_mapping.data_op; if (data_op == nullptr) { - GELOGE(PARAM_INVALID, "data_op is null."); + GELOGE(PARAM_INVALID, "[Check][Param] data_op is null."); return; } auto input_tensor = op_desc->GetInputDescPtr(inner_input_mapping.input_anchor_index); if (input_tensor == nullptr) { - GELOGE(PARAM_INVALID, "input_tensor is null, index: %d, size: %zu.", inner_input_mapping.input_anchor_index, - op_desc->GetInputsSize()); + GELOGE(PARAM_INVALID, "[Get][InputDescPtr] input_tensor in op:%s is null, index:%d, size:%zu.", + op_desc->GetName().c_str(), inner_input_mapping.input_anchor_index, op_desc->GetInputsSize()); return; } @@ -205,7 +205,8 @@ void DataDumper::SaveDumpTask(uint32_t task_id, uint32_t stream_id, const std::s if (AttrUtils::GetInt(input_tensor, ATTR_NAME_INPUT_ORIGIN_SIZE, data_size)) { GELOGI("Get aipp data size according to attr is %ld", data_size); } else if (TensorUtils::GetTensorSizeInBytes(*input_tensor, data_size) != SUCCESS) { - GELOGE(PARAM_INVALID, "Get input size filed"); + GELOGE(PARAM_INVALID, "[Get][InputSize] failed in %s, index:%u", + op_desc->GetName().c_str(), inner_input_mapping.input_anchor_index); return; } @@ -249,7 +250,7 @@ Status DataDumper::GenerateOutput(toolkit::aicpu::dump::Output &output, int64_t output_size = 0; if (TensorUtils::GetTensorSizeInBytes(tensor_descs.at(index), output_size) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Get tensor size fail"); - GELOGE(PARAM_INVALID, "Get output size filed"); + GELOGE(PARAM_INVALID, "[Get][OutputSize] failed"); return PARAM_INVALID; } GELOGD("Get output size in dump is %ld", output_size); @@ -274,34 +275,37 @@ Status DataDumper::DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_inf size_t index; // parser and find which node's input or output tensor desc is chosen for dump info if (!ParseNameIndex(node_name_index, dump_op_name, input_or_output, index)) { - GELOGE(PARAM_INVALID, "Op [%s] output desc[%zu] with invalid ATTR_DATA_DUMP_REF attr[%s].", + GELOGE(PARAM_INVALID, "[Check][Param] Op [%s] output desc[%zu] with invalid ATTR_DATA_DUMP_REF attr[%s].", inner_dump_info.op->GetName().c_str(), i, node_name_index.c_str()); return PARAM_INVALID; } GE_CHECK_NOTNULL(compute_graph_); auto replace_node = compute_graph_->FindNode(dump_op_name); GE_RT_PARAM_INVALID_WITH_LOG_IF_TRUE(replace_node == nullptr, - "Op [%s] output desc[%zu] with invalid ATTR_DATA_DUMP_REF attr[%s]," - " cannot find redirect node[%s].", + "[Check][Param] Op [%s] output desc[%zu] with invalid ATTR_DATA_DUMP_REF " + "attr[%s], cannot find redirect node[%s].", inner_dump_info.op->GetName().c_str(), i, node_name_index.c_str(), dump_op_name.c_str()); auto replace_opdesc = replace_node->GetOpDesc(); GE_CHECK_NOTNULL(replace_opdesc); auto iter = ref_info_.find(replace_opdesc); GE_RT_PARAM_INVALID_WITH_LOG_IF_TRUE(iter == ref_info_.end(), - "Op [%s] output desc[%zu] cannot find any saved redirect node[%s]'s info.", + "[Check][Param] Op [%s] output desc[%zu] cannot find " + "any saved redirect node[%s]'s info.", inner_dump_info.op->GetName().c_str(), i, replace_opdesc->GetName().c_str()); GE_CHECK_NOTNULL(iter->second); auto addr = reinterpret_cast(iter->second); if (input_or_output == kDumpInput) { const auto &replace_input_descs = replace_opdesc->GetAllInputsDesc(); addr += kAddrLen * index; - GE_CHK_STATUS_RET(GenerateOutput(output, replace_input_descs, addr, index), "Generate output failed"); + GE_CHK_STATUS_RET(GenerateOutput(output, replace_input_descs, addr, index), + "[Generate][Output] failed for %s, index:%zu", inner_dump_info.op->GetName().c_str(), index); } else if (input_or_output == kDumpOutput) { const auto &replace_output_descs = replace_opdesc->GetAllOutputsDesc(); const auto replace_input_size = replace_opdesc->GetAllInputsDesc().size(); addr += (index + replace_input_size) * kAddrLen; - GE_CHK_STATUS_RET(GenerateOutput(output, replace_output_descs, addr, index), "Generate output failed"); + GE_CHK_STATUS_RET(GenerateOutput(output, replace_output_descs, addr, index), + "[Generate][Output] failed for %s, index:%zu", inner_dump_info.op->GetName().c_str(), index); } GELOGD("Op [%s] output desc[%zu] dump info is replaced by node[%s] [%s] tensor_desc [%zu]", inner_dump_info.op->GetName().c_str(), i, dump_op_name.c_str(), input_or_output.c_str(), index); @@ -314,9 +318,9 @@ Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, tool std::vector v_memory_type; bool has_mem_type_attr = ge::AttrUtils::GetListInt(inner_dump_info.op, ATTR_NAME_OUTPUT_MEM_TYPE_LIST, v_memory_type); GE_RT_PARAM_INVALID_WITH_LOG_IF_TRUE(has_mem_type_attr && (v_memory_type.size() != output_descs.size()), - "DumpOutputWithTask[%s], output size[%zu], output memory type size[%zu]", - inner_dump_info.op->GetName().c_str(), output_descs.size(), - v_memory_type.size()); + "[Check][Param] DumpOutputWithTask[%s], output size[%zu], " + "output memory type size[%zu]", inner_dump_info.op->GetName().c_str(), + output_descs.size(), v_memory_type.size()); size_t no_need_dump_output_num = 0; for (size_t i = 0; i < output_descs.size(); ++i) { @@ -338,16 +342,16 @@ Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, tool "output which is need to dump.", inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(), no_need_dump_output_num, output_descs.size(), output_addrs.size()); - GELOGE(PARAM_INVALID, "The number of output does not match in op:%s(%s). The size[%zu] of output which is no need" - " to dump should not greater than the size[%zu] of output descs minus the size[%zu] of output which is " - "need to dump.", inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(), - no_need_dump_output_num, output_descs.size(), output_addrs.size()); + GELOGE(PARAM_INVALID, "[Check][Param] The number of output does not match in op:%s(%s). The size[%zu] of output " + "which is no need to dump should not greater than the size[%zu] of output descs minus the size[%zu] " + "of output which is need to dump.", inner_dump_info.op->GetName().c_str(), + inner_dump_info.op->GetType().c_str(), no_need_dump_output_num, output_descs.size(), output_addrs.size()); return PARAM_INVALID; } // check dump output tensor desc is redirected by attr ATTR_DATA_DUMP_REF if (AttrUtils::GetStr(&output_desc, ATTR_DATA_DUMP_REF, node_name_index)) { - GE_CHK_STATUS_RET(DumpRefOutput(inner_dump_info, output, i, node_name_index), "DumpRefOutput failed"); + GE_CHK_STATUS_RET(DumpRefOutput(inner_dump_info, output, i, node_name_index), "[Dump][RefOutput] failed"); task.mutable_output()->Add(std::move(output)); } else { if (IsTensorDescWithSkipDumpAddrType(has_mem_type_attr, v_memory_type, i)) { @@ -356,7 +360,7 @@ Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, tool if (TensorUtils::GetTensorSizeInBytes(output_descs.at(i), output_size) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Get output tensor size fail in op:%s(%s), index:%zu", inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(), i); - GELOGE(PARAM_INVALID, "Get output size failed."); + GELOGE(PARAM_INVALID, "[Get][OutputSize] failed in %s, index:%zu", inner_dump_info.op->GetName().c_str(), i); return PARAM_INVALID; } GELOGI("Get output size of l1_fusion_dump is %ld", output_size); @@ -364,7 +368,8 @@ Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, tool } else { const auto input_size = inner_dump_info.op->GetInputsSize(); auto addr = inner_dump_info.args + (i + input_size) * kAddrLen; - GE_CHK_STATUS_RET(GenerateOutput(output, output_descs, addr, i), "Generate output failed"); + GE_CHK_STATUS_RET(GenerateOutput(output, output_descs, addr, i), + "[Generate][Output] failed for %s, index:%zu", inner_dump_info.op->GetName().c_str(), i); task.mutable_output()->Add(std::move(output)); } } @@ -383,11 +388,11 @@ Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, toolkit::aic auto output_tensor = inner_dump_info.op->GetOutputDescPtr(inner_dump_info.output_anchor_index); const std::vector output_addrs = ModelUtils::GetOutputDataAddrs(*runtime_param_, inner_dump_info.op); if (output_tensor == nullptr) { - REPORT_INNER_ERROR("E19999", "output_desc tensor is nullptr in op:%s(%s), index:%u, " - "check invalid", + REPORT_INNER_ERROR("E19999", "output_desc tensor is nullptr in op:%s(%s), index:%u, check invalid", inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(), inner_dump_info.output_anchor_index); - GELOGE(PARAM_INVALID, "output_tensor is null, index: %d, size: %zu.", inner_dump_info.output_anchor_index, + GELOGE(PARAM_INVALID, "[Get][OutputDescPtr] output_tensor is null in op:%s, index:%d, size:%zu.", + inner_dump_info.op->GetName().c_str(), inner_dump_info.output_anchor_index, inner_dump_info.op->GetOutputsSize()); return PARAM_INVALID; } @@ -413,7 +418,9 @@ Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, toolkit::aic REPORT_INNER_ERROR("E19999", "output_anchor_index:%u >= output addr size:%zu in op:%s(%s), " "check invalid", inner_dump_info.output_anchor_index, output_addrs.size(), inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str()); - GELOGE(FAILED, "Index is out of range."); + GELOGE(FAILED, "[Check][Param] output_anchor_index:%u >= output addr size:%zu in op:%s(%s)", + inner_dump_info.output_anchor_index, output_addrs.size(), + inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str()); return FAILED; } auto data_addr = inner_dump_info.args + kAddrLen * static_cast(inner_dump_info.input_anchor_index); @@ -440,7 +447,7 @@ Status DataDumper::GenerateInput(toolkit::aicpu::dump::Input &input, const OpDes GELOGI("Get aipp input size according to attr is %ld", input_size); } else if (TensorUtils::GetTensorSizeInBytes(tensor_descs.at(index), input_size) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Get tensor size fail"); - GELOGE(PARAM_INVALID, "Get input size filed"); + GELOGE(PARAM_INVALID, "[Get][TensorSize] failed"); return PARAM_INVALID; } GELOGD("Get input size in dump is %ld", input_size); @@ -456,34 +463,37 @@ Status DataDumper::DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info size_t index; // parser and find which node's input or output tensor desc is chosen for dump info if (!ParseNameIndex(node_name_index, dump_op_name, input_or_output, index)) { - GELOGE(PARAM_INVALID, "Op [%s] input desc[%zu] with invalid ATTR_DATA_DUMP_REF attr[%s].", + GELOGE(PARAM_INVALID, "[Call][ParseNameIndex] Op [%s] input desc[%zu] with invalid ATTR_DATA_DUMP_REF attr[%s].", inner_dump_info.op->GetName().c_str(), i, node_name_index.c_str()); return PARAM_INVALID; } GE_CHECK_NOTNULL(compute_graph_); auto replace_node = compute_graph_->FindNode(dump_op_name); GE_RT_PARAM_INVALID_WITH_LOG_IF_TRUE(replace_node == nullptr, - "Op [%s] input desc[%zu] with invalid ATTR_DATA_DUMP_REF attr[%s]," - " cannot find redirect node[%s].", + "[Check][Param] Op [%s] input desc[%zu] with invalid ATTR_DATA_DUMP_REF " + "attr[%s], cannot find redirect node[%s].", inner_dump_info.op->GetName().c_str(), i, node_name_index.c_str(), dump_op_name.c_str()); auto replace_opdesc = replace_node->GetOpDesc(); GE_CHECK_NOTNULL(replace_opdesc); auto iter = ref_info_.find(replace_opdesc); GE_RT_PARAM_INVALID_WITH_LOG_IF_TRUE(iter == ref_info_.end(), - "Op [%s] input desc[%zu] cannot find any saved redirect node[%s]'s info.", + "[Check][Param] Op [%s] input desc[%zu] cannot find " + "any saved redirect node[%s]'s info.", inner_dump_info.op->GetName().c_str(), i, replace_opdesc->GetName().c_str()); GE_CHECK_NOTNULL(iter->second); auto addr = reinterpret_cast(iter->second); if (input_or_output == kDumpInput) { const auto &replace_input_descs = replace_opdesc->GetAllInputsDesc(); addr += kAddrLen * index; - GE_CHK_STATUS_RET(GenerateInput(input, replace_input_descs, addr, index), "Generate input failed"); + GE_CHK_STATUS_RET(GenerateInput(input, replace_input_descs, addr, index), + "[Generate][Input] failed for %s, index:%zu", inner_dump_info.op->GetName().c_str(), index); } else if (input_or_output == kDumpOutput) { const auto &replace_output_descs = replace_opdesc->GetAllOutputsDesc(); const auto replace_input_size = replace_opdesc->GetAllInputsDesc().size(); addr += (index + replace_input_size) * kAddrLen; - GE_CHK_STATUS_RET(GenerateInput(input, replace_output_descs, addr, index), "Generate input failed"); + GE_CHK_STATUS_RET(GenerateInput(input, replace_output_descs, addr, index), + "[Generate][Input] failed for %s, index:%zu", inner_dump_info.op->GetName().c_str(), index); } GELOGD("Op [%s] input desc[%zu] dump info is replaced by node[%s] [%s] tensor_desc [%zu]", inner_dump_info.op->GetName().c_str(), i, dump_op_name.c_str(), input_or_output.c_str(), index); @@ -498,14 +508,14 @@ Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, toolkit::aicp REPORT_INNER_ERROR("E19999", "input_desc size:%zu != input addr size:%zu in op:%s(%s)", input_descs.size(), input_addrs.size(), inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str()); - GELOGE(PARAM_INVALID, "Invalid input desc addrs size %zu, op %s has %zu input desc.", input_addrs.size(), - inner_dump_info.op->GetName().c_str(), input_descs.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Invalid input desc addrs size %zu, op %s has %zu input desc.", + input_addrs.size(), inner_dump_info.op->GetName().c_str(), input_descs.size()); return PARAM_INVALID; } std::vector v_memory_type; bool has_mem_type_attr = ge::AttrUtils::GetListInt(inner_dump_info.op, ATTR_NAME_INPUT_MEM_TYPE_LIST, v_memory_type); GE_RT_PARAM_INVALID_WITH_LOG_IF_TRUE(has_mem_type_attr && (v_memory_type.size() != input_descs.size()), - "DumpInput[%s], input size[%zu], input memory type size[%zu]", + "[Check][Param] DumpInput[%s], input size[%zu], input memory type size[%zu]", inner_dump_info.op->GetName().c_str(), input_descs.size(), v_memory_type.size()); for (size_t i = 0; i < input_descs.size(); ++i) { @@ -513,7 +523,8 @@ Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, toolkit::aicp std::string node_name_index; // check dump input tensor desc is redirected by attr ATTR_DATA_DUMP_REF if (AttrUtils::GetStr(&input_descs.at(i), ATTR_DATA_DUMP_REF, node_name_index)) { - GE_CHK_STATUS_RET(DumpRefInput(inner_dump_info, input, i, node_name_index), "DumpRefInput failed"); + GE_CHK_STATUS_RET(DumpRefInput(inner_dump_info, input, i, node_name_index), + "[Dump][RefInput] failed, node name index:%s", node_name_index.c_str()); task.mutable_input()->Add(std::move(input)); // normal dump without attr } else { @@ -525,14 +536,16 @@ Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, toolkit::aicp } else if (TensorUtils::GetTensorSizeInBytes(input_descs.at(i), input_size) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Get input tensor size fail in op:%s(%s), index:%zu", inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(), i); - GELOGE(PARAM_INVALID, "Get input size failed."); + GELOGE(PARAM_INVALID, "[Get][InputTensorSize] fail in op:%s(%s), index:%zu", + inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(), i); return PARAM_INVALID; } GELOGI("Get input size of l1_fusion_dump is %ld", input_size); GenerateOpBuffer(input_size, task); } else { auto addr = inner_dump_info.args + kAddrLen * i; - GE_CHK_STATUS_RET(GenerateInput(input, input_descs, addr, i), "Generate input failed"); + GE_CHK_STATUS_RET(GenerateInput(input, input_descs, addr, i), + "[Generate][Input] failed for op:%s, index:%zu", inner_dump_info.op->GetName().c_str(), i); task.mutable_input()->Add(std::move(input)); } } @@ -554,7 +567,7 @@ Status DataDumper::ExecuteLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_m bool ret = op_mapping_info.SerializeToString(&proto_str); if (!ret || proto_size == 0) { REPORT_INNER_ERROR("E19999", "Serialize proto to string fail"); - GELOGE(PARAM_INVALID, "Protobuf SerializeToString failed, proto size %zu.", proto_size); + GELOGE(PARAM_INVALID, "[Call][SerializeToString] failed, proto size %zu.", proto_size); return PARAM_INVALID; } @@ -565,25 +578,23 @@ Status DataDumper::ExecuteLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_m rtError_t rt_ret = rtMalloc(&dev_mem_load_, proto_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", - proto_size, rt_ret); - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", proto_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%zu, ret:0x%X", proto_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } GE_PRINT_DYNAMIC_MEMORY(rtMalloc, "load dump information.", proto_size) rt_ret = rtMemcpy(dev_mem_load_, proto_size, proto_str.c_str(), proto_size, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", - proto_size, rt_ret); - GELOGE(RT_FAILED, "Call rtMemcpy failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", proto_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", proto_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtDatadumpInfoLoad(dev_mem_load_, proto_size); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtDatadumpInfoLoad failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rtDatadumpInfoLoad failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtDatadumpInfoLoad failed, length:%zu, ret:0x%X", proto_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtDatadumpInfoLoad] failed, length:%zu, ret:0x%X", proto_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -598,7 +609,7 @@ Status DataDumper::ExecuteUnLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op bool ret = op_mapping_info.SerializeToString(&proto_str); if (!ret || proto_size == 0) { REPORT_INNER_ERROR("E19999", "Serialize proto to string fail"); - GELOGE(PARAM_INVALID, "Protobuf SerializeToString failed, proto size %zu.", proto_size); + GELOGE(PARAM_INVALID, "[Call][SerializeToString] failed, proto size %zu.", proto_size); return PARAM_INVALID; } @@ -609,25 +620,23 @@ Status DataDumper::ExecuteUnLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op rtError_t rt_ret = rtMalloc(&dev_mem_unload_, proto_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", - proto_size, rt_ret); - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", proto_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%zu, ret:0x%X", proto_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } GE_PRINT_DYNAMIC_MEMORY(rtMalloc, "unload dump information.", proto_size) rt_ret = rtMemcpy(dev_mem_unload_, proto_size, proto_str.c_str(), proto_size, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", - proto_size, rt_ret); - GELOGE(RT_FAILED, "Call rtMemcpy failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", proto_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", proto_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtDatadumpInfoLoad(dev_mem_unload_, proto_size); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtDatadumpInfoLoad failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rtDatadumpInfoLoad failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtDatadumpInfoLoad failed, length:%zu, ret:0x%X", proto_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtDatadumpInfoLoad] failed, length:%zu, ret:0x%X", proto_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } load_flag_ = false; @@ -654,7 +663,7 @@ Status DataDumper::LoadDumpInfo() { SetOpMappingLoopAddr(global_step_, loop_per_iter_, loop_cond_, op_mapping_info); auto ret = BuildTaskInfo(op_mapping_info); if (ret != SUCCESS) { - GELOGE(ret, "Build task info failed"); + GELOGE(ret, "[Build][TaskInfo] failed, ret:%u, path:%s", ret, dump_path.c_str()); return ret; } @@ -665,7 +674,7 @@ Status DataDumper::LoadDumpInfo() { if (!op_list_.empty() || is_op_debug_ || is_end_graph_) { ret = ExecuteLoadDumpInfo(op_mapping_info); if (ret != SUCCESS) { - GELOGE(ret, "Execute load dump info failed"); + GELOGE(ret, "[Execute][LoadDumpInfo] failed, ret:%u", ret); return ret; } } @@ -686,7 +695,7 @@ Status DataDumper::BuildTaskInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping if (dump_properties_.GetDumpMode() == kDumpOutput) { Status ret = DumpOutput(op_iter, task); if (ret != SUCCESS) { - GELOGE(ret, "Dump output failed"); + GELOGE(ret, "[Dump][Output] failed, ret:%u, op:%s", ret, op_desc->GetName().c_str()); return ret; } op_mapping_info.mutable_task()->Add(std::move(task)); @@ -696,7 +705,7 @@ Status DataDumper::BuildTaskInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping if (op_iter.is_task) { Status ret = DumpInput(op_iter, task); if (ret != SUCCESS) { - GELOGE(ret, "Dump input failed"); + GELOGE(ret, "[Dump][Input] failed, ret:%u, op:%s", ret, op_desc->GetName().c_str()); return ret; } } @@ -706,13 +715,13 @@ Status DataDumper::BuildTaskInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping if (dump_properties_.GetDumpMode() == kDumpAll || is_op_debug_) { auto ret = DumpOutput(op_iter, task); if (ret != SUCCESS) { - GELOGE(ret, "Dump output failed when in dumping all"); + GELOGE(ret, "[Dump][Output] failed when in dumping all, ret:%u, op:%s", ret, op_desc->GetName().c_str()); return ret; } if (op_iter.is_task) { ret = DumpInput(op_iter, task); if (ret != SUCCESS) { - GELOGE(ret, "Dump input failed when in dumping all"); + GELOGE(ret, "[Dump][Input] failed when in dumping all, ret:%u, op:%s", ret, op_desc->GetName().c_str()); return ret; } } @@ -795,7 +804,7 @@ Status DataDumper::UnloadDumpInfo() { } auto ret = ExecuteUnLoadDumpInfo(op_mapping_info); if (ret != SUCCESS) { - GELOGE(ret, "Execute unload dump info failed"); + GELOGE(ret, "[Execute][UnLoadDumpInfo] failed, ret:%d", ret); return ret; } return SUCCESS; diff --git a/ge/graph/load/model_manager/data_inputer.cc b/ge/graph/load/model_manager/data_inputer.cc index 0fe75465..d286b9b4 100755 --- a/ge/graph/load/model_manager/data_inputer.cc +++ b/ge/graph/load/model_manager/data_inputer.cc @@ -24,7 +24,7 @@ namespace ge { domi::Status InputDataWrapper::Init(const InputData &input, const OutputData &output) { - GE_CHK_BOOL_RET_STATUS(!is_init, domi::INTERNAL_ERROR, "InputDataWrapper is re-initialized"); + GE_CHK_BOOL_RET_STATUS(!is_init, domi::INTERNAL_ERROR, "[Check][Param] InputDataWrapper is re-initialized"); input_ = input; output_ = output; diff --git a/ge/graph/load/model_manager/davinci_model.cc b/ge/graph/load/model_manager/davinci_model.cc index 993400db..5d90d5a1 100755 --- a/ge/graph/load/model_manager/davinci_model.cc +++ b/ge/graph/load/model_manager/davinci_model.cc @@ -69,8 +69,9 @@ } catch (const std::system_error &e) { \ REPORT_CALL_ERROR("E19999", "Create thread fail, ecode:%d, emsg:%s", \ e.code().value(), e.what()); \ - GELOGE(FAILED, "Caught system_error with code:%d, meaning:%s", e.code().value(), e.what()); \ - GELOGE(FAILED, "Thread creat FAIL, Please check the left resource!"); \ + GELOGE(FAILED, "[Create][Thread] Caught system_error with code:%d, meaning:%s", \ + e.code().value(), e.what()); \ + GELOGE(FAILED, "[Create][Thread] FAIL, Please check the left resource!"); \ return FAILED; \ } \ } while (0) @@ -292,14 +293,14 @@ void DavinciModel::UnbindHcomStream() { void DavinciModel::ReleaseTask() { for (const auto &task : cpu_task_list_) { if (task != nullptr) { - GE_CHK_STATUS(task->Release(), "Release task failed."); + GE_CHK_STATUS(task->Release(), "[Release][Task] failed, model id:%u.", model_id_); } } cpu_task_list_.clear(); for (const auto &task : task_list_) { if (task != nullptr) { - GE_CHK_STATUS(task->Release(), "Release task failed."); + GE_CHK_STATUS(task->Release(), "[Release][Task] failed, model id:%u.", model_id_); } } @@ -333,9 +334,8 @@ void DavinciModel::Shrink() { Status DavinciModel::InitWeightMem(void *dev_ptr, void *weight_ptr, size_t weight_size) { if (is_weight_mem_has_inited_) { - REPORT_INNER_ERROR("E19999", "Call InitWeightMem more than once, model_id:%u, check invalid", - model_id_); - GELOGE(FAILED, "call InitWeightMem more than once."); + REPORT_INNER_ERROR("E19999", "Call InitWeightMem more than once, model_id:%u, check invalid", model_id_); + GELOGE(FAILED, "[Check][Param] call InitWeightMem more than once, model id:%u.", model_id_); return FAILED; } is_weight_mem_has_inited_ = true; @@ -347,7 +347,8 @@ Status DavinciModel::InitWeightMem(void *dev_ptr, void *weight_ptr, size_t weigh if ((weight_ptr != nullptr) && (weight_size < weights_size)) { REPORT_INNER_ERROR("E19999", "Param weight_ptr is nullptr or ge_model.weight.size:%zu < param weights_size:%zu, " "model_id:%u, check invalid", weight_size, weights_size, model_id_); - GELOGE(FAILED, "Invalid mem param: weight_size=%zu totalsize=%zu.", weight_size, weights_size); + GELOGE(FAILED, "[Check][Param] Invalid mem param: weight_size=%zu totalsize=%zu, model_id:%u.", + weight_size, weights_size, model_id_); return FAILED; } @@ -362,7 +363,8 @@ Status DavinciModel::InitWeightMem(void *dev_ptr, void *weight_ptr, size_t weigh if (weights_mem_base_ == nullptr) { REPORT_CALL_ERROR("E19999", "MallocWeightsMem fail, weights_size:%zu, model_id:%u, check invalid", weights_size, model_id_); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc weight memory failed. size: %zu", weights_size); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Alloc][Memory] for weight failed. size:%zu, model_id:%u", + weights_size, model_id_); return ACL_ERROR_GE_MEMORY_ALLOCATION; } is_inner_weight_base_ = true; @@ -380,9 +382,8 @@ Status DavinciModel::InitWeightMem(void *dev_ptr, void *weight_ptr, size_t weigh Status DavinciModel::InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size) { if (is_feature_map_mem_has_inited_) { - REPORT_INNER_ERROR("E19999", "Call InitFeatureMapMem more than once, model_id:%u, check invalid", - model_id_); - GELOGE(PARAM_INVALID, "call InitFeatureMapMem more than once"); + REPORT_INNER_ERROR("E19999", "Call InitFeatureMapMem more than once, model_id:%u, check invalid", model_id_); + GELOGE(PARAM_INVALID, "[Check][Param] call InitFeatureMapMem more than once, model_id:%u", model_id_); return PARAM_INVALID; } is_feature_map_mem_has_inited_ = true; @@ -393,7 +394,8 @@ Status DavinciModel::InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size) { if ((dev_ptr != nullptr) && (mem_size < TotalMemSize())) { REPORT_INNER_ERROR("E19999", "Param dev_ptr is nullptr or mem_size:%zu < ge_model.mem_size:%zu, " "model_id:%u, check invalid", mem_size, TotalMemSize(), model_id_); - GELOGE(PARAM_INVALID, "Invalid mem param: mem_size=%zu totalsize=%zu.", mem_size, TotalMemSize()); + GELOGE(PARAM_INVALID, "[Check][Param] Invalid mem param: mem_size=%zu totalsize=%zu, model_id:%u.", + mem_size, TotalMemSize(), model_id_); return PARAM_INVALID; } @@ -406,7 +408,8 @@ Status DavinciModel::InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size) { if (mem_base_ == nullptr) { REPORT_CALL_ERROR("E19999", "MallocFeatureMapMem fail, data_size:%zu, model_id:%u, check invalid", data_size, model_id_); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc feature map memory failed. size: %zu", data_size); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Alloc][Memory] for feature map failed. size:%zu, model_id:%u", + data_size, model_id_); return ACL_ERROR_GE_MEMORY_ALLOCATION; } GEEVENT("[IMAS]InitFeatureMapAndP2PMem graph_%u MallocMemory type[F] memaddr[%p] mem_size[%zu]", @@ -424,7 +427,8 @@ Status DavinciModel::InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size) { if (p2p_mem_base_ == nullptr) { REPORT_CALL_ERROR("E19999", "MallocFeatureMapMem fail, p2p_data_size:%zu, model_id:%u, check invalid", p2p_data_size, model_id_); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc p2p memory failed,size: %zu", p2p_data_size); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Alloc][Memory] for p2p failed, size:%zu, model_id:%u", + p2p_data_size, model_id_); return ACL_ERROR_GE_MEMORY_ALLOCATION; } GELOGI("InitFeatureMapAndP2PMem graph_%u MallocMemory type[F] memaddr[%p] mem_size[%zu]", runtime_param_.graph_id, @@ -432,7 +436,7 @@ Status DavinciModel::InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size) { is_inner_p2p_mem_base_ = true; } - GE_CHK_STATUS_RET(InitVariableMem(), "Init variable memory failed."); + GE_CHK_STATUS_RET(InitVariableMem(), "[Init][VariableMemory] failed, model_id:%u", model_id_); runtime_param_.mem_base = mem_base_; runtime_param_.weight_base = weights_mem_base_; runtime_param_.memory_infos[RT_MEMORY_P2P_DDR].memory_base = p2p_mem_base_; @@ -447,7 +451,7 @@ Status DavinciModel::InitVariableMem() { if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "MallocVarMemory fail, var_size:%zu, model_id:%u, check invalid", TotalVarMemSize(), model_id_); - GELOGE(ret, "Malloc variable memory failed."); + GELOGE(ret, "[Malloc][VarMemory] failed, var_size:%zu, model_id:%u", TotalVarMemSize(), model_id_); return ret; } var_mem_base_ = VarManager::Instance(session_id_)->GetVarMemoryBase(RT_MEMORY_HBM); @@ -555,23 +559,25 @@ Status DavinciModel::DoTaskSink() { GE_CHK_RT_RET(rtGetAicpuDeploy(&deploy_type_)); GELOGI("Do task sink. AiCpu deploy type is: %x.", deploy_type_); - GE_CHK_STATUS_RET(BindModelStream(), "Bind model stream failed."); + GE_CHK_STATUS_RET(BindModelStream(), "[Bind][ModelStream] failed, model_id:%u.", model_id_); if (known_node_) { - GE_CHK_STATUS_RET(MallocKnownArgs(), "Mallloc known node's args failed"); + GE_CHK_STATUS_RET(MallocKnownArgs(), "[Malloc][KnownArgs] failed, model_id:%u.", model_id_); } - GE_CHK_STATUS_RET(InitTaskInfo(*model_task_def.get()), "InitTaskInfo failed"); + GE_CHK_STATUS_RET(InitTaskInfo(*model_task_def.get()), "[Init][TaskInfo] failed, model_id:%u.", model_id_); - GE_CHK_STATUS_RET(ModelManager::GetInstance()->LaunchCustAicpuSo(), "Launch cust aicpu so failed"); + GE_CHK_STATUS_RET(ModelManager::GetInstance()->LaunchCustAicpuSo(), + "[Launch][CustAicpuSo] failed, model_id:%u.", model_id_); - GE_CHK_STATUS_RET(ModelManager::GetInstance()->CheckAicpuOpList(ge_model_), "Check aicpu op type failed"); + GE_CHK_STATUS_RET(ModelManager::GetInstance()->CheckAicpuOpList(ge_model_), + "[Check][AicpuOpList] failed, model_id:%u.", model_id_); - GE_CHK_STATUS_RET(InitEntryTask(), "InitEntryTask failed"); + GE_CHK_STATUS_RET(InitEntryTask(), "[Init][EntryTask] failed, model_id:%u.", model_id_); - GE_CHK_STATUS_RET(InitL1DataDumperArgs(), "InitL1DataDumperArgs failed"); + GE_CHK_STATUS_RET(InitL1DataDumperArgs(), "[Init][L1DataDumperArgs] failed, model_id:%u.", model_id_); - GE_CHK_STATUS_RET(DistributeTask(), "Distribute failed"); + GE_CHK_STATUS_RET(DistributeTask(), "[Distribute][Task] failed, model_id:%u.", model_id_); GE_CHK_RT_RET(rtModelLoadComplete(rt_model_handle_)); @@ -587,9 +593,8 @@ Status DavinciModel::SetTSDevice() { GELOGD("Set TSDevice: %u.", core_type); rtError_t rt_ret = rtSetTSDevice(core_type); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtSetTSDevice failed, core_type:%u, model_id:%u", - core_type, model_id_); - GELOGE(RT_FAILED, "SetTSDevice failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtSetTSDevice failed, core_type:%u, model_id:%u", core_type, model_id_); + GELOGE(RT_FAILED, "[Set][TSDevice] failed, core_type:%u, model_id:%u, ret: 0x%X", core_type, model_id_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } return SUCCESS; @@ -600,7 +605,7 @@ Status DavinciModel::OpDebugRegister() { uint32_t op_debug_mode = GetDumpProperties().GetOpDebugMode(); auto ret = opdebug_register_.RegisterDebugForModel(rt_model_handle_, op_debug_mode, data_dumper_); if (ret != SUCCESS) { - GELOGE(ret,"Register known shape op debug failed, ret: 0x%X",ret); + GELOGE(ret,"[Call][RegisterDebugForModel] Register known shape op debug failed, ret: 0x%X", ret); return ret; } is_op_debug_reg_ = true; @@ -621,17 +626,17 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size // validating params GELOGI("Priority is %d.", priority_); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(priority_ < 0 || priority_ > 7, return PARAM_INVALID, - "Priority must between 0-7, now is %d.", priority_); - GE_CHK_BOOL_RET_STATUS(ge_model_ != nullptr, PARAM_INVALID, "GeModel is null."); + "[Check][Param] Priority must between 0-7, now is %d.", priority_); + GE_CHK_BOOL_RET_STATUS(ge_model_ != nullptr, PARAM_INVALID, "[Check][Param] GeModel is null."); Graph graph = ge_model_->GetGraph(); ComputeGraphPtr compute_graph = GraphUtils::GetComputeGraph(graph); - GE_CHK_BOOL_RET_STATUS(compute_graph != nullptr, INTERNAL_ERROR, "Get compute graph is nullptr."); + GE_CHK_BOOL_RET_STATUS(compute_graph != nullptr, INTERNAL_ERROR, "[Get][ComputeGraph] failed, ret is nullptr."); // Initializing runtime_param_ InitRuntimeParams(); // RTS set aicore or vectorcore - GE_CHK_STATUS_RET(SetTSDevice(), "SetTSDevice failed."); + GE_CHK_STATUS_RET(SetTSDevice(), "[Set][TSDevice] failed, graph:%s.", compute_graph->GetName().c_str()); version_ = ge_model_->GetVersion(); name_ = ge_model_->GetName(); @@ -685,12 +690,16 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size runtime_param_.graph_id = compute_graph->GetGraphID(); // op debug register - GE_CHK_STATUS_RET(OpDebugRegister(), "OpDebugRegister failed"); + GE_CHK_STATUS_RET(OpDebugRegister(), "[Call][OpDebugRegister] failed, model_id:%u.", model_id_); GE_TIMESTAMP_START(TransAllVarData); - GE_CHK_STATUS_RET(TransAllVarData(compute_graph, runtime_param_.graph_id), "TransAllVarData failed"); + GE_CHK_STATUS_RET(TransAllVarData(compute_graph, runtime_param_.graph_id), + "[Call][TransAllVarData] failed, graph:%s, graph_id:%u.", + compute_graph->GetName().c_str(), runtime_param_.graph_id); GE_TIMESTAMP_END(TransAllVarData, "GraphLoader::TransAllVarData"); - GE_CHK_STATUS_RET(TransVarDataUtils::CopyVarData(compute_graph, session_id_, device_id_), "copy var data failed"); + GE_CHK_STATUS_RET(TransVarDataUtils::CopyVarData(compute_graph, session_id_, device_id_), + "[Copy][VarData] failed, graph:%s, session_id:%lu, device_id:%u", + compute_graph->GetName().c_str(), session_id_, device_id_); GE_TIMESTAMP_START(InitModelMem); GELOGD("Known node is %d.", known_node_); @@ -698,7 +707,8 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size if (!known_node_) { GE_CHK_STATUS_RET_NOLOG(InitFeatureMapAndP2PMem(dev_ptr, mem_size)); data_inputer_ = new (std::nothrow) DataInputer(); - GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, MEMALLOC_FAILED, "data_inputer_ is nullptr"); + GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, MEMALLOC_FAILED, + "[Create][DataInputer] data_inputer_ is nullptr"); } fixed_mem_base_ = reinterpret_cast(mem_base_); GE_TIMESTAMP_END(InitModelMem, "GraphLoader::InitModelMem"); @@ -711,10 +721,10 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size (void)ge::AttrUtils::SetStr(op_desc, VAR_ATTR_VAR_IS_BROADCAST, "var_is_restore");); } - GE_CHK_STATUS_RET(InitNodes(compute_graph), "Init nodes failed."); + GE_CHK_STATUS_RET(InitNodes(compute_graph), "[Init][Nodes] failed, graph:%s.", compute_graph->GetName().c_str()); GE_TIMESTAMP_START(DoTaskSink); - GE_CHK_STATUS_RET(DoTaskSink(), "Task sink failed."); + GE_CHK_STATUS_RET(DoTaskSink(), "[Call][DoTaskSink] failed, model_id:%u.", model_id_); GE_TIMESTAMP_END(DoTaskSink, "GraphLoader::DoTaskSink"); /// In zero copy model, if a aicpu operator is connected to the first or last layer, before model execution, @@ -733,10 +743,10 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size // collect profiling for ge auto &profiling_manager = ProfilingManager::Instance(); if (profiling_manager.ProfilingModelLoadOn()) { - GE_CHK_STATUS_RET(InitModelProfile(), "Init model profile failed"); + GE_CHK_STATUS_RET(InitModelProfile(), "[Init][ModelProfile] failed, model_id:%u.", model_id_); Status p_ret = ReportProfilingData(); if (p_ret != SUCCESS) { - GELOGE(p_ret, "Report profiling data failed."); + GELOGE(p_ret, "[Report][ProfilingData] failed, ret:%d, model_id:%u.", p_ret, model_id_); return p_ret; } } @@ -760,7 +770,7 @@ void DavinciModel::SaveSpecifyAttrValues(const OpDescPtr &op_desc) { Status DavinciModel::ReportProfilingData() { ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo()); - GE_CHK_STATUS(SinkModelProfile(), "Sink model profiler failed."); + GE_CHK_STATUS(SinkModelProfile(), "[Sink][ModelProfile] failed, model_id:%u.", model_id_); return SUCCESS; } @@ -870,7 +880,7 @@ Status DavinciModel::InitNodes(const ComputeGraphPtr &compute_graph) { if (IsDataOp(op_desc->GetType())) { if (InitDataOp(compute_graph, node, data_op_index, data_by_index, input_outside_addrs) != SUCCESS) { - GELOGE(PARAM_INVALID, "Data init failed, Name: %s", op_desc->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Init][DataOp] failed, Name:%s", op_desc->GetName().c_str()); return PARAM_INVALID; } data_dumper_.SaveDumpInput(node); @@ -879,11 +889,11 @@ Status DavinciModel::InitNodes(const ComputeGraphPtr &compute_graph) { if (op_desc->GetType() == NETOUTPUT) { if (InitNetOutput(compute_graph, node, output_op_list, output_outside_addrs) != SUCCESS) { - GELOGE(PARAM_INVALID, "NetOutput init failed, Name: %s", op_desc->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Init][NetOutput] failed, Name:%s", op_desc->GetName().c_str()); return PARAM_INVALID; } if (InitRealSizeAndShapeInfo(compute_graph, node) != SUCCESS) { - GELOGE(PARAM_INVALID, "Init real size and shape failed, Name: %s", op_desc->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Init][RealSizeAndShapeInfo] failed, Name:%s", op_desc->GetName().c_str()); return PARAM_INVALID; } continue; @@ -891,7 +901,7 @@ Status DavinciModel::InitNodes(const ComputeGraphPtr &compute_graph) { if (op_desc->GetType() == VARIABLE) { if (InitVariable(op_desc, variable_by_name) != SUCCESS) { - GELOGE(PARAM_INVALID, "Variable init failed, Name: %s", op_desc->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Init][Variable] failed, Name:%s", op_desc->GetName().c_str()); return PARAM_INVALID; } continue; @@ -902,7 +912,7 @@ Status DavinciModel::InitNodes(const ComputeGraphPtr &compute_graph) { auto it = op_desc_handle.find(op_desc->GetType()); if (it != op_desc_handle.end()) { if ((this->*it->second)(op_desc) != SUCCESS) { - GELOGE(PARAM_INVALID, "Node init failed, Name: %s", op_desc->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Init][Node] failed, Name:%s", op_desc->GetName().c_str()); return PARAM_INVALID; } continue; @@ -924,9 +934,8 @@ Status DavinciModel::InitNodes(const ComputeGraphPtr &compute_graph) { rtError_t rt_ret = rtMemcpy(addr, size, tensor_device_addrs.data(), size, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret: 0x%X", - size, rt_ret); - GELOGE(RT_FAILED, "rtMemcpy error, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", size, rt_ret); GE_CHK_RT(rtFree(addr)); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -937,7 +946,7 @@ Status DavinciModel::InitNodes(const ComputeGraphPtr &compute_graph) { if (IsTbeTask(op_desc)) { Status status = InitTbeHandle(op_desc); if (status != SUCCESS) { - GELOGE(status, "TBE init failed. %s", op_desc->GetName().c_str()); + GELOGE(status, "[Init][TbeHandle] failed. op:%s", op_desc->GetName().c_str()); return status; } } @@ -1006,8 +1015,9 @@ Status DavinciModel::InitDataOp(const ComputeGraphPtr &graph, const NodePtr &nod "not equal or has empty, model_id:%u", op_desc->GetName().c_str(), op_desc->GetType().c_str(), output_size_list.size(), virtual_addr_list.size(), output_offset_list.size(), model_id_); - GELOGE(PARAM_INVALID, "Data[%s] init failed: output size is %zu, virtual_addr size is %zu, offset size is %zu.", - op_desc->GetName().c_str(), output_size_list.size(), virtual_addr_list.size(), output_offset_list.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Data[%s] init failed: output size is %zu, " + "virtual_addr size is %zu, offset size is %zu.", op_desc->GetName().c_str(), output_size_list.size(), + virtual_addr_list.size(), output_offset_list.size()); return PARAM_INVALID; } @@ -1017,7 +1027,7 @@ Status DavinciModel::InitDataOp(const ComputeGraphPtr &graph, const NodePtr &nod void *virtual_addr = virtual_addr_list[kDataIndex]; Status ret = zero_copy_offset.InitInputDataInfo(data_size, virtual_addr, op_desc, fusion_flag); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "InitDataInfo of input_info %s failed.", op_desc->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Init][DataInfo] of input_info %s failed.", op_desc->GetName().c_str()); return PARAM_INVALID; } if (input_outside_addrs.count(virtual_addr) == 0) { @@ -1045,11 +1055,16 @@ Status DavinciModel::GenInputOutputInfo(const map &data_by_ GELOGD("Data node is: %s, output addr size: %zu", item.second->GetName().c_str(), output_addrs.size()); input_addrs_list_.emplace_back(output_addrs); - GE_CHK_STATUS_RET(InitAippInfo(item.first, item.second), "Init AIPP Info failed"); - GE_CHK_STATUS_RET(InitAippType(item.first, item.second, data_by_index), "Init AIPP Type failed"); - GE_CHK_STATUS_RET(InitOrigInputInfo(item.first, item.second), "Init Orig input failed"); - GE_CHK_STATUS_RET(InitAippInputOutputDims(item.first, item.second), "Init AIPP dims failed"); - GE_CHK_STATUS_RET(InitInputDescInfo(item.second), "Init input desc info failed"); + GE_CHK_STATUS_RET(InitAippInfo(item.first, item.second), + "[Init][AippInfo] failed, node:%s", item.second->GetName().c_str()); + GE_CHK_STATUS_RET(InitAippType(item.first, item.second, data_by_index), + "[Init][AippType] failed, node:%s", item.second->GetName().c_str()); + GE_CHK_STATUS_RET(InitOrigInputInfo(item.first, item.second), + "[Init][OrigInputInfo] failed, node:%s", item.second->GetName().c_str()); + GE_CHK_STATUS_RET(InitAippInputOutputDims(item.first, item.second), + "[Init][AippInputOutputDims] failed, node:%s", item.second->GetName().c_str()); + GE_CHK_STATUS_RET(InitInputDescInfo(item.second), + "[Init][InputDescInfo] failed, node:%s", item.second->GetName().c_str()); if (item.second->GetType() == AIPP_DATA_TYPE) { GELOGI("This is dynamic aipp model, Node: %s", item.second->GetName().c_str()); is_dynamic_aipp_ = true; @@ -1079,7 +1094,8 @@ Status DavinciModel::GenInputOutputInfo(const map &data_by_ return INTERNAL_ERROR; } - GE_CHK_STATUS_RET(InitOutputDescInfo(op_desc, out_node_name), "Init output desc info failed"); + GE_CHK_STATUS_RET(InitOutputDescInfo(op_desc, out_node_name), + "[Init][OutputDescInfo] failed, node:%s", op_desc->GetName().c_str()); } return SUCCESS; @@ -1123,24 +1139,24 @@ Status DavinciModel::InitNetOutput(const ComputeGraphPtr &graph, const NodePtr & const vector virtual_addr_list = ModelUtils::GetInputDataAddrs(runtime_param_, op_desc); const vector input_offset_list = op_desc->GetInputOffset(); GE_IF_BOOL_EXEC(input_offset_list.size() != virtual_addr_list.size(), - REPORT_INNER_ERROR( - "E19999", "Check data fail in op:%s(%s), input addr size:%zu input offset size:%zu " - "not equal, model_id:%u", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), - virtual_addr_list.size(), input_offset_list.size(), model_id_); - GELOGE(PARAM_INVALID, "virtual_addr size should be equal to offset size."); + REPORT_INNER_ERROR("E19999", "Check data fail in op:%s(%s), input addr size:%zu " + "input offset size:%zu not equal, model_id:%u", op_desc->GetName().c_str(), + op_desc->GetType().c_str(), virtual_addr_list.size(), input_offset_list.size(), + model_id_); + GELOGE(PARAM_INVALID, "[Check][Param] virtual_addr size:%zu should be equal to offset size:%zu, " + "op:%s(%s), model id:%u", virtual_addr_list.size(), input_offset_list.size(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); return PARAM_INVALID;); if (input_size_list.empty() && virtual_addr_list.empty()) { GELOGI("NetOutput[%s] is empty.", op_desc->GetName().c_str()); return SUCCESS; } if (input_size_list.empty() || input_size_list.size() != virtual_addr_list.size()) { - REPORT_INNER_ERROR( - "E19999", "Check data fail in op:%s(%s), input_desc size:%zu input addr size:%zu not equal or has empty, " - "model_id:%u", op_desc->GetName().c_str(), op_desc->GetType().c_str(), - input_size_list.size(), virtual_addr_list.size(), model_id_); - GELOGE(PARAM_INVALID, "NetOutput[%s] init failed: Input size is %zu, Input addr is %zu", op_desc->GetName().c_str(), - input_size_list.size(), virtual_addr_list.size()); + REPORT_INNER_ERROR("E19999", "Check data fail in op:%s(%s), input_desc size:%zu input addr size:%zu " + "not equal or has empty, model_id:%u", op_desc->GetName().c_str(), op_desc->GetType().c_str(), + input_size_list.size(), virtual_addr_list.size(), model_id_); + GELOGE(PARAM_INVALID, "[Check][Param] NetOutput[%s] init failed: Input size is %zu, Input addr is %zu", + op_desc->GetName().c_str(), input_size_list.size(), virtual_addr_list.size()); return PARAM_INVALID; } @@ -1156,8 +1172,9 @@ Status DavinciModel::InitNetOutput(const ComputeGraphPtr &graph, const NodePtr & for (size_t idx = 0; idx < input_count; ++idx) { ZeroCopyOffset zero_copy_offset; Status ret = zero_copy_offset.InitOutputDataInfo(input_size_list, virtual_addr_list, op_desc, idx, fusion_flag); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(PARAM_INVALID, "InitDataInfo of input_info %s failed.", - op_desc->GetName().c_str()); return PARAM_INVALID;); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(PARAM_INVALID, "[Init][DataInfo] of input_info %s failed.", op_desc->GetName().c_str()); + return PARAM_INVALID;); void *addr = virtual_addr_list.at(idx); int64_t input_offset = input_offset_list.at(idx); if (output_outside_addrs.count(addr) == 0) { @@ -1187,13 +1204,18 @@ Status DavinciModel::InitRealSizeAndShapeInfo(const ComputeGraphPtr &compute_gra GetAllGearsInfo(node); if (is_getnext_sink_dynamic_) { GE_IF_BOOL_EXEC(GetGetDynamicDimsNodeInfo(node) != SUCCESS, - GELOGE(PARAM_INVALID, "Failed to get info of getdynamicdims node."); return PARAM_INVALID;); + GELOGE(PARAM_INVALID, "[Get][Info] of getdynamicdims node:%s failed.", node->GetName().c_str()); + return PARAM_INVALID;); } if (is_online_infer_dynamic_) { GE_IF_BOOL_EXEC(GetGearAndRealOutSizeInfo(compute_graph, node) != SUCCESS, - GELOGE(PARAM_INVALID, "Failed to get gear and real out size info."); return PARAM_INVALID;); + GELOGE(PARAM_INVALID, "[Call][GetGearAndRealOutSizeInfo] failed, node:%s.", + node->GetName().c_str()); + return PARAM_INVALID;); GE_IF_BOOL_EXEC(GetGearAndRealOutShapeInfo(compute_graph, node) != SUCCESS, - GELOGE(PARAM_INVALID, "Failed to get gear and real out shape info."); return PARAM_INVALID;); + GELOGE(PARAM_INVALID, "[Call][GetGearAndRealOutShapeInfo] failed, node:%s.", + node->GetName().c_str()); + return PARAM_INVALID;); } return SUCCESS; @@ -1237,9 +1259,9 @@ Status DavinciModel::GetGetDynamicDimsNodeInfo(const NodePtr &node) { auto peer_out_anchor = in_anchor->GetPeerOutAnchor(); if (peer_out_anchor == nullptr) { REPORT_INNER_ERROR("E19999", "In anchor index:%zu in op:%s(%s) peer anchor is nullptr, model_id:%u, check invalid", - get_dynamic_dims_index, - node->GetName().c_str(), node->GetType().c_str(), model_id_); - GELOGE(PARAM_INVALID, "Out anchor of getdynmaicdims node should not be nullptr."); + get_dynamic_dims_index, node->GetName().c_str(), node->GetType().c_str(), model_id_); + GELOGE(PARAM_INVALID, "[Check][Param] In anchor index:%zu in op:%s(%s) peer anchor is nullptr, model_id:%u.", + get_dynamic_dims_index, node->GetName().c_str(), node->GetType().c_str(), model_id_); return PARAM_INVALID; } auto peer_node = peer_out_anchor->GetOwnerNode(); @@ -1253,16 +1275,18 @@ Status DavinciModel::GetGetDynamicDimsNodeInfo(const NodePtr &node) { REPORT_INNER_ERROR("E19999", "input_addr size:%zu or input_length size:%zu in op:%s(%s) has empty, model_id:%u " "check invalid", input_addr.size(), input_size.size(), node->GetName().c_str(), node->GetType().c_str(), model_id_); - GELOGE(PARAM_INVALID, "Not set output of %s", op_desc->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] input_addr size:%zu or input_length size:%zu in op:%s(%s) is empty, " + "model_id:%u", input_addr.size(), input_size.size(), + node->GetName().c_str(), node->GetType().c_str(), model_id_); return PARAM_INVALID; } auto input_desc = node->GetOpDesc()->GetInputDescPtr(get_dynamic_dims_index); GE_CHECK_NOTNULL(input_desc); if (input_desc->GetShape().GetDims().empty()) { REPORT_INNER_ERROR("E19999", "input_desc_index:%zu in op:%s(%s) shape dim is empty, model_id:%u, check invalid", - get_dynamic_dims_index, - node->GetName().c_str(), node->GetType().c_str(), model_id_); - GELOGE(PARAM_INVALID, "Not set output desc shape of %s.", op_desc->GetName().c_str()); + get_dynamic_dims_index, node->GetName().c_str(), node->GetType().c_str(), model_id_); + GELOGE(PARAM_INVALID, "[Check][Param] input_desc_index:%zu in op:%s(%s) shape dim is empty, model_id:%u", + get_dynamic_dims_index, node->GetName().c_str(), node->GetType().c_str(), model_id_); return PARAM_INVALID; } netoutput_last_input_addr_ = input_addr[get_dynamic_dims_index]; @@ -1288,7 +1312,7 @@ Status DavinciModel::GetGearAndRealOutSizeInfo(const ComputeGraphPtr &graph, con GE_CHECK_NOTNULL(op_desc); if ((peer_node->GetType() == CASE) && (op_desc->HasAttr(ATTR_INSERT_BY_MBATCH))) { if (GetRealOutputSizeOfCase(graph, idx, peer_node) != SUCCESS) { - GELOGE(PARAM_INVALID, "Get real output size of %s failed.", peer_node->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Get][RealOutputSizeOfCase] %s failed.", peer_node->GetName().c_str()); return PARAM_INVALID; } } @@ -1308,7 +1332,8 @@ Status DavinciModel::GetRealOutputSizeOfCase(const ComputeGraphPtr &graph, size_ if (subgraph == nullptr) { REPORT_INNER_ERROR("E19999", "Get name:%s subgraph in graph:%s fail, model_id:%u, check invalid", name.c_str(), graph->GetName().c_str(), model_id_); - GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s.", name.c_str()); + GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "[Get][Subgraph] %s in graph:%s failed, model_id:%u.", + name.c_str(), graph->GetName().c_str(), model_id_); return GE_GRAPH_EMPTY_SUBGRAPH; } for (auto &node : subgraph->GetDirectNode()) { @@ -1324,7 +1349,9 @@ Status DavinciModel::GetRealOutputSizeOfCase(const ComputeGraphPtr &graph, size_ "check invalid", batch_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), all_gears_info_.size(), model_id_); - GELOGE(PARAM_INVALID, "The value of ATTR_NAME_BATCH_LABEL is invalid."); + GELOGE(PARAM_INVALID, "[Check][Param] Batch_index:%zu in op:%s(%s) > all_gears_info.size:%zu, " + "model_id:%u.", batch_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), + all_gears_info_.size(), model_id_); return PARAM_INVALID; } @@ -1336,7 +1363,8 @@ Status DavinciModel::GetRealOutputSizeOfCase(const ComputeGraphPtr &graph, size_ REPORT_INNER_ERROR("E19999", "Get input TensorSize in op:%s(%s) failed, input_index:%zu, model_id:%u", op_desc->GetName().c_str(), op_desc->GetType().c_str(), input_index, model_id_); - GELOGE(FAILED, "Get tensor size in bytes failed."); + GELOGE(FAILED, "[Get][TensorSize] in op:%s(%s) failed, input_index:%zu, model_id:%u", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), input_index, model_id_); return FAILED; } gear_and_real_out_size_info[all_gears_info_[batch_index]] = data_size; @@ -1380,7 +1408,8 @@ Status DavinciModel::GetGearAndRealOutShapeInfo(const ComputeGraphPtr &graph, co REPORT_INNER_ERROR("E19999", "gear index:%zu in op:%s(%s) > all_gears_info.size:%zu in model:%u " "check invalid", gear_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), all_gears_info_.size(), model_id_); - GELOGE(PARAM_INVALID, "The value of cur index: %zu is invalid.", static_cast(it[0])); + GELOGE(PARAM_INVALID, "[Check][Param] gear index:%zu in op:%s(%s) > all_gears_info.size:%zu in model:%u.", + gear_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), all_gears_info_.size(), model_id_); return PARAM_INVALID; } @@ -1430,7 +1459,8 @@ Status DavinciModel::GetLabelGotoAddr(uint32_t label_index, rtMemType_t mem_type if (label_index >= label_list_.size()) { REPORT_INNER_ERROR("E19999", "Param label index:%u >= label_list_.size:%zu in model:%u, check invalid", label_index, label_list_.size(), model_id_); - GELOGE(INTERNAL_ERROR, "Invalid label id:%u, label size:%zu", label_index, label_list_.size()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Param label index:%u >= label_list_.size:%zu in model:%u", + label_index, label_list_.size(), model_id_); return INTERNAL_ERROR; } GE_CHECK_NOTNULL(label_list_[label_index]); @@ -1439,17 +1469,16 @@ Status DavinciModel::GetLabelGotoAddr(uint32_t label_index, rtMemType_t mem_type arg_size = label_used.size() * sizeof(rtLabelDevInfo); rtError_t rt_ret = rtMalloc(&arg_addr, arg_size, mem_type); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret: 0x%X", - arg_size, rt_ret); - GELOGE(RT_FAILED, "Call rtMalloc failed, error: %#x", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", arg_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret:0x%X", arg_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } label_goto_args_[label_index] = { arg_addr, arg_size }; rt_ret = rtLabelListCpy(label_used.data(), label_used.size(), arg_addr, arg_size); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtLabelListCpy failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rtLabelListCpy failed, error: %#x", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtLabelListCpy failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtLabelListCpy] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1466,22 +1495,24 @@ Status DavinciModel::InitLabelSet(const OpDescPtr &op_desc) { REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail, model_id:%u, check invalid", ATTR_NAME_LABEL_SWITCH_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); - GELOGE(INTERNAL_ERROR, "InitLabelSet: %s attr [%s] not exist.", op_desc->GetName().c_str(), - ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s in op:%s(%s) fail, model_id:%u", + ATTR_NAME_LABEL_SWITCH_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); return INTERNAL_ERROR; } if (label_index >= LabelNum()) { REPORT_INNER_ERROR("E19999", "label_switch_index:%u in op:%s(%s) >= label_num:%u in model:%u, check invalid", label_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), LabelNum(), model_id_); - GELOGE(INTERNAL_ERROR, "InitLabelSet: label index: %u >= label size: %u.", label_index, LabelNum()); + GELOGE(INTERNAL_ERROR, "[Check][Param] label_switch_index:%u in op:%s(%s) >= label_num:%u in model:%u", + label_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), LabelNum(), model_id_); return INTERNAL_ERROR; } if (label_id_indication_.count(label_index) > 0) { REPORT_INNER_ERROR("E19999", "label_switch_index:%u in op:%s(%s) is already used in model:%u, check invalid", label_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); - GELOGE(INTERNAL_ERROR, "InitLabelSet: %s label index: %u already used.", op_desc->GetName().c_str(), label_index); + GELOGE(INTERNAL_ERROR, "[Check][Param] label_switch_index:%u in op:%s(%s) is already used in model:%u", + label_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); return INTERNAL_ERROR; } @@ -1495,16 +1526,17 @@ Status DavinciModel::InitLabelSet(const OpDescPtr &op_desc) { REPORT_INNER_ERROR("E19999", "stream_id:%u in op:%s(%s) >= stream size:%zu in model:%u, check invalid", stream_id, op_desc->GetName().c_str(), op_desc->GetType().c_str(), stream_list_.size(), model_id_); - GELOGE(INTERNAL_ERROR, "InitLabelSet: stream index: %u >= stream size: %zu.", stream_id, stream_list_.size()); + GELOGE(INTERNAL_ERROR, "[Check][Param] stream_id:%u in op:%s(%s) >= stream size:%zu in model:%u", + stream_id, op_desc->GetName().c_str(), op_desc->GetType().c_str(), stream_list_.size(), model_id_); return INTERNAL_ERROR; } rtLabel_t rt_label = nullptr; rtError_t rt_error = rtLabelCreateExV2(&rt_label, rt_model_handle_, stream); if (rt_error != RT_ERROR_NONE || rt_label == nullptr) { - REPORT_CALL_ERROR("E19999", "Call rtLabelCreateExV2 failed, ret: 0x%X", - rt_error); - GELOGE(INTERNAL_ERROR, "InitLabelSet: %s create label failed, error=0x%x.", op_desc->GetName().c_str(), rt_error); + REPORT_CALL_ERROR("E19999", "Call rtLabelCreateExV2 failed, ret:0x%X", rt_error); + GELOGE(INTERNAL_ERROR, "[Call][RtLabelCreateExV2] InitLabelSet: %s create label failed, ret:0x%x.", + op_desc->GetName().c_str(), rt_error); return INTERNAL_ERROR; } @@ -1542,10 +1574,10 @@ Status DavinciModel::InitVariable(const OpDescPtr &op_desc, map &input_queue_ids, const std::vector &output_queue_ids) { if (input_queue_ids.empty() && output_queue_ids.empty()) { - REPORT_INNER_ERROR("E19999", "Param input_queue_ids.size:%zu or output_queue_ids.size:%zu is empty, model_id:%u," + REPORT_INNER_ERROR("E19999", "Param input_queue_ids.size:%zu and output_queue_ids.size:%zu is empty, model_id:%u," "check invalid", input_queue_ids.size(), output_queue_ids.size(), model_id_); - GELOGE(ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID, "Param is empty"); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID, "[Check][Param] Param is empty, model_id:%u", model_id_); return ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID; } @@ -1570,34 +1602,35 @@ Status DavinciModel::LoadWithQueue() { REPORT_INNER_ERROR("E19999", "Param input_queue_ids_.size:%zu != input_data_info_.size:%zu, model_id:%u," "check invalid", input_queue_ids_.size(), input_data_info_.size(), model_id_); - GELOGE(ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID, "Input queue ids not match model: input_queue=%zu input_data=%zu", - input_queue_ids_.size(), input_data_info_.size()); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID, "[Check][Param] Input queue ids not match model: " + "input_queue=%zu input_data=%zu, model_id:%u", input_queue_ids_.size(), input_data_info_.size(), model_id_); return ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID; } if (output_queue_ids_.size() != output_data_info_.size()) { REPORT_INNER_ERROR("E19999", "Param output_queue_ids_.size:%zu != output_data_info_.size:%zu, model_id:%u," - "check invalid", output_queue_ids_.size(), output_data_info_.size(), - model_id_); + "check invalid", output_queue_ids_.size(), output_data_info_.size(), model_id_); GELOGE(ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID, - "Output queue ids not match model: output_queue=%zu output_data=%zu", - output_queue_ids_.size(), output_data_info_.size()); + "[Check][Param] Output queue ids not match model: output_queue=%zu output_data=%zu, model_id:%u", + output_queue_ids_.size(), output_data_info_.size(), model_id_); return ACL_ERROR_GE_EXEC_MODEL_QUEUE_ID_INVALID; } - GE_CHK_STATUS_RET(AddHeadStream(), "Add head stream failed."); + GE_CHK_STATUS_RET(AddHeadStream(), "[Add][HeadStream] failed, model_id:%u", model_id_); // Binding input_queue and Data Op. - GE_CHK_STATUS_RET(BindInputQueue(), "Launch bind input queue failed."); - GE_CHK_STATUS_RET(CpuTaskModelZeroCopy(input_mbuf_list_, input_data_info_), "Launch zero copy failed."); + GE_CHK_STATUS_RET(BindInputQueue(), "[Bind][InputQueue] failed, model_id:%u", model_id_); + GE_CHK_STATUS_RET(CpuTaskModelZeroCopy(input_mbuf_list_, input_data_info_), + "[Call][CpuTaskModelZeroCopy] failed, model_id:%u", model_id_); // Binding output_queue and NetOutput Op. - GE_CHK_STATUS_RET(BindOutputQueue(), "Launch bind output queue failed."); - GE_CHK_STATUS_RET(CpuTaskModelZeroCopy(output_mbuf_list_, output_data_info_), "Launch zero copy failed."); + GE_CHK_STATUS_RET(BindOutputQueue(), "[Bind][OutputQueue] failed, model_id:%u", model_id_); + GE_CHK_STATUS_RET(CpuTaskModelZeroCopy(output_mbuf_list_, output_data_info_), + "[Call][CpuTaskModelZeroCopy] failed, model_id:%u", model_id_); - GE_CHK_STATUS_RET(CpuActiveStream(), "Launch active entry stream failed."); - GE_CHK_STATUS_RET(CpuWaitEndGraph(), "Launch wait end graph failed."); - GE_CHK_STATUS_RET(BindEnqueue(), "Launch enqueue failed."); - GE_CHK_STATUS_RET(CpuModelRepeat(), "Launch model repeat failed."); + GE_CHK_STATUS_RET(CpuActiveStream(), "[Call][CpuActiveStream] failed, model_id:%u", model_id_); + GE_CHK_STATUS_RET(CpuWaitEndGraph(), "[Call][CpuWaitEndGraph] failed, model_id:%u", model_id_); + GE_CHK_STATUS_RET(BindEnqueue(), "[Call][BindEnqueue] failed, model_id:%u", model_id_); + GE_CHK_STATUS_RET(CpuModelRepeat(), "[Call][CpuModelRepeat] failed, model_id:%u", model_id_); return SUCCESS; } @@ -1610,14 +1643,13 @@ Status DavinciModel::BindInputQueue() { for (size_t i = 0; i < input_queue_ids_.size(); ++i) { auto it = input_data_info_.find(i); if (it == input_data_info_.end()) { - - GELOGE(FAILED, "Input not match: tensor num=%zu, Queue id index=%zu", input_data_info_.size(), i); + GELOGE(FAILED, "[Check][Param] Input not match: tensor num=%zu, Queue id index=%zu", input_data_info_.size(), i); return FAILED; } uint32_t queue_id = input_queue_ids_[i]; if (it->second.GetDataInfo().empty()) { - GELOGE(INTERNAL_ERROR, "the %zu input_queue not set data_info.", i); + GELOGE(INTERNAL_ERROR, "[Check][Param] the %zu input_queue not set data_info.", i); return INTERNAL_ERROR; } uint32_t data_size = static_cast(it->second.GetDataInfo().at(0).first); @@ -1628,7 +1660,7 @@ Status DavinciModel::BindInputQueue() { rtError_t rt_ret = rtModelBindQueue(rt_model_handle_, queue_id, RT_MODEL_INPUT_QUEUE); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtModelBindQueue failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rtModelBindQueue failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtModelBindQueue] failed, ret: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1648,9 +1680,8 @@ Status DavinciModel::CpuModelDequeue(uint32_t queue_id) { GELOGI("Set CpuKernel model dequeue task enter."); std::shared_ptr dequeue_task = MakeShared(rt_entry_stream_); if (dequeue_task == nullptr) { - REPORT_CALL_ERROR("E19999", "New CpuTaskModelDequeue failed, model_id:%u", - model_id_); - GELOGE(MEMALLOC_FAILED, "Make CpuTaskModelDequeue task failed."); + REPORT_CALL_ERROR("E19999", "New CpuTaskModelDequeue failed, model_id:%u", model_id_); + GELOGE(MEMALLOC_FAILED, "[New][CpuTaskModelDequeue] task failed, model_id:%u", model_id_); return MEMALLOC_FAILED; } @@ -1672,9 +1703,8 @@ Status DavinciModel::CpuTaskModelZeroCopy(std::vector &mbuf_list, GELOGI("Set CpuKernel model zero_copy task enter."); std::shared_ptr zero_copy = MakeShared(rt_entry_stream_); if (zero_copy == nullptr) { - REPORT_CALL_ERROR("E19999", "New CpuTaskZeroCopy failed, model_id:%u", - model_id_); - GELOGE(MEMALLOC_FAILED, "Make CpuTaskZeroCopy task failed."); + REPORT_CALL_ERROR("E19999", "New CpuTaskZeroCopy failed, model_id:%u", model_id_); + GELOGE(MEMALLOC_FAILED, "[New][CpuTaskZeroCopy] failed, model_id:%u", model_id_); return MEMALLOC_FAILED; } @@ -1698,15 +1728,16 @@ Status DavinciModel::BindOutputQueue() { if (it == output_data_info_.end()) { REPORT_INNER_ERROR("E19999", "Index:%zu can't find in output_data_info_ size:%zu in model_id:%u, check invalid", i, output_data_info_.size(), model_id_); - GELOGE(FAILED, "Output not match: tensor num=%zu, Queue id index=%zu", output_data_info_.size(), i); + GELOGE(FAILED, "[Check][Param] Index:%zu can't find in output_data_info_ size:%zu in model_id:%u", + i, output_data_info_.size(), model_id_); return FAILED; } uint32_t queue_id = output_queue_ids_[i]; if (it->second.GetDataInfo().empty()) { - REPORT_INNER_ERROR("E19999", "Index:%zu out_data_info in model:%u is empty, check invalid", - i, model_id_); - GELOGE(INTERNAL_ERROR, "the %zu output_queue not set data_info.", i); + REPORT_INNER_ERROR("E19999", "Index:%zu out_data_info in model:%u is empty, check invalid", i, model_id_); + GELOGE(INTERNAL_ERROR, "[Check][Param] Index:%zu out_data_info in model:%u is empty, check invalid", + i, model_id_); return INTERNAL_ERROR; } uint32_t data_size = static_cast(it->second.GetDataInfo().at(0).first); @@ -1716,9 +1747,8 @@ Status DavinciModel::BindOutputQueue() { rtError_t rt_ret = rtModelBindQueue(rt_model_handle_, queue_id, RT_MODEL_OUTPUT_QUEUE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtModelBindQueue failed, queue_id:%u, ret: 0x%X", - queue_id, rt_ret); - GELOGE(RT_FAILED, "Call rtModelBindQueue failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtModelBindQueue failed, queue_id:%u, ret:0x%X", queue_id, rt_ret); + GELOGE(RT_FAILED, "[Call][RtModelBindQueue] failed, queue_id:%u, ret:0x%X", queue_id, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1739,17 +1769,15 @@ Status DavinciModel::BindOutputQueue() { Status DavinciModel::CpuModelPrepareOutput(uintptr_t addr, uint32_t size) { GELOGI("Set CpuKernel model enqueue task enter."); if (input_mbuf_list_.empty()) { - REPORT_INNER_ERROR("E19999", "input_mbuf_list_ is empty, model_id:%u, check invalid", - model_id_); - GELOGE(FAILED, "Need input mbuf for fill output mbuf head info."); + REPORT_INNER_ERROR("E19999", "input_mbuf_list_ is empty, model_id:%u, check invalid", model_id_); + GELOGE(FAILED, "[Check][Param] input_mbuf_list_ is empty, model_id:%u", model_id_); return FAILED; } std::shared_ptr prepare_output = MakeShared(rt_entry_stream_); if (prepare_output == nullptr) { - REPORT_CALL_ERROR("E19999", "New CpuTaskPrepareOutput failed, model_id:%u", - model_id_); - GELOGE(MEMALLOC_FAILED, "Make CpuTaskPrepareOutput task failed."); + REPORT_CALL_ERROR("E19999", "New CpuTaskPrepareOutput failed, model_id:%u", model_id_); + GELOGE(MEMALLOC_FAILED, "[New][CpuTaskPrepareOutput] failed, model_id:%u", model_id_); return MEMALLOC_FAILED; } @@ -1773,9 +1801,8 @@ Status DavinciModel::CpuActiveStream() { GELOGI("Set CpuKernel active stream task enter."); std::shared_ptr active_entry = MakeShared(rt_entry_stream_); if (active_entry == nullptr) { - REPORT_CALL_ERROR("E19999", "New CpuTaskActiveEntry failed, model_id:%u", - model_id_); - GELOGE(MEMALLOC_FAILED, "Make CpuTaskActiveEntry task failed."); + REPORT_CALL_ERROR("E19999", "New CpuTaskActiveEntry failed, model_id:%u", model_id_); + GELOGE(MEMALLOC_FAILED, "[New][CpuTaskActiveEntry] failed, model_id:%u", model_id_); return MEMALLOC_FAILED; } @@ -1796,9 +1823,8 @@ Status DavinciModel::CpuWaitEndGraph() { GELOGI("Set CpuKernel wait end graph task enter."); std::shared_ptr wait_endgraph = MakeShared(rt_entry_stream_); if (wait_endgraph == nullptr) { - REPORT_CALL_ERROR("E19999", "New CpuTaskWaitEndGraph failed, model_id:%u", - model_id_); - GELOGE(MEMALLOC_FAILED, "Make CpuTaskWaitEndGraph task failed."); + REPORT_CALL_ERROR("E19999", "New CpuTaskWaitEndGraph failed, model_id:%u", model_id_); + GELOGE(MEMALLOC_FAILED, "[New][CpuTaskWaitEndGraph] failed, model_id:%u", model_id_); return MEMALLOC_FAILED; } @@ -1818,7 +1844,8 @@ Status DavinciModel::BindEnqueue() { if (it == output_data_info_.end()) { REPORT_INNER_ERROR("E19999", "Index:%zu can't find in output_data_info_ size:%zu in model_id:%u, check invalid", i, output_data_info_.size(), model_id_); - GELOGE(FAILED, "Output not match: tensor num=%zu, Queue id index=%zu", output_data_info_.size(), i); + GELOGE(FAILED, "Index:%zu can't find in output_data_info_ size:%zu in model_id:%u", + i, output_data_info_.size(), model_id_); return FAILED; } @@ -1834,9 +1861,8 @@ Status DavinciModel::CpuModelEnqueue(uint32_t queue_id, uintptr_t out_mbuf) { GELOGI("Set CpuKernel model enqueue task enter."); std::shared_ptr model_enqueue = MakeShared(rt_entry_stream_); if (model_enqueue == nullptr) { - REPORT_CALL_ERROR("E19999", "New CpuTaskModelEnqueue failed, model_id:%u", - model_id_); - GELOGE(MEMALLOC_FAILED, "Make CpuTaskModelEnqueue task failed."); + REPORT_CALL_ERROR("E19999", "New CpuTaskModelEnqueue failed, model_id:%u", model_id_); + GELOGE(MEMALLOC_FAILED, "[New][CpuTaskModelEnqueue] failed, model_id:%u", model_id_); return MEMALLOC_FAILED; } @@ -1856,9 +1882,8 @@ Status DavinciModel::CpuModelRepeat() { GELOGI("Set CpuKernel repeat task enter."); std::shared_ptr model_repeat = MakeShared(rt_entry_stream_); if (model_repeat == nullptr) { - REPORT_CALL_ERROR("E19999", "New CpuTaskModelRepeat failed, model_id:%u", - model_id_); - GELOGE(MEMALLOC_FAILED, "Make CpuTaskModelRepeat task failed."); + REPORT_CALL_ERROR("E19999", "New CpuTaskModelRepeat failed, model_id:%u", model_id_); + GELOGE(MEMALLOC_FAILED, "[New][CpuTaskModelRepeat] failed, model_id:%u", model_id_); return MEMALLOC_FAILED; } @@ -1878,11 +1903,13 @@ Status DavinciModel::GetInputOutputDescInfo(vector &input_d GELOGI("data_op_list_ is empty or input_desc size is not 1."); } else { vector input_formats; - GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats, false), "get input desc info failed."); + GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats, false), + "[Get][InputDescInfo] failed, model_id:%u", model_id_); } vector output_formats; - GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, output_formats), "get output desc info failed"); + GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, output_formats), + "[Get][OutputDescInfo] failed, model_id:%u", model_id_); return SUCCESS; } @@ -1893,13 +1920,15 @@ Status DavinciModel::GetInputOutputDescInfo(vector &input_d if (input_addrs_list_.empty() || input_addrs_list_[0].size() != 1) { REPORT_INNER_ERROR("E19999", "input_addrs_list_ is empty or first member size != 1, model_id:%u, " "check invalid", model_id_); - GELOGE(FAILED, "OP List Pointer is null or input_desc size is not 1!"); + GELOGE(FAILED, "[Check][Param] input_addrs_list_ is empty or first member size != 1, model_id:%u", model_id_); return FAILED; } - GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats, by_dims), "get input desc info failed"); + GE_CHK_STATUS_RET(GetInputDescInfo(input_desc, input_formats, by_dims), + "[Get][InputDescInfo] failed, model_id:%u", model_id_); - GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, output_formats), "get output desc info failed"); + GE_CHK_STATUS_RET(GetOutputDescInfo(output_desc, output_formats), + "[Get][OutputDescInfo] failed, model_id:%u", model_id_); return SUCCESS; } @@ -1955,14 +1984,15 @@ Status DavinciModel::InitAippInfo(uint32_t index, const OpDescPtr &op_desc) { domi::AippOpParams aipp_params; GeAttrValue::NAMED_ATTRS aipp_attr; GE_CHK_BOOL_RET_STATUS(AttrUtils::GetNamedAttrs(op_desc, ATTR_NAME_AIPP, aipp_attr), ACL_ERROR_GE_AIPP_NOT_EXIST, - "Data node do not contain param aipp!"); - GE_CHK_STATUS_RET(OpUtils::ConvertAippParams(aipp_attr, &aipp_params), "get aipp params failed"); + "[Get][NamedAttrs] Data node:%s do not contain param aipp!", op_desc->GetName().c_str()); + GE_CHK_STATUS_RET(OpUtils::ConvertAippParams(aipp_attr, &aipp_params), + "[Convert][AippParams] get aipp params failed, op:%s", op_desc->GetName().c_str()); GELOGI("Node data: %s, type: %s, current index: %u, current node related input rank: %u", op_desc->GetName().c_str(), op_desc->GetType().c_str(), index, aipp_params.related_input_rank()); AippConfigInfo aipp_info; GE_CHK_STATUS_RET(AippUtils::ConvertAippParams2AippInfo(&aipp_params, aipp_info), - "convert aipp params to aipp config info failed"); + "[Call][ConvertAippParams2AippInfo] failed, op:%s", op_desc->GetName().c_str()); aipp_info_list_[index] = aipp_info; return SUCCESS; @@ -2006,8 +2036,9 @@ Status DavinciModel::InitAippType(uint32_t index, const OpDescPtr &op_desc, cons REPORT_INNER_ERROR("E19999", "Attr:%s data_mode:%s in op:%s(%s), model_id:%u, check invalid", ATTR_DATA_RELATED_AIPP_MODE.c_str(), data_mode.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); - GELOGE(ACL_ERROR_GE_AIPP_MODE_INVALID, - "The info of aipp releated info %s is invalid with index %u.", data_mode.c_str(), index); + GELOGE(ACL_ERROR_GE_AIPP_MODE_INVALID, "[Get][Attr] %s data_mode:%s in op:%s(%s), model_id:%u, check invalid", + ATTR_DATA_RELATED_AIPP_MODE.c_str(), data_mode.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); return ACL_ERROR_GE_AIPP_MODE_INVALID; } @@ -2033,7 +2064,8 @@ Status DavinciModel::InitAippType(uint32_t index, const OpDescPtr &op_desc, cons } Status DavinciModel::GetAippType(uint32_t index, InputAippType &aipp_type, size_t &aipp_index) const { - GE_CHK_BOOL_RET_STATUS(index < input_addrs_list_.size(), PARAM_INVALID, "Index %u is invalid", index); + GE_CHK_BOOL_RET_STATUS(index < input_addrs_list_.size(), PARAM_INVALID, + "[Check][Param] Index %u is invalid", index); const auto it = aipp_type_list_.find(index); if (it == aipp_type_list_.end()) { GELOGW("There is no aipp releated info with index %u", index); @@ -2152,7 +2184,8 @@ Status DavinciModel::InitInputDescInfo(const OpDescPtr &op_desc) { input.data_type = op_desc->GetInputDescPtr(0)->GetDataType(); input.name = op_desc->GetName(); int64_t input_size = 0; - GE_CHK_STATUS_RET(TensorUtils::GetSize(*op_desc->GetInputDescPtr(0), input_size), "get input size failed."); + GE_CHK_STATUS_RET(TensorUtils::GetSize(*op_desc->GetInputDescPtr(0), input_size), + "[Get][InputSize] failed in op:%s.", op_desc->GetName().c_str()); input.size = input_size; input_formats_.push_back(format); input_descs_.push_back(input); @@ -2176,9 +2209,10 @@ void DavinciModel::CreateOutput(uint32_t index, const OpDescPtr &op_desc, InputO /// netoutput input tensor desc GE_IF_BOOL_EXEC(op_desc->GetInputDescPtr(index) == nullptr, REPORT_INNER_ERROR("E19999", "input_desc index:%u in op:%s(%s) not exist, model_id:%u, " - "check invalid", index, - op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); - GELOGE(FAILED, "OpDesc GetInputDescPtr is nullptr"); + "check invalid", index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), + model_id_); + GELOGE(FAILED, "[Get][InputDescPtr] input_desc index:%u in op:%s(%s) not exist, model_id:%u", + index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); return); Format format = op_desc->GetInputDescPtr(index)->GetFormat(); GeShape shape = op_desc->GetInputDescPtr(index)->GetShape(); @@ -2239,7 +2273,9 @@ Status DavinciModel::InitOutputDescInfo(const OpDescPtr &op_desc, const vector src_name = op_desc->GetSrcName(); std::vector src_index = op_desc->GetSrcIndex(); GE_CHK_BOOL_RET_STATUS(src_name.size() > i && src_index.size() > i, INTERNAL_ERROR, - "construct output_name failed."); + "[Check][Param] construct output failed, as index:%u >= src name size:%zu, " + "or index >= src index size:%zu, op:%s.", + i, src_name.size(), src_index.size(), op_desc->GetName().c_str()); // forward compatbility, if old om has no out_node_name, need to return output follow origin way if (out_size == out_node_name.size()) { // neweast plan, the index will add to name during generate model. @@ -2269,8 +2305,8 @@ Status DavinciModel::CopyInputData(const InputData &input_data) { if (data.first >= blobs.size()) { REPORT_INNER_ERROR("E19999", "index:%u in input_data_info_ >= input_data.blobs.size:%zu, model_id:%u, " "check invalid", data.first, blobs.size(), model_id_); - GELOGE(FAILED, "Blobs not match: blobs=%zu, tensor=%zu, index=%u, size=%ld, op_name(%s)", blobs.size(), - input_data_info_.size(), data.first, data.second.GetDataInfo().at(0).first, + GELOGE(FAILED, "[Check][Param] Blobs not match: blobs=%zu, tensor=%zu, index=%u, size=%ld, op_name(%s)", + blobs.size(), input_data_info_.size(), data.first, data.second.GetDataInfo().at(0).first, data.second.GetOpName().c_str()); return FAILED; } @@ -2284,8 +2320,8 @@ Status DavinciModel::CopyInputData(const InputData &input_data) { } uint64_t data_size = data.second.GetDataSize(); GE_CHK_BOOL_RET_STATUS(data_size >= data_buf.length, PARAM_INVALID, - "input data size(%lu) does not match model required size(%lu), op_name(%s) ret failed.", - data_buf.length, data_size, data.second.GetOpName().c_str()); + "[Check][Param] input data size(%lu) does not match model required size(%lu), " + "op_name(%s), ret failed.", data_buf.length, data_size, data.second.GetOpName().c_str()); void *mem_addr = data.second.GetBasicAddr(); void *data_buf_addr = reinterpret_cast(reinterpret_cast(data_buf.data)); uint64_t data_buf_length = data_buf.length; @@ -2338,7 +2374,11 @@ Status DavinciModel::InitModelProfile() { } const auto &op_desc = GetOpByIndex(fusion_op_info->op_index); - GE_CHK_BOOL_EXEC(op_desc != nullptr, return FAILED, "index: %u out of range", fusion_op_info->op_index); + GE_CHK_BOOL_EXEC(op_desc != nullptr, + REPORT_INNER_ERROR("E19999", "Get op by index failed, as index:%u out of range", + fusion_op_info->op_index); + return FAILED, + "[Get][Op] failed, as index:%u out of range", fusion_op_info->op_index); ProfileInfo profile; profile.fusion_info = *fusion_op_info; @@ -2422,11 +2462,10 @@ Status DavinciModel::SinkModelProfile() { } catch (std::exception &e) { REPORT_INNER_ERROR("E19999", "Convert model_load_info JSON to string failed, model_id:%u, reason:%s", model_id_, e.what()); - GELOGE(FAILED, "Failed to convert JSON to string, reason: %s.", e.what()); + GELOGE(FAILED, "[Convert][JSON] to string failed, model_id:%u, reason:%s.", model_id_, e.what()); } catch (...) { - REPORT_INNER_ERROR("E19999", "Convert model_load_info JSON to string failed, model_id:%u", - model_id_); - GELOGE(FAILED, "Failed to convert JSON to string."); + REPORT_INNER_ERROR("E19999", "Convert model_load_info JSON to string failed, model_id:%u", model_id_); + GELOGE(FAILED, "[Convert][JSON] to string failed, model_id:%u.", model_id_); } reported_data.append(",") .append("\n"); @@ -2462,11 +2501,10 @@ Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { } catch (std::exception &e) { REPORT_INNER_ERROR("E19999", "Convert model_time_info JSON to string failed, model_id:%u, reason:%s", model_id_, e.what()); - GELOGE(FAILED, "Failed to convert JSON to string, reason: %s.", e.what()); + GELOGE(FAILED, "[Convert][JSON] to string failed, model_id:%u, reason:%s.", model_id_, e.what()); } catch (...) { - REPORT_INNER_ERROR("E19999", "Convert model_time_info JSON to string failed, model_id:%u", - model_id_); - GELOGE(FAILED, "Failed to convert JSON to string."); + REPORT_INNER_ERROR("E19999", "Convert model_time_info JSON to string failed, model_id:%u", model_id_); + GELOGE(FAILED, "[Convert][JSON] to string failed, model_id:%u.", model_id_); } reported_data.append(",") .append("\n"); @@ -2533,10 +2571,9 @@ Status DavinciModel::CopyOutputData(uint32_t data_id, OutputData &output_data, r output_data.model_id = model_id_; if (output_data.blobs.size() != output_data_info_.size()) { REPORT_INNER_ERROR("E19999", "output_data.blobs.size:%zu != output_data_info.size:%zu, model_id:%u, " - "check invalid", - output_data.blobs.size(), output_data_info_.size(), model_id_); - GELOGE(FAILED, "Output data buffer num=%zu not equal model data num=%zu", output_data.blobs.size(), - output_data_info_.size()); + "check invalid", output_data.blobs.size(), output_data_info_.size(), model_id_); + GELOGE(FAILED, "[Check][Param] output_data.blobs.size:%zu != output_data_info.size:%zu, model_id:%u", + output_data.blobs.size(), output_data_info_.size(), model_id_); return FAILED; } @@ -2546,8 +2583,8 @@ Status DavinciModel::CopyOutputData(uint32_t data_id, OutputData &output_data, r if (output.first >= blobs.size()) { REPORT_INNER_ERROR("E19999", "index:%u in output_data_info_ >= output_data.blobs.size:%zu, model_id:%u, " "check invalid", output.first, blobs.size(), model_id_); - GELOGE(FAILED, "Blobs not match: blobs=%zu, tensor=%zu, index=%u, size=%ld", blobs.size(), - input_data_info_.size(), output.first, output.second.GetDataInfo().at(0).first); + GELOGE(FAILED, "[Check][Param] index:%u in output_data_info_ >= output_data.blobs.size:%zu, model_id:%u", + output.first, blobs.size(), model_id_); return FAILED; } @@ -2565,9 +2602,9 @@ Status DavinciModel::CopyOutputData(uint32_t data_id, OutputData &output_data, r GELOGI("No need to check output data size."); } else if (buffer.length < mem_size) { REPORT_INNER_ERROR("E19999", "Buffer.length:%lu in output blob < mem_size:%lu in output_data_info, index:%u, " - "model_id:%u, check invalid", buffer.length, mem_size, output.first, - model_id_); - GELOGE(FAILED, "Tensor data size=%lu, buffer size=%lu", mem_size, buffer.length); + "model_id:%u, check invalid", buffer.length, mem_size, output.first, model_id_); + GELOGE(FAILED, "[Check][Param] Buffer.length:%lu in output blob < mem_size:%lu in output_data_info, index:%u, " + "model_id:%u", buffer.length, mem_size, output.first, model_id_); return FAILED; } else if (buffer.length > mem_size) { GELOGW("Tensor data size=%lu, buffer size=%lu", mem_size, buffer.length); @@ -2604,10 +2641,10 @@ Status DavinciModel::InitOutputTensorInfo(const OpDescPtr &op_desc) { auto ret = TensorUtils::GetTensorSizeInBytes(*input_desc, size); GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, REPORT_INNER_ERROR("E19999", "Get input TensorSize in op:%s(%s) failed, input_index:%zu, " - "model_id:%u", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), i, + "model_id:%u", op_desc->GetName().c_str(), op_desc->GetType().c_str(), i, model_id_); - GELOGE(ret, "Get size from TensorDesc failed, op:%s, input id:%zu", op_desc->GetName().c_str(), i); + GELOGE(ret, "[Get][InputTensorSize] in op:%s(%s) failed, input_index:%zu, model_id:%u", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), i, model_id_); return ret); const GeShape &shape = input_desc->GetShape(); GELOGI("Output size is %ld, output shape is %s.", size, formats::JoinToString(shape.GetDims()).c_str()); @@ -2679,7 +2716,9 @@ Status DavinciModel::GenOutputTensorInfo(OutputData *output_data, vector outputs; // return result is not required @@ -2689,17 +2728,20 @@ Status DavinciModel::ReturnResult(uint32_t data_id, const bool rslt_flg, const b GE_CHECK_NOTNULL(model_manager); auto exception_infos = model_manager->GetExceptionInfos(); if (exception_infos.size() > 0) { - GE_CHK_STATUS_RET(DumpExceptionInfo(exception_infos), "[Dump][Exception] Dump exception info failed."); + GE_CHK_STATUS_RET(DumpExceptionInfo(exception_infos), + "[Dump][Exception] Dump exception info failed, model_id:%u.", model_id_); } else { GELOGI("[Dump][Exception] Exception info is null."); } - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed."); + GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), + "[Call][OnComputeDone] failed, model_id:%u, data_id:%u.", model_id_, data_id); return INTERNAL_ERROR; } if (!has_output_node_) { GELOGW("Output tensor list is empty, model id: %u", model_id_); - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed."); + GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), + "[Call][OnComputeDone] failed, model_id:%u, data_id:%u.", model_id_, data_id); return INTERNAL_ERROR; } @@ -2722,16 +2764,19 @@ Status DavinciModel::ReturnResult(uint32_t data_id, const bool rslt_flg, const b } if (CopyOutputData(data_id, *output_data, RT_MEMCPY_DEVICE_TO_HOST) != SUCCESS) { - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), "OnComputeDone failed"); + GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, INTERNAL_ERROR, outputs), + "[Call][OnComputeDone] failed, model_id:%u, data_id:%u.", model_id_, data_id); return INTERNAL_ERROR; } if (seq_end_flag) { GELOGW("End of sequence, model id: %u", model_id_); - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, END_OF_SEQUENCE, outputs), "OnCompute Done failed."); + GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, END_OF_SEQUENCE, outputs), + "[Call][OnComputeDone] failed, model_id:%u, data_id:%u.", model_id_, data_id); return END_OF_SEQUENCE; } - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, SUCCESS, outputs), "OnComputeDone failed"); + GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, SUCCESS, outputs), + "[Call][OnComputeDone] failed, model_id:%u, data_id:%u.", model_id_, data_id); return SUCCESS; } /// @@ -2743,15 +2788,18 @@ Status DavinciModel::ReturnResult(uint32_t data_id, const bool rslt_flg, const b Status DavinciModel::ReturnNoOutput(uint32_t data_id) { GELOGI("ReturnNoOutput model id:%u.", model_id_); - GE_CHK_BOOL_EXEC(listener_ != nullptr, return PARAM_INVALID, "listener_ is null!"); + GE_CHK_BOOL_EXEC(listener_ != nullptr, + REPORT_INNER_ERROR("E19999", "listener_ is nullptr, check invalid."); + return PARAM_INVALID, "[Check][Param] listener_ is null!"); std::vector outputs; - GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, SUCCESS, outputs), "OnComputeDone failed."); + GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_id, SUCCESS, outputs), + "[Call][OnComputeDone] failed, model_id:%u, data_id:%u.", model_id_, data_id); return SUCCESS; } void *DavinciModel::Run(DavinciModel *model) { GE_CHK_BOOL_EXEC(model != nullptr, - return nullptr, "model_pointer is null!") + return nullptr, "[Check][Param] model_pointer is null!") bool seq_end_flag = false; uint32_t model_id = model->Id(); uint32_t device_id = model->GetDeviceId(); @@ -2761,7 +2809,7 @@ void *DavinciModel::Run(DavinciModel *model) { rtError_t rt_ret = rtSetDevice(static_cast(device_id)); if (rt_ret != RT_ERROR_NONE) { - GELOGE(FAILED, "Model run rtsetdevice failed."); + GELOGE(FAILED, "[Run][Rtsetdevice] failed, model_id:%u, device_id:%u.", model_id, device_id); return nullptr; } // DeviceReset before thread run finished! @@ -2794,7 +2842,8 @@ void *DavinciModel::Run(DavinciModel *model) { ret = model->SyncVarData(); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); - continue, "Copy input data to model failed."); // [No need to check value] + continue, + "[Call][SyncVarData] Copy input data to model failed, model_id:%u.", model_id); // [No need to check value] GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(Model_SyncVarData, "Model Run SyncVarData")); GELOGI("Copy input data, model id:%u", model_id); @@ -2803,7 +2852,8 @@ void *DavinciModel::Run(DavinciModel *model) { ret = model->CopyInputData(current_data); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( ret != SUCCESS, (void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); - continue, "Copy input data to model failed."); // [No need to check value] + continue, + "[Call][CopyInputData] Copy input data to model failed, model_id:%u.", model_id); // [No need to check value] if (model->is_online_infer_dynamic_ && !model->is_getnext_sink_dynamic_) { model->cur_dynamic_dims_.clear(); GE_IF_BOOL_EXEC(current_data.blobs.empty(), break); @@ -2902,12 +2952,13 @@ Status DavinciModel::DestroyThread() { /// @author /// Status DavinciModel::ModelRunStart() { - GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, INTERNAL_ERROR, "data_inputer_ is nullptr."); + GE_CHK_BOOL_RET_STATUS(data_inputer_ != nullptr, INTERNAL_ERROR, + "[Check][Param] data_inputer_ is nullptr, model id:%u.", model_id_); LockRunFlg(); GE_MAKE_GUARD(tmp_lock, [&] { UnlockRunFlg(); }); - GE_CHK_BOOL_RET_STATUS(!run_flg_, INTERNAL_ERROR, "Model already started."); + GE_CHK_BOOL_RET_STATUS(!run_flg_, INTERNAL_ERROR, "[Check][Param] Model already started, model id:%u.", model_id_); run_flg_ = true; @@ -2937,7 +2988,7 @@ Status DavinciModel::ModelRunStop() { LockRunFlg(); GE_MAKE_GUARD(tmp_lock, [&] { UnlockRunFlg(); }); - GE_CHK_STATUS_RET(DestroyThread(), "DestoyThead failed."); + GE_CHK_STATUS_RET(DestroyThread(), "[Destoy][Thead] failed, model id:%u.", model_id_); return SUCCESS; } @@ -2991,7 +3042,10 @@ Status DavinciModel::CreateKnownZeroCopyMap(const vector &inputs, const GELOGI("in, inputs size: %zu, input addr size: %zu, outputs size: %zu, output addr size: %zu", inputs.size(), input_addrs_list_.size(), outputs.size(), output_addrs_list_.size()); if (inputs.size() > input_addrs_list_.size()) { - GELOGE(FAILED, "input data addr %zu should less than input op num %zu.", inputs.size(), input_addrs_list_.size()); + REPORT_INNER_ERROR("E19999", "input data addr %zu should less than input op num %zu.", + inputs.size(), input_addrs_list_.size()); + GELOGE(FAILED, "[Check][Param] input data addr %zu should less than input op num %zu.", + inputs.size(), input_addrs_list_.size()); return FAILED; } // remove zero copy addr in last iteration @@ -3057,19 +3111,21 @@ Status DavinciModel::UpdateKnownZeroCopyAddr(vector &total_io_addrs, boo Status DavinciModel::UpdateKnownNodeArgs(const vector &inputs, const vector &outputs) { GELOGI("DavinciModel::UpdateKnownNodeArgs in"); GE_CHK_STATUS_RET(CreateKnownZeroCopyMap(inputs, outputs), - "DavinciModel::UpdateKnownNodeArgs create map for input/output zero copy."); + "[Call][CreateKnownZeroCopyMap] failed, model_id:%u.", model_id_); total_io_addrs_.clear(); for (size_t task_index = 0; task_index < task_list_.size(); ++task_index) { auto &task = task_list_[task_index]; if (task != nullptr) { Status ret = task->UpdateArgs(); if (ret != SUCCESS) { - GELOGE(FAILED, "task %zu created by davinci model is nullptr.", task_index); + REPORT_CALL_ERROR("E19999", "task %zu update args failed, model_id:%u", task_index, model_id_); + GELOGE(FAILED, "[Update][Args] to task %zu failed, model_id:%u.", task_index, model_id_); return FAILED; } } } - GE_CHK_STATUS_RET(UpdateKnownZeroCopyAddr(total_io_addrs_, false), "DavinciModel::UpdateKnownZeroCopyAddr failed."); + GE_CHK_STATUS_RET(UpdateKnownZeroCopyAddr(total_io_addrs_, false), + "[Call][UpdateKnownZeroCopyAddr] failed, model_id:%u.", model_id_); if (total_args_size_ == 0) { GELOGW("DavinciModel::UpdateKnownNodeArgs device args %p, dst size %u, pass rtMemcpy.", args_, total_args_size_); @@ -3080,7 +3136,10 @@ Status DavinciModel::UpdateKnownNodeArgs(const vector &inputs, const vec Status rt_ret = rtMemcpy(args_, total_args_size_, total_io_addrs_.data(), total_addr_size, RT_MEMCPY_HOST_TO_DEVICE); - GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(rt_ret, "rtMemcpy error, ret: Ox%X", rt_ret); return FAILED;) + GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", total_args_size_ , rt_ret); + GELOGE(rt_ret, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", total_args_size_ , rt_ret); + return FAILED;) } GELOGI("DavinciModel::UpdateKnownNodeArgs success"); @@ -3099,7 +3158,8 @@ Status DavinciModel::InitTaskInfo(domi::ModelTaskDef &model_task_def) { GE_CHECK_NOTNULL(task_list_[i]); Status ret = task_list_[i]->Init(task, this); if (ret != SUCCESS) { - GELOGE(ret, "Task index %d init failed.", i); + REPORT_CALL_ERROR("E19999", "Task index:%d init failed, ret:%d.", i, ret); + GELOGE(ret, "[Init][Task] index:%d failed, ret:%d.", i, ret); return ret; } } @@ -3110,7 +3170,7 @@ Status DavinciModel::InitTaskInfo(domi::ModelTaskDef &model_task_def) { Status DavinciModel::CheckCapability(rtFeatureType_t featureType, int32_t featureInfo, bool &is_support) const { int64_t value = RT_CAPABILITY_SUPPORT; auto rt_ret = rtGetRtCapability(featureType, featureInfo, &value); - GE_CHK_BOOL_RET_STATUS(rt_ret == RT_ERROR_NONE, FAILED, "call rtGetRtCapability failed!"); + GE_CHK_BOOL_RET_STATUS(rt_ret == RT_ERROR_NONE, FAILED, "[Call][RtGetRtCapability] failed, ret:0x%X", rt_ret); is_support = (value == RT_CAPABILITY_SUPPORT) ? true : false; return SUCCESS; } @@ -3129,7 +3189,8 @@ Status DavinciModel::MallocKnownArgs() { GE_CHECK_NOTNULL(task_list_[i]); Status ret = task_list_[i]->CalculateArgs(taskdef, this); if (ret != SUCCESS) { - GELOGE(ret, "TaskInfo CalculateArgs failed."); + REPORT_CALL_ERROR("E19999", "task index:%d CalculateArgs failed, ret:%d", i, ret); + GELOGE(ret, "[Calculate][Args] for taskdef index:%d failed, ret:%d", i, ret); return ret; } } @@ -3141,9 +3202,8 @@ Status DavinciModel::MallocKnownArgs() { if (total_args_size_ != 0) { rt_ret = rtMalloc(&args_, total_args_size_, mem_type); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret: 0x%X", - total_args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret: 0x%X", total_args_size_, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret: 0x%X", total_args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -3151,9 +3211,8 @@ Status DavinciModel::MallocKnownArgs() { if (total_hybrid_args_size_ != 0) { rt_ret = rtMalloc(&hybrid_addrs_, total_hybrid_args_size_, mem_type); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret: 0x%X", - total_hybrid_args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret: 0x%X", total_hybrid_args_size_, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret: 0x%X", total_hybrid_args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -3162,9 +3221,8 @@ Status DavinciModel::MallocKnownArgs() { GELOGI("Begin to allocate fixed addr."); rt_ret = rtMalloc(&fixed_addrs_, total_fixed_addr_size_, mem_type); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret: 0x%X", - total_hybrid_args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret: 0x%X", total_hybrid_args_size_, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret: 0x%X", total_hybrid_args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -3249,7 +3307,7 @@ Status DavinciModel::DistributeTask() { auto &task_def = model_task_def->task(task_index); auto &task = task_list_.at(task_index); GE_CHECK_NOTNULL(task); - GE_CHK_STATUS_RET(task->Distribute(), "Task[%zu] distribute fail", task_index); + GE_CHK_STATUS_RET(task->Distribute(), "[Call][Distribute] for Task[%zu] fail", task_index); // for data dump auto op_index = std::max(task_def.kernel().context().op_index(), task_def.kernel_ex().op_index()); @@ -3272,7 +3330,7 @@ Status DavinciModel::DistributeTask() { SaveProfilingTaskDescInfo(op, task, task_def, task_index); } // launch dump kernel to aicpu - GE_CHK_STATUS_RET(data_dumper_.LoadDumpInfo(), "Load dump info failed."); + GE_CHK_STATUS_RET(data_dumper_.LoadDumpInfo(), "[Load][DumpInfo] failed, model_id:%u.", model_id_); return SUCCESS; } @@ -3415,7 +3473,8 @@ bool DavinciModel::CheckInputAndModelSize(const int64_t &input_size, const int64 "check invalid", input_size, kDataMemAlignSizeCompare, op_size, model_id_); GELOGE(ACL_ERROR_GE_PARAM_INVALID, - "Input size [%ld] can not be smaller than op size [%ld] after 64-byte alignment", input_size, op_size); + "[Check][Param] input size:%ld from user add align:%u > input_op_size:%ld in model, model_id:%u", + input_size, kDataMemAlignSizeCompare, op_size, model_id_); return false; } return true; @@ -3431,18 +3490,21 @@ bool DavinciModel::CheckInputAndModelSize(const int64_t &input_size, const int64 /// Status DavinciModel::CopyModelData(const InputData &input_data, OutputData &output_data, bool is_dynamic) { if (UpdateIoTaskArgs(input_data_info_, true, input_data.blobs, is_dynamic, input_data.batch_label) != SUCCESS) { - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[ZCPY] Update input data to model failed."); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Call][UpdateIoTaskArgs] [ZCPY] Update input data to model:%u failed.", + model_id_); return ACL_ERROR_GE_PARAM_INVALID; } if (UpdateIoTaskArgs(output_data_info_, false, output_data.blobs, is_dynamic, input_data.batch_label) != SUCCESS) { - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[ZCPY] Update output data to model failed."); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Call][UpdateIoTaskArgs] [ZCPY] Update output data to model:%u failed.", + model_id_); return ACL_ERROR_GE_PARAM_INVALID; } for (ZeroCopyTask &task : zero_copy_tasks_) { - GE_CHK_STATUS_RET(task.DistributeParam(is_async_mode_, rt_model_stream_), "[ZCPY] Update args failed."); + GE_CHK_STATUS_RET(task.DistributeParam(is_async_mode_, rt_model_stream_), + "[Call][DistributeParam] [ZCPY] Update args failed, model_id:%u.", model_id_); } output_data.index = input_data.index; @@ -3464,36 +3526,35 @@ Status DavinciModel::UpdateIoTaskArgs(const std::map & const vector &blobs, bool is_dynamic, const string &batch_label) { if (blobs.size() != data_info.size()) { REPORT_INNER_ERROR("E19999", "is_input:%d blob size:%ld from user != op_size:%ld in model, mode_id:%u" - "check invalid", is_input, - blobs.size(), data_info.size(), model_id_); - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "Verify %s data num failed: model requires %zu, but user actually feeds %zu", - is_input ? "input" : "output", data_info.size(), blobs.size()); + "check invalid", is_input, blobs.size(), data_info.size(), model_id_); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Param] is_input:%d blob size:%ld " + "from user != op_size:%ld in model, mode_id:%u", + is_input, blobs.size(), data_info.size(), model_id_); return ACL_ERROR_GE_PARAM_INVALID; } for (const auto &data : data_info) { if (data.first >= blobs.size()) { // check data index. REPORT_INNER_ERROR("E19999", "is_input:%d, data index:%u from model >= blobs.size:%zu from user, mode_id:%u" - "check invalid", is_input, - data.first, blobs.size(), model_id_); + "check invalid", is_input, data.first, blobs.size(), model_id_); GELOGE(ACL_ERROR_GE_PARAM_INVALID, - "Verify %s data num failed: can not find No.%u data, because user only feeds %zu", - is_input ? "input" : "output", data.first, blobs.size()); + "[Check][Param] is_input:%d, data index:%u from model >= blobs.size:%zu from user, mode_id:%u", + is_input, data.first, blobs.size(), model_id_); return ACL_ERROR_GE_PARAM_INVALID; } const DataBuffer &buffer = blobs[data.first]; // index of data. if (buffer.data == nullptr) { REPORT_INNER_ERROR("E19999", "is_input:%d buffer from user is nullptr, index:%u, mode_id:%u" - "check invalid", is_input, - data.first, model_id_); - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "data_buf.data is nullptr, index=%u", data.first); + "check invalid", is_input, data.first, model_id_); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Param] data_buf.data is nullptr, " + "index=%u, mode_id:%u", data.first, model_id_); return ACL_ERROR_GE_PARAM_INVALID; } if (!CheckInputAndModelSize(buffer.length, data.second.GetDataSize(), is_dynamic)) { - GELOGE(ACL_ERROR_GE_PARAM_INVALID, - "Check input size and model size failed, op[%s]", data.second.GetOpName().c_str()); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Call][CheckInputAndModelSize] failed, op[%s]", + data.second.GetOpName().c_str()); return ACL_ERROR_GE_PARAM_INVALID; } @@ -3504,9 +3565,8 @@ Status DavinciModel::UpdateIoTaskArgs(const std::map & GELOGI("[IMAS] Find addr %p need direct copy from user malloc input %p", basic_addr, buffer.data); rtError_t rt_ret = rtMemcpy(basic_addr, data_size, buffer.data, buffer.length, RT_MEMCPY_DEVICE_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%lu, model_id:%u", - data_size, model_id_); - GELOGE(rt_ret, "Non-zero copy data node copy failed"); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%lu, model_id:%u", data_size, model_id_); + GELOGE(rt_ret, "[Call][RtMemcpy] failed, size:%lu, model_id:%u", data_size, model_id_); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -3598,9 +3658,10 @@ Status DavinciModel::InitConstant(const OpDescPtr &op_desc) { GE_IF_BOOL_EXEC(static_cast(v_output_size[0]) < tensor->GetData().size(), REPORT_INNER_ERROR("E19999", "Output size:%zu < weight size:%zu in op:%s(%s) model_id:%u, " "check invalid", v_output_size[0], tensor->GetData().size(), - op_desc->GetName().c_str(), op_desc->GetType().c_str() ,model_id_); - GELOGE(PARAM_INVALID, "output size:%ld less than weight data size:%zu", v_output_size[0], - tensor->GetData().size()); + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); + GELOGE(PARAM_INVALID, "[Check][Param] Output size:%zu < weight size:%zu in op:%s(%s), model_id:%u", + v_output_size[0], tensor->GetData().size(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); return PARAM_INVALID;); GE_IF_BOOL_EXEC(tensor->GetData().size() == 0, GELOGW("const op:%s has no weight data.", op_desc->GetName().c_str()); @@ -3619,7 +3680,7 @@ Status DavinciModel::InitConstant(const OpDescPtr &op_desc) { } uint64_t *buff = reinterpret_cast(tensor->MutableData().data()); if (ge::CheckInt64Uint32MulOverflow(elem_num, kBytes * kStringHeadElems) != SUCCESS) { - GELOGE(FAILED, "Shape size is invalid"); + GELOGE(FAILED, "[Call][CheckInt64Uint32MulOverflow] Shape size:%ld is invalid", elem_num); return FAILED; } uint64_t offset = elem_num * kBytes * kStringHeadElems; @@ -3649,8 +3710,8 @@ Status DavinciModel::InitTbeHandle(const OpDescPtr &op_desc) { auto tbe_kernel = (kernel != nullptr) ? kernel : op_desc->TryGetExtAttr(OP_EXTATTR_NAME_TBE_KERNEL, TBEKernelPtr()); if (tbe_kernel == nullptr) { REPORT_INNER_ERROR("E19999", "Get tbe_kernel for op:%s(%s) fail, model_id:%u", - op_desc->GetName().c_str(), op_desc->GetType().c_str() ,model_id_); - GELOGE(INTERNAL_ERROR, "TBE: %s can't find tvm bin file!", op_desc->GetName().c_str()); + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); + GELOGE(INTERNAL_ERROR, "[Check][Param] TBE: %s can't find tvm bin file!", op_desc->GetName().c_str()); return INTERNAL_ERROR; } @@ -3678,8 +3739,10 @@ Status DavinciModel::InitTbeHandle(const OpDescPtr &op_desc) { } else { REPORT_INNER_ERROR("E19999", "Attr:%s value:%s in op:%s(%s), model_id:%u, check invalid", TVM_ATTR_NAME_MAGIC.c_str(), json_string.c_str(), - op_desc->GetName().c_str(), op_desc->GetType().c_str() ,model_id_); - GELOGE(PARAM_INVALID, "TBE: Invalid parameter magic number! json: %s", json_string.c_str()); + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s value:%s in op:%s(%s), model_id:%u, check invalid", + TVM_ATTR_NAME_MAGIC.c_str(), json_string.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); return PARAM_INVALID; } @@ -3752,7 +3815,12 @@ Status DavinciModel::InitStreamActive(const OpDescPtr &op_desc) { if (op_desc->HasAttr(ATTR_NAME_SWITCH_BRANCH_NODE_LABEL)) { std::vector active_stream_list; GE_CHK_BOOL_EXEC(AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list), - return INTERNAL_ERROR, "StreamActiveOp get attr ACTIVE_STREAM failed."); + REPORT_INNER_ERROR("E19999", "[Get][Attr] %s in op:%s(%s) failed, model_id:%u.", + ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); + return INTERNAL_ERROR, + "[Get][Attr] %s in op:%s(%s) failed, model_id:%u.", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); for (size_t j = 0; j < active_stream_list.size(); ++j) { active_stream_indication_.insert(active_stream_list[j]); @@ -3773,7 +3841,9 @@ Status DavinciModel::InitStreamSwitch(const OpDescPtr &op_desc) { ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), active_stream_list.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), kTrueBranchStreamNum, model_id_); - GELOGE(INTERNAL_ERROR, "Stream num of switch true branch must be %u.", kTrueBranchStreamNum); + GELOGE(INTERNAL_ERROR, "[Check][Param] Attr:%s active_stream_list.size:%zu in op:%s(%s) != %u, model_id:%u", + ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), active_stream_list.size(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), kTrueBranchStreamNum, model_id_); return INTERNAL_ERROR; } @@ -3787,10 +3857,10 @@ Status DavinciModel::InitStreamSwitch(const OpDescPtr &op_desc) { Status DavinciModel::InitStreamSwitchN(const OpDescPtr &op_desc) { std::vector active_stream_list; if (!AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s from op:%s(%s) fail, model_id:%u", - ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s from op:%s(%s) fail, model_id:%u", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); - GELOGE(INTERNAL_ERROR, "StreamSwitchNOp get attr ACTIVE_STREAM failed."); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s from op:%s(%s) fail, model_id:%u", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); return INTERNAL_ERROR; } @@ -3801,10 +3871,10 @@ Status DavinciModel::InitStreamSwitchN(const OpDescPtr &op_desc) { uint32_t batch_num = 0; if (!AttrUtils::GetInt(op_desc, ATTR_NAME_BATCH_NUM, batch_num)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s from op:%s(%s) fail, model_id:%u", - ATTR_NAME_BATCH_NUM.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s from op:%s(%s) fail, model_id:%u", ATTR_NAME_BATCH_NUM.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); - GELOGE(FAILED, "Failed to get attr ATTR_NAME_BATCH_NUM, StreamSwitchN: %s.", op_desc->GetName().c_str()); + GELOGE(FAILED, "[Get][Attr] %s from op:%s(%s) fail, model_id:%u", ATTR_NAME_BATCH_NUM.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); return FAILED; } @@ -3821,10 +3891,10 @@ Status DavinciModel::SetDynamicBatchInfo(const OpDescPtr &op_desc, uint32_t batc std::vector batch_shape; const std::string attr_name = ATTR_NAME_PRED_VALUE + "_" + std::to_string(i); if (!AttrUtils::GetListInt(op_desc, attr_name, batch_shape)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s from op:%s(%s) fail, model_id:%u", - attr_name.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s from op:%s(%s) fail, model_id:%u", attr_name.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); - GELOGE(FAILED, "Get attr ATTR_NAME_PRED_VALUE failed, Node: %s", op_desc->GetName().c_str()); + GELOGE(FAILED, "[Get][Attr] %s from op:%s(%s) fail, model_id:%u", attr_name.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), model_id_); batch_info_.clear(); return FAILED; } @@ -3873,7 +3943,7 @@ bool DavinciModel::IsBroadCastOpData(const ge::NodePtr &var_node) { Status DavinciModel::InitModelStream(rtStream_t stream) { ExecuteMode curr_mode = is_async_mode_ ? ASYNCHRONIZATION : SYNCHRONIZATION; GE_CHK_BOOL_RET_STATUS((curr_mode == last_execute_mode_) || (last_execute_mode_ == INITIALIZATION), INTERNAL_ERROR, - "NnExecute not support mix execute."); + "[Check][Param] NnExecute not support mix execute."); last_execute_mode_ = curr_mode; // asynchronize mode, use user input stream. @@ -3914,15 +3984,15 @@ Status DavinciModel::NnExecute(rtStream_t stream, bool async_mode, const InputDa OutputData &output_data) { is_async_mode_ = async_mode; GELOGD("Model Run begin, model id:%u, data index:%u, flag:%d.", model_id_, input_data.index, is_async_mode_); - GE_CHK_STATUS_RET(InitModelStream(stream), "Init model stream failed."); + GE_CHK_STATUS_RET(InitModelStream(stream), "[Init][ModelStream] failed, model_id:%u.", model_id_); is_dynamic_ = input_data.is_dynamic_batch; bool profiling_model_execute_on = ProfilingManager::Instance().ProfilingModelExecuteOn(); bool profiling_model_load_on = ProfilingManager::Instance().ProfilingModelLoadOn(); GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_PRE_PROC_START)); Status ret = CopyModelData(input_data, output_data, is_dynamic_); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Copy input data to model failed. model id: %u", - model_id_); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, + "[Copy][ModelData] failed. model id: %u", model_id_); GELOGD("current_data.index=%u", input_data.index); GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_PRE_PROC_END)); @@ -3955,7 +4025,7 @@ Status DavinciModel::NnExecute(rtStream_t stream, bool async_mode, const InputDa GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_AFTER_PROC_START)); ret = CopyOutputData(input_data.index, output_data, RT_MEMCPY_DEVICE_TO_DEVICE); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ACL_ERROR_GE_INTERNAL_ERROR, - "Copy Output data to user failed."); + "[Copy][OutputData] to user failed, ret:%d, model_id:%u.", ret, model_id_); GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_AFTER_PROC_END)); } @@ -3968,10 +4038,8 @@ Status DavinciModel::NnExecute(rtStream_t stream, bool async_mode, const InputDa // Add active entry stream for special env. Status DavinciModel::AddHeadStream() { if (active_stream_list_.empty()) { - REPORT_INNER_ERROR("E19999", "active_stream_list is empty in model:%u, check invalid", - model_id_); - GELOGE(INTERNAL_ERROR, "Active stream is empty, stream list size: %zu, stream indication size: %zu.", - stream_list_.size(), active_stream_indication_.size()); + REPORT_INNER_ERROR("E19999", "active_stream_list is empty in model:%u, check invalid", model_id_); + GELOGE(INTERNAL_ERROR, "[Check][Param] active_stream_list is empty in model:%u, check invalid", model_id_); return INTERNAL_ERROR; } @@ -3989,9 +4057,8 @@ Status DavinciModel::AddHeadStream() { for (auto s : active_stream_list_) { std::shared_ptr active_entry = MakeShared(rt_head_stream_); if (active_entry == nullptr) { - REPORT_CALL_ERROR("E19999", "New CpuTaskActiveEntry failed, model_id:%u", - model_id_); - GELOGE(MEMALLOC_FAILED, "Make CpuTaskActiveEntry task failed."); + REPORT_CALL_ERROR("E19999", "New CpuTaskActiveEntry failed, model_id:%u", model_id_); + GELOGE(MEMALLOC_FAILED, "[New][CpuTaskActiveEntry] task failed, model_id:%u", model_id_); return MEMALLOC_FAILED; } @@ -4012,7 +4079,7 @@ Status DavinciModel::AddHeadStream() { Status DavinciModel::InitEntryTask() { if (deploy_type_ == AICPU_DEPLOY_CROSS_THREAD) { - GE_CHK_STATUS_RET(AddHeadStream(), "Add head stream failed."); + GE_CHK_STATUS_RET(AddHeadStream(), "[Add][HeadStream] failed."); return CpuActiveStream(); } else { return LoadWithQueue(); @@ -4073,13 +4140,13 @@ void DavinciModel::FreeFeatureMapMem() { string weight_memory_key = std::to_string(0) + "_f"; if (MemManager::Instance(RT_MEMORY_HBM)->GetMemoryAddr(weight_memory_key) != nullptr) { GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(weight_memory_key, GetDeviceId()), - "failed to free weight memory"); + "[Free][Memory] failed, model_id:%u", model_id_); } mem_base_ = nullptr; } else { GE_IF_BOOL_EXEC(mem_base_ != nullptr && is_inner_mem_base_, GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(mem_base_, GetDeviceId()), - "failed to free feature_map memory"); + "[Free][Memory] failed, model_id:%u", model_id_); mem_base_ = nullptr); } } @@ -4089,13 +4156,13 @@ void DavinciModel::FreeP2PMem() { std::string p2p_memory_key = std::to_string(0) + "_p"; if (MemManager::Instance(RT_MEMORY_P2P_DDR)->GetMemoryAddr(p2p_memory_key) != nullptr) { GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_P2P_DDR)->FreeMemory(p2p_memory_key, GetDeviceId()), - "failed to free p2p memory"); + "[Free][Memory] failed, model_id:%u", model_id_); } p2p_mem_base_ = nullptr; } else { GE_IF_BOOL_EXEC(p2p_mem_base_ != nullptr && is_inner_mem_base_, GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_P2P_DDR)->FreeMemory(p2p_mem_base_, GetDeviceId()), - "failed to free p2p memory"); + "[Free][Memory] failed, model_id:%u", model_id_); p2p_mem_base_ = nullptr); } } @@ -4107,13 +4174,13 @@ void DavinciModel::FreeWeightsMem() { string memory_key = std::to_string(0) + "_w"; if (MemManager::Instance(RT_MEMORY_HBM)->GetMemoryAddr(memory_key) != nullptr) { GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(memory_key, GetDeviceId()), - "failed to free feature_map memory"); + "[Free][Memory] failed, model_id:%u", model_id_); } weights_mem_base_ = nullptr; } else { GE_IF_BOOL_EXEC(weights_mem_base_ != nullptr && weights_mem_base_ != mem_base_ && is_inner_weight_base_, GE_CHK_STATUS(MemManager::Instance(RT_MEMORY_HBM)->FreeMemory(weights_mem_base_, GetDeviceId()), - "failed to free weight memory"); + "[Free][Memory] failed, model_id:%u", model_id_); weights_mem_base_ = nullptr); } } @@ -4122,9 +4189,8 @@ Status DavinciModel::TransAllVarData(ComputeGraphPtr &graph, uint32_t graph_id) rtContext_t ctx = nullptr; rtError_t rt_ret = rtCtxGetCurrent(&ctx); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, model_id:%u", - model_id_); - GELOGE(RT_FAILED, "Failed to get current context, error_code is: 0x%X.", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, model_id:%u", model_id_); + GELOGE(RT_FAILED, "[Call][RtCtxGetCurrent] failed, ret:0x%X, model_id:%u.", rt_ret, model_id_); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -4158,7 +4224,7 @@ void DavinciModel::SetDataDumperArgs(const ComputeGraphPtr &graph, const mapGetName().c_str(), op_desc->GetType().c_str(), infos.size(), kAippInfoNum, model_id_); - GELOGE(ACL_ERROR_GE_AIPP_MODE_INVALID, "origin input str is invalid[%zu, %u].", infos.size(), kAippInfoNum); + GELOGE(ACL_ERROR_GE_AIPP_MODE_INVALID, "[Check][Param] Attr:%s in op:%s(%s), " + "aipp input size:%zu != kAippInfoNum:%u, model_id:%u", ATTR_NAME_AIPP_INPUTS.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), infos.size(), kAippInfoNum, model_id_); return ACL_ERROR_GE_AIPP_MODE_INVALID; } @@ -4244,9 +4312,9 @@ Status DavinciModel::InitOrigInputInfo(uint32_t index, const OpDescPtr &op_desc) Status DavinciModel::GetOrigInputInfo(uint32_t index, OriginInputInfo &orig_input_info) const { const auto it = orig_input_info_.find(index); if (it == orig_input_info_.end()) { - REPORT_INNER_ERROR("E19999", "Get index:%u from orig_input_info_ fail, model_id:%u", - index, model_id_); - GELOGE(ACL_ERROR_GE_AIPP_NOT_EXIST, "there is not AIPP related with index %u.", index); + REPORT_INNER_ERROR("E19999", "Get index:%u from orig_input_info_ fail, model_id:%u", index, model_id_); + GELOGE(ACL_ERROR_GE_AIPP_NOT_EXIST, "[Check][Param] Get index:%u from orig_input_info_ fail, model_id:%u", + index, model_id_); return ACL_ERROR_GE_AIPP_NOT_EXIST; } @@ -4265,7 +4333,8 @@ void DavinciModel::ParseAIPPInfo(std::string in_out_info, InputOutputDims &dims_ REPORT_INNER_ERROR("E19999", "in_out_info:%s size:%zu != kAippInfoNum:%u, model_id:%u, " "check invalid", in_out_info.c_str(), infos.size(), kAippInfoNum, model_id_); - GELOGE(ACL_ERROR_GE_AIPP_MODE_INVALID, "origin input str is invalid[%zu, %u].", infos.size(), kAippInfoNum); + GELOGE(ACL_ERROR_GE_AIPP_MODE_INVALID, "[Check][Param] in_out_info:%s size:%zu != kAippInfoNum:%u, model_id:%u", + in_out_info.c_str(), infos.size(), kAippInfoNum, model_id_); return; } dims_info.name = infos[kAippInfoTensorName]; @@ -4327,9 +4396,9 @@ Status DavinciModel::GetAllAippInputOutputDims(uint32_t index, vector &output_dims) const { const auto it = aipp_dims_info_.find(index); if (it == aipp_dims_info_.end()) { - REPORT_INNER_ERROR("E19999", "Get index:%u from aipp_dims_info_ fail, model_id:%u", - index, model_id_); - GELOGE(ACL_ERROR_GE_AIPP_NOT_EXIST, "there is not AIPP related with index %u.", index); + REPORT_INNER_ERROR("E19999", "Get index:%u from aipp_dims_info_ fail, model_id:%u", index, model_id_); + GELOGE(ACL_ERROR_GE_AIPP_NOT_EXIST, "[Check][Param] Get index:%u from aipp_dims_info_ fail, model_id:%u", + index, model_id_); return ACL_ERROR_GE_AIPP_NOT_EXIST; } @@ -4360,9 +4429,8 @@ Status DavinciModel::InitL1DataDumperArgs() { if (rtDumpAddrSet(rt_model_handle_, l1_fusion_addr_, kDumpL1FusionOpMByteSize, kDumpFlagOfL1Fusion) != RT_ERROR_NONE) { // l1_fusion_addr_ will be free when DavinciModel destruct - REPORT_CALL_ERROR("E19999", "Call rtDumpAddrSet failed, model_id:%u", - model_id_); - GELOGE(FAILED, "Call rtDumpAddrSet failed"); + REPORT_CALL_ERROR("E19999", "Call rtDumpAddrSet failed, model_id:%u", model_id_); + GELOGE(FAILED, "[Call][RtDumpAddrSet] failed, model_id:%u", model_id_); return FAILED; } diff --git a/ge/graph/load/model_manager/task_info/end_graph_task_info.cc b/ge/graph/load/model_manager/task_info/end_graph_task_info.cc index d3c98684..c714d586 100644 --- a/ge/graph/load/model_manager/task_info/end_graph_task_info.cc +++ b/ge/graph/load/model_manager/task_info/end_graph_task_info.cc @@ -28,13 +28,13 @@ Status EndGraphTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin GELOGI("InitEndGraphTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } davinci_model_ = davinci_model; Status ret = SetStream(task_def.stream_id(), davinci_model->GetStreamList()); if (ret != SUCCESS) { - GELOGE(ret, "SetStream fail, stream_id:%u", task_def.stream_id()); + GELOGE(ret, "[Set][Stream] fail, stream_id:%u", task_def.stream_id()); return ret; } @@ -51,7 +51,7 @@ Status EndGraphTaskInfo::Distribute() { rtError_t rt_ret = rtEndGraphEx(model_, stream_, kDumpFlag); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtEndGraphEx failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rtEndGraphEx failed, ret: 0x%x", rt_ret); + GELOGE(RT_FAILED, "[Call][RtEndGraphEx] failed, ret:0x%x", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } else { @@ -59,7 +59,7 @@ Status EndGraphTaskInfo::Distribute() { rtError_t rt_ret = rtEndGraph(model_, stream_); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtEndGraph failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rtEndGraph failed, ret: 0x%x", rt_ret); + GELOGE(RT_FAILED, "[Call][RtEndGraph] failed, ret:0x%x", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -68,9 +68,8 @@ Status EndGraphTaskInfo::Distribute() { uint32_t stream_id = 0; rtError_t rt_ret = rtModelGetTaskId(davinci_model_->GetRtModelHandle(), &task_id, &stream_id); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtModelGetTaskId] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } task_id_ = task_id; diff --git a/ge/graph/load/model_manager/task_info/event_record_task_info.cc b/ge/graph/load/model_manager/task_info/event_record_task_info.cc index 13dae9ee..6af7e20d 100755 --- a/ge/graph/load/model_manager/task_info/event_record_task_info.cc +++ b/ge/graph/load/model_manager/task_info/event_record_task_info.cc @@ -24,7 +24,7 @@ Status EventRecordTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da GELOGI("EventRecordTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } @@ -37,7 +37,7 @@ Status EventRecordTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da if (task_def.event_id() >= eventList.size()) { REPORT_INNER_ERROR("E19999", "Task event_id:%u > model event size:%zu, check invalid", task_def.event_id(), eventList.size()); - GELOGE(INTERNAL_ERROR, "event list size:%zu, cur:%u!", eventList.size(), task_def.event_id()); + GELOGE(INTERNAL_ERROR, "[Check][Param] event list size:%zu, cur:%u!", eventList.size(), task_def.event_id()); return INTERNAL_ERROR; } @@ -50,9 +50,8 @@ Status EventRecordTaskInfo::Distribute() { GELOGI("EventRecordTaskInfo Distribute Start."); rtError_t rt_ret = rtEventRecord(event_, stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtEventRecord failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtEventRecord failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtEventRecord] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/event_wait_task_info.cc b/ge/graph/load/model_manager/task_info/event_wait_task_info.cc index 8fae9225..52dc660d 100755 --- a/ge/graph/load/model_manager/task_info/event_wait_task_info.cc +++ b/ge/graph/load/model_manager/task_info/event_wait_task_info.cc @@ -24,7 +24,7 @@ Status EventWaitTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davi GELOGI("EventWaitTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } @@ -37,7 +37,7 @@ Status EventWaitTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davi if (task_def.event_id() >= eventList.size()) { REPORT_INNER_ERROR("E19999", "Task event_id:%u > model event size:%zu, check invalid", task_def.event_id(), eventList.size()); - GELOGE(INTERNAL_ERROR, "event list size:%zu, cur:%u!", eventList.size(), task_def.event_id()); + GELOGE(INTERNAL_ERROR, "[Check][Param] event list size:%zu, cur:%u!", eventList.size(), task_def.event_id()); return INTERNAL_ERROR; } @@ -51,17 +51,15 @@ Status EventWaitTaskInfo::Distribute() { GELOGI("EventWaitTaskInfo Distribute Start."); rtError_t rt_ret = rtStreamWaitEvent(stream_, event_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamWaitEvent failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamWaitEvent failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtStreamWaitEvent] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtEventReset(event_, stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtEventReset failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtEventReset failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtEventReset] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/fusion_start_task_info.cc b/ge/graph/load/model_manager/task_info/fusion_start_task_info.cc index b47ac097..c9fc5424 100755 --- a/ge/graph/load/model_manager/task_info/fusion_start_task_info.cc +++ b/ge/graph/load/model_manager/task_info/fusion_start_task_info.cc @@ -24,7 +24,7 @@ Status FusionStartTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da GELOGI("FusionStartTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } @@ -40,9 +40,8 @@ Status FusionStartTaskInfo::Distribute() { GELOGI("FusionStartTaskInfo Distribute Start."); rtError_t rt_ret = rtKernelFusionStart(stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtKernelFusionStart failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtKernelFusionStart failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtKernelFusionStart] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/fusion_stop_task_info.cc b/ge/graph/load/model_manager/task_info/fusion_stop_task_info.cc index 6188cfc8..efce62ad 100755 --- a/ge/graph/load/model_manager/task_info/fusion_stop_task_info.cc +++ b/ge/graph/load/model_manager/task_info/fusion_stop_task_info.cc @@ -24,7 +24,7 @@ Status FusionStopTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *dav GELOGI("FusionStopTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } @@ -41,7 +41,7 @@ Status FusionStopTaskInfo::Distribute() { rtError_t rt_ret = rtKernelFusionEnd(stream_); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtKernelFusionEnd failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtKernelFusionEnd] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/hccl_task_info.cc b/ge/graph/load/model_manager/task_info/hccl_task_info.cc index 7a435f91..c3c5c8b7 100644 --- a/ge/graph/load/model_manager/task_info/hccl_task_info.cc +++ b/ge/graph/load/model_manager/task_info/hccl_task_info.cc @@ -31,7 +31,7 @@ HcclTaskInfo::~HcclTaskInfo() { rtError_t ret = rtFreeHost(private_def_); if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtFreeHost failed, ret:0x%X", ret); - GELOGE(RT_FAILED, "Call rtFree Fail, ret = 0x%X.", ret); + GELOGE(RT_FAILED, "[Call][RtFree] Fail, ret = 0x%X.", ret); } private_def_ = nullptr; } @@ -43,7 +43,7 @@ Status HcclTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_m GELOGI("HcclTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } davinci_model_ = davinci_model; @@ -71,21 +71,21 @@ Status HcclTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_m if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Call GetHorovodInputs fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(ret, "davinci_model: GetHorovodInputs fail! domi error: %u", ret); + GELOGE(ret, "[Get][HorovodInputs] fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return ret; } Status dmrt = HcomOmeUtil::GetHcclDataType(op_desc, kernel_hccl_infos_); if (dmrt != SUCCESS) { REPORT_CALL_ERROR("E19999", "Call GetHcclDataType fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(dmrt, "davinci_model: GetHcomDataType fail! domi error: %u", dmrt); + GELOGE(dmrt, "[Get][HcomDataType] fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return dmrt; } dmrt = HcomOmeUtil::GetHcclCount(op_desc, kernel_hccl_infos_); if (dmrt != SUCCESS) { REPORT_CALL_ERROR("E19999", "Call GetHcclCount fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(dmrt, "davinci_model: GetHcomCount fail! domi error: %u", dmrt); + GELOGE(dmrt, "[Get][HcomCount] fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return dmrt; } // Only HCOMBROADCAST and HVDCALLBACKBROADCAST need to get the rootId @@ -93,14 +93,14 @@ Status HcclTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_m if (dmrt != SUCCESS) { REPORT_CALL_ERROR("E19999", "Call GetAllRootId fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(dmrt, "davinci_model: Get rootId fail! domi error: %u", dmrt); + GELOGE(dmrt, "[Get][RootId] fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return dmrt; } // GE's new process: hccl declares the number of streams required, creates a stream by GE, and sends it to hccl ret = SetFollowStream(op_desc, davinci_model); if (ret != SUCCESS) { - GELOGE(ret, "SetStream Fail."); + GELOGE(ret, "[Set][Stream] Fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return ret; } @@ -111,13 +111,13 @@ Status HcclTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_m ret = SetAddrs(op_desc, kernel_hccl_infos_); if (ret != SUCCESS) { - GELOGE(ret, "Setaddrs Fail."); + GELOGE(ret, "[Set][Addrs] Fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return ret; } // GE's new process: hccl declares the need for Workspace size, and GE allocates Workspace ret = SetWorkspace(op_desc, kernel_hccl_infos_); if (ret != SUCCESS) { - GELOGE(ret, "SetWorkspace Fail."); + GELOGE(ret, "[Set][Workspace] Fail for op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return ret; } @@ -156,7 +156,8 @@ Status HcclTaskInfo::SetFollowStream(const ge::ConstOpDescPtr &op_desc, DavinciM } ret = CreateStream(hccl_stream_num - created_stream_num, davinci_model, main_stream_id); if (ret != SUCCESS) { - GELOGE(RT_FAILED, "Create hccl stream failed."); + GELOGE(RT_FAILED, "[Create][Stream] for %s failed, stream id:%ld, stream num:%ld.", + op_desc->GetName().c_str(), main_stream_id, hccl_stream_num - created_stream_num); return RT_ERROR_TO_GE_STATUS(ret); } } @@ -165,7 +166,8 @@ Status HcclTaskInfo::SetFollowStream(const ge::ConstOpDescPtr &op_desc, DavinciM GELOGI("need to create follow stream for %s with new mainstream %ld.", op_desc->GetName().c_str(), main_stream_id); ret = CreateStream(hccl_stream_num, davinci_model, main_stream_id); if (ret != SUCCESS) { - GELOGE(RT_FAILED, "Create hccl stream failed."); + GELOGE(RT_FAILED, "[Create][Stream] for %s failed, stream id:%ld, stream num:%ld.", + op_desc->GetName().c_str(), main_stream_id, hccl_stream_num); return RT_ERROR_TO_GE_STATUS(ret); } } @@ -181,7 +183,8 @@ Status HcclTaskInfo::CreateStream(int64_t stream_num, DavinciModel *davinci_mode if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtStreamCreateWithFlags failed, ret:0x%X, stream_idx:%ld, stream_num:%ld", rt_ret, i, stream_num); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtStreamCreateWithFlags] failed, ret:0x%X, stream_idx:%ld, stream_num:%ld", + rt_ret, i, stream_num); return RT_ERROR_TO_GE_STATUS(rt_ret); } // Create slave stream, inactive by default, activated by hccl @@ -189,7 +192,8 @@ Status HcclTaskInfo::CreateStream(int64_t stream_num, DavinciModel *davinci_mode if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtModelBindStream failed, ret:0x%X, stream_idx:%ld, stream_num:%ld", rt_ret, i, stream_num); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtModelBindStream] failed, ret:0x%X, stream_idx:%ld, stream_num:%ld", + rt_ret, i, stream_num); (void)rtStreamDestroy(stream); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -207,7 +211,7 @@ Status HcclTaskInfo::Distribute() { GELOGI("HcclTaskInfo Distribute Start. begin to call function LoadTask in hccl."); if (ops_kernel_store_ == nullptr) { REPORT_INNER_ERROR("E19999", "Check param ops_kernel_store_ nullptr"); - GELOGE(INTERNAL_ERROR, "ops kernel store is null."); + GELOGE(INTERNAL_ERROR, "[Check][Param] ops kernel store is null."); return INTERNAL_ERROR; } OpsKernelInfoStore *ops_kernel_info_store = reinterpret_cast(ops_kernel_store_); @@ -217,7 +221,7 @@ Status HcclTaskInfo::Distribute() { auto result = ops_kernel_info_store->LoadTask(ge_task); if (result != HCCL_SUCCESS) { REPORT_CALL_ERROR("E19999", "Call ops_kernel_info_store LoadTask fail"); - GELOGE(INTERNAL_ERROR, "davinci_model : load task fail, return ret: %u", result); + GELOGE(INTERNAL_ERROR, "[Load][Task] fail, return ret:%u", result); return INTERNAL_ERROR; } GELOGI("HcclTaskInfo Distribute Success."); @@ -265,8 +269,9 @@ Status HcclTaskInfo::SetAddrs(const std::shared_ptr &op_desc, std::vector &kernel_hccl_infos) { GE_CHECK_NOTNULL(op_desc); GE_CHK_STATUS_RET(HcomOmeUtil::CheckKernelHcclInfo(op_desc, kernel_hccl_infos), - "HcomOmeUtil:: the number of GETaskKernelHcclInfo is invalid."); - GELOGI("Set hccl task input output address, node[%s}, type[%s] kernel_hccl_infos.size[%zu].", + "[Check][Param] HcomOmeUtil:: the number of GETaskKernelHcclInfo is invalid, node:%s(%s).", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GELOGI("Set hccl task input output address, node[%s], type[%s] kernel_hccl_infos.size[%zu].", op_desc->GetName().c_str(), op_desc->GetType().c_str(), kernel_hccl_infos.size()); if (op_desc->GetType() == HVDWAIT) { return SUCCESS; @@ -300,7 +305,7 @@ Status HcclTaskInfo::SetAddrs(const std::shared_ptr &op_desc, } else if (hccl_type == HCOMALLREDUCE || hccl_type == HCOMREDUCESCATTER || hccl_type == HVDCALLBACKALLREDUCE || hccl_type == HCOMREDUCE) { GE_CHK_STATUS_RET(HcomOmeUtil::GetHcclOperationType(op_desc, op_type), - "davinci_model: GetHcomOperationType fail!"); + "[Get][HcomOperationType] fail! op:%s", op_desc->GetName().c_str()); kernel_hccl_infos[i].outputDataAddr = output_data_addr; kernel_hccl_infos[i].opType = op_type; } @@ -332,18 +337,16 @@ void HcclTaskInfo::GetPrivateDefByTaskDef(const domi::TaskDef &task) { private_def_len_ = private_def_temp.size(); rtError_t ret = rtMallocHost(&private_def_, private_def_len_); if (ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, ret:0x%X, size:%u", - ret, private_def_len_); - GELOGE(RT_FAILED, "Call rtMallocHost Fail, ret = 0x%X.", ret); + REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, ret:0x%X, size:%u", ret, private_def_len_); + GELOGE(RT_FAILED, "[Call][RtMallocHost] Fail, ret:0x%X, size:%u", ret, private_def_len_); return; } ret = rtMemcpy(private_def_, private_def_len_, task.private_def().c_str(), private_def_len_, RT_MEMCPY_HOST_TO_HOST); if (ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%u", - ret, private_def_len_); - GELOGE(RT_FAILED, "Call rtMemcpy Fail, ret = 0x%X.", ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%u", ret, private_def_len_); + GELOGE(RT_FAILED, "[Call][RtMemcpy] Fail, ret:0x%X, size:%u", ret, private_def_len_); return; } GELOGI("The first address of the custom info, privateDef=%p.", private_def_); diff --git a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc index 8fc8cc6c..7c26d23d 100644 --- a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc +++ b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc @@ -48,10 +48,10 @@ Status KernelExTaskInfo::InitTaskExtInfo(const std::string &ext_info, const OpDe num_inputs, num_outputs, unknown_type)); - GE_CHK_BOOL_RET_STATUS(ext_handle != nullptr, FAILED, "Malloc aicpu_ext_handle mem failed!"); + GE_CHK_BOOL_RET_STATUS(ext_handle != nullptr, FAILED, "[Malloc][Memory] for aicpu_ext_handle failed!"); GE_CHK_STATUS_RET(ext_handle->Parse(ext_info), - "Parse kernel ext info failed, kernel_ext_info_size=%zu.", ext_info.size()); - GE_CHK_STATUS_RET(ext_handle->UpdateExecuteMode(true), "UpdateExecuteMode failed."); + "[Parse][KernelExtInfo] failed, kernel_ext_info_size=%zu.", ext_info.size()); + GE_CHK_STATUS_RET(ext_handle->UpdateExecuteMode(true), "[Update][ExecuteMode] failed."); GELOGD("Update aicpu_task ext_info bit_map execute mode to 1."); topic_type_flag_ = ext_handle->GetTopicTypeFlag(); @@ -63,29 +63,30 @@ Status KernelExTaskInfo::InitTaskExtInfo(const std::string &ext_info, const OpDe auto input_desc = op_desc->MutableInputDesc(i); GE_CHECK_NOTNULL(input_desc); GE_CHK_STATUS_RET(ext_handle->UpdateInputShapeAndType(i, *input_desc), - "Input[%u] update input shape failed.", i); + "[Call][UpdateInputShapeAndType] Input[%u] update input shape failed, op:%s.", + i, op_desc->GetName().c_str()); } if (unknown_type != DEPEND_COMPUTE) { for (uint32_t j = 0; j < num_outputs; j++) { auto output_desc = op_desc->MutableOutputDesc(j); GE_CHECK_NOTNULL(output_desc); GE_CHK_STATUS_RET(ext_handle->UpdateOutputShapeAndType(j, *output_desc), - "Output[%u] update output shape failed.", j); + "[Call][UpdateOutputShapeAndType] Output[%u] update output shape failed, op:%s.", + j, op_desc->GetName().c_str()); } } } auto rt_ret = rtMalloc(&ext_info_addr_, ext_handle->GetExtInfoLen(), RT_MEMORY_HBM); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", - ext_info.size(), rt_ret); - GELOGE(RT_FAILED, "rtMalloc ext_info error: 0x%X, size=%zu", rt_ret, ext_info.size()); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", ext_info.size(), rt_ret); + GELOGE(RT_FAILED, "[RtMalloc][ExtInfo] error:0x%X, size=%zu", rt_ret, ext_info.size()); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(ext_info_addr_, ext_handle->GetExtInfoLen(), ext_handle->GetExtInfo(), ext_handle->GetExtInfoLen(), RT_MEMCPY_HOST_TO_DEVICE); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", ext_handle->GetExtInfoLen(), rt_ret); - GELOGE(RT_FAILED, "rtMemcpy ext_info error: 0x%X, size=%zu", rt_ret, ext_info.size()); + GELOGE(RT_FAILED, "[RtMemcpy][ExtInfo] error:0x%X, size=%zu", rt_ret, ext_info.size()); return RT_ERROR_TO_GE_STATUS(rt_ret);) return SUCCESS; } @@ -106,9 +107,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin uint32_t op_index = kernel_ex_def.op_index(); OpDescPtr op_desc = davinci_model_->GetOpByIndex(op_index); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - op_index); - GELOGE(INTERNAL_ERROR, "Init aicpu task info error, index is out of range!"); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", op_index); + GELOGE(INTERNAL_ERROR, "[Get][Op] by index failed, index:%u is out of range!", op_index); return INTERNAL_ERROR; } @@ -117,22 +117,22 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin if (sizeof(STR_FWK_OP_KERNEL) < kernel_ex_def.args_size()) { REPORT_INNER_ERROR("E19999", "Param kernel_ex_def.args_size():%u > sizeof(STR_FWK_OP_KERNEL):%zu, " "check invalid", kernel_ex_def.args_size(), sizeof(STR_FWK_OP_KERNEL)); - GELOGE(FAILED, "sizeof STR_FWK_OP_KERNEL is: %zu, but args_size is: %u", sizeof(STR_FWK_OP_KERNEL), + GELOGE(FAILED, "[Check][Param] sizeof STR_FWK_OP_KERNEL is: %zu, but args_size is: %u", sizeof(STR_FWK_OP_KERNEL), kernel_ex_def.args_size()); return FAILED; } errno_t sec_ret = memcpy_s(&fwk_op_kernel, sizeof(STR_FWK_OP_KERNEL), kernel_ex_def.args().data(), kernel_ex_def.args_size()); if (sec_ret != EOK) { - REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%zu, ret:0x%X", - sizeof(STR_FWK_OP_KERNEL), sec_ret); - GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%zu, ret:0x%X", sizeof(STR_FWK_OP_KERNEL), sec_ret); + GELOGE(FAILED, "[Call][Memcpy] failed, size:%zu, ret: %d", sizeof(STR_FWK_OP_KERNEL), sec_ret); return FAILED; } const auto &ext_info = kernel_ex_def.kernel_ext_info(); GE_CHK_STATUS_RET(InitTaskExtInfo(ext_info, op_desc), - "Init aicpu tf_task ext info failed, ext_info size=%zu", ext_info.size()); + "[Init][TaskExtInfo] failed, ext_info size=%zu, op:%s", + ext_info.size(), op_desc->GetName().c_str()); GELOGI("Node[%s] type[%s] kernel_ext_info size=%zu, ext_info_addr_=%p", op_desc->GetName().c_str(), op_desc->GetType().c_str(), ext_info.size(), ext_info_addr_); @@ -149,15 +149,15 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin davinci_model->SubModelId(), kernel_id) != SUCCESS, REPORT_CALL_ERROR("E19999", "CreateAicpuKernel fail, session_id:%lu, model_id:%u, kernel_id:%lu", session_id, davinci_model->Id(), kernel_id); - GELOGE(FAILED, "CreateAicpuKernel error."); + GELOGE(FAILED, "[Create][AicpuKernel] fail, session_id:%lu, model_id:%u, kernel_id:%lu", + session_id, davinci_model->Id(), kernel_id); return FAILED;) // 2.3 Create session GE_CHECK_NOTNULL(ModelManager::GetInstance()); ret = ModelManager::GetInstance()->CreateAicpuSession(session_id); GE_IF_BOOL_EXEC(ret != SUCCESS, - REPORT_CALL_ERROR("E19999", "CreateAicpuSession fail, session_id:%lu", - session_id); - GELOGE(ret, "CreateAicpuSession error. session id: %lu", session_id); + REPORT_CALL_ERROR("E19999", "CreateAicpuSession fail, session_id:%lu", session_id); + GELOGE(ret, "[Create][AicpuSession] error. session id:%lu", session_id); return ret;) kernel_buf_size_ = sizeof(STR_FWK_OP_KERNEL); @@ -170,7 +170,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", kernel_ex_def.task_info_size(), rt_ret); - GELOGE(RT_FAILED, "rtMalloc error, ret: Ox%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret:0x%X", + kernel_ex_def.task_info_size(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);); rt_ret = rtMemcpy(workspace_base_addr, kernel_ex_def.task_info_size(), kernel_ex_def.task_info().data(), kernel_ex_def.task_info_size(), RT_MEMCPY_HOST_TO_DEVICE); @@ -182,17 +183,15 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin rt_ret = rtMalloc(&kernel_buf_, kernel_buf_size_, RT_MEMORY_HBM); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, ret:0x%X, size:%u", - rt_ret, kernel_buf_size_); - GELOGE(RT_FAILED, "rtMalloc error: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, ret:0x%X, size:%u", rt_ret, kernel_buf_size_); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, ret:0x%X, size:%u", rt_ret, kernel_buf_size_); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(kernel_buf_, kernel_buf_size_, static_cast(&fwk_op_kernel), kernel_buf_size_, RT_MEMCPY_HOST_TO_DEVICE); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%u", - rt_ret, kernel_buf_size_); - GELOGE(RT_FAILED, "rtMemcpy error, ret: Ox%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%u", rt_ret, kernel_buf_size_); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, ret:0x%X, size:%u", rt_ret, kernel_buf_size_); return RT_ERROR_TO_GE_STATUS(rt_ret);) SetIoAddrs(op_desc); @@ -204,7 +203,7 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin // 3. Set workspaceaddr, inputOutputDataAddr Status ge_ret = CopyTaskInfo(kernel_ex_def, rts_param, op_desc); if (ge_ret != SUCCESS) { - GELOGE(ge_ret, "copy task info to workspace failed."); + GELOGE(ge_ret, "[Copy][TaskInfo] to workspace failed, op:%s.", op_desc->GetName().c_str()); return ge_ret; } @@ -212,7 +211,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin if (workspace_data_addrs.empty()) { REPORT_CALL_ERROR("E19999", "workspace_data_addrs is empty in op:%s(%s), check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "workspace_data_addrs is empty."); + GELOGE(FAILED, "[Check][Param] workspace_data_addrs is empty in op:%s(%s).", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } @@ -227,16 +227,14 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin if (addrs_size > 0) { rtError_t rt_ret = rtMalloc(&input_output_addr_, addrs_size, RT_MEMORY_HBM); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, ret:0x%X, size:%lu", - rt_ret, addrs_size); - GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, ret:0x%X, size:%lu", rt_ret, addrs_size); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, ret:0x%X, size:%lu", rt_ret, addrs_size); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(input_output_addr_, addrs_size, io_addrs.data(), addrs_size, RT_MEMCPY_HOST_TO_DEVICE); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%lu", - rt_ret, addrs_size); - GELOGE(RT_FAILED, "rtMemcpy to input_output_addr_ error: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%lu", rt_ret, addrs_size); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, ret:0x%X, size:%lu", rt_ret, addrs_size); return RT_ERROR_TO_GE_STATUS(rt_ret);) InitDumpTask(input_output_addr_, op_desc); @@ -254,7 +252,7 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, ret:0x%X, size:%zu", rt_ret, sizeof(STR_FWK_OP_KERNEL)); - GELOGE(RT_FAILED, "rtMalloc error: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, ret:0x%X, size:%zu", rt_ret, sizeof(STR_FWK_OP_KERNEL)); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(kernel_buf_, sizeof(STR_FWK_OP_KERNEL), static_cast(&fwk_op_kernel), @@ -262,7 +260,7 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%zu", rt_ret, sizeof(STR_FWK_OP_KERNEL)); - GELOGE(RT_FAILED, "rtMemcpy error, ret: Ox%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, ret:0x%X, size:%zu", rt_ret, sizeof(STR_FWK_OP_KERNEL)); return RT_ERROR_TO_GE_STATUS(rt_ret);) davinci_model_->SetZeroCopyAddr(op_desc, io_addrs, io_addrs.data(), input_output_addr_, addrs_size, 0); @@ -288,9 +286,8 @@ Status KernelExTaskInfo::CalculateArgs(const domi::TaskDef &task_def, DavinciMod uint32_t op_index = kernel_ex_def.op_index(); OpDescPtr op_desc = davinci_model->GetOpByIndex(op_index); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - op_index); - GELOGE(INTERNAL_ERROR, "Init aicpu task info error, index is out of range!"); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", op_index); + GELOGE(INTERNAL_ERROR, "[Get][Op] By Index, index:%u is out of range!", op_index); return INTERNAL_ERROR; } args_offset_ = davinci_model->GetTotalArgsSize(); @@ -310,7 +307,8 @@ Status KernelExTaskInfo::CalculateArgs(const domi::TaskDef &task_def, DavinciMod REPORT_INNER_ERROR("E19999", "The output size[%zu] and output index[%u] in op:%s(%s) are inconsistent, " "check invalid", outputs_size, output_index, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "The output size[%zu] and output index[%u] are inconsistent.", outputs_size, output_index); + GELOGE(FAILED, "[Check][Param] The output size[%zu] and output index[%u] in op:%s(%s) are inconsistent.", + outputs_size, output_index, op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } fixed_addr_offset_ = davinci_model->GetFixedAddrsSize(peer_input_name); @@ -336,11 +334,12 @@ void KernelExTaskInfo::SetIoAddrs(const OpDescPtr &op_desc) { if (AttrUtils::GetStr(op_desc, ATTR_DYNAMIC_SHAPE_FIXED_ADDR, peer_input_name)) { uint32_t output_index = davinci_model_->GetFixedAddrOutputIndex(peer_input_name); if (output_index > output_data_addrs.size()) { - REPORT_INNER_ERROR("E19999", "The output data addr size[%zu] and output index[%u] in op:%s(%s) are inconsistent" - ", check invalid", output_data_addrs.size(), output_index, + REPORT_INNER_ERROR("E19999", "The output data addr size[%zu] and output index[%u] in op:%s(%s) " + "are inconsistent, check invalid", output_data_addrs.size(), output_index, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "The output data addr size[%zu] and output index[%u] are inconsistent.", - output_data_addrs.size(), output_index); + GELOGE(FAILED, "[Check][Param] The output data addr size[%zu] and output index[%u] in op:%s(%s) " + "are inconsistent.", output_data_addrs.size(), output_index, + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return; } io_addrs_.insert(io_addrs_.end(), input_data_addrs.begin(), input_data_addrs.end()); @@ -373,7 +372,7 @@ Status KernelExTaskInfo::CopyTaskInfo(const domi::KernelExDef &kernel_def, const REPORT_INNER_ERROR("E19999", "Node:%s(%s) workspace addr:%zu or size:%zu empty, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), workspace_data_addrs.size(), workspace_data_sizes.size()); - GELOGE(FAILED, "Node:%s invalid workspace, addrs is %zu, size is %zu.", op_desc->GetName().c_str(), + GELOGE(FAILED, "[Check][Param] Node:%s invalid workspace, addrs is %zu, size is %zu.", op_desc->GetName().c_str(), workspace_data_addrs.size(), workspace_data_sizes.size()); return FAILED; } @@ -381,7 +380,7 @@ Status KernelExTaskInfo::CopyTaskInfo(const domi::KernelExDef &kernel_def, const if (workspace_data_addrs[0] == nullptr) { REPORT_INNER_ERROR("E19999", "Node:%s(%s) workspace addr is nullptr, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Node:%s workspace addrs is null.", op_desc->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Node:%s workspace addrs is null.", op_desc->GetName().c_str()); return FAILED; } @@ -389,7 +388,7 @@ Status KernelExTaskInfo::CopyTaskInfo(const domi::KernelExDef &kernel_def, const REPORT_INNER_ERROR("E19999", "Node:%s(%s) workspace size:%ld < task info size:%d, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), workspace_data_sizes[0], kernel_def.task_info_size()); - GELOGE(FAILED, "Node:%s workspace size is %ld, task info size is %d.", op_desc->GetName().c_str(), + GELOGE(FAILED, "[Check][Param] Node:%s workspace size is %ld, task info size is %d.", op_desc->GetName().c_str(), workspace_data_sizes[0], kernel_def.task_info_size()); return FAILED; } @@ -397,9 +396,8 @@ Status KernelExTaskInfo::CopyTaskInfo(const domi::KernelExDef &kernel_def, const rtError_t rt_ret = rtMemcpy(workspace_data_addrs[0], kernel_def.task_info_size(), kernel_def.task_info().data(), kernel_def.task_info_size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%d", - rt_ret, kernel_def.task_info_size()); - GELOGE(RT_FAILED, "rtMemcpy error: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%d", rt_ret, kernel_def.task_info_size()); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, ret:0x%X, size:%d", rt_ret, kernel_def.task_info_size()); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -418,15 +416,14 @@ Status KernelExTaskInfo::Distribute() { } rtError_t rt_ret = rtKernelLaunchEx(kernel_buf_, kernel_buf_size_, dump_flag_, stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchEx failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchEx failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtKernelLaunchEx] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } if (davinci_model_ == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model_ is null."); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model_ is null."); return PARAM_INVALID; } @@ -434,9 +431,8 @@ Status KernelExTaskInfo::Distribute() { uint32_t stream_id = 0; // for profiling rt_ret = rtModelGetTaskId(davinci_model_->GetRtModelHandle(), &task_id, &stream_id); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtModelGetTaskId] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } task_id_ = task_id; diff --git a/ge/graph/load/model_manager/task_info/kernel_task_info.cc b/ge/graph/load/model_manager/task_info/kernel_task_info.cc index f3213308..c72bfeef 100755 --- a/ge/graph/load/model_manager/task_info/kernel_task_info.cc +++ b/ge/graph/load/model_manager/task_info/kernel_task_info.cc @@ -95,11 +95,10 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci rtError_t rt_ret = rtGetFunctionByName(const_cast(kernel_def.stub_func().c_str()), &stub_func_); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtGetFunctionByName failed for op:%s(%s), " - "bin_file_key:%s, ret:0x%X", - op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), - kernel_def.stub_func().c_str(), rt_ret); - GELOGE(RT_FAILED, "execute rtGetFunctionByName failed. stub_func: %s", - kernel_def.stub_func().c_str()); + "bin_file_key:%s, ret:0x%X", op_desc_->GetName().c_str(), + op_desc_->GetType().c_str(), kernel_def.stub_func().c_str(), rt_ret); + GELOGE(RT_FAILED, "[Execute][RtGetFunctionByName] failed for op:%s(%s). stub_func:%s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), kernel_def.stub_func().c_str()); return RT_ERROR_TO_GE_STATUS(rt_ret);); } else if (kernel_type_ == ccKernelType::TE) { // get bin_file_key @@ -109,18 +108,20 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci rtError_t rt_ret = rtGetFunctionByName(bin_file_key, &stub_func_); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtGetFunctionByName failed for op:%s(%s), " - "bin_file_key:%s, ret:0x%X", - op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), - bin_file_key, rt_ret); - GELOGE(RT_FAILED, "execute rtGetFunctionByName failed. bin_file_key: %s", bin_file_key); + "bin_file_key:%s, ret:0x%X", op_desc_->GetName().c_str(), + op_desc_->GetType().c_str(), bin_file_key, rt_ret); + GELOGE(RT_FAILED, "[Execute][RtGetFunctionByName] failed for op:%s(%s), bin_file_key:%s", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), bin_file_key); return RT_ERROR_TO_GE_STATUS(rt_ret);); } if (context.origin_op_index_size() > CC_FUSION_OP_MAX) { - REPORT_INNER_ERROR("E19999", "context.origin_op_index_size():%d is more than CC_FUSION_OP_MAX(%d), op:%s(%s) ," + REPORT_INNER_ERROR("E19999", "context.origin_op_index_size():%d is more than CC_FUSION_OP_MAX(%d), op:%s(%s), " "check invalid", context.origin_op_index_size(), CC_FUSION_OP_MAX, op_desc_->GetName().c_str(), op_desc_->GetType().c_str()); - GELOGE(PARAM_INVALID, "context.origin_op_index_size() is more than CC_FUSION_OP_MAX(%d)", CC_FUSION_OP_MAX); + GELOGE(PARAM_INVALID, "[Check][Param] context.origin_op_index_size():%d is more than CC_FUSION_OP_MAX(%d), " + "op:%s(%s)", context.origin_op_index_size(), CC_FUSION_OP_MAX, + op_desc_->GetName().c_str(), op_desc_->GetType().c_str()); return PARAM_INVALID; } @@ -132,10 +133,11 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci ctx_.opIndex = context.op_index(); uint16_t *args_offset_tmp = reinterpret_cast(const_cast(context.args_offset().data())); if (context.args_offset().size() / sizeof(uint16_t) < 1) { - REPORT_INNER_ERROR("E19999", "context.args_offset().size():%zu / sizeof(uint16_t) less than 1, op:%s(%s) ," + REPORT_INNER_ERROR("E19999", "context.args_offset().size():%zu / sizeof(uint16_t) less than 1, op:%s(%s), " "check invalid", context.args_offset().size(), op_desc_->GetName().c_str(), op_desc_->GetType().c_str()); - GELOGE(FAILED, "context.args_offset().size() / sizeof(uint16_t) less than 1"); + GELOGE(FAILED, "[Check][Param] context.args_offset().size() / sizeof(uint16_t) less than 1, op:%s(%s)", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str()); return FAILED; } @@ -149,7 +151,8 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci if (kernel_def.args().empty() || args_size_ == 0) { REPORT_INNER_ERROR("E19999", "kernel_def.args() is empty, op:%s(%s), check invalid", op_desc_->GetName().c_str(), op_desc_->GetType().c_str()); - GELOGE(FAILED, "args is null."); + GELOGE(FAILED, "[Check][Param] args is empty, op:%s(%s)", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str()); return FAILED; } ret = InitCceTask(kernel_def); @@ -181,9 +184,8 @@ void KernelTaskInfo::UpdateSKTTaskId() { if (davinci_model_ != nullptr) { rtError_t rt_ret = rtModelGetTaskId(davinci_model_->GetRtModelHandle(), &task_id, &stream_id); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtModelGetTaskId] failed, ret:0x%X", rt_ret); return; } SuperKernelTaskInfo &skt_info = davinci_model_->GetSuperKernelTaskInfo(); @@ -201,9 +203,8 @@ void KernelTaskInfo::UpdateTaskId() { if (davinci_model_ != nullptr) { rtError_t rt_ret = rtModelGetTaskId(davinci_model_->GetRtModelHandle(), &task_id, &stream_id); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtModelGetTaskId] failed, ret:0x%X", rt_ret); return; } task_id_ = task_id; @@ -214,7 +215,7 @@ void KernelTaskInfo::UpdateTaskId() { Status KernelTaskInfo::SKTFinalize() { UpdateSKTTaskId(); - GE_CHK_STATUS_RET(SaveSKTDumpInfo(), "skt save dump info failed"); + GE_CHK_STATUS_RET(SaveSKTDumpInfo(), "[Save][SKTDumpInfo] failed"); GELOGI("SuperKernel Distribute [skt_id:%u]", skt_id_); SuperKernelTaskInfo &skt_info = davinci_model_->GetSuperKernelTaskInfo(); skt_info.kernel_list.clear(); @@ -258,13 +259,12 @@ Status KernelTaskInfo::SuperKernelLaunch() { static_cast(skt_info.last_sm_desc), skt_info.last_stream, skt_info.last_dump_flag); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "SuperKernelLaunch: Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtKernelLaunchWithFlag] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } call_save_dump_ = true; - GE_CHK_STATUS_RET(SKTFinalize(), "Skt finalize failed"); + GE_CHK_STATUS_RET(SKTFinalize(), "[Call][SKTFinalize] failed"); return SUCCESS; } // Create super kernel factory @@ -272,27 +272,24 @@ Status KernelTaskInfo::SuperKernelLaunch() { // Init super kernel factory Status ge_ret = factory->Init(); if (ge_ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", "Call SuperKernelFactory init fail, ret:0x%X", - ge_ret); - GELOGE(ge_ret, "SuperKernelLaunch: SuperKernelFactory init failed"); + REPORT_CALL_ERROR("E19999", "Call SuperKernelFactory init fail, ret:0x%X", ge_ret); + GELOGE(ge_ret, "[Init][SuperKernelFactory] failed, ret:0x%X", ge_ret); return ge_ret; } // Call the fuse API std::unique_ptr superKernel = nullptr; ge_ret = factory->FuseKernels(skt_kernel_list, skt_arg_list, skt_info.last_block_dim, superKernel); if (ge_ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", "Call SuperKernelFactory FuseKernels fail, ret:0x%X", - ge_ret); - GELOGE(ge_ret, "SuperKernelLaunch: fuse call failed"); + REPORT_CALL_ERROR("E19999", "Call SuperKernelFactory FuseKernels fail, ret:0x%X", ge_ret); + GELOGE(ge_ret, "[Call][FuseKernels] failed, ret:0x%X", ge_ret); return ge_ret; } // Launch a super kernel skt_dump_flag_ = GetDumpFlag(); ge_ret = superKernel->Launch(skt_info.last_stream, skt_dump_flag_); if (ge_ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", "Call SuperKernelFactory Launch fail, ret:0x%X", - ge_ret); - GELOGE(ge_ret, "SuperKernelLaunch: launch failed"); + REPORT_CALL_ERROR("E19999", "Call SuperKernelFactory Launch fail, ret:0x%X", ge_ret); + GELOGE(ge_ret, "[Call][Launch] failed, ret:0x%X", ge_ret); return ge_ret; } GELOGI("SuperKernelLaunch: success[skt_kernel_list size[%zu] skt_arg_list[%zu]]", skt_kernel_list.size(), @@ -300,7 +297,7 @@ Status KernelTaskInfo::SuperKernelLaunch() { // record skt addr for release superkernel_dev_nav_table_ = superKernel->GetNavTablePtr(); superkernel_device_args_addr_ = superKernel->GetDeviceArgsPtr(); - GE_CHK_STATUS_RET(SKTFinalize(), "Skt finalize failed"); + GE_CHK_STATUS_RET(SKTFinalize(), "[Call][SKTFinalize] failed"); return SUCCESS; } @@ -331,14 +328,13 @@ Status KernelTaskInfo::SaveSuperKernelInfo() { bool KernelTaskInfo::IsMarkedLastNode() { if (davinci_model_ == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return false; } OpDescPtr op_desc = davinci_model_->GetOpByIndex(ctx_.opIndex); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - ctx_.opIndex); - GELOGE(INTERNAL_ERROR, "InitTVMTaskInfo error, index is out of range!"); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", ctx_.opIndex); + GELOGE(INTERNAL_ERROR, "[Get][Op] by index failed, index:%u is out of range!", ctx_.opIndex); return false; } bool is_last_node = false; @@ -349,14 +345,13 @@ bool KernelTaskInfo::IsMarkedLastNode() { bool KernelTaskInfo::IsMarkedFirstNode() { if (davinci_model_ == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return false; } OpDescPtr op_desc = davinci_model_->GetOpByIndex(ctx_.opIndex); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - ctx_.opIndex); - GELOGE(INTERNAL_ERROR, "InitTVMTaskInfo error, index is out of range!"); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", ctx_.opIndex); + GELOGE(INTERNAL_ERROR, "[Get][Op] by index failed, index:%u is out of range!", ctx_.opIndex); return false; } bool is_first_node = false; @@ -381,7 +376,7 @@ Status KernelTaskInfo::SuperKernelDistribute() { if (FirstCallSKTLaunchCheck()) { ret = SuperKernelLaunch(); if (ret != SUCCESS) { - GELOGE(FAILED, "Call SuperKernelLaunch failed!"); + GELOGE(FAILED, "[Call][SuperKernelLaunch] failed, taskid:%u", task_id_); return FAILED; } } @@ -389,16 +384,15 @@ Status KernelTaskInfo::SuperKernelDistribute() { // 1.launch before ret = SuperKernelLaunch(); if (ret != SUCCESS) { - GELOGE(ret, "Call SuperKernelLaunch failed!"); + GELOGE(ret, "[Call][SuperKernelLaunch] failed, taskid:%u", task_id_); return ret; } // 2.launch current rtError_t rt_ret = rtKernelLaunchWithFlag(stub_func_, block_dim_, args_, args_size_, static_cast(sm_desc_), stream_, dump_flag_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtKernelLaunchWithFlag] failed, ret:0x%X", rt_ret); return rt_ret; } call_save_dump_ = true; @@ -407,7 +401,7 @@ Status KernelTaskInfo::SuperKernelDistribute() { } else { ret = SaveSuperKernelInfo(); if (ret != SUCCESS) { - GELOGE(ret, "Call SuperKernelLaunch failed!"); + GELOGE(ret, "[Call][SaveSuperKernelInfo] failed, taskid:%u", task_id_); return ret; } } @@ -473,7 +467,7 @@ Status KernelTaskInfo::Distribute() { if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag or rtCpuKernelLaunchWithFlag failed, " "ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtApi] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } // set for task_id_ @@ -509,18 +503,16 @@ Status KernelTaskInfo::CopyNoncontinuousArgs(uint16_t offset) { // copy io addr errno_t sec_ret = memcpy_s(args_addr.get() + offset, addr_size, io_addrs.data(), addr_size); if (sec_ret != EOK) { - REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%zu, ret:0x%X", - addr_size, sec_ret); - GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%zu, ret:0x%X", addr_size, sec_ret); + GELOGE(FAILED, "[Call][Memcpy] failed, size:%zu, ret:%d", addr_size, sec_ret); return FAILED; } // copy args to device rtError_t rt_ret = rtMemcpy(args_, args_size_, args_addr.get(), args_size_, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", - args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rt api(rtMemcpy) failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", args_size_, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } GELOGD("Copy noncontinuous args success, kernel type %d.", kernel_type_); @@ -570,7 +562,7 @@ Status KernelTaskInfo::Release() { ret = (sm_desc_ != nullptr) ? rtMemFreeManaged(sm_desc_) : RT_ERROR_NONE; if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemFreeManaged failed, ret:0x%X", ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", static_cast(ret)); + GELOGE(RT_FAILED, "[Call][RtMemFreeManaged] failed, ret:0x%X", static_cast(ret)); return RT_ERROR_TO_GE_STATUS(ret); } sm_desc_ = nullptr; @@ -600,17 +592,15 @@ Status KernelTaskInfo::UpdateL2Data(const domi::KernelDef &kernel_def) { rtError_t rt_ret = rtMemAllocManaged(&sm_desc_, sm_desc.size(), RT_MEMORY_SPM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemAllocManaged failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemAllocManaged failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemAllocManaged] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtMemcpy(sm_desc_, sm_desc.size(), sm_desc.data(), sm_desc.size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", - sm_desc.size(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", sm_desc.size(), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", sm_desc.size(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -656,9 +646,8 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne args_addr = std::unique_ptr(new (std::nothrow) uint8_t[args_size_]); errno_t sec_ret = memcpy_s(args_addr.get(), args_size_, kernel_def.args().data(), args_size_); if (sec_ret != EOK) { - REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%u, ret:0x%X", - args_size_, sec_ret); - GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%u, ret:0x%X", args_size_, sec_ret); + GELOGE(FAILED, "[Call][Memcpy] failed, size:%u, ret:0x%X", args_size_, sec_ret); return FAILED; } @@ -700,26 +689,24 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne // malloc args memory rt_ret = rtMalloc(&args_, args_size_, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", - args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", args_size_, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret:0x%X", args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } // copy orign args rt_ret = rtMemcpy(args_, args_size_, kernel_def.args().data(), args_size_, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", - args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", args_size_, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } if ((args_size_ <= offset) || (args_size_ - offset < kAddrLen * tensor_device_addrs.size())) { REPORT_INNER_ERROR("E19999", "offset:%u >= kernelInfo.argsSize:%u or copy content:%zu beyond applied memory:%u, " - "check invalid", - offset, args_size_, kAddrLen * tensor_device_addrs.size(), args_size_ - offset); - GELOGE(FAILED, "offset >= kernelInfo.argsSize or copy content beyond applied memory."); + "check invalid", offset, args_size_, kAddrLen * tensor_device_addrs.size(), args_size_ - offset); + GELOGE(FAILED, "[Check][Param] offset:%u >= kernelInfo.argsSize:%u or copy content:%zu beyond applied memory:%u, " + "check invalid", offset, args_size_, kAddrLen * tensor_device_addrs.size(), args_size_ - offset); return FAILED; } @@ -727,17 +714,15 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne rt_ret = rtMemcpy(static_cast(args_) + offset, args_size_ - offset, tensor_device_addrs.data(), kAddrLen * tensor_device_addrs.size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", - args_size_ - offset, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", args_size_ - offset, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", args_size_ - offset, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } sec_ret = memcpy_s(args_addr.get() + offset, args_size_ - offset, tensor_device_addrs.data(), kAddrLen * tensor_device_addrs.size()); if (sec_ret != EOK) { - REPORT_CALL_ERROR("E19999", "Call memcpy_s failed, size:%u, ret:0x%X", - args_size_ - offset, sec_ret); - GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); + REPORT_CALL_ERROR("E19999", "Call memcpy_s failed, size:%u, ret:0x%X", args_size_ - offset, sec_ret); + GELOGE(FAILED, "[Call][Memcpy] failed, size:%u, ret:0x%X", args_size_ - offset, sec_ret); return FAILED; } skt_dump_args_ = static_cast(args_) + offset; @@ -778,9 +763,8 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel GELOGI("Do InitAICPUCustomTask"); OpDescPtr op_desc = davinci_model_->GetOpByIndex(op_index); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - op_index); - GELOGE(INTERNAL_ERROR, "index is out of range, index: %u", op_index); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", op_index); + GELOGE(INTERNAL_ERROR, "[Get][Op] index is out of range, index:%u", op_index); return INTERNAL_ERROR; } @@ -792,16 +776,18 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel if (ctx_.argsOffset == nullptr) { REPORT_CALL_ERROR("E19999", "New ctx_.argsOffset fail, size:%u, op:%s(%s)", kCustomAicpuArgsLen, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "ctx_.argsOffset is null!"); + GELOGE(PARAM_INVALID, "[Malloc][Memory] ctx_.argsOffset is null, size:%u, op:%s(%s)", + kCustomAicpuArgsLen, op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } if (context.args_offset().size() / sizeof(uint16_t) < kCustomAicpuArgsLen) { REPORT_INNER_ERROR("E19999", "context.args_offset().size():%zu / sizeof(uint16_t) is less than " - "kCustomAicpuArgsLen:%u, op:%s(%s), check invalid", - context.args_offset().size(), kCustomAicpuArgsLen, - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "context.args_offset().size() / sizeof(uint16_t) is less than kCustomAicpuArgsLen"); + "kCustomAicpuArgsLen:%u, op:%s(%s), check invalid", context.args_offset().size(), + kCustomAicpuArgsLen, op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] context.args_offset().size():%zu / sizeof(uint16_t) is less than " + "kCustomAicpuArgsLen:%u, op:%s(%s)", context.args_offset().size(), kCustomAicpuArgsLen, + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } @@ -814,7 +800,7 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel Status ret = StoreInputOutputTensor(input_data_addrs, output_data_addrs, ModelUtils::GetInputDescs(op_desc), ModelUtils::GetOutputDescs(op_desc)); if (ret != SUCCESS) { - GELOGE(ret, "StoreInputOutputTensor Failed"); + GELOGE(ret, "[Store][InputOutputTensor] Failed, op:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return ret; } @@ -823,7 +809,8 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel if (!AttrUtils::GetBytes(op_desc, ATTR_NAME_OPATTR, buffer)) { REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_OPATTR.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "can't find opattr bytes!."); + GELOGE(FAILED, "[Get][Attr] %s in op:%s(%s) fail", ATTR_NAME_OPATTR.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } @@ -831,7 +818,7 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel if (op_attr_size == 0) { REPORT_INNER_ERROR("E19999", "Attr:%s in op:%s(%s) size is 0, check invalid", ATTR_NAME_OPATTR.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "param op_attr_size is out of range"); + GELOGE(PARAM_INVALID, "[Check][Param] param op_attr_size is out of range, op:%s", op_desc->GetName().c_str()); return PARAM_INVALID; } @@ -839,7 +826,8 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMalloc failed for op:%s(%s), size:%u, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_attr_size, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed for op:%s(%s), size:%u, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_attr_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -847,7 +835,8 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed for op:%s(%s), size:%u, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_attr_size, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed for op:%s(%s), size:%u, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_attr_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -860,7 +849,7 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel "op:%s(%s) check invalid", i, (uint32_t)ctx_.argsOffset[i], sizeof(uint64_t), kernel_def.args().size(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "ctx.argsOffset[%u]: %u + sizeof(uint64_t): %zu >= kernelDef.args().size():%zu", i, + GELOGE(FAILED, "[Check][Param] ctx.argsOffset[%u]:%u + sizeof(uint64_t):%zu >= kernelDef.args().size():%zu", i, (uint32_t)ctx_.argsOffset[i], sizeof(uint64_t), kernel_def.args().size()); return FAILED; } @@ -880,7 +869,8 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMalloc failed for op:%s(%s), size:%u, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed for op:%s(%s), size:%u, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -888,9 +878,9 @@ Status KernelTaskInfo::InitAICPUCustomTask(uint32_t op_index, const domi::Kernel RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed for op:%s(%s), size:%u, ret:0x%X", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), - kernel_def.args_size(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + op_desc->GetName().c_str(), op_desc->GetType().c_str(), kernel_def.args_size(), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed for op:%s(%s), size:%u, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), kernel_def.args_size(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -905,12 +895,12 @@ Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { GELOGI("Do InitCCETask"); if (davinci_model_ == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } Status ret = SetContext(kernel_def); if (ret != SUCCESS) { - GELOGE(ret, "SetContext Fail."); + GELOGE(ret, "[Set][Context] Fail."); return ret; } @@ -920,7 +910,7 @@ Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { if (context.is_flowtable()) { if (flowtable.empty()) { REPORT_INNER_ERROR("E19999", "kernel_def.flowtable is empty, check invalid"); - GELOGE(FAILED, "flowtable is null."); + GELOGE(FAILED, "[Check][Param] flowtable is null."); return FAILED; } } @@ -940,23 +930,22 @@ Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { ret = UpdateCceArgs(sm_desc, flowtable, kernel_def); if (ret != SUCCESS) { - GELOGE(ret, "update cce args fail"); + GELOGE(ret, "[Update][CceArgs] fail"); return ret; } // flowtable ret = SetFlowtable(flowtable, kernel_def); if (ret != SUCCESS) { - GELOGE(ret, "SetFlowtable Fail"); + GELOGE(ret, "[Set][Flowtable] Fail"); return ret; } // args rtError_t rt_ret = rtMalloc(&args_, kernel_def.args_size(), RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", - kernel_def.args_size(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X", kernel_def.args_size(), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret:0x%X", kernel_def.args_size(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } GE_PRINT_DYNAMIC_MEMORY(rtMalloc, "cce task physical memory.", kernel_def.args_size()) @@ -964,9 +953,8 @@ Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { rt_ret = rtMemcpy(args_, kernel_def.args_size(), kernel_def.args().data(), kernel_def.args_size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", - kernel_def.args_size(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X", kernel_def.args_size(), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%u, ret:0x%X", kernel_def.args_size(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -974,17 +962,15 @@ Status KernelTaskInfo::InitCceTask(const domi::KernelDef &kernel_def) { if (!sm_desc.empty()) { rt_ret = rtMemAllocManaged(&sm_desc_, sm_desc.size(), RT_MEMORY_SPM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemAllocManaged failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemAllocManaged failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemAllocManaged] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtMemcpy(sm_desc_, sm_desc.size(), sm_desc.data(), sm_desc.size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", - sm_desc.size(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", sm_desc.size(), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", sm_desc.size(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -998,9 +984,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k OpDescPtr op_desc = davinci_model_->GetOpByIndex(op_index); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - op_index); - GELOGE(INTERNAL_ERROR, "index is out of range, index: %u", op_index); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", op_index); + GELOGE(INTERNAL_ERROR, "[Get][Op] index is out of range, index:%u", op_index); return INTERNAL_ERROR; } GELOGI("node[%s] test so name %s, kernel name %s", op_desc->GetName().c_str(), so_name_.c_str(), @@ -1009,7 +994,7 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k if (kernel_type_ == ccKernelType::CUST_AI_CPU) { bool loaded = false; GE_CHK_STATUS_RET(ModelManager::GetInstance()->LoadCustAicpuSo(op_desc, so_name_, loaded), - "launch cust aicpu so failed"); + "[Launch][CustAicpuSo] failed"); } // copy args to new host memory @@ -1017,9 +1002,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k GE_PRINT_DYNAMIC_MEMORY(new, "cce task physical memory.", sizeof(uint8_t) * args_size_) errno_t sec_ret = memcpy_s(args_addr.get(), args_size_, kernel_def.args().data(), args_size_); if (sec_ret != EOK) { - REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%u, ret:0x%X", - args_size_, sec_ret); - GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%u, ret:0x%X", args_size_, sec_ret); + GELOGE(FAILED, "[Call][Memcpy] failed, size:%u, ret:0x%X", args_size_, sec_ret); return FAILED; } @@ -1027,7 +1011,7 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k const auto &ext_info = kernel_def.kernel_ext_info(); auto init_ret = InitAicpuTaskExtInfo(ext_info); if (init_ret != SUCCESS) { - GELOGE(init_ret, "Init aicpu task ext info failed, ext_info size=%zu", ext_info.size()); + GELOGE(init_ret, "[Init][AicpuTaskExtInfo] failed, ext_info size=%zu", ext_info.size()); return init_ret; } GELOGI("Node[%s] type[%s] kernel_ext_info size=%zu, aicpu_ext_info_addr_=%p", op_desc->GetName().c_str(), @@ -1053,9 +1037,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k auto addrs_size = sizeof(uint64_t) * io_addrs.size(); sec_ret = memcpy_s(reinterpret_cast(io_addr), addrs_size, io_addrs.data(), addrs_size); if (sec_ret != EOK) { - REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%lu, ret:0x%X", - addrs_size, sec_ret); - GELOGE(FAILED, "memcpy failed, ret: %d", sec_ret); + REPORT_CALL_ERROR("E19999", "Call memcpy_s fail, size:%lu, ret:0x%X", addrs_size, sec_ret); + GELOGE(FAILED, "[Call][Memcpy] failed, size:%lu, ret:0x%X", addrs_size, sec_ret); return FAILED; } } @@ -1065,7 +1048,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMalloc failed for op:%s(%s), size:%u, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rt api(rtMalloc) failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed for op:%s(%s), size:%u, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } GE_PRINT_DYNAMIC_MEMORY(rtMalloc, "cce task physical memory.", args_size_) @@ -1075,7 +1059,8 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed for op:%s(%s), size:%u, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rt api(rtMemcpy) failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed for op:%s(%s), size:%u, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } InitDumpTask(sizeof(aicpu::AicpuParamHead)); @@ -1120,13 +1105,15 @@ Status KernelTaskInfo::InitAicpuTaskExtInfo(const std::string &ext_info) { num_inputs, num_outputs, unknown_type)); - GE_CHK_BOOL_RET_STATUS(ext_handle != nullptr, FAILED, "Malloc aicpu_ext_handle mem failed!"); + GE_CHK_BOOL_RET_STATUS(ext_handle != nullptr, FAILED, "[Malloc][Memory] for aicpu_ext_handle failed!"); GE_CHK_STATUS_RET(ext_handle->Parse(ext_info), - "Parse kernel ext info failed, kernel_ext_info_size=%zu.", ext_info.size()); + "[Parse][KernelExtInfo] failed, kernel_ext_info_size=%zu, op:%s.", + ext_info.size(), op_desc_->GetName().c_str()); GE_CHK_STATUS_RET(ext_handle->UpdateSessionInfoSessionId(davinci_model_->GetSessionId()), - "Update session info session id failed."); + "[Update][SessionInfoSessionId] failed, op:%s", op_desc_->GetName().c_str()); GELOGD("Update aicpu_task ext_info session_info session_id is %lu", davinci_model_->GetSessionId()); - GE_CHK_STATUS_RET(ext_handle->UpdateExecuteMode(true), "UpdateExecuteMode failed."); + GE_CHK_STATUS_RET(ext_handle->UpdateExecuteMode(true), + "[Update][ExecuteMode] failed, op:%s", op_desc_->GetName().c_str()); GELOGD("Update aicpu_task ext_info bit_map execute mode to 1."); topic_type_flag_ = ext_handle->GetTopicTypeFlag(); @@ -1138,13 +1125,15 @@ Status KernelTaskInfo::InitAicpuTaskExtInfo(const std::string &ext_info) { auto input_desc = op_desc_->MutableInputDesc(i); GE_CHECK_NOTNULL(input_desc); GE_CHK_STATUS_RET(ext_handle->UpdateInputShapeAndType(i, *input_desc), - "Input[%u] update input shape failed.", i); + "[Call][UpdateInputShapeAndType] Input[%u] update input shape failed, op:%s.", + i, op_desc_->GetName().c_str()); } for (uint32_t j = 0; j < num_outputs; j++) { auto output_desc = op_desc_->MutableOutputDesc(j); GE_CHECK_NOTNULL(output_desc); GE_CHK_STATUS_RET(ext_handle->UpdateOutputShapeAndType(j, *output_desc), - "Output[%u] update output shape failed.", j); + "[Call][UpdateOutputShapeAndType] Output[%u] update output shape failed, op:%s.", + j, op_desc_->GetName().c_str()); } } auto rt_ret = rtMalloc(&aicpu_ext_info_addr_, ext_handle->GetExtInfoLen(), RT_MEMORY_HBM); @@ -1152,7 +1141,8 @@ Status KernelTaskInfo::InitAicpuTaskExtInfo(const std::string &ext_info) { REPORT_CALL_ERROR("E19999", "Call rtMalloc failed for op:%s(%s), size:%zu, ret:0x%X", op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), ext_handle->GetExtInfoLen(), rt_ret); - GELOGE(RT_FAILED, "rtMalloc ext_info error: 0x%X, size=%zu", rt_ret, ext_info.size()); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed for op:%s(%s), size:%zu, ret:0x%X", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), ext_handle->GetExtInfoLen(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtMemcpy(aicpu_ext_info_addr_, ext_handle->GetExtInfoLen(), ext_handle->GetExtInfo(), @@ -1161,7 +1151,8 @@ Status KernelTaskInfo::InitAicpuTaskExtInfo(const std::string &ext_info) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed for op:%s(%s), size:%zu, ret:0x%X", op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), ext_handle->GetExtInfoLen(), rt_ret); - GELOGE(RT_FAILED, "rtMemcpy ext_info error: 0x%X, size=%zu", rt_ret, ext_info.size()); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed for op:%s(%s), size:%zu, ret:0x%X", + op_desc_->GetName().c_str(), op_desc_->GetType().c_str(), ext_handle->GetExtInfoLen(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1178,9 +1169,8 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d // inputDescs rtError_t rt_ret = rtMalloc(&custom_info_.input_descs, sizeof(opTensor_t) * input_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", - sizeof(opTensor_t) * input_size, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", sizeof(opTensor_t) * input_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%zu, ret:0x%X", sizeof(opTensor_t) * input_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1188,9 +1178,8 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d rt_ret = rtMemcpy(static_cast(custom_info_.input_descs) + i, sizeof(opTensor_t), const_cast(&input_descs[i]), sizeof(opTensor_t), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", - sizeof(opTensor_t), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", sizeof(opTensor_t), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", sizeof(opTensor_t), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -1198,9 +1187,8 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d // inputAddrs rt_ret = rtMalloc(&custom_info_.input_addrs, sizeof(opTensor_t) * input_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", - sizeof(opTensor_t) * input_size, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", sizeof(opTensor_t) * input_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%zu, ret:0x%X", sizeof(opTensor_t) * input_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1208,9 +1196,8 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d rt_ret = rtMemcpy(custom_info_.input_addrs, kAddrLen * input_size, &input_data_addrs[0], kAddrLen * input_size, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", - kAddrLen * input_size, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", kAddrLen * input_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", kAddrLen * input_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -1218,18 +1205,16 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d // outputDescs rt_ret = rtMalloc(&custom_info_.output_descs, sizeof(opTensor_t) * output_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", - sizeof(opTensor_t) * output_size, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", sizeof(opTensor_t) * output_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%zu, ret:0x%X", sizeof(opTensor_t) * output_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } for (std::size_t i = 0; i < output_size; ++i) { rt_ret = rtMemcpy(static_cast(custom_info_.output_descs) + i, sizeof(opTensor_t), const_cast(&input_descs[i]), sizeof(opTensor_t), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", - sizeof(opTensor_t), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", sizeof(opTensor_t), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", sizeof(opTensor_t), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -1237,9 +1222,8 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d // outputAddrs rt_ret = rtMalloc(&custom_info_.output_addrs, sizeof(opTensor_t) * output_size, RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", - sizeof(opTensor_t) * output_size, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", sizeof(opTensor_t) * output_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%zu, ret:0x%X", sizeof(opTensor_t) * output_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1247,9 +1231,8 @@ Status KernelTaskInfo::StoreInputOutputTensor(const std::vector &input_d rt_ret = rtMemcpy(custom_info_.output_addrs, kAddrLen * output_size, &output_data_addrs[0], kAddrLen * output_size, RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", - kAddrLen * output_size, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", kAddrLen * output_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", kAddrLen * output_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } @@ -1266,7 +1249,7 @@ Status KernelTaskInfo::SetContext(const domi::KernelDef &kernel_def) { ctx_.argsCount = context.args_count(); if (ctx_.argsCount == 0) { REPORT_INNER_ERROR("E19999", "kernel_def.context.args_count is 0, check invalid"); - GELOGE(INTERNAL_ERROR, "check argsCount fail:%u.", ctx_.argsCount); + GELOGE(INTERNAL_ERROR, "[Check][Param] argsCount is %u.", ctx_.argsCount); return INTERNAL_ERROR; } @@ -1274,16 +1257,16 @@ Status KernelTaskInfo::SetContext(const domi::KernelDef &kernel_def) { REPORT_INNER_ERROR("E19999", "param [context.args_offset().size():%zu / sizeof(uint16_t)] " "is less than [ctx_.argsCount:%u], check invalid", context.args_offset().size(), ctx_.argsCount); - GELOGE(PARAM_INVALID, "param [context.args_offset().size() / sizeof(uint16_t)] is less than [ctx_.argsCount]"); + GELOGE(PARAM_INVALID, "[Check][Param] [context.args_offset().size():%zu / sizeof(uint16_t)] " + "is less than [ctx_.argsCount:%u], check invalid", context.args_offset().size(), ctx_.argsCount); return PARAM_INVALID; } // ctx_.argsOffset stores the offset of the internal information of agrs_, equal to the ctx_.argsCount ctx_.argsOffset = new (std::nothrow) uint16_t[ctx_.argsCount](); if (ctx_.argsOffset == nullptr) { - REPORT_CALL_ERROR("E19999", "New ctx_.argsOffset fail, size:%u", - ctx_.argsCount); - GELOGE(PARAM_INVALID, "(param [ctx_.argsOffset] must not be null."); + REPORT_CALL_ERROR("E19999", "New ctx_.argsOffset fail, size:%u", ctx_.argsCount); + GELOGE(PARAM_INVALID, "[Malloc][Memory] failed, ctx_.argsOffset must not be null, size:%u", ctx_.argsCount); return PARAM_INVALID; } @@ -1301,7 +1284,7 @@ void KernelTaskInfo::FreeRtMem(void **ptr) { rtError_t ret = rtFree(*ptr); if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret:0x%X", ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", ret); + GELOGE(RT_FAILED, "[Call][RtFree] failed, ret:0x%X", ret); } *ptr = nullptr; @@ -1319,7 +1302,7 @@ Status KernelTaskInfo::UpdateCceArgs(std::string &sm_desc, std::string &flowtabl Status status = CceUpdateKernelArgs(context, data_base_addr, weight_base_addr, var_base_addr, sm_desc, flowtable, kernel_def); if (status != SUCCESS) { - GELOGE(status, "Call cce api failed"); + GELOGE(status, "[Call][CceUpdateKernelArgs] failed, ret:%d", status); return status; } return SUCCESS; @@ -1348,9 +1331,8 @@ Status KernelTaskInfo::CceUpdateKernelArgs(const domi::KernelContext &context, u if (handle == nullptr) { error = mmDlerror(); GE_IF_BOOL_EXEC(error == nullptr, error = ""); - REPORT_INNER_ERROR("E19999", "Failed in dlopen:%s, dlerror:%s", - canonicalPath.c_str(), error); - GELOGE(GE_PLGMGR_SO_NOT_EXIST, "Failed in dlopen %s! ", error); + REPORT_INNER_ERROR("E19999", "Failed in dlopen:%s, dlerror:%s", canonicalPath.c_str(), error); + GELOGE(GE_PLGMGR_SO_NOT_EXIST, "[Open][File] %s failed, reason:%s! ", canonicalPath.c_str(), error); return FAILED; } ccStatus_t cc_ret; @@ -1360,7 +1342,7 @@ Status KernelTaskInfo::CceUpdateKernelArgs(const domi::KernelContext &context, u if (cceUpdateKernelArgs == nullptr) { REPORT_INNER_ERROR("E19999", "No symbol:%s in %s, check invalid", update_kernel_args.c_str(), canonicalPath.c_str()); - GELOGE(FAILED, "Failed to invoke function ccUpdateKernelArgs"); + GELOGE(FAILED, "[Invoke][Function] ccUpdateKernelArgs failed."); if (mmDlclose(handle) != 0) { error = mmDlerror(); GE_IF_BOOL_EXEC(error == nullptr, error = ""); @@ -1384,9 +1366,8 @@ Status KernelTaskInfo::CceUpdateKernelArgs(const domi::KernelContext &context, u return FAILED; } if (cc_ret != CC_STATUS_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Call cceUpdateKernelArgs fail, ret:0x%X", - cc_ret); - GELOGE(CCE_FAILED, "Call cce api failed, ret: 0x%X", cc_ret); + REPORT_CALL_ERROR("E19999", "Call cceUpdateKernelArgs fail, ret:0x%X", cc_ret); + GELOGE(CCE_FAILED, "[Call][CceUpdateKernelArgs] failed, ret:0x%X", cc_ret); return CCE_FAILED; } @@ -1399,18 +1380,16 @@ Status KernelTaskInfo::SetFlowtable(std::string &flowtable, const domi::KernelDe if (context.is_flowtable()) { rtError_t rt_ret = rtMalloc(&flowtable_, flowtable.size(), RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", - flowtable.size(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", flowtable.size(), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%zu, ret:0x%X", flowtable.size(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } GE_PRINT_DYNAMIC_MEMORY(rtMalloc, "flowtable refresh of cce scence.", flowtable.size()) rt_ret = rtMemcpy(flowtable_, flowtable.size(), flowtable.data(), flowtable.size(), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", - flowtable.size(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", flowtable.size(), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", flowtable.size(), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -1424,7 +1403,8 @@ Status KernelTaskInfo::SetFlowtable(std::string &flowtable, const domi::KernelDe "kernelDef.args().size():%zu, check invalid", (uint32_t)((reinterpret_cast(const_cast(context.args_offset().data())))[0]), sizeof(uint64_t), kernel_def.args().size()); - GELOGE(FAILED, "(context.args_offset().data()))[0]:%u + sizeof(uint64_t):%zu > kernelDef.args().size():%zu", + GELOGE(FAILED, "[Check][Param] (context.args_offset().data()))[0]:%u + sizeof(uint64_t):%zu > " + "kernelDef.args().size():%zu", (uint32_t)((reinterpret_cast(const_cast(context.args_offset().data())))[0]), sizeof(uint64_t), kernel_def.args().size()); return FAILED; diff --git a/ge/graph/load/model_manager/task_info/label_goto_ex_task_info.cc b/ge/graph/load/model_manager/task_info/label_goto_ex_task_info.cc index b858259e..861056b3 100755 --- a/ge/graph/load/model_manager/task_info/label_goto_ex_task_info.cc +++ b/ge/graph/load/model_manager/task_info/label_goto_ex_task_info.cc @@ -40,7 +40,7 @@ Status LabelGotoExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da if (op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", label_goto.op_index()); - GELOGE(INTERNAL_ERROR, "Task op index:%u out of range!", label_goto.op_index()); + GELOGE(INTERNAL_ERROR, "[Get][Op] Task op index:%u out of range!", label_goto.op_index()); return INTERNAL_ERROR; } @@ -49,8 +49,8 @@ Status LabelGotoExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_LABEL_SWITCH_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "LabelGotoExTaskInfo: %s attr [%s] not exist.", - op_desc->GetName().c_str(), ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s in op:%s(%s) fail.", + ATTR_NAME_LABEL_SWITCH_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } @@ -63,7 +63,8 @@ Status LabelGotoExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMalloc failed for op:%s(%s), size:%lu, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), sizeof(uint64_t), rt_ret); - GELOGE(RT_FAILED, "Call rtMalloc failed, error: %#x", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed for op:%s(%s), size:%lu, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), sizeof(uint64_t), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -72,7 +73,8 @@ Status LabelGotoExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed for op:%s(%s), size:%lu, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), sizeof(uint64_t), rt_ret); - GELOGE(RT_FAILED, "Call rtMemcpy failed, error: %#x", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed for op:%s(%s), size:%lu, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), sizeof(uint64_t), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -86,15 +88,14 @@ Status LabelGotoExTaskInfo::Distribute() { GE_CHECK_NOTNULL(index_value_); if (args_size_ == 0) { REPORT_INNER_ERROR("E19999", "Param args_size_ is 0, check fail"); - GELOGE(PARAM_INVALID, "branch max: %u, args size: %u invalid.", kGotoBranchMax, args_size_); + GELOGE(PARAM_INVALID, "[Check][Param] branch max:%u, args size:%u invalid.", kGotoBranchMax, args_size_); return PARAM_INVALID; } rtError_t rt_ret = rtLabelSwitchByIndex(index_value_, kGotoBranchMax, args_, stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtLabelSwitchByIndex failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtLabelSwitchByIndex failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtLabelSwitchByIndex] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/label_set_task_info.cc b/ge/graph/load/model_manager/task_info/label_set_task_info.cc index c8cb7975..df88b3c8 100644 --- a/ge/graph/load/model_manager/task_info/label_set_task_info.cc +++ b/ge/graph/load/model_manager/task_info/label_set_task_info.cc @@ -32,9 +32,8 @@ Status LabelSetTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin const domi::LabelSetDef &label_set = task_def.label_set(); OpDescPtr op_desc = davinci_model->GetOpByIndex(label_set.op_index()); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - label_set.op_index()); - GELOGE(INTERNAL_ERROR, "Task op index:%u out of range!", label_set.op_index()); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", label_set.op_index()); + GELOGE(INTERNAL_ERROR, "[Get][Op] Task op index:%u out of range!", label_set.op_index()); return INTERNAL_ERROR; } @@ -43,7 +42,7 @@ Status LabelSetTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_LABEL_SWITCH_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSetTaskInfo: %s attr [%s] not exist.", + GELOGE(INTERNAL_ERROR, "[Get][Attr] LabelSetTaskInfo:%s attr [%s] not exist.", op_desc->GetName().c_str(), ATTR_NAME_LABEL_SWITCH_INDEX.c_str()); return INTERNAL_ERROR; } @@ -53,7 +52,8 @@ Status LabelSetTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin REPORT_INNER_ERROR("E19999", "lable_index:%u >= label_list.size():%zu in model, op:%s(%s), " "check invalid", label_index, label_list.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSetTaskInfo: Invalid label id:%u, label size:%zu", label_index, label_list.size()); + GELOGE(INTERNAL_ERROR, "[Check][Param] LabelSetTaskInfo: Invalid label id:%u, label size:%zu, op:%s(%s)", + label_index, label_list.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } label_ = label_list[label_index]; @@ -66,9 +66,8 @@ Status LabelSetTaskInfo::Distribute() { GELOGI("LabelSetTaskInfo Distribute Start."); rtError_t rt_ret = rtLabelSet(label_, stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtLabelSet failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtLabelSet failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtLabelSet] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/label_switch_by_index_task_info.cc b/ge/graph/load/model_manager/task_info/label_switch_by_index_task_info.cc index b7ffdb84..cd001092 100644 --- a/ge/graph/load/model_manager/task_info/label_switch_by_index_task_info.cc +++ b/ge/graph/load/model_manager/task_info/label_switch_by_index_task_info.cc @@ -39,9 +39,8 @@ Status LabelSwitchByIndexTaskInfo::Init(const domi::TaskDef &task_def, DavinciMo const domi::LabelSwitchByIndexDef &label_switch = task_def.label_switch_by_index(); OpDescPtr op_desc = davinci_model->GetOpByIndex(label_switch.op_index()); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - label_switch.op_index()); - GELOGE(INTERNAL_ERROR, "Task op index:%u out of range!", label_switch.op_index()); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", label_switch.op_index()); + GELOGE(INTERNAL_ERROR, "[Get][Op] Task op index:%u out of range!", label_switch.op_index()); return INTERNAL_ERROR; } @@ -52,7 +51,7 @@ Status LabelSwitchByIndexTaskInfo::Init(const domi::TaskDef &task_def, DavinciMo REPORT_INNER_ERROR("E19999", "input_data_addr size:%zu != kLabelSwitchIndexNum:%u, op:%s(%s), " "check invalid", input_data_addr.size(), kLabelSwitchIndexNum, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndexTaskInfo: %s invalid addr size: %zu, num: %u!", + GELOGE(INTERNAL_ERROR, "[Check][Param] %s invalid addr size:%zu, num:%u!", op_desc->GetName().c_str(), input_data_addr.size(), kLabelSwitchIndexNum); return INTERNAL_ERROR; } @@ -70,17 +69,16 @@ Status LabelSwitchByIndexTaskInfo::Init(const domi::TaskDef &task_def, DavinciMo REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_LABEL_SWITCH_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndexTaskInfo: %s Get attr %s failed.", op_desc->GetName().c_str(), - ATTR_NAME_LABEL_SWITCH_LIST.c_str()); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s in op:%s(%s) failed.", ATTR_NAME_LABEL_SWITCH_LIST.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } if (label_idx_list.empty() || label_idx_list.size() != branch_max_) { REPORT_INNER_ERROR("E19999", "label_idx_list in op:%s(%s) is empty, or size:%zu != branch_max_:%u" - "check invalid", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), + "check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), label_idx_list.size(), branch_max_); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndexTaskInfo: %s label index size: %zu, task branch max: %u.", + GELOGE(INTERNAL_ERROR, "[Check][Param] %s label index size:%zu, task branch max:%u.", op_desc->GetName().c_str(), label_idx_list.size(), branch_max_); return INTERNAL_ERROR; } @@ -93,7 +91,7 @@ Status LabelSwitchByIndexTaskInfo::Init(const domi::TaskDef &task_def, DavinciMo REPORT_INNER_ERROR("E19999", "label_id:%u in op:%s(%s) >= label_list.size():%zu in model" "check invalid", label_id, op_desc->GetName().c_str(), op_desc->GetType().c_str(), label_list.size()); - GELOGE(INTERNAL_ERROR, "LabelSwitchByIndexTaskInfo: %s index: %zu, label index: %u, model label size: %zu.", + GELOGE(INTERNAL_ERROR, "[Check][Param] %s index:%zu, label index:%u, model label size:%zu.", op_desc->GetName().c_str(), idx, label_id, label_list.size()); return INTERNAL_ERROR; } @@ -108,15 +106,15 @@ Status LabelSwitchByIndexTaskInfo::Init(const domi::TaskDef &task_def, DavinciMo if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMalloc failed for op:%s(%s), size:%u, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed for op:%s(%s), size:%u, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtLabelListCpy(label_used.data(), label_used.size(), args_, args_size_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtLabelListCpy failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtLabelListCpy failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtLabelListCpy] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -129,17 +127,15 @@ Status LabelSwitchByIndexTaskInfo::Distribute() { GE_CHECK_NOTNULL(args_); GE_CHECK_NOTNULL(index_value_); if (branch_max_ == 0 || args_size_ == 0) { - REPORT_INNER_ERROR("E19999", "branch_max_:%u or args_size_:%u is 0" - "check invalid", branch_max_, args_size_); - GELOGE(PARAM_INVALID, "branch max: %u, args size: %u invalid.", branch_max_, args_size_); + REPORT_INNER_ERROR("E19999", "branch_max_:%u or args_size_:%u is 0, check invalid", branch_max_, args_size_); + GELOGE(PARAM_INVALID, "[Check][Param] branch max:%u, args size:%u invalid.", branch_max_, args_size_); return PARAM_INVALID; } rtError_t rt_ret = rtLabelSwitchByIndex(index_value_, branch_max_, args_, stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtLabelSwitchByIndex failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtLabelSwitchByIndex failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtLabelSwitchByIndex] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -159,7 +155,8 @@ Status LabelSwitchByIndexTaskInfo::CalculateArgs(const domi::TaskDef &task_def, REPORT_INNER_ERROR("E19999", "input size:%zu in op:%s(%s) != kLabelSwitchIndexNum" "check invalid", op_desc->GetInputsSize(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Label switch op only have one data input. Now input size is %zu", op_desc->GetInputsSize()); + GELOGE(FAILED, "[Check][Param] Label switch op:%s(%s) only have one data input. Now input size is %zu", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_desc->GetInputsSize()); return FAILED; } string input_tensor_name = op_desc->GetName(); diff --git a/ge/graph/load/model_manager/task_info/memcpy_addr_async_task_info.cc b/ge/graph/load/model_manager/task_info/memcpy_addr_async_task_info.cc index 960862b4..56ddbaf5 100755 --- a/ge/graph/load/model_manager/task_info/memcpy_addr_async_task_info.cc +++ b/ge/graph/load/model_manager/task_info/memcpy_addr_async_task_info.cc @@ -36,9 +36,8 @@ Status MemcpyAddrAsyncTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel const auto &memcpy_async = task_def.memcpy_async(); OpDescPtr op_desc = davinci_model->GetOpByIndex(memcpy_async.op_index()); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - memcpy_async.op_index()); - GELOGE(INTERNAL_ERROR, "Task op index:%u out of range", memcpy_async.op_index()); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", memcpy_async.op_index()); + GELOGE(INTERNAL_ERROR, "[Get][Op] Task op index:%u out of range", memcpy_async.op_index()); return INTERNAL_ERROR; } @@ -66,7 +65,8 @@ Status MemcpyAddrAsyncTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel REPORT_CALL_ERROR("E19999", "Call rtMalloc failed for op:%s(%s), size:%lu, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size + kAlignBytes, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed for op:%s(%s), size:%lu, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size + kAlignBytes, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -78,7 +78,8 @@ Status MemcpyAddrAsyncTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed for op:%s(%s), size:%zu, ret:0x%X", op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size, rt_ret); - GELOGE(RT_FAILED, "Call rt api for src failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] for src failed for op:%s(%s), size:%zu, ret:0x%X", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -98,9 +99,8 @@ Status MemcpyAddrAsyncTaskInfo::Distribute() { rtError_t rt_ret = rtMemcpyAsync(reinterpret_cast(reinterpret_cast(args_align_) + sizeof(void *)), dst_max_, args_align_, count_, static_cast(kind_), stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpyAsync failed, size:%lu, ret:0x%X", - dst_max_, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpyAsync failed, size:%lu, ret:0x%X", dst_max_, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpyAsync] failed, size:%lu, ret:0x%X", dst_max_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/memcpy_async_task_info.cc b/ge/graph/load/model_manager/task_info/memcpy_async_task_info.cc index 0bc8fb8d..5c017544 100755 --- a/ge/graph/load/model_manager/task_info/memcpy_async_task_info.cc +++ b/ge/graph/load/model_manager/task_info/memcpy_async_task_info.cc @@ -36,9 +36,8 @@ Status MemcpyAsyncTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da dst_max_ = memcpy_async.dst_max(); OpDescPtr op_desc = davinci_model_->GetOpByIndex(memcpy_async.op_index()); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - memcpy_async.op_index()); - GELOGE(INTERNAL_ERROR, "Task op index:%u out of range", memcpy_async.op_index()); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", memcpy_async.op_index()); + GELOGE(INTERNAL_ERROR, "[Get][Op] Task op index:%u out of range", memcpy_async.op_index()); return INTERNAL_ERROR; } @@ -47,7 +46,7 @@ Status MemcpyAsyncTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da dst_ = reinterpret_cast(reinterpret_cast(src_) + sizeof(void *)); // for zero copy kind_ = RT_MEMCPY_ADDR_DEVICE_TO_DEVICE; - GE_CHK_STATUS_RET(SetIoAddrs(op_desc, memcpy_async), "Set addrs failed"); + GE_CHK_STATUS_RET(SetIoAddrs(op_desc, memcpy_async), "[Set][Addrs] failed, op:%s", op_desc->GetName().c_str()); GELOGI("MemcpyAsyncTaskInfo op name %s, src_ %p, dst_ %p, args_offset %u.", op_desc->GetName().c_str(), src_, dst_, args_offset_); return SUCCESS; @@ -77,7 +76,7 @@ Status MemcpyAsyncTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da davinci_model_->DisableZeroCopy(src_); davinci_model_->DisableZeroCopy(dst_); - GE_CHK_STATUS_RET(SetIoAddrs(op_desc, memcpy_async), "Set addrs failed"); + GE_CHK_STATUS_RET(SetIoAddrs(op_desc, memcpy_async), "[Set][Addrs] failed, op:%s", op_desc->GetName().c_str()); GELOGI("MemcpyAsyncTaskInfo Init Success, logic[0x%lx, 0x%lx], src:%p, dst:%p, max:%lu, count:%lu", memcpy_async.src(), memcpy_async.dst(), src_, dst_, dst_max_, count_); return SUCCESS; @@ -88,9 +87,8 @@ Status MemcpyAsyncTaskInfo::Distribute() { rtError_t rt_ret = rtMemcpyAsync(dst_, dst_max_, src_, count_, static_cast(kind_), stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpyAsync failed, size:%lu, ret:0x%X", - dst_max_, rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpyAsync failed, size:%lu, ret:0x%X", dst_max_, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpyAsync] failed, size:%lu, ret:0x%X", dst_max_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/model_exit_task_info.cc b/ge/graph/load/model_manager/task_info/model_exit_task_info.cc index f0e3dfb7..fddb142a 100644 --- a/ge/graph/load/model_manager/task_info/model_exit_task_info.cc +++ b/ge/graph/load/model_manager/task_info/model_exit_task_info.cc @@ -25,13 +25,13 @@ Status ModelExitTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davi GELOGI("InitModelExitTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } Status ret = SetStream(task_def.stream_id(), davinci_model->GetStreamList()); if (ret != SUCCESS) { - GELOGE(ret, "SetStream fail, stream_id:%u", task_def.stream_id()); + GELOGE(ret, "[Set][Stream] fail, stream_id:%u", task_def.stream_id()); return ret; } @@ -44,9 +44,8 @@ Status ModelExitTaskInfo::Distribute() { GELOGI("ModelExitTaskInfo Distribute Start."); rtError_t rt_ret = rtModelExit(model_, stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtModelExit failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rtModelExit failed, ret: 0x%x", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtModelExit failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtModelExit] failed, ret:0x%x", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } GELOGI("ModelExitTaskInfo Distribute Success."); diff --git a/ge/graph/load/model_manager/task_info/profiler_trace_task_info.cc b/ge/graph/load/model_manager/task_info/profiler_trace_task_info.cc index 4e829182..ce696978 100755 --- a/ge/graph/load/model_manager/task_info/profiler_trace_task_info.cc +++ b/ge/graph/load/model_manager/task_info/profiler_trace_task_info.cc @@ -24,7 +24,7 @@ Status ProfilerTraceTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel * GELOGI("ProfilerTraceTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } @@ -47,9 +47,9 @@ Status ProfilerTraceTaskInfo::Distribute() { rtError_t rt_ret = rtProfilerTrace(log_id_, notify_, flat_, stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtProfilerTrace failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtProfilerTrace failed, ret:0x%X, logid:%lu. notify:%d", + rt_ret, log_id_, notify_); + GELOGE(RT_FAILED, "[Call][RtProfilerTrace] failed, ret:0x%X, logid:%lu. notify:%d", rt_ret, log_id_, notify_); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/stream_active_task_info.cc b/ge/graph/load/model_manager/task_info/stream_active_task_info.cc index 4ab4951d..4767913c 100755 --- a/ge/graph/load/model_manager/task_info/stream_active_task_info.cc +++ b/ge/graph/load/model_manager/task_info/stream_active_task_info.cc @@ -27,7 +27,7 @@ Status StreamActiveTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *d GELOGI("StreamActiveTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } @@ -46,10 +46,10 @@ Status StreamActiveTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *d GE_CHECK_NOTNULL(op_desc); std::vector active_stream_index_list; if (!AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_index_list)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "StreamActiveOp get attr ACTIVE_STREAM fail, node name:%s.", op_desc->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s in op:%s(%s) fail", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } @@ -57,8 +57,8 @@ Status StreamActiveTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *d REPORT_INNER_ERROR("E19999", "flowctrl index:%u >= active_stream_list size:%zu in op:%s(%s), " "check invalid", internal_index, active_stream_index_list.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "InitStreamSwitchTaskInfo stream id index invalid. index:%u, list size:%zu.", internal_index, - active_stream_index_list.size()); + GELOGE(INTERNAL_ERROR, "[Check][Param] stream id index invalid. index:%u, list size:%zu, op:%s(%s).", + internal_index, active_stream_index_list.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } @@ -66,8 +66,9 @@ Status StreamActiveTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *d REPORT_INNER_ERROR("E19999", "active_stream_index:%u in op:%s(%s) >= stream size:%zu in model, " "check invalid", active_stream_index_list[internal_index], op_desc->GetName().c_str(), op_desc->GetType().c_str(), davinci_model->GetStreamList().size()); - GELOGE(INTERNAL_ERROR, "InitStreamSwitchTaskInfo stream index invalid. index:%u, stream list size:%zu.", - active_stream_index_list[internal_index], davinci_model->GetStreamList().size()); + GELOGE(INTERNAL_ERROR, "[Check][Param] active_stream_index:%u in op:%s(%s) >= stream size:%zu in model", + active_stream_index_list[internal_index], op_desc->GetName().c_str(), op_desc->GetType().c_str(), + davinci_model->GetStreamList().size()); return INTERNAL_ERROR; } @@ -83,9 +84,8 @@ Status StreamActiveTaskInfo::Distribute() { GELOGI("StreamActiveTaskInfo Distribute Start."); rtError_t rt_ret = rtStreamActive(active_stream_, stream_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamActive failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamActive failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtStreamActive] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } diff --git a/ge/graph/load/model_manager/task_info/stream_switch_task_info.cc b/ge/graph/load/model_manager/task_info/stream_switch_task_info.cc index 33dfacf7..dcab4d2f 100644 --- a/ge/graph/load/model_manager/task_info/stream_switch_task_info.cc +++ b/ge/graph/load/model_manager/task_info/stream_switch_task_info.cc @@ -32,7 +32,7 @@ Status StreamSwitchTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *d GELOGI("StreamSwitchTaskInfo Init Start."); if (davinci_model == nullptr) { REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr"); - GELOGE(PARAM_INVALID, "davinci_model is null!"); + GELOGE(PARAM_INVALID, "[Check][Param] davinci_model is null!"); return PARAM_INVALID; } @@ -50,10 +50,10 @@ Status StreamSwitchTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *d SetInputAndValuePtr(davinci_model, input_data_addr); uint32_t cond = 0; if (!AttrUtils::GetInt(op_desc, ATTR_NAME_STREAM_SWITCH_COND, cond)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - ATTR_NAME_STREAM_SWITCH_COND.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_STREAM_SWITCH_COND.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "StreamSwitchOp get attr STREAM_SWITCH_COND fail."); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s in op:%s(%s) fail", + ATTR_NAME_STREAM_SWITCH_COND.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } cond_ = static_cast(cond); @@ -63,17 +63,18 @@ Status StreamSwitchTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *d REPORT_INNER_ERROR("E19999", "input_data_addr.size():%zu or input size:%zu != STREAM_SWITCH_INPUT_NUM:%u " "in op:%s(%s), check invalid", input_data_addr.size(), input_size, STREAM_SWITCH_INPUT_NUM, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Input num should be %u. inputAddr size:%zu, inputDesc size:%zu.", - STREAM_SWITCH_INPUT_NUM, input_data_addr.size(), input_size); + GELOGE(INTERNAL_ERROR, "[Check][Param] Input num should be %u. inputAddr size:%zu, inputDesc size:%zu, op:%s(%s).", + STREAM_SWITCH_INPUT_NUM, input_data_addr.size(), input_size, + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } vector active_stream_list; if (!AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_stream_list)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "StreamSwitchOp get attr ACTIVE_STREAM_LIST fail."); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s in op:%s(%s) fail", + ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } @@ -81,17 +82,19 @@ Status StreamSwitchTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *d REPORT_INNER_ERROR("E19999", "active_stream_list.size():%zu in op:%s(%s) != kTrueBranchStreamNum:%u, " "check invalid", active_stream_list.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), kTrueBranchStreamNum); - GELOGE(FAILED, "Stream num of switch true branch must be %u.", kTrueBranchStreamNum); + GELOGE(FAILED, "[Check][Param] active_stream_list.size():%zu in op:%s(%s) must be equal %u", + active_stream_list.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), kTrueBranchStreamNum); return FAILED; } size_t true_stream_index = active_stream_list.front(); if (true_stream_index >= davinci_model->GetStreamList().size()) { REPORT_INNER_ERROR("E19999", "active_stream_index:%zu in op:%s(%s) >= stream list size:%zu in model," - "check invalid", true_stream_index, - op_desc->GetName().c_str(), op_desc->GetType().c_str(), davinci_model->GetStreamList().size()); - GELOGE(INTERNAL_ERROR, "InitStreamSwitchTaskInfo stream index invalid. index:%zu, stream list size:%zu.", - true_stream_index, davinci_model->GetStreamList().size()); + "check invalid", true_stream_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), + davinci_model->GetStreamList().size()); + GELOGE(INTERNAL_ERROR, "[Check][Param] active_stream_index:%zu in op:%s(%s) >= stream list size:%zu in model", + true_stream_index, op_desc->GetName().c_str(), op_desc->GetType().c_str(), + davinci_model->GetStreamList().size()); return INTERNAL_ERROR; } @@ -103,10 +106,10 @@ Status StreamSwitchTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *d if (op_desc->HasAttr(ATTR_NAME_SWITCH_DATA_TYPE)) { int64_t data_type = 0; if (!AttrUtils::GetInt(op_desc, ATTR_NAME_SWITCH_DATA_TYPE, data_type)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - ATTR_NAME_SWITCH_DATA_TYPE.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_SWITCH_DATA_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "StreamSwitchOp[node:%s] get attr SWITCH_DATA_TYPE fail.", op_desc->GetName().c_str()); + GELOGE(FAILED, "[Get][Attr] %s in op:%s(%s) fail", + ATTR_NAME_SWITCH_DATA_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } data_type_ = static_cast(data_type); @@ -122,9 +125,8 @@ Status StreamSwitchTaskInfo::Distribute() { GELOGI("StreamSwitchTaskInfo Distribute Start."); rtError_t rt_ret = rtStreamSwitchEx(input_ptr_, cond_, value_ptr_, true_stream_, stream_, data_type_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamSwitchEx fail, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamSwitchEx fail, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtStreamSwitchEx] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -143,7 +145,8 @@ Status StreamSwitchTaskInfo::CalculateArgs(const domi::TaskDef &task_def, Davinc REPORT_INNER_ERROR("E19999", "input size:%zu in op:%s(%s) != STREAM_SWITCH_INPUT_NUM:%u," "check invalid", op_desc->GetInputsSize(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), STREAM_SWITCH_INPUT_NUM); - GELOGE(FAILED, "Stream switch op only have one data input. Now input size is %zu", op_desc->GetInputsSize()); + GELOGE(FAILED, "[Check][Param] Stream switch op:%s only have one data input. Now input size is %zu", + op_desc->GetName().c_str(), op_desc->GetInputsSize()); return FAILED; } for (uint32_t i = 0; i < STREAM_SWITCH_INPUT_NUM; ++i) { diff --git a/ge/graph/load/model_manager/task_info/stream_switchn_task_info.cc b/ge/graph/load/model_manager/task_info/stream_switchn_task_info.cc index 40bbff02..0daf4626 100755 --- a/ge/graph/load/model_manager/task_info/stream_switchn_task_info.cc +++ b/ge/graph/load/model_manager/task_info/stream_switchn_task_info.cc @@ -36,9 +36,8 @@ Status StreamSwitchNTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel * auto stream_switchn_def = task_def.stream_switch_n(); OpDescPtr op_desc = davinci_model->GetOpByIndex(stream_switchn_def.op_index()); if (op_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", - stream_switchn_def.op_index()); - GELOGE(FAILED, "Index is out of range, index: %u", stream_switchn_def.op_index()); + REPORT_INNER_ERROR("E19999", "Can't get op_desc from davinci_model by index:%u", stream_switchn_def.op_index()); + GELOGE(FAILED, "[Get][Op] failed, as Index is out of range, index:%u", stream_switchn_def.op_index()); return FAILED; } @@ -51,7 +50,8 @@ Status StreamSwitchNTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel * REPORT_INNER_ERROR("E19999", "task_Def.stream_switch_n.target_value:%d in op:%s(%s) is 0," "check invalid", value.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "The number of gears in dynamic batch scenario can not be 0."); + GELOGE(FAILED, "[Check][Param] The number of gears in dynamic batch scenario can not be 0, op:%s.", + op_desc->GetName().c_str()); return FAILED; } for (int i = 0; i < value.size(); ++i) { @@ -62,15 +62,15 @@ Status StreamSwitchNTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel * // set element_size_ if (!AttrUtils::GetInt(op_desc, ATTR_NAME_BATCH_NUM, element_size_)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - ATTR_NAME_BATCH_NUM.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_BATCH_NUM.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Get ATTR_NAME_BATCH_NUM of switchN op failed."); + GELOGE(FAILED, "[Get][Attr] %s in op:%s(%s) fail", + ATTR_NAME_BATCH_NUM.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } if (GetTrueStreamPtr(op_desc, davinci_model) != SUCCESS) { - GELOGE(FAILED, "Get true stream ptr of switchN op failed."); + GELOGE(FAILED, "[Get][TrueStreamPtr] of switchN op:%s failed.", op_desc->GetName().c_str()); return FAILED; } @@ -92,9 +92,8 @@ Status StreamSwitchNTaskInfo::Distribute() { rtError_t rt_ret = rtStreamSwitchN(input_ptr_, input_size_, value_ptr_, true_stream_ptr_, element_size_, stream_, data_type_); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamSwitchN failed, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamSwitchN failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtStreamSwitchN] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -106,10 +105,10 @@ Status StreamSwitchNTaskInfo::Distribute() { Status StreamSwitchNTaskInfo::GetTrueStreamPtr(const OpDescPtr &op_desc, DavinciModel *davinci_model) { vector true_stream_id_list; if (!AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, true_stream_id_list)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "StreamSwitchNOp get attr ACTIVE_STREAM_LIST fail."); + GELOGE(FAILED, "[Get][Attr] %s in op:%s(%s) fail", + ATTR_NAME_ACTIVE_STREAM_LIST.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } @@ -118,9 +117,8 @@ Status StreamSwitchNTaskInfo::GetTrueStreamPtr(const OpDescPtr &op_desc, Davinci "check invalid", true_stream_id_list.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), davinci_model->GetStreamList().size()); GELOGE(FAILED, - "InitStreamSwitchNTaskInfo get true stream id list failed. true stream size:%zu, " - "stream list size:%zu.", - true_stream_id_list.size(), davinci_model->GetStreamList().size()); + "[Check][Param] InitStreamSwitchNTaskInfo get true stream id list failed. true stream size:%zu, " + "stream list size:%zu.", true_stream_id_list.size(), davinci_model->GetStreamList().size()); return FAILED; } @@ -131,8 +129,8 @@ Status StreamSwitchNTaskInfo::GetTrueStreamPtr(const OpDescPtr &op_desc, Davinci REPORT_INNER_ERROR("E19999", "active_stream_id:%u in op:%s(%s) >= stream list size:%zu in model," "check invalid", true_stream_id, op_desc->GetName().c_str(), op_desc->GetType().c_str(), davinci_model->GetStreamList().size()); - GELOGE(FAILED, "InitStreamSwitchNTaskInfo stream id invalid. id:%u, stream list size:%zu.", true_stream_id, - davinci_model->GetStreamList().size()); + GELOGE(FAILED, " [Check][Param] stream id:%u in op:%s invalid, stream list size:%zu.", + true_stream_id, op_desc->GetName().c_str(), davinci_model->GetStreamList().size()); return FAILED; } rtStream_t true_stream = davinci_model->GetStreamList()[true_stream_id]; @@ -144,7 +142,7 @@ Status StreamSwitchNTaskInfo::GetTrueStreamPtr(const OpDescPtr &op_desc, Davinci REPORT_INNER_ERROR("E19999", "active_stream_list.size():%zu in op:%s(%s) is empty, " "check invalid", true_stream_id_list.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "true stream list is null."); + GELOGE(FAILED, "[Check][Param] true stream list is null, op:%s.", op_desc->GetName().c_str()); return FAILED; } true_stream_ptr_ = &true_stream_list_[0]; @@ -160,10 +158,11 @@ Status StreamSwitchNTaskInfo::CalculateArgs(const domi::TaskDef &task_def, Davin GE_CHECK_NOTNULL(op_desc); GELOGI("Calc opType[%s] args size. Node name is [%s]", op_desc->GetType().c_str(), op_desc->GetName().c_str()); if (op_desc->GetInputsSize() != kStreamSwitchnInputNum) { - REPORT_INNER_ERROR("E19999", "input size:%zu in op:%s(%s) != kStreamSwitchnInputNum:%u ," + REPORT_INNER_ERROR("E19999", "input size:%zu in op:%s(%s) != kStreamSwitchnInputNum:%u, " "check invalid", op_desc->GetInputsSize(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), kStreamSwitchnInputNum); - GELOGE(FAILED, "Stream switchn op only have one data input. Now input size is %zu", op_desc->GetInputsSize()); + GELOGE(FAILED, "[Check][Param] Stream switchn op:%s only have one data input. Now input size is %zu", + op_desc->GetName().c_str(), op_desc->GetInputsSize()); return FAILED; } string input_tensor_name = op_desc->GetInputNameByIndex(0); @@ -187,7 +186,8 @@ Status StreamSwitchNTaskInfo::InputPtrUpdate(const OpDescPtr &op_desc, DavinciMo REPORT_INNER_ERROR("E19999", "input_offset size:%zu or input_length.size:%zu in op:%s(%s) is empty," "check invalid", input_offset.size(), input_legnth.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "input offset size %zu, input legnth size: %zu", input_offset.size(), input_legnth.size()); + GELOGE(FAILED, "[Check][Param] op:%s input offset size %zu, input legnth size:%zu", + op_desc->GetName().c_str(), input_offset.size(), input_legnth.size()); return FAILED; } const RuntimeParam &rts_param = davinci_model->GetRuntimeParam(); @@ -201,7 +201,7 @@ Status StreamSwitchNTaskInfo::InputPtrUpdate(const OpDescPtr &op_desc, DavinciMo REPORT_INNER_ERROR("E19999", "input_data_addr size:%zu in op:%s(%s) is empty," "check invalid", input_data_addr.size(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "input data addr is empty"); + GELOGE(FAILED, "[Check][Param] input data addr is empty in op:%s", op_desc->GetName().c_str()); return FAILED; } input_ptr_ = input_data_addr[0]; diff --git a/ge/graph/load/model_manager/task_info/super_kernel/super_kernel.cc b/ge/graph/load/model_manager/task_info/super_kernel/super_kernel.cc index 66bf5ab7..44aac465 100644 --- a/ge/graph/load/model_manager/task_info/super_kernel/super_kernel.cc +++ b/ge/graph/load/model_manager/task_info/super_kernel/super_kernel.cc @@ -27,23 +27,21 @@ Status SuperKernel::Launch(rtStream_t stream, uint32_t dump_flag) { rtError_t rt_ret = rtMalloc(reinterpret_cast(&device_args_addr_), sizeof(args), RT_MEMORY_HBM); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%lu, ret:0x%X", - sizeof(args), rt_ret); - GELOGE(RT_FAILED, "rtMalloc failied. error: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%lu, ret:0x%X", sizeof(args), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failied, size:%lu, ret:0x%X", sizeof(args), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(reinterpret_cast(device_args_addr_), sizeof(args), reinterpret_cast(args), sizeof(args), RT_MEMCPY_HOST_TO_DEVICE); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%lu, ret:0x%X", - sizeof(args), rt_ret); - GELOGE(RT_FAILED, "rtMemcpy failied. error: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%lu, ret:0x%X", sizeof(args), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failied, size:%lu, ret:0x%X", sizeof(args), rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtKernelLaunchWithFlag((void *const)func_stub_, block_dim_, device_args_addr_, sizeof(args), NULL, stream, dump_flag); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchWithFlag failed, dump_flag:%u, ret:0x%X", dump_flag, rt_ret); - GELOGE(RT_FAILED, "rtKernelLaunchWithFlag failied. error: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtKernelLaunchWithFlag] failied. error: 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) return SUCCESS; } diff --git a/ge/graph/load/model_manager/task_info/super_kernel/super_kernel_factory.cc b/ge/graph/load/model_manager/task_info/super_kernel/super_kernel_factory.cc index 2a3e3a17..07dc5d19 100644 --- a/ge/graph/load/model_manager/task_info/super_kernel/super_kernel_factory.cc +++ b/ge/graph/load/model_manager/task_info/super_kernel/super_kernel_factory.cc @@ -33,20 +33,20 @@ Status SuperKernelFactory::Init() { if (handle_ == nullptr) { const char* error = mmDlerror(); GE_IF_BOOL_EXEC(error == nullptr, error = ""); - GELOGE(FAILED, "SKT: open skt lib failed, please check LD_LIBRARY_PATH. errmsg:%s", error); + GELOGE(FAILED, "[Open][SktLib] failed, please check LD_LIBRARY_PATH. errmsg:%s", error); } rtError_t rt_ret; rt_ret = rtGetFunctionByName(this->sk_stub_name_.c_str(), &this->func_stub_); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtGetFunctionByName failed, stub_func:%s, ret:0x%X", this->sk_stub_name_.c_str(), rt_ret); - GELOGE(RT_FAILED, "rtGetFunctionByName failed. stub_func: %s, please export LD_LIBRARY_PATH for " - "libcce_aicore.so", this->sk_stub_name_.c_str()); + GELOGE(RT_FAILED, "[Call][RtGetFunctionByName] failed. stub_func:%s, " + "please export LD_LIBRARY_PATH for libcce_aicore.so", this->sk_stub_name_.c_str()); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtGetAddrByFun(this->func_stub_, &this->func_ptr_); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtGetAddrByFun failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "rtGetAddrByFun failed. error: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtGetAddrByFun] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) GELOGD( "SKT: fuseKernels super_kernel_template subFunc %p, device func " @@ -104,7 +104,7 @@ Status SuperKernelFactory::FuseKernels(const std::vector &stub_func_list rt_ret = rtGetAddrByFun(stub_func_list[i], &sub_device_func); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, REPORT_CALL_ERROR("E19999", "Call rtGetAddrByFun failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "rtGetAddrByFun failed. error: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtGetAddrByFun] failed, ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) GELOGD("SKT: fuseKernels subFunc %p, device func address %p", stub_func_list[i], sub_device_func); // store two uint64_t address @@ -116,16 +116,14 @@ Status SuperKernelFactory::FuseKernels(const std::vector &stub_func_list } rt_ret = rtMalloc(reinterpret_cast(&hbm_nav_table_addr), nav_table_size, RT_MEMORY_HBM); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%lu, ret:0x%X", - nav_table_size, rt_ret); - GELOGE(RT_FAILED, "rtMalloc failed. error: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%lu, ret:0x%X", nav_table_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%lu, ret:0x%X", nav_table_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(reinterpret_cast(hbm_nav_table_addr), nav_table_size, reinterpret_cast(nav_table.get()), nav_table_size, RT_MEMCPY_HOST_TO_DEVICE); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%lu, ret:0x%X", - nav_table_size, rt_ret); - GELOGE(RT_FAILED, "rtMemcpy failed. error: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%lu, ret:0x%X", nav_table_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%lu, ret:0x%X", nav_table_size, rt_ret); GE_CHK_RT(rtFree(hbm_nav_table_addr)); return RT_ERROR_TO_GE_STATUS(rt_ret);) // Create the necessary metadata for the super kernel h = diff --git a/ge/graph/load/model_manager/task_info/task_info.cc b/ge/graph/load/model_manager/task_info/task_info.cc index fb446bf7..fad18d37 100755 --- a/ge/graph/load/model_manager/task_info/task_info.cc +++ b/ge/graph/load/model_manager/task_info/task_info.cc @@ -27,7 +27,7 @@ Status TaskInfo::SetStream(uint32_t stream_id, const std::vector &st } else { REPORT_INNER_ERROR("E19999", "stream_id:%u >= stream_list.size(): %zu, check invalid", stream_id, stream_list.size()); - GELOGE(FAILED, "index: %u >= stream_list.size(): %zu.", stream_id, stream_list.size()); + GELOGE(FAILED, "[Check][Param] index:%u >= stream_list.size():%zu.", stream_id, stream_list.size()); return FAILED; }