diff --git a/ge/graph/load/model_manager/davinci_model.cc b/ge/graph/load/model_manager/davinci_model.cc index 2430ae3d..0a92447b 100755 --- a/ge/graph/load/model_manager/davinci_model.cc +++ b/ge/graph/load/model_manager/davinci_model.cc @@ -4021,14 +4021,18 @@ Status DavinciModel::GetComputeGraphInfo(vector &graph_des } else { compute_graph_info.model_name = name_; } + + std::vector format = { FORMAT_NULL }; + std::vector> shape = { {0} }; + std::vector data_type = { DT_UNDEFINED }; compute_graph_info.op_name = op_desc.op_name; compute_graph_info.op_type = op_desc.op_type; - compute_graph_info.input_format = op_desc.input_format; - compute_graph_info.input_shape = op_desc.input_shape; - compute_graph_info.input_data_type = op_desc.input_data_type; - compute_graph_info.output_format = op_desc.output_format; - compute_graph_info.output_shape = op_desc.output_shape; - compute_graph_info.output_data_type = op_desc.output_data_type; + compute_graph_info.input_format = op_desc.input_format.empty() ? format : op_desc.input_format; + compute_graph_info.input_shape = op_desc.input_shape.empty() ? shape : op_desc.input_shape; + compute_graph_info.input_data_type = op_desc.input_data_type.empty() ? data_type : op_desc.input_data_type; + compute_graph_info.output_format = op_desc.output_format.empty() ? format : op_desc.output_format; + compute_graph_info.output_shape = op_desc.output_shape.empty() ? shape : op_desc.output_shape; + compute_graph_info.output_data_type = op_desc.output_data_type.empty() ? data_type : op_desc.output_data_type; uint32_t task_id = 0; uint32_t stream_id = 0; auto iter = profiler_report_op_info_.find(op_desc.op_name); diff --git a/ge/hybrid/executor/worker/execution_engine.cc b/ge/hybrid/executor/worker/execution_engine.cc index 5e9d3607..44f7d87f 100755 --- a/ge/hybrid/executor/worker/execution_engine.cc +++ b/ge/hybrid/executor/worker/execution_engine.cc @@ -171,43 +171,9 @@ Status NodeDoneCallback::GetGraphDescInfo(const NodePtr node, const HybridModel GE_CHECK_NOTNULL(model); GELOGD("GetComputeGraphInfo of node [%s] start.", node->GetName().c_str()); + compute_graph_info = context_->GetProfilingGraphDescInfo(); + context_->ClearProfilingGraphDescInfo(); - std::string dynamic_model_name = model->GetModelName(); - auto op_desc = node->GetOpDesc(); - if (op_desc == nullptr) { - GELOGE(PARAM_INVALID, "op_desc is nullptr."); - return PARAM_INVALID; - } - - auto op_mode = static_cast(domi::ImplyType::INVALID); - if (AttrUtils::GetInt(op_desc, ATTR_NAME_IMPLY_TYPE, op_mode) && - op_mode == static_cast(domi::ImplyType::TVM)) { - ComputeGraphDescInfo tmp_compute_graph_info; - tmp_compute_graph_info.model_name = dynamic_model_name; - tmp_compute_graph_info.op_name = op_desc->GetName(); - tmp_compute_graph_info.op_type = op_desc->GetType(); - - for (size_t i = 0; i < op_desc->GetAllInputsSize(); ++i) { - GeTensorDescPtr input_desc = op_desc->MutableInputDesc(i); - if (input_desc == nullptr) { - continue; - } - tmp_compute_graph_info.input_format.emplace_back(input_desc->GetFormat()); - tmp_compute_graph_info.input_shape.emplace_back(input_desc->GetShape().GetDims()); - tmp_compute_graph_info.input_data_type.emplace_back(input_desc->GetDataType()); - } - - for (size_t j = 0; j < op_desc->GetOutputsSize(); ++j) { - GeTensorDesc output_desc = op_desc->GetOutputDesc(j); - tmp_compute_graph_info.output_format.emplace_back(output_desc.GetFormat()); - tmp_compute_graph_info.output_shape.emplace_back(output_desc.GetShape().GetDims()); - tmp_compute_graph_info.output_data_type.emplace_back(output_desc.GetDataType()); - } - tmp_compute_graph_info.task_id = context_->GetTaskId(); - tmp_compute_graph_info.stream_id = context_->GetStreamId(); - compute_graph_info.emplace_back(tmp_compute_graph_info); - GELOGD("GetComputeGraphInfo of node [%s] end.", node->GetName().c_str()); - } return SUCCESS; } diff --git a/ge/hybrid/node_executor/aicore/aicore_node_executor.cc b/ge/hybrid/node_executor/aicore/aicore_node_executor.cc index a8736154..cb5a7d4c 100755 --- a/ge/hybrid/node_executor/aicore/aicore_node_executor.cc +++ b/ge/hybrid/node_executor/aicore/aicore_node_executor.cc @@ -183,7 +183,16 @@ Status AiCoreNodeTask::ExecuteAsync(TaskContext &context, std::function RECORD_EXECUTION_EVENT(context.GetExecutionContext(), context.GetNodeName(), "[AiCoreNodeLaunchKernel] Start"); GE_CHK_STATUS_RET_NOLOG((*it)->LaunchKernel(context.GetStream())); // save profiling data - (void)context.SaveProfilingTaskDescInfo(kTaskTypeAicore, (*it)->GetBlockDim()); + uint32_t task_id = 0; + uint32_t stream_id = 0; + rtError_t rt_ret = rtGetTaskIdAndStreamID(&task_id, &stream_id); // must be called after Launch kernel + if (rt_ret != RT_ERROR_NONE) { + GELOGE(rt_ret, "Get task_id and stream_id failed."); + return FAILED; + } + GELOGD("Aicore node[%s] task_id: %u, stream_id: %u.", context.GetNodeName(), task_id, stream_id); + (void)context.SaveProfilingTaskDescInfo(task_id, stream_id, kTaskTypeAicore, (*it)->GetBlockDim()); + (void)context.SaveProfilingGraphDescInfo(task_id, stream_id); RECORD_EXECUTION_EVENT(context.GetExecutionContext(), context.GetNodeName(), "[AiCoreNodeLaunchKernel] End"); RECORD_EXECUTION_EVENT(context.GetExecutionContext(), context.GetNodeName(), "[AiCoreNodeLaunchKernel] End"); } diff --git a/ge/hybrid/node_executor/aicpu/aicpu_node_executor.cc b/ge/hybrid/node_executor/aicpu/aicpu_node_executor.cc index 109939d9..a2e610b4 100755 --- a/ge/hybrid/node_executor/aicpu/aicpu_node_executor.cc +++ b/ge/hybrid/node_executor/aicpu/aicpu_node_executor.cc @@ -191,8 +191,16 @@ Status AicpuNodeTaskBase::ExecuteAsync(TaskContext &context, std::functionSynchronize(GetStream()); } -Status TaskContext::SaveProfilingTaskDescInfo(uint32_t task_type, uint32_t block_dim) { +Status TaskContext::SaveProfilingTaskDescInfo(uint32_t task_id, uint32_t stream_id, + uint32_t task_type, uint32_t block_dim) { if (ProfilingManager::Instance().ProfilingModelExecuteOn()) { const NodeItem &node_item = GetNodeItem(); auto op_desc = node_item.GetOpDesc(); GE_CHECK_NOTNULL(op_desc); - - uint32_t task_id = 0; - uint32_t stream_id = 0; - rtError_t rt_ret = rtGetTaskIdAndStreamID(&task_id, &stream_id); // must be called after Launch kernel - if (rt_ret != RT_ERROR_NONE) { - GELOGE(rt_ret, "Get task_id and stream_id failed."); - return rt_ret; - } - GELOGD("Node[%s] task_id: %u, stream_id: %u.", GetNodeName(), task_id, stream_id); - const GraphExecutionContext * graph_context = GetExecutionContext(); GE_CHECK_NOTNULL(graph_context); const HybridModel *model = graph_context->model; @@ -544,5 +535,59 @@ Status TaskContext::SaveProfilingTaskDescInfo(uint32_t task_type, uint32_t block return SUCCESS; } + +Status TaskContext::SaveProfilingGraphDescInfo(uint32_t task_id, uint32_t stream_id) { + if (ProfilingManager::Instance().ProfilingModelExecuteOn()) { + const NodeItem &node_item = GetNodeItem(); + auto op_desc = node_item.GetOpDesc(); + GE_CHECK_NOTNULL(op_desc); + const GraphExecutionContext * graph_context = GetExecutionContext(); + GE_CHECK_NOTNULL(graph_context); + const HybridModel *model = graph_context->model; + GE_CHECK_NOTNULL(model); + + std::string dynamic_model_name = model->GetModelName(); + auto op_mode = static_cast(domi::ImplyType::INVALID); + if (AttrUtils::GetInt(op_desc, ATTR_NAME_IMPLY_TYPE, op_mode) && + op_mode == static_cast(domi::ImplyType::TVM)) { + ComputeGraphDescInfo tmp_compute_graph_info; + tmp_compute_graph_info.model_name = dynamic_model_name; + tmp_compute_graph_info.op_name = op_desc->GetName(); + tmp_compute_graph_info.op_type = op_desc->GetType(); + // default + if (op_desc->GetAllInputsSize() == 0) { + tmp_compute_graph_info.input_format = { FORMAT_NULL }; + tmp_compute_graph_info.input_shape = { {0} }; + tmp_compute_graph_info.input_data_type = { DT_UNDEFINED }; + } + for (size_t i = 0; i < op_desc->GetAllInputsSize(); ++i) { + GeTensorDescPtr input_desc = op_desc->MutableInputDesc(i); + if (input_desc == nullptr) { + continue; + } + tmp_compute_graph_info.input_format.emplace_back(input_desc->GetFormat()); + tmp_compute_graph_info.input_shape.emplace_back(input_desc->GetShape().GetDims()); + tmp_compute_graph_info.input_data_type.emplace_back(input_desc->GetDataType()); + } + + if (op_desc->GetOutputsSize() == 0) { + tmp_compute_graph_info.output_format = { FORMAT_NULL }; + tmp_compute_graph_info.output_shape = { {0} }; + tmp_compute_graph_info.output_data_type = { DT_UNDEFINED }; + } + for (size_t j = 0; j < op_desc->GetOutputsSize(); ++j) { + GeTensorDesc output_desc = op_desc->GetOutputDesc(j); + tmp_compute_graph_info.output_format.emplace_back(output_desc.GetFormat()); + tmp_compute_graph_info.output_shape.emplace_back(output_desc.GetShape().GetDims()); + tmp_compute_graph_info.output_data_type.emplace_back(output_desc.GetDataType()); + } + tmp_compute_graph_info.task_id = task_id; + tmp_compute_graph_info.stream_id = stream_id; + compute_graph_info.emplace_back(tmp_compute_graph_info); + } + } + return SUCCESS; +} + } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/node_executor/task_context.h b/ge/hybrid/node_executor/task_context.h index 9a668f8c..e7ee4fc8 100644 --- a/ge/hybrid/node_executor/task_context.h +++ b/ge/hybrid/node_executor/task_context.h @@ -110,9 +110,13 @@ class TaskContext { void *handle_ = nullptr; const std::vector& GetProfilingTaskDescInfo() const { return task_desc_info; } - Status SaveProfilingTaskDescInfo(uint32_t task_type, uint32_t block_dim); + Status SaveProfilingTaskDescInfo(uint32_t task_id, uint32_t stream_id, uint32_t task_type, uint32_t block_dim); void ClearProfilingTaskDescInfo() { task_desc_info.clear(); } + const std::vector& GetProfilingGraphDescInfo() const { return compute_graph_info; } + Status SaveProfilingGraphDescInfo(uint32_t task_id, uint32_t stream_id); + void ClearProfilingGraphDescInfo() { compute_graph_info.clear(); } + private: TaskContext(GraphExecutionContext *execution_context, const NodeItem *node_item, @@ -133,6 +137,7 @@ class TaskContext { uint32_t task_id_ = 0; uint32_t stream_id_ = 0; std::vector task_desc_info; + std::vector compute_graph_info; }; } // namespace hybrid } // namespace ge