From 3fc37e5a2a97e35daef6be01ce7e953addfe267f Mon Sep 17 00:00:00 2001 From: wangxiaotian22 Date: Fri, 5 Mar 2021 15:37:30 +0800 Subject: [PATCH] add stage --- ge/client/ge_api.cc | 23 +++++++++++- ge/generator/ge_generator.cc | 8 ++++ ge/graph/build/graph_builder.cc | 6 +++ ge/graph/build/model_builder.cc | 5 +++ ge/graph/load/model_manager/davinci_model.cc | 39 ++++++++++---------- ge/graph/manager/graph_manager.cc | 37 ++++++++++++++----- ge/graph/passes/flow_ctrl_pass.cc | 2 +- ge/init/gelib.cc | 7 ++++ ge/ir_build/ge_ir_build.cc | 11 ++++++ ge/offline/main.cc | 10 +++++ metadef | 2 +- 11 files changed, 119 insertions(+), 31 deletions(-) diff --git a/ge/client/ge_api.cc b/ge/client/ge_api.cc index f1760062..f0cf9e03 100644 --- a/ge/client/ge_api.cc +++ b/ge/client/ge_api.cc @@ -79,6 +79,7 @@ Status CheckOptionsValid(const std::map &options) { // Initialize GE, prepare for execution, call GELib::Initialize Status GEInitializeImpl(const std::map &options) { + ErrorManager::GetInstance().GenWorkStreamIdDefault(); GELOGT(TRACE_INIT, "GEInitialize start"); std::string path_base = ge::GELib::GetPath(); auto ret = ErrorManager::GetInstance().Init(path_base); @@ -87,12 +88,12 @@ Status GEInitializeImpl(const std::map &options) { return ret; } - ErrorManager::GetInstance().GenWorkStreamIdDefault(); // 0.check init status if (g_ge_initialized) { GELOGW("GEInitialize is called more than once"); return SUCCESS; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOpsProtoInit); // Load OpsProto lib plugin std::string opsproto_path; GetOpsProtoPath(opsproto_path); @@ -107,6 +108,7 @@ Status GEInitializeImpl(const std::map &options) { return FAILED; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); // check options is valid GE_TIMESTAMP_START(CheckOptionsValid); if (CheckOptionsValid(options) != SUCCESS) { @@ -114,11 +116,13 @@ Status GEInitializeImpl(const std::map &options) { } GE_TIMESTAMP_END(CheckOptionsValid, "GEInitialize::CheckOptionsValid"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOpsProtoInit); GE_TIMESTAMP_START(InitPreparation); TBEPluginManager::Instance().InitPreparation(options); GE_TIMESTAMP_END(InitPreparation, "GEInitialize::InitPreparation"); // call Initialize GELOGT(TRACE_RUNNING, "Initializing environment"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); GE_TIMESTAMP_START(GELibInitialize); ret = ge::GELib::Initialize(options); GE_TIMESTAMP_END(GELibInitialize, "GEInitialize::GELibInitialize"); @@ -139,6 +143,7 @@ Status GEInitializeImpl(const std::map &options) { // Initialize GE, prepare for execution, call GELib::Initialize Status GEInitialize(const std::map &options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); if (DlogReportInitialize() != SUCCESS) { GELOGW("Dlog report device log initialize failed."); } @@ -146,6 +151,7 @@ Status GEInitialize(const std::map &options) { } Status GEInitialize(const std::map &options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); std::map str_options; for (auto &option : options) { if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) { @@ -165,6 +171,7 @@ Status GEInitialize(const std::map &options) { // GE finalize, releasing all resources Status GEFinalize() { + ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize); GELOGT(TRACE_INIT, "GEFinalize start"); ErrorManager::GetInstance().GenWorkStreamIdDefault(); @@ -223,6 +230,7 @@ std::string GEGetWarningMsg() { // Initialize session，which calls innerSession Session::Session(const std::map &options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); GELOGT(TRACE_INIT, "Session Constructor start"); ErrorManager::GetInstance().GenWorkStreamIdDefault(); @@ -255,6 +263,7 @@ Session::Session(const std::map &options) { } Session::Session(const std::map &options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); GELOGT(TRACE_INIT, "Session Constructor start"); ErrorManager::GetInstance().GenWorkStreamIdDefault(); @@ -298,6 +307,7 @@ Session::Session(const std::map &options) { // session destructor Session::~Session() { + ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize); GELOGT(TRACE_INIT, "Session Destructor start"); // 0.check init status if (!g_ge_initialized) { @@ -333,12 +343,14 @@ Session::~Session() { } Status Session::AddGraph(uint32_t graph_id, const Graph &graph) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); std::map options; ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); return AddGraph(graph_id, graph, options); } Status Session::AddGraph(uint32_t graph_id, const Graph &graph, const std::map &options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGT(TRACE_INIT, "Start to add graph in Session. graph_id: %u, session_id: %lu.", graph_id, sessionId_); ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); std::shared_ptr instance_ptr = ge::GELib::GetInstance(); @@ -358,6 +370,7 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph, const std::map &options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGT(TRACE_INIT, "Start to add graph in Session. graph_id: %u, session_id: %lu.", graph_id, sessionId_); ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); std::shared_ptr instance_ptr = ge::GELib::GetInstance(); @@ -386,6 +399,7 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph, } Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); std::map options; return AddGraphWithCopy(graph_id, graph, options); @@ -393,6 +407,7 @@ Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph) { Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph, const std::map &options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGT(TRACE_INIT, "Start to add graph in Session. graph_id: %u, session_id: %lu.", graph_id, sessionId_); ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); std::shared_ptr instance_ptr = ge::GELib::GetInstance(); @@ -415,6 +430,7 @@ Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph, } Status Session::RemoveGraph(uint32_t graph_id) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGT(TRACE_INIT, "Session RemoveGraph start"); ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); @@ -484,6 +500,7 @@ void PrintOutputResult(std::vector &outputs) { } Status Session::RunGraph(uint32_t graph_id, const std::vector &inputs, std::vector &outputs) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGT(TRACE_INIT, "Session RunGraph start"); ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); @@ -527,6 +544,7 @@ Status Session::RegisterCallBackFunc(const char *key, const session::pCallBackFu } Status Session::BuildGraph(uint32_t graph_id, const std::vector &inputs) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { @@ -544,6 +562,7 @@ Status Session::BuildGraph(uint32_t graph_id, const std::vector Status Session::RunGraphAsync(uint32_t graph_id, const std::vector &inputs, RunAsyncCallback callback) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute); ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { @@ -563,6 +582,7 @@ Status Session::RunGraphAsync(uint32_t graph_id, const std::vector &var_names, std::vector &var_values) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute); ErrorManager::GetInstance().GenWorkStreamIdDefault(); auto instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { @@ -579,6 +599,7 @@ Status Session::GetVariables(const std::vector &var_names, std::vec } Status Session::GetVariables(const std::vector &var_names, std::vector &var_values) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute); ErrorManager::GetInstance().GenWorkStreamIdDefault(); auto instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { diff --git a/ge/generator/ge_generator.cc b/ge/generator/ge_generator.cc index 32d9e5a1..354b22f7 100644 --- a/ge/generator/ge_generator.cc +++ b/ge/generator/ge_generator.cc @@ -326,6 +326,8 @@ Status GeGenerator::Initialize(const map &options, OmgContext &o GELOGE(MEMALLOC_FAILED, "Make shared failed"); return MEMALLOC_FAILED; } + + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOpsProtoInit); string opsproto_path; GetOpsProtoPath(opsproto_path); GELOGI("Get opsproto path is %s", opsproto_path.c_str()); @@ -374,6 +376,7 @@ Status GeGenerator::Initialize(const map &options, OmgContext &o } Status GeGenerator::Finalize() { + ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize); GE_CHECK_NOTNULL_EXEC(impl_, return PARAM_INVALID); Status ret = impl_->graph_manager_.Finalize(); if (ret != SUCCESS) { @@ -385,12 +388,14 @@ Status GeGenerator::Finalize() { Status GeGenerator::GenerateOfflineModel(const Graph &graph, const string &file_name_prefix, const vector &inputs) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGI("Start to generate offline model."); ModelBufferData model; return GenerateModel(graph, file_name_prefix, inputs, model, true); } Status GeGenerator::GenerateOnlineModel(const Graph &graph, const vector &inputs, ModelBufferData &model) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); return GenerateModel(graph, "online", inputs, model, false); } @@ -754,6 +759,7 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector &in */ Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector &inputs, const vector &outputs, const string &model_file_name) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGI("Start to build single op offline model, input size: %zu, output size: %zu", inputs.size(), outputs.size()); ModelBufferData model_buff; OpEngineType engine_type = ENGINE_SYS; @@ -775,6 +781,7 @@ Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector &inputs, const vector &outputs, OpEngineType engine_type, ModelBufferData &model_buff) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGI("Start to build single op online, input size: %zu, output size: %zu", inputs.size(), outputs.size()); Status status = BuildSingleOp(op_desc, inputs, outputs, kFileNameSuffix, engine_type, model_buff, false); GELOGI("Finish build single online model, status: %u", status); @@ -912,6 +919,7 @@ Status GeGenerator::Impl::BuildModel(const Graph &graph, const vector ret = graph_manager_.BuildGraph(graph_id, inputs, ge_root_model, session_id); } + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); if (ret != SUCCESS) { GELOGE(GE_GENERATOR_GRAPH_MANAGER_BUILD_GRAPH_FAILED, "GraphManager build graph fail, graph id: %u", graph_id); VarManagerPool::Instance().RemoveVarManager(session_id); diff --git a/ge/graph/build/graph_builder.cc b/ge/graph/build/graph_builder.cc index 2731e076..0883d895 100644 --- a/ge/graph/build/graph_builder.cc +++ b/ge/graph/build/graph_builder.cc @@ -222,6 +222,7 @@ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph, return SUCCESS; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kPreBuild); GELOGI("Begin to build known shape graph[%s].", comp_graph->GetName().c_str()); Status ret = SecondPartition(comp_graph); GE_CHK_STATUS_RET(ret, "Graph[%s] second partition Failed.", comp_graph->GetName().c_str()); @@ -252,6 +253,7 @@ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph, GE_TIMESTAMP_END(BuildModelForGetTask, "GraphBuilder::BuildModelForGetTask"); GE_DUMP(comp_graph, "AfterBuildModel"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kTaskGenerate); GE_TIMESTAMP_START(GetTaskInfo); ret = GetTaskInfo(builder, model_ptr, comp_graph, subgraph_map, session_id); GE_TIMESTAMP_END(GetTaskInfo, "GraphBuilder::GetTaskInfo"); @@ -261,6 +263,7 @@ Status GraphBuilder::BuildForKnownShapeGraph(ComputeGraphPtr &comp_graph, return ret; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); ge_model_ptr = MakeShared(); if (ge_model_ptr == nullptr) { return MEMALLOC_FAILED; @@ -320,6 +323,7 @@ Status GraphBuilder::SetConstantInputOffset(ComputeGraphPtr &comp_graph) { Status GraphBuilder::BuildForUnknownShapeGraph(ComputeGraphPtr &comp_graph, GeModelPtr &ge_model_ptr, uint64_t session_id) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kPreBuild); GELOGI("Begin to build unknown shape graph[%s].", comp_graph->GetName().c_str()); Graph2SubGraphInfoList subgraph_map; ge::ModelBuilder builder(session_id, comp_graph, subgraph_map, stream_max_parallel_num_, hcom_parallel_, build_mode_); @@ -352,9 +356,11 @@ Status GraphBuilder::BuildForUnknownShapeGraph(ComputeGraphPtr &comp_graph, GeMo GE_CHK_STATUS_RET(builder.BuildModelForGetDynShapeTask(*model_ptr), "Graph[%s] builder BuildModelForGetDynShapeTask() return fail.", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(BuildModelForGetDynShapeTask, "GraphBuilder::BuildModelForGetDynShapeTask"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kTaskGenerate); GE_TIMESTAMP_START(GetTaskInfo); Status ret = GetTaskInfo(builder, model_ptr, comp_graph, subgraph_map, session_id); GE_TIMESTAMP_END(GetTaskInfo, "GraphBuilder::GetTaskInfo"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GraphUtils::DumpGEGraph(comp_graph, "AfterGetTask"); GraphUtils::DumpGEGraphToOnnx(*comp_graph, "AfterGetTask"); diff --git a/ge/graph/build/model_builder.cc b/ge/graph/build/model_builder.cc index 78c49057..7256c7be 100755 --- a/ge/graph/build/model_builder.cc +++ b/ge/graph/build/model_builder.cc @@ -684,6 +684,7 @@ Status ModelBuilder::PreBuildModel() { Status ModelBuilder::BuildModelForGetTask(ge::Model &model) { GE_CHK_STATUS_RET(AdjustInputTensorFlag(), "AdjustInputTensorFlag failed!"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kStreamAlloc); // Assign logical streams. StreamAllocator stream_allocator(compute_graph_, subgraphs_); GE_TIMESTAMP_START(AssignLogicalStreams); @@ -691,6 +692,7 @@ Status ModelBuilder::BuildModelForGetTask(ge::Model &model) { "Assign logical streams failed."); GE_TIMESTAMP_END(AssignLogicalStreams, "GraphBuilder::AssignLogicalStreams"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kMemoryAlloc); // Assign functional op labels. auto root_graph = GraphUtils::FindRootGraph(compute_graph_); (void)AttrUtils::GetInt(*root_graph, ATTR_MODEL_LABEL_NUM, label_num_); @@ -701,6 +703,7 @@ Status ModelBuilder::BuildModelForGetTask(ge::Model &model) { "Assign Memory Failed!"); GE_TIMESTAMP_END(AssignMemory, "GraphBuilder::AssignMemory"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GE_TIMESTAMP_START(SetInputOutputOffset); SetInputOutputOffsetPass input_output_offset; GE_CHK_STATUS_RET(input_output_offset.Run(compute_graph_), "Set input output offset failed."); @@ -711,12 +714,14 @@ Status ModelBuilder::BuildModelForGetTask(ge::Model &model) { GE_CHK_STATUS_RET(CompileSingleOp(), "ATC builder CompileSingleOp() return fail."); GE_TIMESTAMP_EVENT_END(CompileSingleOp, "GraphBuilder::CompileSingleOp"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kStreamAlloc); // Refresh real streams and insert event nodes. GE_TIMESTAMP_START(RefreshRealStream); GE_CHK_STATUS_RET(stream_allocator.RefreshRealStream(stream_num_, event_num_), "RefreshRealStream failed."); huge_streams_ = stream_allocator.GetHugeStreams(); GE_TIMESTAMP_END(RefreshRealStream, "GraphBuilder::RefreshRealStream"); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GE_TIMESTAMP_START(MergeWeights); GE_CHK_STATUS_RET(MergeWeights(), "MergeWeights Failed!"); GE_TIMESTAMP_END(MergeWeights, "GraphBuilder::MergeWeights"); diff --git a/ge/graph/load/model_manager/davinci_model.cc b/ge/graph/load/model_manager/davinci_model.cc index 7355714b..d228f2fa 100755 --- a/ge/graph/load/model_manager/davinci_model.cc +++ b/ge/graph/load/model_manager/davinci_model.cc @@ -214,12 +214,12 @@ DavinciModel::~DavinciModel() { UnbindTaskSinkStream(); for (size_t i = 0; i < label_list_.size(); ++i) { if (label_list_[i] != nullptr) { - GE_LOGW_IF(rtLabelDestroy(label_list_[i]) != RT_ERROR_NONE, "Destroy label failed, index:%zu", i); + GE_LOGW_IF(rtLabelDestroy(label_list_[i]) != RT_ERROR_NONE, "Destroy label failed, index:%zu.", i); } } for (size_t i = 0; i < stream_list_.size(); ++i) { - GE_LOGW_IF(rtStreamDestroy(stream_list_[i]) != RT_ERROR_NONE, "Destroy stream failed, index:%zu", i); + GE_LOGW_IF(rtStreamDestroy(stream_list_[i]) != RT_ERROR_NONE, "Destroy stream failed, index:%zu.", i); } for (size_t i = 0; i < event_list_.size(); ++i) { @@ -278,7 +278,7 @@ void DavinciModel::UnbindHcomStream() { for (size_t i = 0; i < all_hccl_stream_list_.size(); i++) { GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, all_hccl_stream_list_[i]) != RT_ERROR_NONE, "Unbind hccl stream from model failed! Index: %zu", i); - GE_LOGW_IF(rtStreamDestroy(all_hccl_stream_list_[i]) != RT_ERROR_NONE, "Destroy hccl stream for rt_model failed!") + GE_LOGW_IF(rtStreamDestroy(all_hccl_stream_list_[i]) != RT_ERROR_NONE, "Destroy hccl stream for rt_model failed") } } return; @@ -364,7 +364,7 @@ Status DavinciModel::InitWeightMem(void *dev_ptr, void *weight_ptr, size_t weigh Status DavinciModel::InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size) { if (is_feature_map_mem_has_inited_) { - GELOGE(PARAM_INVALID, "call InitFeatureMapMem more than once."); + GELOGE(PARAM_INVALID, "call InitFeatureMapMem more than once"); return PARAM_INVALID; } is_feature_map_mem_has_inited_ = true; @@ -387,7 +387,7 @@ Status DavinciModel::InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size) { GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc feature map memory failed. size: %zu", data_size); return ACL_ERROR_GE_MEMORY_ALLOCATION; } - GEEVENT("[IMAS]InitFeatureMapAndP2PMem graph_%u MallocMemory type[F] memaddr[%p] mem_size[%zu].", + GEEVENT("[IMAS]InitFeatureMapAndP2PMem graph_%u MallocMemory type[F] memaddr[%p] mem_size[%zu]", runtime_param_.graph_id, mem_base_, data_size); if (!is_inner_weight_base_) { @@ -408,7 +408,7 @@ Status DavinciModel::InitFeatureMapAndP2PMem(void *dev_ptr, size_t mem_size) { is_inner_p2p_mem_base_ = true; } - GE_CHK_STATUS_RET(InitVariableMem(), "Init variable memory failed"); + GE_CHK_STATUS_RET(InitVariableMem(), "Init variable memory failed."); runtime_param_.mem_base = mem_base_; runtime_param_.weight_base = weights_mem_base_; runtime_param_.memory_infos[RT_MEMORY_P2P_DDR].memory_base = p2p_mem_base_; @@ -480,7 +480,7 @@ void DavinciModel::CheckHasHcomOp(const ComputeGraphPtr &compute_graph) { for (const auto &node : compute_graph->GetAllNodes()) { OpDescPtr op_desc = node->GetOpDesc(); - GE_IF_BOOL_EXEC(op_desc == nullptr, GELOGW("Node OpDesc is nullptr"); continue); + GE_IF_BOOL_EXEC(op_desc == nullptr, GELOGW("Node OpDesc is nullptr."); continue); if (hcom_opp_types.count(op_desc->GetType()) > 0) { uint32_t stream_id = static_cast(op_desc->GetStreamId()); hcom_streams_.emplace(stream_id); @@ -527,25 +527,25 @@ Status DavinciModel::DoTaskSink() { } GE_CHK_RT_RET(rtGetAicpuDeploy(&deploy_type_)); - GELOGI("do task_sink. AiCpu deploy type is: %x", deploy_type_); + GELOGI("do task_sink. AiCpu deploy type is: %x.", deploy_type_); - GE_CHK_STATUS_RET(BindModelStream(), "Bind model stream failed."); + GE_CHK_STATUS_RET(BindModelStream(), "Bind model stream failed"); if (known_node_) { - GE_CHK_STATUS_RET(MallocKnownArgs(), "Mallloc known node args failed."); + GE_CHK_STATUS_RET(MallocKnownArgs(), "Mallloc known node args failed"); } - GE_CHK_STATUS_RET(InitTaskInfo(*model_task_def.get()), "InitTaskInfo failed."); + GE_CHK_STATUS_RET(InitTaskInfo(*model_task_def.get()), "InitTaskInfo failed"); - GE_CHK_STATUS_RET(ModelManager::GetInstance()->LaunchCustAicpuSo(), "Launch cust aicpu so failed."); + GE_CHK_STATUS_RET(ModelManager::GetInstance()->LaunchCustAicpuSo(), "Launch cust aicpu so failed"); - GE_CHK_STATUS_RET(ModelManager::GetInstance()->CheckAicpuOpList(ge_model_), "Check aicpu op type failed."); + GE_CHK_STATUS_RET(ModelManager::GetInstance()->CheckAicpuOpList(ge_model_), "Check aicpu op type failed"); - GE_CHK_STATUS_RET(InitEntryTask(), "InitEntryTask failed."); + GE_CHK_STATUS_RET(InitEntryTask(), "InitEntryTask failed"); - GE_CHK_STATUS_RET(InitL1DataDumperArgs(), "InitL1DataDumperArgs failed."); + GE_CHK_STATUS_RET(InitL1DataDumperArgs(), "InitL1DataDumperArgs failed"); - GE_CHK_STATUS_RET(DistributeTask(), "Distribute failed."); + GE_CHK_STATUS_RET(DistributeTask(), "Distribute failed"); GE_CHK_RT_RET(rtModelLoadComplete(rt_model_handle_)); @@ -558,7 +558,7 @@ Status DavinciModel::SetTSDevice() { int64_t value = 0; bool ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_CORE_TYPE, value); uint32_t core_type = ret ? static_cast(value) : 0; - GELOGD("SetTSDevice: %u.", core_type); + GELOGD("SetTSDevice: %u", core_type); rtError_t rt_ret = rtSetTSDevice(core_type); if (rt_ret != RT_ERROR_NONE) { GELOGE(RT_FAILED, "SetTSDevice failed, ret: 0x%X", rt_ret); @@ -570,7 +570,7 @@ Status DavinciModel::SetTSDevice() { Status DavinciModel::OpDebugRegister() { bool is_op_debug = false; (void)ge::AttrUtils::GetBool(ge_model_, ATTR_OP_DEBUG_FLAG, is_op_debug); - GELOGD("The value of op debug in ge_model is %d.", is_op_debug); + GELOGD("The value of op debug in ge_model is %d", is_op_debug); if (is_op_debug) { debug_reg_mutex_.lock(); rtError_t rt_ret = rtMalloc(&op_debug_addr_, kOpDebugMemorySize, RT_MEMORY_DDR); @@ -2575,7 +2575,7 @@ Status DavinciModel::ReturnResult(uint32_t data_id, const bool rslt_flg, const b /// @return Status result /// Status DavinciModel::ReturnNoOutput(uint32_t data_id) { - GELOGI("ReturnNoOutput model id:%u", model_id_); + GELOGI("ReturnNoOutput model id:%u.", model_id_); GE_CHK_BOOL_EXEC(listener_ != nullptr, return PARAM_INVALID, "listener_ is null!"); std::vector outputs; @@ -2601,6 +2601,7 @@ void *DavinciModel::Run(DavinciModel *model) { // DeviceReset before thread run finished! GE_MAKE_GUARD(not_used_var, [&] { GE_CHK_RT(rtDeviceReset(device_id)); }); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute); while (model->RunFlag()) { bool rslt_flg = true; if (model->GetDataInputer() == nullptr) { diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index 170460b9..131765ed 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -142,7 +142,7 @@ ge::Status CheckFpCeilingMode() { GELOGI("The parameter fp_ceiling_mode is set to %s.", mode.c_str()); return ge::SUCCESS; } - GELOGW("The parameter fp_ceiling_mode is not set."); + GELOGW("The parameter fp_ceiling_mode is not set"); return ge::SUCCESS; } } // namespace @@ -155,6 +155,7 @@ GraphManager::GraphManager() } Status GraphManager::Initialize(const std::map &options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); if (init_flag_) { GELOGW("[Initialize] GraphManager already initialized."); return SUCCESS; @@ -294,7 +295,7 @@ Status GraphManager::InitDynamicParams(ComputeGraphPtr &compute_graph) { return FAILED; } if ((op_desc->GetType() == DATA) || (op_type == kGetNextName)) { - GELOGI("Need to process multi batch for compute graph. op_type:%s", op_desc->GetType().c_str()); + GELOGI("Need to process multi batch for compute graph. op_type:%s.", op_desc->GetType().c_str()); GetLocalOmgContext().need_multi_batch = true; break; } @@ -349,14 +350,14 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, for (auto &subgraph : compute_graph->GetAllSubgraphs()) { (void)AttrUtils::SetStr(*subgraph, ATTR_NAME_SESSION_GRAPH_ID, session_graph_id); } - GELOGD("Get graph session_graph_id attr failed, set session id to default value: [0]."); + GELOGD("Get graph session_graph_id attr failed, set session id to default value: [0]"); } GraphNodePtr graph_node = MakeShared(graph_id); - GE_IF_BOOL_EXEC(graph_node == nullptr, GELOGE(FAILED, "GraphNode make shared failed."); + GE_IF_BOOL_EXEC(graph_node == nullptr, GELOGE(FAILED, "GraphNode make shared failed"); return FAILED); std::shared_ptr graph_ptr = MakeShared(graph); - GE_IF_BOOL_EXEC(graph_ptr == nullptr, GELOGE(FAILED, "GraphPtr make shared failed."); + GE_IF_BOOL_EXEC(graph_ptr == nullptr, GELOGE(FAILED, "GraphPtr make shared failed"); return FAILED); graph_node->SetGraph(graph_ptr); @@ -666,6 +667,7 @@ Status GraphManager::SetSubgraph(uint64_t session_id, ComputeGraphPtr compute_gr Status GraphManager::PreRunOptimizeOriginalGraph(const GraphNodePtr &graph_node, const std::vector &inputs, ge::ComputeGraphPtr &compute_graph, uint64_t session_id) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kPrepareOptimize); GE_CHECK_NOTNULL(graph_node); GE_CHECK_NOTNULL(compute_graph); @@ -674,8 +676,10 @@ Status GraphManager::PreRunOptimizeOriginalGraph(const GraphNodePtr &graph_node, GM_RUN_AND_DUMP_PERF("HandleSummaryOp", stages.optimizer.HandleSummaryOp, compute_graph); GM_RUN_AND_DUMP_PERF("Prepare", stages.preparer.PrepareDynShape, graph_node, inputs, compute_graph, session_id); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOriginOptimize); GM_RUN_AND_DUMP_PERF("OptimizeOriginalGraph", stages.optimizer.OptimizeOriginalGraph, compute_graph); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kPrepareOptimize); GM_RUN_AND_DUMP_PERF("PrepareRunningFormatRefiner", stages.preparer.PrepareRunningFormatRefiner); GM_RUN_AND_DUMP_PERF("RefineRunningFormat", stages.optimizer.OptimizeOriginalGraphJudgeInsert, compute_graph); GM_RUN_AND_DUMP_PERF("SubexpressionMigration", SubexpressionMigration, compute_graph); @@ -718,6 +722,7 @@ Status GraphManager::PreRunAfterOptimizeSubGraph(const GraphNodePtr &graph_node, GE_CHECK_NOTNULL(graph_node); GE_CHECK_NOTNULL(compute_graph); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kMergeGraphOptimize); CompilerStages &stages = GetCompilerStages(graph_node->GetGraphId()); GM_RUN_AND_DUMP_PERF("OptimizeWholeGraph", stages.optimizer.OptimizeWholeGraph, compute_graph); GM_RUN_AND_DUMP_PERF("Optimize2", OptimizeStage2, compute_graph); @@ -761,7 +766,7 @@ Status GraphManager::RunCustomPass(const GraphNodePtr &graph_node) { GE_TIMESTAMP_START(RunCustomPass); GraphPtr graph = std::const_pointer_cast(const_graph); - GE_CHK_STATUS_RET(CustomPassHelper::Instance().Run(graph), "Graph[%s] run custom pass fail", + GE_CHK_STATUS_RET(CustomPassHelper::Instance().Run(graph), "Graph[%s] run custom pass fail.", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(RunCustomPass, "GraphBuilder::RunCustomPass"); return SUCCESS; @@ -779,7 +784,7 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorBuildJsonObject(session_id, compute_graph->GetGraphID()), "BuildJsonObject Failed") - GEEVENT("PreRun start: graph node size %zu, session id %lu, graph id %u, graph name %s.", + GEEVENT("PreRun start: graph node size %zu, session id %lu, graph id %u, graph name %s", compute_graph->GetDirectNodesSize(), session_id, compute_graph->GetGraphID(), compute_graph->GetName().c_str()); GE_DUMP(compute_graph, "PreRunBegin"); @@ -800,11 +805,12 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorGetName().c_str()); + GELOGE(ret, "Run PreRunOptimizeOriginalGraph failed for graph:%s", compute_graph->GetName().c_str()); return ret; } } + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kPrepareOptimize); ret = PreRunOptimizeSubGraph(graph_node, compute_graph, session_id); if (ret != SUCCESS) { GELOGE(ret, "Run PreRunOptimizeSubGraph failed for graph:%s.", compute_graph->GetName().c_str()); @@ -826,6 +832,7 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorGetGraphId(), compute_graph, ge_model); @@ -857,6 +864,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: // it will not execute graph prreprocess, optimize, parition, build if the graph has built successful. Status ret = SUCCESS; if (IsGraphNeedBuild(graph_node)) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); if (graph_node->GetBuildFlag()) { GELOGE(PARAM_INVALID, "The graph %u need to re-build, you should remove it from GE " @@ -876,6 +884,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: return ret; } } + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelLoad, ErrorMessage::kModelLoad); if (!graph_node->IsAsync()) { ret = LoadGraph(ge_root_model, graph_node); } else { @@ -888,6 +897,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: graph_node->SetBuildFlag(true); var_acc_ctrl_.SetGraphBuildEnd(graph_node->GetGraphId()); } else if (!graph_node->GetLoadFlag()) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelLoad, ErrorMessage::kModelLoad); GeRootModelPtr ge_root_model_ptr = graph_node->GetGeRootModel(); if (!graph_node->IsAsync()) { ret = LoadGraph(ge_root_model_ptr, graph_node); @@ -1045,6 +1055,7 @@ Status GraphManager::InnerRunGraph(GraphNodePtr &graph_node, const GraphId &grap Status GraphManager::RunGraph(const GraphId &graph_id, const std::vector &inputs, std::vector &outputs, uint64_t session_id) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); std::lock_guard lock(run_mutex_); GELOGI("[RunGraph] start to run graph, graph_id = %u, is_train_graph: %d", graph_id, GetTrainFlag()); @@ -1097,6 +1108,7 @@ Status GraphManager::RunGraph(const GraphId &graph_id, const std::vector &inputs, GeRootModelPtr &ge_root_model, uint64_t session_id, bool async) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGD("[BuildGraph] start to build graph, graph_id:%u", graph_id); if (inputs.empty()) { GELOGW("[BuildGraph] BuildGraph warning: empty GeTensor inputs"); @@ -1511,7 +1524,7 @@ Status GraphManager::ParseOptions(const std::map &opti Status GraphManager::ParseTrainGraphFlag(bool &options, bool &option) { std::shared_ptr ge_instance_ptr = ge::GELib::GetInstance(); if (ge_instance_ptr == nullptr) { - GELOGW("[Initialize] set train_graph_flag to 0 when GE is not initialized or finalized."); + GELOGW("[Initialize] set train_graph_flag to 0 when GE is not initialized or finalized"); option = false; } else if (!ge_instance_ptr->isTrainMode()) { option = false; @@ -2560,6 +2573,7 @@ Status GraphManager::ProcessSubGraphWithMultiThreads(GraphManager *graph_manager // run graph async on session Status GraphManager::RunGraphAsync(const GraphId &graph_id, const std::vector &inputs, uint64_t session_id, RunAsyncCallback callback) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute); GELOGI("[GraphManager] Start to run graph async, graph_id=%u, inputsSize=%zu.", graph_id, inputs.size()); bool ret = prerun_args_q_.Push(PreRunArgs({graph_id, inputs, session_id, @@ -2652,6 +2666,7 @@ void GraphManager::PreRunThread(GraphManager *graph_manager) { GELOGI("A new loop start."); ErrorManager::GetInstance().SetErrorContext(args.error_context); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GetContext().SetSessionId(args.session_id); GetThreadLocalContext() = args.context; graph_manager->UpdateLocalOmgContext(args.graph_id); @@ -2820,6 +2835,7 @@ Status GraphManager::ParseInputsDims(const std::vector &input_t } void GraphManager::RunThread(GraphManager *graph_manager) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute); if (prctl(PR_SET_NAME, ("GE_Run")) != 0) { GELOGW("Set thread name failed."); } @@ -2851,6 +2867,7 @@ void GraphManager::RunThread(GraphManager *graph_manager) { } if (!args.graph_node->GetLoadFlag()) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelLoad, ErrorMessage::kModelLoad); ret = graph_manager->LoadGraphAsync(args.ge_root_model, args.graph_node); if (ret != SUCCESS || args.ge_root_model == nullptr) { StopQueue(graph_manager); @@ -2863,6 +2880,7 @@ void GraphManager::RunThread(GraphManager *graph_manager) { args.ge_root_model->GetModelId()); } + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute); if (graph_manager->GetTrainFlag()) { ret = graph_manager->graph_executor_.SetGraphContext(graph_manager->GetGraphContext()); if (ret != SUCCESS) { @@ -3110,6 +3128,7 @@ Status GraphManager::ConvertGraphToFile(ComputeGraphPtr &compute_graph, GraphPar Status GraphManager::Build(const GraphNodePtr &graph_node, ComputeGraphPtr &compute_graph, GeRootModelPtr &ge_root_model, uint64_t session_id) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); // build if (compute_graph != nullptr) { std::string graph_name = compute_graph->GetName(); diff --git a/ge/graph/passes/flow_ctrl_pass.cc b/ge/graph/passes/flow_ctrl_pass.cc index 52a570cb..5fc1733a 100755 --- a/ge/graph/passes/flow_ctrl_pass.cc +++ b/ge/graph/passes/flow_ctrl_pass.cc @@ -87,7 +87,7 @@ Status FlowCtrlPass::Run(ComputeGraphPtr compute_graph) { assign_add_node_in_fpbp_loop_->GetInControlAnchor()); if (ret != GRAPH_SUCCESS) { GELOGW("add control edge between iter_loop_node:%s and fpbp_loop_node:%s fail, may cause block", - active_node->GetName().c_str(), assign_add_node_in_fpbp_loop_->GetName().c_str()); + active_node->GetName().c_str(), assign_add_node_in_fpbp_loop_->GetName().c_str()); } } GELOGI("FlowCtrl pass end, graph is %s.", graph_change ? "changed" : "not changed"); diff --git a/ge/init/gelib.cc b/ge/init/gelib.cc index faa06962..354ee0c2 100755 --- a/ge/init/gelib.cc +++ b/ge/init/gelib.cc @@ -71,6 +71,7 @@ Status GELib::Initialize(const map &options) { return GE_CLI_INIT_FAILED; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kSystemInit); map new_options; Status ret = instancePtr_->SetRTSocVersion(options, new_options); if (ret != SUCCESS) { @@ -110,6 +111,7 @@ Status GELib::InnerInitialize(const map &options) { return SUCCESS; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kSystemInit); GELOGI("GE System initial."); GE_TIMESTAMP_START(SystemInitialize); Status initSystemStatus = SystemInitialize(options); @@ -120,6 +122,7 @@ Status GELib::InnerInitialize(const map &options) { return initSystemStatus; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kEngineInit); GELOGI("engineManager initial."); GE_TIMESTAMP_START(EngineInitialize); Status initEmStatus = engineManager_.Initialize(options); @@ -130,6 +133,7 @@ Status GELib::InnerInitialize(const map &options) { return initEmStatus; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOpsKernelInit); GELOGI("opsManager initial."); GE_TIMESTAMP_START(OpsManagerInitialize); Status initOpsStatus = opsManager_.Initialize(options); @@ -140,6 +144,7 @@ Status GELib::InnerInitialize(const map &options) { return initOpsStatus; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOpsKernelBuilderInit); GELOGI("opsBuilderManager initial."); GE_TIMESTAMP_START(OpsKernelBuilderManagerInitialize); Status initOpsBuilderStatus = OpsKernelBuilderManager::Instance().Initialize(options); @@ -150,6 +155,7 @@ Status GELib::InnerInitialize(const map &options) { return initOpsBuilderStatus; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); GELOGI("sessionManager initial."); GE_TIMESTAMP_START(SessionManagerInitialize); Status initSmStatus = sessionManager_.Initialize(options); @@ -423,6 +429,7 @@ string GELib::GetPath() { return PluginManager::GetPath(); } // Finalize all modules Status GELib::Finalize() { + ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize); GELOGI("finalization start"); // Finalization is not allowed before initialization if (!init_flag_) { diff --git a/ge/ir_build/ge_ir_build.cc b/ge/ir_build/ge_ir_build.cc index d7daaede..747a1ec7 100644 --- a/ge/ir_build/ge_ir_build.cc +++ b/ge/ir_build/ge_ir_build.cc @@ -201,10 +201,12 @@ graphStatus aclgrphBuildInitializeImpl(std::map &globa } graphStatus aclgrphBuildInitialize(std::map global_options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); return aclgrphBuildInitializeImpl(global_options); } graphStatus aclgrphBuildInitialize(std::map &global_options) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); std::map tmp_global_options; for (auto &option : global_options) { if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) { @@ -219,6 +221,7 @@ graphStatus aclgrphBuildInitialize(std::map &global_ } void aclgrphBuildFinalize() { + ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize); if (ge::GELib::GetInstance() != nullptr && ge::GELib::GetInstance()->InitFlag()) { (void)ge::GELib::GetInstance()->Finalize(); return; @@ -563,6 +566,7 @@ graphStatus Impl::InitDomiOmgContext(const string &input_shape, const string &in graphStatus aclgrphBuildModel(const ge::Graph &graph, const std::map &build_options, ModelBufferData &model) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGD("Enter aclmdlBuildModel process!"); Impl builder; return builder.BuildModel(graph, build_options, model); @@ -570,6 +574,7 @@ graphStatus aclgrphBuildModel(const ge::Graph &graph, const std::map &build_options, ModelBufferData &model) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); GELOGD("Enter aclmdlBuildModel process!"); std::map tmp_build_options; for (auto &option : build_options) { @@ -587,6 +592,7 @@ graphStatus aclgrphBuildModel(const ge::Graph &graph, const std::map PATH_MAX || len != strlen(file) || strlen(file) == 0) { @@ -678,6 +687,7 @@ graphStatus aclgrphDumpGraph(const ge::Graph &graph, const char *file, const siz graphStatus aclgrphGenerateForOp(const AscendString &op_type, const vector &inputs, const vector &outputs, Graph &graph) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); auto op_type_str = std::string(op_type.GetString()); auto op_name = op_type_str + "_" + std::to_string(ge::GetCurrentTimestamp()); auto op_desc = ge::MakeShared(op_name, op_type_str); @@ -737,6 +747,7 @@ static std::string AttrTypeToSerialString(aclgrphAttrType attr_type) { } graphStatus aclgrphSetOpAttr(Graph &graph, aclgrphAttrType attr_type, const char *cfg_path) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); auto compute_graph = GraphUtils::GetComputeGraph(graph); GE_CHECK_NOTNULL(compute_graph); if (cfg_path == nullptr) { diff --git a/ge/offline/main.cc b/ge/offline/main.cc index 069ec769..3d6dbf13 100755 --- a/ge/offline/main.cc +++ b/ge/offline/main.cc @@ -949,6 +949,7 @@ domi::Status GenerateModel(std::map &options, std::string output ge::Graph graph; std::vector inputs; if (FLAGS_framework == domi::MINDSPORE) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); // load model from file ge::Model load_model = ge::Model("loadmodel", "version2"); auto ret1 = load_model.LoadFromFile(FLAGS_model); @@ -987,10 +988,12 @@ domi::Status GenerateModel(std::map &options, std::string output atc_params.insert(std::pair(string(ge::OUTPUT_DATATYPE), FLAGS_output_type)); atc_params.insert(std::pair("output", output)); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kParser); Status ret = ParseGraph(graph, atc_params, FLAGS_model.c_str(), FLAGS_weight.c_str(), (domi::FrameworkType)FLAGS_framework, FLAGS_op_name_map.c_str(), FLAGS_target.c_str(), (ge::RunMode)FLAGS_mode, is_dynamic_input); + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); // in ONLY_PRE_CHECK mode, pre-checking report has already saved in ParseGraph if (FLAGS_mode == ge::ONLY_PRE_CHECK) { (void)ge_generator.Finalize(); @@ -1088,6 +1091,7 @@ domi::Status GenerateSingleOp(const std::string& json_file_path) { return domi::FAILED; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kParser); vector build_params; if (ge::SingleOpParser::ParseSingleOpList(json_file_path, build_params) != ge::SUCCESS) { DOMI_LOGE("parse single op json file failed"); @@ -1220,6 +1224,7 @@ domi::Status GenerateOmModel() { return domi::FAILED; } + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); if (FLAGS_display_model_info == "1") { GELOGI("need to display model info."); return ge::ConvertOm(FLAGS_output.c_str(), "", false); @@ -1229,6 +1234,7 @@ domi::Status GenerateOmModel() { } domi::Status ConvertModelToJson() { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); Status ret = GFlagUtils::CheckConverJsonParamFlags(); GE_CHK_BOOL_EXEC(ret == domi::SUCCESS, return domi::FAILED, "Check convert json params flags failed!"); @@ -1239,6 +1245,7 @@ domi::Status ConvertModelToJson() { } domi::Status DisplayModelInfo() { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); // No model path passed in GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(FLAGS_om == "", ErrorManager::GetInstance().ATCReportErrMessage("E10004", {"parameter"}, {"om"}); @@ -1287,6 +1294,7 @@ bool CheckRet(domi::Status ret) { } domi::Status ConvertPbtxtToJson() { + ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther); Status ret = GFlagUtils::CheckConverJsonParamFlags(); if (ret != domi::SUCCESS) { GELOGE(ge::FAILED, "Check convert json params flags failed!"); @@ -1374,6 +1382,7 @@ bool CheckMemInfo() { } int main(int argc, char* argv[]) { + ErrorManager::GetInstance().SetStage(ErrorMessage::kInitialize, ErrorMessage::kOther); Status ret = domi::SUCCESS; std::cout << "ATC start working now, please wait for a moment." << std::endl; @@ -1414,6 +1423,7 @@ int main(int argc, char* argv[]) { } } while (0); + ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize); if (!CheckRet(ret)) { std::cout << "ATC run failed, Please check the detail log, Try \'atc --help\' for more information" << std::endl; int result = ErrorManager::GetInstance().OutputErrMessage(STDOUT_FILENO); diff --git a/metadef b/metadef index 781bdcdf..7a51997c 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit 781bdcdff510f62fe1c5ca6b1b18c5a8e15724c4 +Subproject commit 7a51997cbd34e1869b9fb4ea5597a021e6427272