!1555 code_sync_0420

From: @ding_fei_fei
Reviewed-by: @xsmq,@liucunwei,@liucunwei
Signed-off-by: @liucunwei,@liucunwei

build.sh

@@ -229,7 +229,7 @@ if [[ "X$ENABLE_GE_UT" = "Xon" || "X$ENABLE_GE_COV" = "Xon" ]]; then
     rm -rf ${BASEPATH}/cov
     mkdir ${BASEPATH}/cov
     lcov -c -d build/tests/ut/ge -d build/tests/ut/common/graph/ -o cov/tmp.info
-    lcov -r cov/tmp.info '*/output/*' '*/build/opensrc/*' '*/build/proto/*' '*/third_party/*' '*/tests/*' '/usr/local/*' -o cov/coverage.info
+    lcov -r cov/tmp.info '*/output/*' '*/build/opensrc/*' '*/build/proto/*' '*/third_party/*' '*/tests/*' '/usr/local/*' '/usr/include/*' '*/metadef/*' '*/parser/*' -o cov/coverage.info
     cd ${BASEPATH}/cov
     genhtml coverage.info
 fi

ge/CMakeLists.txt

@@ -31,6 +31,7 @@ set(PROTO_HEADER_LIST
 protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
 protobuf_generate(ge PROTO_CLIENT_SRCS PROTO_CLIENT_HDRS ${PROTO_CLIENT_LIST})
 protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST})
+protobuf_generate(ge_client PROTO_CLIENT_HEADER_SRCS PROTO_CLIENT_HEADER_HDRS ${PROTO_HEADER_LIST})
 
 if (NOT ENABLE_D AND NOT ENABLE_ACL AND NOT ENABLE_MS_TESTCASES)
 ############ libge_proto_common.a ############
@@ -56,7 +57,7 @@ target_link_libraries(ge_proto_common PRIVATE
 
 ############ libge_proto_client.a ############
 add_library(ge_proto_client STATIC
-    ${PROTO_HEADER_HDRS}
+    ${PROTO_CLIENT_HEADER_HDRS}
     ${PROTO_CLIENT_SRCS}
 )
 
@@ -65,6 +66,11 @@ target_compile_definitions(ge_proto_client PRIVATE
     google=ascend_private
 )
 
+target_include_directories(ge_proto_client PRIVATE
+    ${CMAKE_BINARY_DIR}/proto/ge_client
+    ${CMAKE_BINARY_DIR}/proto/ge_client/proto
+)
+
 target_compile_options(ge_proto_client PRIVATE
     -O2
     -fno-common
@@ -102,6 +108,7 @@ set(TRAIN_SRC_LIST
     "common/helper/model_cache_helper.cc"
     "common/profiling/profiling_manager.cc"
     "common/dump/dump_manager.cc"
+    "common/dump/exception_dumper.cc"
     "common/dump/dump_properties.cc"
     "common/dump/opdebug_register.cc"
     "common/dump/dump_op.cc"
@@ -189,6 +196,7 @@ set(TRAIN_SRC_LIST
     "graph/passes/atomic_addr_clean_pass.cc"
     "graph/passes/mark_same_addr_pass.cc"
     "graph/passes/mark_graph_unknown_status_pass.cc"
+    "graph/passes/mark_node_unknown_shape_pass.cc"
     "graph/passes/mark_agnostic_pass.cc"
     "graph/partition/dynamic_shape_partition.cc"
     "graph/partition/stage_partition.cc"
@@ -209,6 +217,7 @@ set(TRAIN_SRC_LIST
     "graph/passes/dimension_compute_pass.cc"
     "graph/passes/dropout_pass.cc"
     "graph/passes/hccl_group_pass.cc"
+    "graph/passes/hccl_tailing_optimization_pass.cc"
     "graph/passes/enter_pass.cc"
     "graph/passes/assign_remove_pass.cc"
     "graph/passes/inplace_support_check_pass.cc"
@@ -320,7 +329,9 @@ set(TRAIN_SRC_LIST
     "graph/passes/variable_ref_useless_control_out_delete_pass.cc"
     "graph/passes/end_of_sequence_add_control_pass.cc"
     "graph/passes/memcpy_addr_async_pass.cc"
+    "graph/passes/parallel_group_pass.cc"
     "graph/passes/set_input_output_offset_pass.cc"
+    "graph/passes/buffer_pool_memory_pass.cc"
     "graph/preprocess/graph_preprocess.cc"
     "graph/preprocess/insert_op/ge_aipp_op.cc"
     "graph/preprocess/insert_op/util_insert_aipp_op.cc"
@@ -399,6 +410,7 @@ set(TRAIN_SRC_LIST
     "graph/build/memory/hybrid_mem_assigner.cc"
     "graph/build/memory/max_block_mem_assigner.cc"
     "graph/build/memory/var_mem_assign_util.cc"
+    "graph/build/memory/buffer_pool_mem_assigner.cc"
 )
 
 set(INFER_SRC_LIST
@@ -426,6 +438,7 @@ set(INFER_SRC_LIST
     "common/formats/formats.cc"
     "common/profiling/profiling_manager.cc"
     "common/dump/dump_properties.cc"
+    "common/dump/exception_dumper.cc"
     "common/dump/dump_manager.cc"
     "common/dump/dump_op.cc"
     "common/dump/opdebug_register.cc"
@@ -499,6 +512,7 @@ set(INFER_SRC_LIST
     "graph/passes/atomic_addr_clean_pass.cc"
     "graph/passes/mark_same_addr_pass.cc"
     "graph/passes/mark_graph_unknown_status_pass.cc"
+    "graph/passes/mark_node_unknown_shape_pass.cc"
     "graph/passes/mark_agnostic_pass.cc"
     "graph/common/omg_util.cc"
     "graph/common/bcast.cc"
@@ -605,8 +619,11 @@ set(INFER_SRC_LIST
     "graph/passes/link_gen_mask_nodes_pass.cc"
     "graph/passes/replace_with_empty_const_pass.cc"
     "graph/passes/hccl_group_pass.cc"
+    "graph/passes/hccl_tailing_optimization_pass.cc"
     "graph/passes/memcpy_addr_async_pass.cc"
     "graph/passes/set_input_output_offset_pass.cc"
+    "graph/passes/parallel_group_pass.cc"
+    "graph/passes/buffer_pool_memory_pass.cc"
     "graph/manager/model_manager/event_manager.cc"
     "graph/manager/util/rt_context_util.cc"
     "graph/manager/util/variable_accelerate_ctrl.cc"
@@ -670,6 +687,7 @@ set(INFER_SRC_LIST
     "graph/build/memory/hybrid_mem_assigner.cc"
     "graph/build/memory/max_block_mem_assigner.cc"
     "graph/build/memory/var_mem_assign_util.cc"
+    "graph/build/memory/buffer_pool_mem_assigner.cc"
 )
 
 if (NOT ENABLE_D AND NOT ENABLE_ACL AND NOT ENABLE_MS_TESTCASES)
@@ -700,6 +718,7 @@ target_compile_definitions(ge_runner PRIVATE
     DAVINCI_CLOUD
     google=ascend_private
     FUNC_VISIBILITY
+    $<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
 )
 
 target_compile_options(ge_runner PRIVATE
@@ -775,6 +794,7 @@ target_compile_definitions(ge_compiler PRIVATE
     COMPILE_OMG_PACKAGE
     google=ascend_private
     FUNC_VISIBILITY
+    $<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
 )
 
 target_compile_options(ge_compiler PRIVATE
@@ -937,6 +957,10 @@ add_library(atc_stub_ge_compiler SHARED
 
 add_dependencies(atc_stub_ge_compiler ge_stub)
 
+target_compile_options(atc_stub_ge_compiler PRIVATE
+    -fno-common
+)
+
 target_link_libraries(atc_stub_ge_compiler PRIVATE
     $<BUILD_INTERFACE:intf_pub>
 )
@@ -973,6 +997,10 @@ add_library(fwk_stub_ge_runner SHARED
 
 add_dependencies(fwk_stub_ge_runner ge_stub)
 
+target_compile_options(fwk_stub_ge_runner PRIVATE
+    -fno-common
+)
+
 target_link_libraries(fwk_stub_ge_runner PRIVATE
     $<BUILD_INTERFACE:intf_pub>
 )

ge/analyzer/analyzer.cc

@@ -155,12 +155,12 @@ std::shared_ptr<GraphInfo> Analyzer::GetJsonObject(uint64_t session_id, uint64_t
   std::lock_guard<std::recursive_mutex> lg(mutex_);
   auto iter = graph_infos_.find(session_id);
   if (iter == graph_infos_.end()) {
-    GELOGE(PARAM_INVALID, "[Check][Session_id]session_id:%lu does not exist! graph_id:%lu.", session_id, graph_id);
+    GELOGE(PARAM_INVALID, "[Check][SessionId]session_id:%lu does not exist! graph_id:%lu", session_id, graph_id);
     return nullptr;
   } else {
     auto iter1 = (iter->second).find(graph_id);
     if (iter1 == (iter->second).end()) {
-      GELOGE(PARAM_INVALID, "[Check][Graph_id]graph_id:%lu does not exist! session_id:%lu.", graph_id, session_id);
+      GELOGE(PARAM_INVALID, "[Check][GraphId]graph_id:%lu does not exist! session_id:%lu.", graph_id, session_id);
       return nullptr;
     }
     GELOGI("GetJsonObject Success!session_id:%lu graph_id:%lu", session_id, graph_id);
@@ -200,7 +200,7 @@ ge::Status Analyzer::CreateAnalyzerFile() {
 }
 
 ge::Status Analyzer::SaveAnalyzerDataToFile(uint64_t session_id, uint64_t graph_id) {
-  GELOGD("start to save analyze file.");
+  GELOGD("start to save analyze file");
 
   auto graph_info = GetJsonObject(session_id, graph_id);
   GE_CHECK_NOTNULL(graph_info);
@@ -221,7 +221,10 @@ ge::Status Analyzer::SaveAnalyzerDataToFile(uint64_t session_id, uint64_t graph_
   try {
     json_file_ << jsn.dump(kJsonDumpLevel) << std::endl;
   } catch (nlohmann::detail::type_error &e) {
-    GELOGE(FAILED, "[Json.dump][GraphInfo]json.dump to analyze file [%s] failed because [%s], session_id:%lu, graph_id:%lu", json_file_name_.c_str(), e.what(), session_id, graph_id);
+    GELOGE(FAILED, 
+           "[Json.dump][GraphInfo]json.dump to analyze file [%s] failed because [%s],"
+	   "session_id:%lu, graph_id:%lu",
+           json_file_name_.c_str(), e.what(), session_id, graph_id);
     ret_failed = true;
   }
   json_file_.close();
@@ -229,7 +232,7 @@ ge::Status Analyzer::SaveAnalyzerDataToFile(uint64_t session_id, uint64_t graph_
 }
 
 ge::Status Analyzer::DoAnalyze(DataInfo &data_info) {
-  GELOGD("start to do analyzer process!");
+  GELOGD("start to do analyzer process");
 
   auto pnode = data_info.node_ptr;
   GE_CHECK_NOTNULL(pnode);
@@ -241,7 +244,9 @@ ge::Status Analyzer::DoAnalyze(DataInfo &data_info) {
   GE_CHECK_NOTNULL(graph_info);
   auto status = SaveOpInfo(desc, data_info, graph_info);
   if (status != SUCCESS) {
-    GELOGE(status, "[Check][SaveOpInfo]save op info: desc_name [%s] desc_type [%s] failed!", desc->GetName().c_str(), desc->GetType().c_str());
+    GELOGE(status, 
+           "[Check][SaveOpInfo]save op info: desc_name [%s] desc_type [%s] failed!",
+           desc->GetName().c_str(), desc->GetType().c_str());
     return FAILED;
   }
   // create json file

ge/client/ge_api.cc

@@ -69,7 +69,11 @@ Status CheckOptionsValid(const std::map<string, string> &options) {
   auto job_id_iter = options.find(OPTION_EXEC_JOB_ID);
   if (job_id_iter != options.end()) {
     if (job_id_iter->second.length() > kMaxStrLen) {
-      GELOGE(PARAM_INVALID, "CheckOptionsValid job_id failed, string len > %d", kMaxStrLen);
+      GELOGE(PARAM_INVALID,"[Check][JobId]Failed,"
+             "the job_id [%s] string length: %zu > max string length: %d",
+             job_id_iter->second.c_str(), job_id_iter->second.length(), kMaxStrLen);
+      REPORT_INPUT_ERROR("E10051", std::vector<std::string>({"id","length"}),
+                         std::vector<std::string>({job_id_iter->second, std::to_string(kMaxStrLen)}));
       return FAILED;
     }
   }
@@ -84,7 +88,8 @@ Status GEInitializeImpl(const std::map<string, string> &options) {
   std::string path_base = ge::GELib::GetPath();
   auto ret = ErrorManager::GetInstance().Init(path_base);
   if (ret != SUCCESS) {
-    GELOGE(GE_CLI_INIT_FAILED, "ErrorManager init fail");
+    GELOGE(GE_CLI_INIT_FAILED,
+           "[Init][PathBase]Init failed when pass param path_base:%s", path_base.c_str());
     return ret;
   }
 
@@ -104,7 +109,9 @@ Status GEInitializeImpl(const std::map<string, string> &options) {
   bool is_proto_init = manager->Initialize(option_tmp);
   GE_TIMESTAMP_END(GEInitialize, "GEInitialize::ManagerInitialize");
   if (!is_proto_init) {
-    GELOGE(GE_CLI_INIT_FAILED, "geInitialize failed, ops proto path is invalid.");
+    GELOGE(GE_CLI_INIT_FAILED,
+           "[Init][OpsProtoPath]Loading OpsProto lib plugin failed, OpsProtoPath:%s invalid.",
+	   opsproto_path.c_str());
     return FAILED;
   }
 
@@ -127,7 +134,7 @@ Status GEInitializeImpl(const std::map<string, string> &options) {
   ret = ge::GELib::Initialize(options);
   GE_TIMESTAMP_END(GELibInitialize, "GEInitialize::GELibInitialize");
   if (ret != SUCCESS) {
-    GELOGE(GE_CLI_INIT_FAILED, "geInitialize failed, error code = %u", ret);
+    GELOGE(GE_CLI_INIT_FAILED, "[Init][GELib]Failed, error code = %u", ret);
     return FAILED;
   }
 
@@ -155,7 +162,9 @@ Status GEInitialize(const std::map<AscendString, AscendString> &options) {
   std::map<std::string, std::string> str_options;
   for (auto &option : options) {
     if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) {
-      GELOGE(FAILED, "GEInitialize options is nullptr.");
+      GELOGE(FAILED, "[Check][Param]Options invalid, first or second option is nullptr.");
+      REPORT_INNER_ERROR("E19999", "Check parameter's options invalid,"
+                         "the first or second option is nullptr.");
       return FAILED;
     }
     std::string key = option.first.GetString();
@@ -171,17 +180,17 @@ Status GEInitialize(const std::map<AscendString, AscendString> &options) {
 
 // GE finalize, releasing all resources
 Status GEFinalize() {
-  ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize);
-  GELOGT(TRACE_INIT, "GEFinalize start");
-
-  ErrorManager::GetInstance().GenWorkStreamIdDefault();
+  std::lock_guard<std::mutex> lock(g_ge_release_mutex);
   // check init status
   if (!g_ge_initialized) {
-    GELOGW("GEFinalize is called before GEInitialize");
+    GELOGW("[FINAL][FINAL]GEFinalize is called before GEInitialize");
     return SUCCESS;
   }
 
-  std::lock_guard<std::mutex> lock(g_ge_release_mutex);
+  ErrorManager::GetInstance().SetStage(ErrorMessage::kFinalize, ErrorMessage::kFinalize);
+  ErrorManager::GetInstance().GenWorkStreamIdDefault();
+  GELOGT(TRACE_INIT, "GEFinalize start");
+
   // call Finalize
   Status ret = SUCCESS;
   Status middle_ret;
@@ -237,13 +246,17 @@ Session::Session(const std::map<string, string> &options) {
   // check init status
   sessionId_ = 0;
   if (!g_ge_initialized) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized.");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Construct][Session]Failed because lack GEInitialize call before.");
+    REPORT_INNER_ERROR("E19999",
+                       "Creating session failed because lack GEInitialize call before.");
     return;
   }
   // call Initialize
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Session Constructor failed");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED, 
+           "[Construct][Session]Failed, GELib instance is nullptr or it is not InitFlag");
     return;
   }
 
@@ -256,7 +269,7 @@ Session::Session(const std::map<string, string> &options) {
   if (ret == SUCCESS) {
     sessionId_ = session_id;
   } else {
-    GELOGE(ret, "Session constructor failed, session Id not initialized");
+    GELOGE(ret, "[Construct][Session]Failed, error code:%u.", ret);
     return;
   }
   GELOGT(TRACE_STOP, "Session Constructor finished");
@@ -270,13 +283,17 @@ Session::Session(const std::map<AscendString, AscendString> &options) {
   // check init status
   sessionId_ = 0;
   if (!g_ge_initialized) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized.");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Construct][Session]Failed because lack GEInitialize call before.");
+    REPORT_INNER_ERROR("E19999",
+                       "Creating session failed because lack GEInitialize call before.");
     return;
   }
   // call Initialize
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Session Constructor failed");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Construct][Session]Failed, the GELib instance is nullptr or is not InitFlag");
     return;
   }
 
@@ -284,7 +301,9 @@ Session::Session(const std::map<AscendString, AscendString> &options) {
   std::map<std::string, std::string> str_options;
   for (auto &option : options) {
     if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) {
-      GELOGE(FAILED, "Session options is nullptr.");
+      GELOGE(FAILED, "[Construct][Session]Failed, the first or second option is nullptr.");
+      REPORT_INNER_ERROR("E19999", "Creating session's options invalid,"
+                         "the first or second option is nullptr.");
       return;
     }
     std::string key = option.first.GetString();
@@ -299,7 +318,7 @@ Session::Session(const std::map<AscendString, AscendString> &options) {
   if (ret == SUCCESS) {
     sessionId_ = session_id;
   } else {
-    GELOGE(ret, "Session constructor failed, session Id not initialized");
+    GELOGE(ret, "[Construct][Session]Failed, error code:%u.", ret);
     return;
   }
   GELOGT(TRACE_STOP, "Session Constructor finished");
@@ -331,17 +350,18 @@ Session::~Session() {
 
     ret = instance_ptr->SessionManagerObj().DestroySession(session_id);
   } catch (google::protobuf::FatalException &e) {
-    GELOGE(GE_CLI_SESS_DESTROY_FAILED, "SessionDestructor throws FatalException");
+    GELOGE(GE_CLI_SESS_DESTROY_FAILED, "[Destruct][Session]Failed because get fatalException.");
   }
 
   // check return status, return, update session id if success
   if (ret != SUCCESS) {
-    GELOGE(ret, "Session Destructor failed");
+    GELOGE(ret, "[Destruct][Session]Failed, error code:%u.", ret);
   }
 
   GELOGT(TRACE_STOP, "Session Destructor finished");
 }
 
+// Add Graph
 Status Session::AddGraph(uint32_t graph_id, const Graph &graph) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
   std::map<std::string, std::string> options;
@@ -349,25 +369,32 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph) {
   return AddGraph(graph_id, graph, options);
 }
 
+// Add Graph
 Status Session::AddGraph(uint32_t graph_id, const Graph &graph, const std::map<std::string, std::string> &options) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
   GELOGT(TRACE_INIT, "Start to add graph in Session. graph_id: %u, session_id: %lu.", graph_id, sessionId_);
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "AddGraph failed in Session.");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Add][Graph]Failed because GELib instance is nullptr or it is not InitFlag.");
+    REPORT_INNER_ERROR("E19999",
+		       "AddGraph Failed, GELib instance is nullptr or it is not InitFlag.");
     return FAILED;
   }
   GELOGD("Adding graph to session");
   Status ret = instance_ptr->SessionManagerObj().AddGraph(sessionId_, graph_id, graph, options);
   if (ret != SUCCESS) {
-    GELOGE(ret, "AddGraph failed in Session.");
+    GELOGE(ret, 
+           "[Add][Graph]Failed, error code:%u, session_id:%lu, graph_id:%u.",
+	   ret, sessionId_, graph_id);
     return FAILED;
   }
   GELOGD("AddGraph finished in Session.");
   return ret;
 }
 
+//Add Graph
 Status Session::AddGraph(uint32_t graph_id, const Graph &graph,
                          const std::map<AscendString, AscendString> &options) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
@@ -375,14 +402,19 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph,
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "AddGraph failed in Session.");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Add][Graph]Failed, the GELib instance is nullptr or is not InitFlag.");
+    REPORT_INNER_ERROR("E19999",
+                       "AddGraph Failed, GELib instance is nullptr or it is not InitFlag.");
     return FAILED;
   }
   GELOGD("Adding graph to session");
   std::map<std::string, std::string> str_options;
   for (auto &option : options) {
     if (option.first.GetString() == nullptr || option.second.GetString() == nullptr) {
-      GELOGE(FAILED, "AddGraph options is nullptr.");
+      GELOGE(FAILED, "[Add][Graph]Failed, the first or second option is nullptr.");
+      REPORT_INNER_ERROR("E19999",
+		         "Add Graph Failed, the first or second option is nullptr.");
       return FAILED;
     }
     std::string key = option.first.GetString();
@@ -391,7 +423,9 @@ Status Session::AddGraph(uint32_t graph_id, const Graph &graph,
   }
   Status ret = instance_ptr->SessionManagerObj().AddGraph(sessionId_, graph_id, graph, str_options);
   if (ret != SUCCESS) {
-    GELOGE(ret, "AddGraph failed in Session.");
+    GELOGE(ret,
+           "[Add][Graph]Failed, error code:%u, session_id:%lu, graph_id:%u.",
+	   ret, sessionId_, graph_id);
     return FAILED;
   }
   GELOGD("AddGraph finished in Session.");
@@ -405,6 +439,7 @@ Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph) {
   return AddGraphWithCopy(graph_id, graph, options);
 }
 
+// Add Graph With Copy
 Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph,
                                  const std::map<AscendString, AscendString> &options) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
@@ -412,7 +447,10 @@ Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph,
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "AddGraph failed in Session.");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Add][Graph]Failed, the GELib instance is nullptr or is not InitFlag.");
+    REPORT_INNER_ERROR("E19999",
+		       "AddGraph Failed, GELib instance is nullptr or is not InitFlag.");
     return FAILED;
   }
   std::map<std::string, std::string> str_options;
@@ -422,13 +460,16 @@ Status Session::AddGraphWithCopy(uint32_t graph_id, const Graph &graph,
   GELOGD("Adding graph to session");
   Status ret = instance_ptr->SessionManagerObj().AddGraphWithCopy(sessionId_, graph_id, graph, str_options);
   if (ret != SUCCESS) {
-    GELOGE(ret, "AddGraph failed in Session.");
+    GELOGE(ret,
+           "[Add][Graph]Failed, error code:%u, session_id:%lu, graph_id:%u.",
+	   ret, sessionId_, graph_id);
     return FAILED;
   }
   GELOGD("AddGraph finished in Session.");
   return ret;
 }
 
+// Remove Graph
 Status Session::RemoveGraph(uint32_t graph_id) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
   GELOGT(TRACE_INIT, "Session RemoveGraph start");
@@ -437,7 +478,10 @@ Status Session::RemoveGraph(uint32_t graph_id) {
   // call RemoveGraph
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (!instance_ptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Session RemoveGraph failed");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Remove][Graph]Failed, GELib instance is nullptr or is not InitFlag ");
+    REPORT_INNER_ERROR("E19999",
+		       "RemoveGraph Failed, GELib instance is nullptr or is not InitFlag.");
     return FAILED;
   }
 
@@ -445,13 +489,16 @@ Status Session::RemoveGraph(uint32_t graph_id) {
   Status ret = instance_ptr->SessionManagerObj().RemoveGraph(sessionId_, graph_id);
   // check return status, return
   if (ret != SUCCESS) {
-    GELOGE(ret, "session RemoveGraph failed");
+    GELOGE(ret,
+           "[Remove][Graph]Failed, error code:%u, session_id:%lu, graph_id:%u.",
+	   ret, sessionId_, graph_id);
     return FAILED;
   }
   GELOGT(TRACE_STOP, "Session RemoveGraph finished");
   return ret;
 }
 
+// Print Output Result
 void PrintOutputResult(std::vector<Tensor> &outputs) {
   if (outputs.empty() || outputs[0].GetData() == nullptr) {
     GELOGW("outputs is empty or data is nullptr.");
@@ -499,6 +546,7 @@ void PrintOutputResult(std::vector<Tensor> &outputs) {
   }
 }
 
+// Run Graph
 Status Session::RunGraph(uint32_t graph_id, const std::vector<Tensor> &inputs, std::vector<Tensor> &outputs) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
   GELOGT(TRACE_INIT, "Session RunGraph start");
@@ -508,14 +556,19 @@ Status Session::RunGraph(uint32_t graph_id, const std::vector<Tensor> &inputs, s
   // call RunGraph
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Session RunGraph failed");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Run][Graph]Failed, GELib instance is nullptr or is not InitFlag.");
+    REPORT_INNER_ERROR("E19999",
+	               "RunGraph Failed, GELib instance is nullptr or is not InitFlag.");
     return FAILED;
   }
   GELOGT(TRACE_RUNNING, "Running Graph");
   Status ret = instance_ptr->SessionManagerObj().RunGraph(sessionId_, graph_id, graph_inputs, outputs);
   // check return status
   if (ret != SUCCESS) {
-    GELOGE(ret, "Session RunGraph failed");
+    GELOGE(ret,
+           "[Run][Graph]Failed, error code:%u, session_id:%lu, graph_id:%u.",
+	   ret, sessionId_, graph_id);
     return FAILED;
   }
 
@@ -529,6 +582,7 @@ Status Session::RunGraph(uint32_t graph_id, const std::vector<Tensor> &inputs, s
   return ret;
 }
 
+// Register Call Back
 Status Session::RegisterCallBackFunc(const std::string &key, const pCallBackFunc &callback) {
   ErrorManager::GetInstance().GenWorkStreamIdDefault();
   return ge::GELib::GetInstance()->SessionManagerObj().RegisterCallBackFunc(sessionId_, key, callback);
@@ -543,30 +597,40 @@ Status Session::RegisterCallBackFunc(const char *key, const session::pCallBackFu
   return ge::GELib::GetInstance()->SessionManagerObj().RegisterCallBackFunc(sessionId_, str_key, callback);
 }
 
+// Build Graph
 Status Session::BuildGraph(uint32_t graph_id, const std::vector<InputTensorInfo> &inputs) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "SessionConstructor failed");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED, 
+           "[Build][Graph]Failed, the GELib instance is nullptr or is not InitFlag.");
+    REPORT_INNER_ERROR("E19999",
+		       "Build graph failed, the GELib instance is nullptr or is not InitFlag.");
     return FAILED;
   }
   GELOGT(TRACE_RUNNING, "Building Graph");
   Status ret = instance_ptr->SessionManagerObj().BuildGraph(sessionId_, graph_id, inputs);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Session BuildGraph failed");
+    GELOGE(ret,
+           "[Build][Graph]Failed, error code:%u, session_id:%lu, graph_id:%u.",
+	   ret, sessionId_, graph_id);
     return FAILED;
   }
   return SUCCESS;
 }
 
+// Run Graph Asynchronously
 Status Session::RunGraphAsync(uint32_t graph_id, const std::vector<InputTensorInfo> &inputs,
                               RunAsyncCallback callback) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute);
   ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id);
   std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "SessionConstructor failed");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Run][Graph]RunGraphAsyncFailed, the GELib instance is nullptr or is not InitFlag.");
+    REPORT_INNER_ERROR("E19999",
+		       "RunGraphAsync Failed, the GELib instance is nullptr or is not InitFlag.");
     return FAILED;
   }
   GELOGT(TRACE_RUNNING, "Run Graph Asynchronously");
@@ -575,49 +639,59 @@ Status Session::RunGraphAsync(uint32_t graph_id, const std::vector<InputTensorIn
 
   Status ret = ge::GELib::GetInstance()->SessionManagerObj().RunGraphAsync(sessionId_, graph_id, inputs, callback);
   if (ret != SUCCESS) {
-    GELOGE(ret, "SessionManager RunGraphAsync failed");
+    GELOGE(ret, "[Run][Graph]RunGraphAsync Failed, error code:%u, session_id:%lu, graph_id:%u.",
+           ret, sessionId_, graph_id);
     return FAILED;
   }
   return SUCCESS;
 }
 
+// Get Variables
 Status Session::GetVariables(const std::vector<std::string> &var_names, std::vector<Tensor> &var_values) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute);
   ErrorManager::GetInstance().GenWorkStreamIdDefault();
   auto instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "SessionConstructor failed");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED, 
+           "[Get][Variables]Failed, the GELib instance is nullptr or is not InitFlag.");
+    REPORT_INNER_ERROR("E19999",
+                       "GetVariables failed, the GELib instance is nullptr or is not InitFlag.");
     return FAILED;
   }
   GELOGT(TRACE_RUNNING, "Get Variables");
   Status ret = ge::GELib::GetInstance()->SessionManagerObj().GetVariables(sessionId_, var_names, var_values);
   if (ret != SUCCESS) {
-    GELOGE(ret, "SessionManager RunGraphAsync failed");
+    GELOGE(ret, "[Get][Variables]Failed, error code:%u, session_id:%lu.", ret, sessionId_);
     return FAILED;
   }
   return SUCCESS;
 }
 
+// Get Variables
 Status Session::GetVariables(const std::vector<AscendString> &var_names, std::vector<Tensor> &var_values) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelExecute, ErrorMessage::kModelExecute);
   ErrorManager::GetInstance().GenWorkStreamIdDefault();
   auto instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
-    GELOGE(GE_CLI_GE_NOT_INITIALIZED, "SessionConstructor failed");
+    GELOGE(GE_CLI_GE_NOT_INITIALIZED,
+           "[Get][Variables]Failed, the GELib instance is nullptr or is not InitFlag.");
+    REPORT_INNER_ERROR("E19999",
+                       "GetVariables failed, the GELib instance is nullptr or is not InitFlag.");
     return FAILED;
   }
   GELOGT(TRACE_RUNNING, "Get Variables");
   std::vector<ge::string> str_var_names;
   for (auto &var_name : var_names) {
     if (var_name.GetString() == nullptr) {
-      GELOGE(FAILED, "GetVariables name is nullptr.");
+      GELOGE(FAILED, "[Get][Variable]Failed, variables' names are nullptr.");
+      REPORT_INNER_ERROR("E19999", "GetVariables failed, variables' names are nullptr.");
       return FAILED;
     }
     str_var_names.emplace_back(var_name.GetString());
   }
   Status ret = ge::GELib::GetInstance()->SessionManagerObj().GetVariables(sessionId_, str_var_names, var_values);
   if (ret != SUCCESS) {
-    GELOGE(ret, "SessionManager RunGraphAsync failed");
+    GELOGE(ret, "[Get][Variables]Failed, error code:%u, session_id:%lu.", ret, sessionId_);
     return FAILED;
   }
   return SUCCESS;

ge/client/proto/insert_op.proto

@@ -88,6 +88,7 @@ message AippOpParams {
 	int32 right_padding_size = 69;
 	int32 top_padding_size = 70;
 	int32 bottom_padding_size = 71;
+	float padding_value = 72;
 
 	int32 mean_chn_0 = 10;
 	int32 mean_chn_1 = 11;

ge/common/CMakeLists.txt

@@ -16,6 +16,7 @@ set(PROTO_LIST
 )
 
 protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
+protobuf_generate(ge_static PROTO_STATIC_SRCS PROTO_STATIC_HDRS ${PROTO_LIST})
 
 set(SRC_LIST
     "context/ctx.cc"
@@ -127,7 +128,7 @@ target_link_libraries(ge_common PRIVATE
 )
 
 ############ libge_common.a ############
-add_library(ge_common_static STATIC ${SRC_LIST} ${PROTO_HDRS})
+add_library(ge_common_static STATIC ${SRC_LIST} ${PROTO_STATIC_HDRS})
 target_compile_definitions(ge_common_static PRIVATE
     PROTOBUF_INLINE_NOT_IN_HEADERS=0
     HOST_VISIBILITY
@@ -158,7 +159,7 @@ target_include_directories(ge_common_static PRIVATE
     ${METADEF_DIR}/inc/external/graph
     ${METADEF_DIR}/inc/graph
     ${CMAKE_BINARY_DIR}
-    ${CMAKE_BINARY_DIR}/proto/ge
+    ${CMAKE_BINARY_DIR}/proto/ge_static
     #### yellow zone ####
     ${GE_DEPEND_DIR}/inc
     ${GE_DEPEND_DIR}/inc/cce

ge/common/auth/file_saver.cc

@@ -33,7 +33,8 @@ const int kFileOpSuccess = 0;
 namespace ge {
 Status FileSaver::OpenFile(int32_t &fd, const std::string &file_path) {
   if (CheckPath(file_path) != SUCCESS) {
-    GELOGE(FAILED, "Check output file failed.");
+    GELOGE(FAILED, "[Check][FilePath]Check output file failed, file_path:%s.", file_path.c_str());
+    REPORT_CALL_ERROR("E19999", "Check output file failed, file_path:%s.", file_path.c_str());
     return FAILED;
   }
 
@@ -45,7 +46,8 @@ Status FileSaver::OpenFile(int32_t &fd, const std::string &file_path) {
   fd = mmOpen2(real_path, M_RDWR | M_CREAT | O_TRUNC, mode);
   if (fd == EN_INVALID_PARAM || fd == EN_ERROR) {
     // -1: Failed to open file; - 2: Illegal parameter
-    GELOGE(FAILED, "Open file failed. mmpa_errno = %d, %s", fd, strerror(errno));
+    GELOGE(FAILED, "[Open][File]Failed. mmpa_errno = %d, %s", fd, strerror(errno));
+    REPORT_INNER_ERROR("E19999", "Open file failed, mmpa_errno = %d, error:%s.", fd, strerror(errno));
     return FAILED;
   }
   return SUCCESS;
@@ -62,7 +64,9 @@ Status FileSaver::WriteData(const void *data, uint32_t size, int32_t fd) {
     while (size > size_1g) {
       write_count = mmWrite(fd, reinterpret_cast<void *>(seek), size_1g);
       if (write_count == EN_INVALID_PARAM || write_count == EN_ERROR) {
-        GELOGE(FAILED, "Write data failed. mmpa_errorno = %ld, %s", write_count, strerror(errno));
+        GELOGE(FAILED, "[Write][Data]Failed, mmpa_errorno = %ld, error:%s", write_count, strerror(errno));
+	REPORT_INNER_ERROR("E19999", "Write data failed, mmpa_errorno = %ld, error:%s.",
+			   write_count, strerror(errno));
         return FAILED;
       }
       size -= size_1g;
@@ -75,7 +79,9 @@ Status FileSaver::WriteData(const void *data, uint32_t size, int32_t fd) {
 
   // -1: Failed to write to file; - 2: Illegal parameter
   if (write_count == EN_INVALID_PARAM || write_count == EN_ERROR) {
-    GELOGE(FAILED, "Write data failed. mmpa_errorno = %ld, %s", write_count, strerror(errno));
+    GELOGE(FAILED, "[Write][Data]Failed. mmpa_errorno = %ld, error:%s", write_count, strerror(errno));
+    REPORT_INNER_ERROR("E19999", "Write data failed, mmpa_errorno = %ld, error:%s.",
+                       write_count, strerror(errno));
     return FAILED;
   }
 
@@ -85,7 +91,8 @@ Status FileSaver::WriteData(const void *data, uint32_t size, int32_t fd) {
 Status FileSaver::SaveWithFileHeader(const std::string &file_path, const ModelFileHeader &file_header, const void *data,
                                      int len) {
   if (data == nullptr || len <= 0) {
-    GELOGE(FAILED, "Model_data is null or the length[%d] less than 1.", len);
+    GELOGE(FAILED, "[Check][Param]Failed, model_data is null or the length[%d] is less than 1.", len);
+    REPORT_INNER_ERROR("E19999", "Save file failed, model_data is null or the length:%d is less than 1.", len);
     return FAILED;
   }
 
@@ -104,7 +111,8 @@ Status FileSaver::SaveWithFileHeader(const std::string &file_path, const ModelFi
   } while (0);
   // Close file
   if (mmClose(fd) != 0) {  // mmClose 0: success
-    GELOGE(FAILED, "Close file failed.");
+    GELOGE(FAILED, "[Close][File]Failed, error_code:%u errmsg:%s", ret, strerror(errno));
+    REPORT_INNER_ERROR("E19999", "Close file failed, error_code:%u errmsg:%s", ret, strerror(errno));
     ret = FAILED;
   }
   return ret;
@@ -140,60 +148,95 @@ Status FileSaver::SaveWithFileHeader(const std::string &file_path, const ModelFi
     }
   } while (0);
   // Close file
-  GE_CHK_BOOL_RET_STATUS(mmClose(fd) == EN_OK, FAILED, "Close file failed.");
+  if (mmClose(fd) != EN_OK) {
+    GELOGE(FAILED, "[Close][File]Failed, error_code:%u errmsg:%s", ret, strerror(errno));
+    REPORT_CALL_ERROR("E19999", "Close file failed, error_code:%u errmsg:%s", ret, strerror(errno));
+    ret = FAILED;
+  }
   return ret;
 }
 
 Status FileSaver::SaveToBuffWithFileHeader(const ModelFileHeader &file_header,
                                            ModelPartitionTable &model_partition_table,
-                                           const std::vector<ModelPartition> &partitionDatas,
+                                           const std::vector<ModelPartition> &partition_datas,
+                                           ge::ModelBufferData &model) {
+  const vector<ModelPartitionTable *> model_partition_tables = { &model_partition_table };
+  const std::vector<std::vector<ModelPartition>> all_partition_datas = { partition_datas };
+  return SaveToBuffWithFileHeader(file_header, model_partition_tables, all_partition_datas, model);
+}
+
+Status FileSaver::SaveToBuffWithFileHeader(const ModelFileHeader &file_header,
+                                           const vector<ModelPartitionTable *> &model_partition_tables,
+                                           const std::vector<std::vector<ModelPartition>> &all_partition_datas,
                                            ge::ModelBufferData &model) {
-  GE_CHK_BOOL_RET_STATUS(
-      !partitionDatas.empty() && model_partition_table.num != 0 && model_partition_table.num == partitionDatas.size(),
-      FAILED, "Invalid param:partition data size is (%u), model_partition_table.num is (%zu).",
-      model_partition_table.num, partitionDatas.size());
-  uint32_t model_header_size = sizeof(ModelFileHeader);
-  uint32_t table_size = static_cast<uint32_t>(SIZE_OF_MODEL_PARTITION_TABLE(model_partition_table));
-  uint32_t total_size = model_header_size + table_size;
-
-  for (const auto &partitionData : partitionDatas) {
-    auto ret = ge::CheckUint32AddOverflow(total_size, partitionData.size);
-    GE_CHK_BOOL_RET_STATUS(ret == SUCCESS, FAILED, "add uint32 overflow!");
-    total_size = total_size + partitionData.size;
+  GE_CHK_BOOL_RET_STATUS(model_partition_tables.size() == all_partition_datas.size(), PARAM_INVALID,
+                         "Model table size %zu does not match partition size %zu.",
+                         model_partition_tables.size(), all_partition_datas.size());
+  for (size_t index = 0; index < model_partition_tables.size(); ++index) {
+    auto &cur_partiton_data = all_partition_datas[index];
+    auto &cur_model_partition_table = *model_partition_tables[index];
+    GE_CHK_BOOL_RET_STATUS(!cur_partiton_data.empty() && cur_model_partition_table.num != 0
+                           && cur_model_partition_table.num == cur_partiton_data.size(), FAILED,
+                           "Invalid param: partition data size is (%zu), model_partition_table.num is (%u).",
+                           cur_partiton_data.size(), cur_model_partition_table.num);
   }
+
+  uint64_t model_header_size = sizeof(ModelFileHeader);
+  uint64_t total_size = model_header_size;
+  for (size_t index = 0; index < model_partition_tables.size(); ++index) {
+    auto &cur_model_partition_table = *model_partition_tables[index];
+    total_size += static_cast<uint64_t>(SIZE_OF_MODEL_PARTITION_TABLE(cur_model_partition_table));
+    auto &cur_partition_data = all_partition_datas[index];
+    for (const auto &partition_data : cur_partition_data) {
+      auto ret = ge::CheckUint64AddOverflow(total_size, partition_data.size);
+      GE_CHK_BOOL_RET_STATUS(ret == SUCCESS, FAILED, "Add uint64 overflow!");
+      total_size += partition_data.size;
+    }
+  }
+  // save to buff
   auto buff = reinterpret_cast<uint8_t *>(malloc(total_size));
-  GE_CHK_BOOL_RET_STATUS(buff != nullptr, FAILED, "malloc failed!");
-  GE_PRINT_DYNAMIC_MEMORY(malloc, "file buffer.", total_size)
+  GE_CHK_BOOL_RET_STATUS(buff != nullptr, FAILED, "Malloc failed!");
+  GE_PRINT_DYNAMIC_MEMORY(malloc, "File buffer.", total_size)
   model.data.reset(buff, [](uint8_t *buff) {
     GELOGD("Free online model memory.");
     free(buff);
     buff = nullptr;
   });
   model.length = total_size;
-  uint32_t left_space = total_size;
-  auto ret_mem1 = memcpy_s(buff, left_space, reinterpret_cast<void *>(const_cast<ModelFileHeader *>(&file_header)),
-                           model_header_size);
-  GE_CHK_BOOL_RET_STATUS(ret_mem1 == 0, FAILED, "memcpy_s failed!");
+  uint64_t left_space = total_size;
+  auto ret_mem = memcpy_s(buff, left_space, reinterpret_cast<void *>(const_cast<ModelFileHeader *>(&file_header)),
+                          model_header_size);
+  GE_CHK_BOOL_RET_STATUS(ret_mem == EOK, FAILED, "Memcpy_s failed!");
   buff += model_header_size;
   left_space -= model_header_size;
-  auto ret_mem2 = memcpy_s(buff, left_space, reinterpret_cast<void *>(&model_partition_table), table_size);
-  GE_CHK_BOOL_RET_STATUS(ret_mem2 == 0, FAILED, "memcpy_s failed!");
-  buff += table_size;
-  left_space -= table_size;
-  for (const auto &partitionData : partitionDatas) {
-    auto ret_mem3 = memcpy_s(buff, left_space, reinterpret_cast<void *>(const_cast<uint8_t *>(partitionData.data)),
-                             partitionData.size);
-    GE_CHK_BOOL_RET_STATUS(ret_mem3 == 0, FAILED, "memcpy failed!");
-    buff += partitionData.size;
-    left_space -= partitionData.size;
+
+  for (size_t index = 0; index < model_partition_tables.size(); ++index) {
+    auto &cur_tabel = *model_partition_tables[index];
+    uint64_t table_size = static_cast<uint64_t>(SIZE_OF_MODEL_PARTITION_TABLE(cur_tabel));
+    ret_mem = memcpy_s(buff, left_space, reinterpret_cast<void *>(&cur_tabel), table_size);
+    GE_CHK_BOOL_RET_STATUS(ret_mem == EOK, FAILED, "Memcpy_s failed!");
+    buff += table_size;
+    left_space -= table_size;
+    auto &cur_partition_data = all_partition_datas[index];
+    for (const auto &partition_data : cur_partition_data) {
+      ret_mem = memcpy_s(buff, left_space, reinterpret_cast<void *>(const_cast<uint8_t *>(partition_data.data)),
+                         partition_data.size);
+      GE_CHK_BOOL_RET_STATUS(ret_mem == EOK, FAILED, "Memcpy_s failed!");
+      buff += partition_data.size;
+      left_space -= partition_data.size;
+    }
   }
+
   return SUCCESS;
 }
 
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status FileSaver::CheckPath(const std::string &file_path) {
   // Determine file path length
   if (file_path.size() >= MMPA_MAX_PATH) {
-    GELOGE(FAILED, "Path is too long:%zu", file_path.size());
+    GELOGE(FAILED, "[Check][FilePath]Failed, file path's length:%zu > mmpa_max_path:%d",
+           file_path.size(), MMPA_MAX_PATH);
+    REPORT_INNER_ERROR("E19999", "Check file path failed, file path's length:%zu > mmpa_max_path:%d",
+                       file_path.size(), MMPA_MAX_PATH);
     return FAILED;
   }
 
@@ -212,7 +255,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status FileSaver::CheckPath(con
   // If there is a path before the file name, create the path
   if (path_split_pos != -1) {
     if (CreateDirectory(std::string(file_path).substr(0, static_cast<size_t>(path_split_pos))) != kFileOpSuccess) {
-      GELOGE(FAILED, "CreateDirectory failed, file path:%s.", file_path.c_str());
+      GELOGE(FAILED, "[Create][Directory]Failed, file path:%s.", file_path.c_str());
       return FAILED;
     }
   }
@@ -223,7 +266,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status FileSaver::CheckPath(con
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status
 FileSaver::SaveToFile(const string &file_path, const ge::ModelData &model, const ModelFileHeader *model_file_header) {
   if (file_path.empty() || model.model_data == nullptr || model.model_len == 0) {
-    GELOGE(FAILED, "Incorrected input param. file_path.empty() || model.model_data == nullptr || model.model_len == 0");
+    GELOGE(FAILED, "[Save][File]Incorrect input param, file_path is empty or model_data is nullptr or model_len is 0");
+    REPORT_INNER_ERROR("E19999", "Save file failed, at least one of the input parameters(file_path, model_data, model_len) is incorrect");
     return FAILED;
   }
 
@@ -240,7 +284,8 @@ FileSaver::SaveToFile(const string &file_path, const ge::ModelData &model, const
 
   const Status ret = SaveWithFileHeader(file_path, file_header, model.model_data, file_header.length);
   if (ret != SUCCESS) {
-    GELOGE(FAILED, "Save file failed, file_path:%s, file header len:%u.", file_path.c_str(), file_header.length);
+    GELOGE(FAILED, "[Save][File]Failed, file_path:%s, file_header_len:%u, error_code:%u.",
+		    file_path.c_str(), file_header.length, ret);
     return FAILED;
   }
 
@@ -305,7 +350,7 @@ Status FileSaver::SaveWithFileHeader(const std::string &file_path, const ModelFi
       // Write partition data
       auto &cur_partition_datas = all_partition_datas[index];
       for (const auto &partition_data : cur_partition_datas) {
-        GELOGI("GC:size[%u]", partition_data.size);
+        GELOGI("part_size[%u]", partition_data.size);
         GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
             WriteData(static_cast<const void *>(partition_data.data), partition_data.size, fd) != SUCCESS, ret = FAILED;
             break);
@@ -313,14 +358,19 @@ Status FileSaver::SaveWithFileHeader(const std::string &file_path, const ModelFi
     }
   } while (0);
   // Close file
-  GE_CHK_BOOL_RET_STATUS(mmClose(fd) == EN_OK, FAILED, "Close file failed.");
+  if (mmClose(fd) != 0) {  // mmClose 0: success
+    GELOGE(FAILED, "[Close][File]Failed, error_code:%u errmsg:%s", ret, strerror(errno));
+    REPORT_CALL_ERROR("E19999", "Close file failed, error_code:%u errmsg:%s", ret, strerror(errno));
+    ret = FAILED;
+  }
   return ret;
 }
 
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status FileSaver::SaveToFile(const string &file_path, const void *data,
                                                                               int len) {
   if (data == nullptr || len <= 0) {
-    GELOGE(FAILED, "Model_data is null or the length[%d] less than 1.", len);
+    GELOGE(FAILED, "[Check][Param]Failed, model_data is null or the length[%d] is less than 1.", len);
+    REPORT_INNER_ERROR("E19999", "Save file failed, the model_data is null or its length:%d is less than 1.", len);
     return FAILED;
   }
 
@@ -335,7 +385,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status FileSaver::SaveToFile(co
 
   // Close file
   if (mmClose(fd) != 0) {  // mmClose 0: success
-    GELOGE(FAILED, "Close file failed.");
+    GELOGE(FAILED, "[Close][File]Failed, error_code:%u errmsg:%s", ret, strerror(errno));
+    REPORT_CALL_ERROR("E19999", "Close file failed, error_code:%u errmsg:%s", ret, strerror(errno));
     ret = FAILED;
   }
   return ret;

ge/common/auth/file_saver.h

@@ -80,9 +80,14 @@ class FileSaver {
 
   static Status SaveToBuffWithFileHeader(const ModelFileHeader &file_header,
                                             ModelPartitionTable &model_partition_table,
-                                            const std::vector<ModelPartition> &partitionDatas,
+                                            const std::vector<ModelPartition> &partition_datas,
                                             ge::ModelBufferData& model);
 
+  static Status SaveToBuffWithFileHeader(const ModelFileHeader &file_header,
+                                         const std::vector<ModelPartitionTable *> &model_partition_tables,
+                                         const std::vector<std::vector<ModelPartition>> &all_partition_datas,
+                                         ge::ModelBufferData &model);
+
   static Status SaveToFile(const string &file_path, const void *data, int len);
 
  protected:
@@ -113,8 +118,8 @@ class FileSaver {
                                    ModelPartitionTable &model_partition_table,
                                    const std::vector<ModelPartition> &partition_datas);
   static Status SaveWithFileHeader(const std::string &file_path, const ModelFileHeader &file_header,
-                                       vector<ModelPartitionTable *> &model_partition_tables,
-                                       const vector<vector<ModelPartition>> &all_partition_datas);
+                                   std::vector<ModelPartitionTable *> &model_partition_tables,
+                                   const std::vector<std::vector<ModelPartition>> &all_partition_datas);
 };
 }  // namespace ge
 #endif  // GE_COMMON_AUTH_FILE_SAVER_H_

ge/common/debug/memory_dumper.cc

@@ -41,14 +41,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::DumpToFile
   GE_CHECK_NOTNULL(filename);
   GE_CHECK_NOTNULL(data);
   if (len == 0) {
-    GELOGE(FAILED, "len is 0.");
+    GELOGE(FAILED, "[Check][Param]Failed, data length is 0.");
+    REPORT_INNER_ERROR("E19999", "Check param failed, data length is 0.");
     return PARAM_INVALID;
   }
 
   // Open the file
   int fd = OpenFile(filename);
   if (fd == kInvalidFd) {
-    GELOGE(FAILED, "Open file failed.");
+    GELOGE(FAILED, "[Open][File]Failed, filename:%s.", filename);
+    REPORT_INNER_ERROR("E19999", "Opne file failed, filename:%s.", filename);
     return FAILED;
   }
 
@@ -57,13 +59,15 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::DumpToFile
   int32_t mmpa_ret = mmWrite(fd, data, len);
   // mmWrite return -1:Failed to write data to file；return -2:Invalid parameter
   if (mmpa_ret == EN_ERROR || mmpa_ret == EN_INVALID_PARAM) {
-    GELOGE(FAILED, "Write to file failed. errno = %d, %s", mmpa_ret, strerror(errno));
+    GELOGE(FAILED, "[Write][Data]Failed, errno = %d, error:%s", mmpa_ret, strerror(errno));
+    REPORT_INNER_ERROR("E19999", "Write data failed, errno = %d, error:%s.", mmpa_ret, strerror(errno));
     ret = FAILED;
   }
 
   // Close the file
   if (mmClose(fd) != EN_OK) {  // mmClose return 0: success
-    GELOGE(FAILED, "Close file failed.");
+    GELOGE(FAILED, "[Close][File]Failed, error_code:%u, filename:%s.", ret, filename);
+    REPORT_INNER_ERROR("E19999", "Close file failed, error_code:%u, filename:%s.", ret, filename);
     ret = FAILED;
   }
 
@@ -89,7 +93,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::Open(const
 
   fd_ = OpenFile(filename);
   if (fd_ == kInvalidFd) {
-    GELOGE(FAILED, "Open %s failed.", filename);
+    GELOGE(FAILED, "[Open][File]Failed, filename:%s.", filename);
+    REPORT_INNER_ERROR("E19999", "Open file:%s failed.", filename);
     return FAILED;
   }
 
@@ -104,7 +109,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status MemoryDumper::Dump(void
   int32_t mmpa_ret = mmWrite(fd_, data, len);
   // mmWrite return -1:failed to write data to file；return -2:invalid parameter
   if (mmpa_ret == EN_ERROR || mmpa_ret == EN_INVALID_PARAM) {
-    GELOGE(FAILED, "Write to file failed. errno = %d, %s", mmpa_ret, strerror(errno));
+    GELOGE(FAILED, "[Write][Data]Failed, errno = %d, error:%s", mmpa_ret, strerror(errno));
+    REPORT_INNER_ERROR("E19999", "Write data to file failed, errno = %d, error:%s.", mmpa_ret, strerror(errno));
     return FAILED;
   }
 
@@ -155,9 +161,10 @@ int MemoryDumper::OpenFile(const char *filename) {
   // Using the O_EXCL, if the file already exists,return failed to avoid privilege escalation vulnerability.
   mmMode_t mode = M_IRUSR | M_IWUSR;
 
-  int32_t fd = mmOpen2(real_path.c_str(), M_RDWR | M_CREAT | O_TRUNC, mode);
+  int32_t fd = mmOpen2(real_path.c_str(), M_RDWR | M_CREAT | M_APPEND, mode);
   if (fd == EN_ERROR || fd == EN_INVALID_PARAM) {
-    GELOGE(kInvalidFd, "open file failed. errno = %d, %s", fd, strerror(errno));
+    GELOGE(kInvalidFd, "[Open][File]Failed. errno = %d, error:%s, filename:%s.",
+           fd, strerror(errno), filename);
     return kInvalidFd;
   }
   return fd;

ge/common/dump/dump_manager.cc

@@ -96,7 +96,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf
   dump_mode = dump_config.dump_mode;
   GELOGI("Dump mode is %s", dump_mode.c_str());
   dump_properties.SetDumpMode(dump_mode);
-  dump_properties_map_.emplace(kInferSessionId, dump_properties);
+  dump_properties_map_[kInferSessionId] = dump_properties;
 
   return SUCCESS;
 }

ge/common/dump/dump_op.cc

@@ -20,6 +20,7 @@
 #include "common/ge/datatype_util.h"
 #include "framework/common/debug/ge_log.h"
 #include "framework/common/util.h"
+#include "framework/common/types.h"
 #include "graph/anchor.h"
 #include "graph/ge_tensor.h"
 #include "graph/op_desc.h"
@@ -55,8 +56,10 @@ void DumpOp::SetLoopAddr(void *global_step, void *loop_per_iter, void *loop_cond
   loop_cond_ = reinterpret_cast<uintptr_t>(loop_cond);
 }
 
-void DumpOp::SetDynamicModelInfo(const string &dynamic_model_name, uint32_t dynamic_model_id) {
+void DumpOp::SetDynamicModelInfo(const string &dynamic_model_name, const string &dynamic_om_name,
+                                 uint32_t dynamic_model_id) {
   dynamic_model_name_ = dynamic_model_name;
+  dynamic_om_name_ = dynamic_om_name;
   dynamic_model_id_ = dynamic_model_id;
 }
 
@@ -200,6 +203,32 @@ Status DumpOp::ExecutorDumpOp(aicpu::dump::OpMappingInfo &op_mapping_info) {
   return SUCCESS;
 }
 
+Status DumpOp::SetDumpModelName(aicpu::dump::OpMappingInfo &op_mapping_info) {
+  if (dynamic_model_name_.empty() && dynamic_om_name_.empty()) {
+    GELOGI("Single op dump, no need set model name");
+    return SUCCESS;
+  }
+  std::set<std::string> model_list = dump_properties_.GetAllDumpModel();
+  bool not_find_by_omname = model_list.find(dynamic_om_name_) == model_list.end();
+  bool not_find_by_modelname = model_list.find(dynamic_model_name_) == model_list.end();
+  std::string dump_model_name = not_find_by_omname ? dynamic_model_name_ : dynamic_om_name_;
+  if (model_list.find(DUMP_ALL_MODEL) == model_list.end()) {
+    if (not_find_by_omname && not_find_by_modelname) {
+      std::string model_list_str;
+      for (auto &model : model_list) {
+        model_list_str += "[" + model + "].";
+      }
+      GELOGW("Model %s will not be set to dump, dump list: %s", dump_model_name.c_str(), model_list_str.c_str());
+      return FAILED;
+    }
+  }
+  if (!dump_model_name.empty() && dump_properties_.IsDumpOpen()) {
+    GELOGI("Dump model name is %s", dump_model_name.c_str());
+    op_mapping_info.set_model_name(dump_model_name);
+  }
+  return SUCCESS;
+}
+
 Status DumpOp::LaunchDumpOp() {
   GELOGI("Start to launch dump op %s", op_desc_->GetName().c_str());
   int32_t device_id = 0;
@@ -209,8 +238,7 @@ Status DumpOp::LaunchDumpOp() {
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
   if (device_id < 0) {
-    GELOGE(ACL_ERROR_GE_INTERNAL_ERROR,
-           "Check device_id failed, device_id = %d, which should be not less than 0.",
+    GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "Check device_id failed, device_id = %d, which should be not less than 0.",
            device_id);
     return ACL_ERROR_GE_INTERNAL_ERROR;
   }
@@ -220,11 +248,12 @@ Status DumpOp::LaunchDumpOp() {
   op_mapping_info.set_flag(kAicpuLoadFlag);
   op_mapping_info.set_dump_step(dump_properties_.GetDumpStep());
   op_mapping_info.set_model_id(dynamic_model_id_);
-  if (!dynamic_model_name_.empty() && dump_properties_.IsDumpOpen()) {
-    op_mapping_info.set_model_name(dynamic_model_name_);
+
+  if (SetDumpModelName(op_mapping_info) != SUCCESS) {
+    return SUCCESS;
   }
   SetOpMappingLoopAddr(global_step_, loop_per_iter_, loop_cond_, op_mapping_info);
-  GELOGI("Dump step is %s ,dump path is %s ,in Launch dump op", dump_properties_.GetDumpStep().c_str(),
+  GELOGI("Dump step is %s ,dump path is %s in Launch dump op", dump_properties_.GetDumpStep().c_str(),
          dump_path.c_str());
   uint32_t task_id = 0;
   uint32_t stream_id = 0;
@@ -273,4 +302,4 @@ Status DumpOp::LaunchDumpOp() {
   }
   return SUCCESS;
 }
-}  // namesapce ge
+}  // namespace ge

ge/common/dump/dump_op.h

@@ -34,12 +34,13 @@ class DumpOp {
                    vector<uintptr_t> output_addrs, rtStream_t stream);
   Status LaunchDumpOp();
   void SetLoopAddr(void *global_step, void *loop_per_iter, void *loop_cond);
-  void SetDynamicModelInfo(const string &dynamic_model_name, uint32_t dynamic_model_id);
+  void SetDynamicModelInfo(const string &dynamic_model_name, const string &dynamic_om_name, uint32_t dynamic_model_id);
 
  private:
   Status ExecutorDumpOp(aicpu::dump::OpMappingInfo &op_mapping_info);
   Status DumpOutput(aicpu::dump::Task &task);
   Status DumpInput(aicpu::dump::Task &task);
+  Status SetDumpModelName(aicpu::dump::OpMappingInfo &op_mapping_info);
 
   DumpProperties dump_properties_;
   OpDescPtr op_desc_;
@@ -54,6 +55,7 @@ class DumpOp {
   uintptr_t loop_cond_;
 
   std::string dynamic_model_name_;
+  std::string dynamic_om_name_;
   std::uint32_t dynamic_model_id_;
 };
 }  // namespace ge

ge/common/dump/dump_properties.cc

@@ -35,14 +35,14 @@ const std::string kDumpStatusOpen = "on";
 const uint32_t kAicoreOverflow = (0x1 << 0);
 const uint32_t kAtomicOverflow = (0x1 << 1);
 const uint32_t kAllOverflow = (kAicoreOverflow | kAtomicOverflow);
-}
+}  // namespace
 namespace ge {
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties::DumpProperties(const DumpProperties &other) {
   CopyFrom(other);
 }
 
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties &DumpProperties::operator=(
-    const DumpProperties &other) {
+  const DumpProperties &other) {
   CopyFrom(other);
   return *this;
 }
@@ -97,7 +97,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::InitByOpti
 
 // The following is the new dump scenario of the fusion operator
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::AddPropertyValue(
-    const std::string &model, const std::set<std::string> &layers) {
+  const std::string &model, const std::set<std::string> &layers) {
   for (const std::string &layer : layers) {
     GELOGI("This model %s config to dump layer %s", model.c_str(), layer.c_str());
   }
@@ -138,7 +138,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set<std::string> DumpPrope
 }
 
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set<std::string> DumpProperties::GetPropertyValue(
-    const std::string &model) const {
+  const std::string &model) const {
   auto iter = model_dump_properties_map_.find(model);
   if (iter != model_dump_properties_map_.end()) {
     return iter->second;
@@ -147,8 +147,9 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set<std::string> DumpPrope
 }
 
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool DumpProperties::IsLayerNeedDump(
-    const std::string &model, const std::string &om_name, const std::string &op_name) const {
+  const std::string &model, const std::string &om_name, const std::string &op_name) const {
   // if dump all
+  GELOGD("model name is %s om name is %s op is %s in layer need dump", model.c_str(), om_name.c_str(), op_name.c_str());
   if (model_dump_properties_map_.find(DUMP_ALL_MODEL) != model_dump_properties_map_.end()) {
     return true;
   }
@@ -203,7 +204,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const std::string &DumpProperti
 }
 
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::SetDumpOpSwitch(
-    const std::string &dump_op_switch) {
+  const std::string &dump_op_switch) {
   dump_op_switch_ = dump_op_switch;
 }
 
@@ -270,4 +271,4 @@ void DumpProperties::SetDumpDebugOptions() {
     GELOGI("ge.exec.enableDumpDebug is false or is not set.");
   }
 }
-}  // namespace
+}  // namespace ge

ge/common/dump/exception_dumper.cc

@@ -0,0 +1,241 @@
+/**
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common/dump/exception_dumper.h"
+
+#include "common/ge/datatype_util.h"
+#include "common/debug/memory_dumper.h"
+#include "framework/common/debug/log.h"
+#include "graph/manager/util/debug.h"
+#include "graph/utils/tensor_utils.h"
+#include "graph/load/model_manager/model_utils.h"
+#include "proto/dump_task.pb.h"
+
+namespace {
+static uint64_t GetNowTime() {
+  uint64_t ret = 0;
+  mmTimeval tv;
+  if (mmGetTimeOfDay(&tv, nullptr) == 0) {
+    ret = tv.tv_sec * 1000000ULL + tv.tv_usec;
+  }
+
+  return ret;
+}
+
+static void ReplaceStringElem(std::string &str) {
+  for_each(str.begin(), str.end(), [](char &ch) {
+    if ((ch == ' ') || (ch == '.') || (ch == '/') || (ch == '\\')) {
+      ch = '_';
+    }
+  });
+}
+
+static void SetDumpData(const ge::OpDescInfo &op_desc_info, toolkit::dumpdata::DumpData &dump_data) {
+  dump_data.set_version("2.0");
+  dump_data.set_dump_time(GetNowTime());
+  dump_data.set_op_name(op_desc_info.op_name);
+  for (size_t i = 0; i < op_desc_info.input_format.size(); ++i) {
+    toolkit::dumpdata::OpInput input;
+    input.set_data_type(toolkit::dumpdata::OutputDataType(
+        ge::DataTypeUtil::GetIrDataType(op_desc_info.input_data_type[i])));
+    input.set_format(toolkit::dumpdata::OutputFormat(op_desc_info.input_format[i]));
+    for (auto dim : op_desc_info.input_shape[i]) {
+      input.mutable_shape()->add_dim(dim);
+    }
+    input.set_size(op_desc_info.input_size[i]);
+    GELOGI("[Set][DumpData] The input size int exception is %ld", op_desc_info.input_size[i]);
+    dump_data.mutable_input()->Add(std::move(input));
+  }
+
+  for (size_t j = 0; j < op_desc_info.output_format.size(); ++j) {
+    toolkit::dumpdata::OpOutput output;
+    output.set_data_type(toolkit::dumpdata::OutputDataType(
+        ge::DataTypeUtil::GetIrDataType(op_desc_info.output_data_type[j])));
+    output.set_format(toolkit::dumpdata::OutputFormat(op_desc_info.output_format[j]));
+    for (auto dim : op_desc_info.output_shape[j]) {
+      output.mutable_shape()->add_dim(dim);
+    }
+    output.set_size(op_desc_info.output_size[j]);
+    GELOGI("[Set][DumpData] The output size int exception is %ld", op_desc_info.output_size[j]);
+    dump_data.mutable_output()->Add(std::move(output));
+  }
+}
+}  // namespace
+
+namespace ge {
+ExceptionDumper::~ExceptionDumper() {}
+
+void ExceptionDumper::SaveDumpOpInfo(const OpDescPtr &op, uint32_t task_id, uint32_t stream_id,
+                                     vector<void *> &input_addrs, vector<void *> &output_addrs) {
+  OpDescInfo op_desc_info;
+  SaveOpDescInfo(op, task_id, stream_id, op_desc_info);
+  op_desc_info.input_addrs = input_addrs;
+  op_desc_info.output_addrs = output_addrs;
+  op_desc_info_.emplace_back(std::move(op_desc_info));
+}
+
+void ExceptionDumper::SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op,
+                                     uint32_t task_id, uint32_t stream_id) {
+  OpDescInfo op_desc_info;
+  SaveOpDescInfo(op, task_id, stream_id, op_desc_info);
+  op_desc_info.input_addrs = ModelUtils::GetInputDataAddrs(model_param, op);
+  op_desc_info.output_addrs = ModelUtils::GetOutputDataAddrs(model_param, op);
+  op_desc_info_.emplace_back(std::move(op_desc_info));
+}
+
+void ExceptionDumper::SaveOpDescInfo(const OpDescPtr &op, uint32_t task_id, uint32_t stream_id,
+                                     OpDescInfo &op_desc_info) {
+  if (op == nullptr) {
+    GELOGW("[Save][OpExceptionInfo] op desc ptr is null.");
+    return;
+  }
+  GELOGD("[Save][OpExceptionInfo] Start to save dump op [%s] info of task_id: %u, stream_id: %u",
+         op->GetName().c_str(), task_id, stream_id);
+  op_desc_info.op_name = op->GetName();
+  op_desc_info.op_type = op->GetType();
+  op_desc_info.task_id = task_id;
+  op_desc_info.stream_id = stream_id;
+  for (size_t i = 0; i < op->GetAllInputsSize(); ++i) {
+    GeTensorDescPtr input_tensor_desc = op->MutableInputDesc(i);
+    if (input_tensor_desc == nullptr) {
+      continue;
+    }
+    op_desc_info.input_format.emplace_back(input_tensor_desc->GetFormat());
+    op_desc_info.input_shape.emplace_back(input_tensor_desc->GetShape().GetDims());
+    op_desc_info.input_data_type.emplace_back(input_tensor_desc->GetDataType());
+    int64_t input_size = 0;
+
+    if (TensorUtils::GetTensorSizeInBytes(*input_tensor_desc, input_size) != SUCCESS) {
+      GELOGW("[Save][OpExceptionInfo] Op [%s] get input size failed.", op->GetName().c_str());
+      return;
+    }
+    GELOGD("[Save][OpExceptionInfo] Save dump op info, the input size is %ld", input_size);
+    op_desc_info.input_size.emplace_back(input_size);
+  }
+  for (size_t j = 0; j < op->GetOutputsSize(); ++j) {
+    GeTensorDescPtr output_tensor_desc = op->MutableOutputDesc(j);
+    if (output_tensor_desc == nullptr) {
+      continue;
+    }
+    op_desc_info.output_format.emplace_back(output_tensor_desc->GetFormat());
+    op_desc_info.output_shape.emplace_back(output_tensor_desc->GetShape().GetDims());
+    op_desc_info.output_data_type.emplace_back(output_tensor_desc->GetDataType());
+    int64_t output_size = 0;
+    if (TensorUtils::GetTensorSizeInBytes(*output_tensor_desc, output_size) != SUCCESS) {
+      GELOGW("[Save][OpExceptionInfo] Op [%s] get output size failed.", op->GetName().c_str());
+      return;
+    }
+    GELOGD("[Save][OpExceptionInfo] Save dump op info, the output size is %ld.", output_size);
+    op_desc_info.output_size.emplace_back(output_size);
+  }
+}
+
+Status ExceptionDumper::DumpExceptionInfo(const std::vector<rtExceptionInfo> &exception_infos) const {
+  GELOGI("[Dump][Exception] Start to dump exception info");
+  for (const rtExceptionInfo &iter : exception_infos) {
+    OpDescInfo op_desc_info;
+    if (GetOpDescInfo(iter.streamid, iter.taskid, op_desc_info)) {
+      toolkit::dumpdata::DumpData dump_data;
+      SetDumpData(op_desc_info, dump_data);
+      uint64_t now_time = GetNowTime();
+      std::string op_name = op_desc_info.op_name;
+      std::string op_type = op_desc_info.op_type;
+      ReplaceStringElem(op_name);
+      ReplaceStringElem(op_type);
+      string dump_file_path =
+        "./" + op_type + "." + op_name + "." + std::to_string(op_desc_info.task_id) + "." + std::to_string(now_time);
+      GELOGI("[Dump][Exception] The exception dump file path is %s", dump_file_path.c_str());
+
+      uint64_t proto_size = dump_data.ByteSizeLong();
+      std::unique_ptr<char[]> proto_msg(new (std::nothrow) char[proto_size]);
+      bool ret = dump_data.SerializeToArray(proto_msg.get(), proto_size);
+      if (!ret || proto_size == 0) {
+        REPORT_INNER_ERROR("E19999", "Serialize proto to string fail");
+        GELOGE(PARAM_INVALID, "[Dump][Exception] Dump data proto serialize failed");
+        return PARAM_INVALID;
+      }
+
+      GE_CHK_STATUS_RET(MemoryDumper::DumpToFile(dump_file_path.c_str(), &proto_size, sizeof(uint64_t)),
+                        "Failed to dump proto size");
+      GE_CHK_STATUS_RET(MemoryDumper::DumpToFile(dump_file_path.c_str(), proto_msg.get(), proto_size),
+                        "Failed to dump proto msg");
+      if (DumpExceptionInput(op_desc_info, dump_file_path) != SUCCESS) {
+        GELOGE(PARAM_INVALID, "[Dump][Exception] Dump exception input failed");
+        return PARAM_INVALID;
+      }
+
+      if (DumpExceptionOutput(op_desc_info, dump_file_path) != SUCCESS) {
+        GELOGE(PARAM_INVALID, "[Dump][Exception] Dump exception output failed");
+        return PARAM_INVALID;
+      }
+      GELOGI("[Dump][Exception] Dump exception info SUCCESS");
+    } else {
+      GELOGE(PARAM_INVALID, "[Dump][Exception] Get op desc info failed,task id:%u,stream id:%u",
+             iter.taskid, iter.streamid);
+      return PARAM_INVALID;
+    }
+  }
+  return SUCCESS;
+}
+
+bool ExceptionDumper::GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const {
+  GELOGI("[Get][OpDescInfo] There are %zu op need to dump.", op_desc_info_.size());
+  for (size_t index = 0; index < op_desc_info_.size(); ++index) {
+    OpDescInfo dump_op_info = op_desc_info_.at(index);
+    if (dump_op_info.task_id == task_id && dump_op_info.stream_id == stream_id) {
+      GELOGI("[Get][OpDescInfo] Find exception op [%s] of task_id: %u, stream_id: %u.",
+             dump_op_info.op_name.c_str(), task_id, stream_id);
+      op_desc_info = dump_op_info;
+      return true;
+    }
+  }
+  return false;
+}
+
+Status ExceptionDumper::DumpExceptionInput(const OpDescInfo &op_desc_info, const string &dump_file) const {
+  GELOGI("[Dump][ExceptionInput] Start to dump exception input");
+  for (size_t i = 0; i < op_desc_info.input_addrs.size(); i++) {
+    if (Debug::DumpDevMem(dump_file.data(), op_desc_info.input_addrs.at(i), op_desc_info.input_size.at(i)) != SUCCESS) {
+      GELOGE(PARAM_INVALID, "[Dump][ExceptionInput] Dump the %zu input data of op [%s] failed",
+             i, op_desc_info.op_name.c_str());
+      return PARAM_INVALID;
+    }
+  }
+  return SUCCESS;
+}
+
+Status ExceptionDumper::DumpExceptionOutput(const OpDescInfo &op_desc_info, const string &dump_file) const {
+  GELOGI("[Dump][ExceptionOutput] Start to dump exception output");
+  for (size_t i = 0; i < op_desc_info.output_addrs.size(); i++) {
+    if (Debug::DumpDevMem(dump_file.data(), op_desc_info.output_addrs.at(i), op_desc_info.output_size.at(i)) !=
+        SUCCESS) {
+      GELOGE(PARAM_INVALID, "[Dump][ExceptionInput] Dump the %zu input data of op [%s] failed",
+             i, op_desc_info.op_name.c_str());
+      return PARAM_INVALID;
+    }
+  }
+  return SUCCESS;
+}
+
+OpDescInfo *ExceptionDumper::MutableOpDescInfo(uint32_t task_id, uint32_t stream_id) {
+  for (OpDescInfo &op_desc_info : op_desc_info_) {
+    if (op_desc_info.task_id == task_id && op_desc_info.stream_id == stream_id) {
+      return &op_desc_info;
+    }
+  }
+  return nullptr;
+}
+}  // namespace ge

ge/common/dump/exception_dumper.h

@@ -0,0 +1,48 @@
+/**
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef GE_COMMON_DUMP_EXCEPTION_DUMPER_H_
+#define GE_COMMON_DUMP_EXCEPTION_DUMPER_H_
+
+#include <vector>
+
+#include "graph/op_desc.h"
+#include "framework/common/ge_types.h"
+#include "graph/load/model_manager/task_info/task_info.h"
+
+namespace ge {
+class ExceptionDumper {
+ public:
+  ExceptionDumper() = default;
+  ~ExceptionDumper();
+
+  void SaveDumpOpInfo(const OpDescPtr &op, uint32_t task_id, uint32_t stream_id,
+                      std::vector<void *> &input_addrs, std::vector<void *> &output_addrs);
+  void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id);
+  Status DumpExceptionInfo(const std::vector<rtExceptionInfo> &exception_infos) const;
+  bool GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const;
+  OpDescInfo *MutableOpDescInfo(uint32_t task_id, uint32_t stream_id);
+
+ private:
+  void SaveOpDescInfo(const OpDescPtr &op, uint32_t task_id, uint32_t stream_id, OpDescInfo &op_desc_info);
+  Status DumpExceptionInput(const OpDescInfo &op_desc_info, const std::string &dump_file) const;
+  Status DumpExceptionOutput(const OpDescInfo &op_desc_info, const std::string &dump_file) const;
+
+  std::vector<OpDescInfo> op_desc_info_;
+};
+}  // namespace ge
+
+#endif // GE_COMMON_DUMP_EXCEPTION_DUMPER_H_

ge/common/dump/opdebug_register.cc

@@ -80,13 +80,11 @@ Status OpdebugRegister::RegisterDebugForStream(rtStream_t stream, uint32_t op_de
 
   uint32_t debug_stream_id = 0;
   uint32_t debug_task_id = 0;
-#ifdef ONLY_COMPILE_OPEN_SRC
   auto rt_ret = rtDebugRegisterForStream(stream, op_debug_mode, op_debug_addr_, &debug_stream_id, &debug_task_id);
   if (rt_ret != RT_ERROR_NONE) {
     GELOGE(RT_FAILED, "rtDebugRegisterForStream error, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
-#endif
   GELOGD("debug_task_id:%u, debug_stream_id:%u in stream overflow.", debug_task_id, debug_stream_id);
   data_dumper.SaveOpDebugId(debug_task_id, debug_stream_id, p2p_debug_addr_, true);
   return SUCCESS;
@@ -94,7 +92,6 @@ Status OpdebugRegister::RegisterDebugForStream(rtStream_t stream, uint32_t op_de
 
 void OpdebugRegister::UnregisterDebugForStream(rtStream_t stream) {
   rtError_t rt_ret = RT_ERROR_NONE;
-#ifdef ONLY_COMPILE_OPEN_SRC
   if (stream != nullptr) {
     GELOGD("start call rtDebugUnRegisterForStream in unknown shape over flow.");
     rt_ret = rtDebugUnRegisterForStream(stream);
@@ -102,7 +99,6 @@ void OpdebugRegister::UnregisterDebugForStream(rtStream_t stream) {
       GELOGW("rtDebugUnRegisterForStream failed, ret: 0x%X", rt_ret);
     }
   }
-#endif
 
   if (op_debug_addr_ != nullptr) {
     rt_ret = rtFree(op_debug_addr_);
@@ -145,4 +141,4 @@ Status OpdebugRegister::MallocMemForOpdebug() {
   return SUCCESS;
 }
 
-}  // namespace ge
+}  // namespace ge

ge/common/formats/format_transfers/datatype_transfer.cc

@@ -154,7 +154,11 @@ Status DataTypeTransfer::TransDataType(const CastArgs &args, TransResult &result
 
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to alloc the memory for dst buf %zu, data size %zu", total_size, args.src_data_size);
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION,
+           "[Allocate][DSTMemory]Failed, memory for dst buf %zu, data size %zu",
+           total_size, args.src_data_size);
+    REPORT_CALL_ERROR("E19999", "Failed to allocate memory for dst buf %zu, data size %zu",
+                      total_size, args.src_data_size);
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 

ge/common/formats/format_transfers/format_transfer_c1hwncoc0_hwcn.cc

@@ -49,11 +49,15 @@ Status CheckArgsForC1hwncoc0ToHwcn(const TransArgs &args) {
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShapeValid(src_shape, kC1hwncoc0DimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check src shape %s", ShapeToString(src_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][SrcShape]Failed, src shape %s",
+           ShapeToString(src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to check src shape %s", ShapeToString(src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   if (!CheckShapeValid(dst_shape, kHwcnDimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s.", ShapeToString(dst_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][DSTShape]Failed, dst shape %s.",
+           ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to check dst shape %s", ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   auto cube_size = GetCubeSizeByDataType(args.src_data_type);
@@ -73,9 +77,17 @@ Status CheckArgsForC1hwncoc0ToHwcn(const TransArgs &args) {
 Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, int size, int64_t total_size) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld, shape %s",
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION,
+           "[Allocate][DSTMemory]Failed to allcoate memory for dst buf %ld, "
+           "shape %s when trans format from %s to %s",
+           total_size, ShapeToString(args.dst_shape).c_str(),
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), total_size, ShapeToString(args.dst_shape).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to allcoate memory for dst buf %ld, "
+                      "shape %s when trans format from %s to %s",
+                      total_size, ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -115,10 +127,16 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, int size
                               static_cast<size_t>(size));
           if (ret != EOK) {
             GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                   "Failed to copy data from C1HWNCoC0[%ld, %ld, %ld, %ld, %ld, %ld] offset %ld to "
+                   "[Operate][Memory]Failed to copy data from "
+                   "C1HWNCoC0[%ld, %ld, %ld, %ld, %ld, %ld] offset %ld to "
                    "HWCN[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
-                   c1_idx, h_idx, w_idx, n_idx, co_idx, c0_idx, src_offset, h_idx, w_idx, c_idx, n_idx, dst_offset,
-                   ret);
+                   c1_idx, h_idx, w_idx, n_idx, co_idx, c0_idx, src_offset,
+                   h_idx, w_idx, c_idx, n_idx, dst_offset, ret);
+            REPORT_CALL_ERROR("E19999", "Failed to copy data from "
+                              "C1HWNCoC0[%ld, %ld, %ld, %ld, %ld, %ld] offset %ld to "
+                              "HWCN[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                              c1_idx, h_idx, w_idx, n_idx, co_idx, c0_idx, src_offset,
+                              h_idx, w_idx, c_idx, n_idx, dst_offset, ret);
             return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
@@ -144,8 +162,13 @@ Status FormatTransferC1hwncoc0Hwcn::TransFormat(const TransArgs &args, TransResu
       result.length = static_cast<size_t>(total_size);
       return SUCCESS;
     }
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Get %ld total size from dst shape %s, src shape %s.", total_size,
-           ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Get][Shape]Failed, total size %ld from dst shape %s, "
+           "src shape %s.",
+           total_size, ShapeToString(args.dst_shape).c_str(),
+           ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Get shape faield, total size %ld from dst shape %s, src shape %s.",
+                      total_size, ShapeToString(args.dst_shape).c_str(),
+                      ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   GELOGD("Begin to trans format from C1HWNCoC0 to HWCN, src shape %s, data type %s, dst shape %s, memory size %ld.",
@@ -153,9 +176,16 @@ Status FormatTransferC1hwncoc0Hwcn::TransFormat(const TransArgs &args, TransResu
          ShapeToString(args.dst_shape).c_str(), total_size);
   ret = GetDstDataAfterTrans(args, result, size, total_size);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
-           ShapeToString(args.dst_shape).c_str(), total_size);
+    GELOGE(ret, "[Get][Data]Failed when after trans, src shape %s, data type %s, dst shape %s, "
+           "memory size %ld, error_code %u",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+           ShapeToString(args.dst_shape).c_str(), total_size, ret);
+    REPORT_CALL_ERROR("E19999", "Failed to get data after trans, src shape %s, data type %s, "
+                      "dst shape %s, memory size %ld, error_code %u",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                      ShapeToString(args.dst_shape).c_str(), total_size, ret);
     return ret;
   }
   return SUCCESS;

ge/common/formats/format_transfers/format_transfer_dhwcn_fracz3D.cc

@@ -94,9 +94,14 @@ Status TransFormatDhwckToFz3D(const TransArgs &args, TransResult &result) {
 
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-           TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to allcoate memory "
+           "for dst buf %ld when trans format from %s to %s",
+           dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to allcoate memory for dst buf %ld "
+                      "when trans format from %s to %s",
+                      dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -122,8 +127,10 @@ Status TransFormatDhwckToFz3D(const TransArgs &args, TransResult &result) {
                                args.data + src_idx * data_size, static_cast<size_t>(data_size));
               }
               if (ret != EOK) {
-                GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d, pad mode %d",
-                       dst_offset, ret, pad_zero);
+                GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed at "
+                       "offset %ld, error-code %d, pad mode %d", dst_offset, ret, pad_zero);
+                REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, "
+                                  "error-code %d, pad mode %d", dst_offset, ret, pad_zero);
                 return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
               }
             }

ge/common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.cc

@@ -95,9 +95,14 @@ Status TransFormatDhwncToFz3DTranspose(const TransArgs &args, TransResult &resul
 
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-           TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to allcoate memory "
+           "for dst buf %ld when trans format from %s to %s",
+           dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to allcoate memory for dst buf %ld "
+                      "when trans format from %s to %s",
+                      dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -123,8 +128,10 @@ Status TransFormatDhwncToFz3DTranspose(const TransArgs &args, TransResult &resul
                                args.data + src_idx * data_size, static_cast<size_t>(data_size));
               }
               if (ret != EOK) {
-                GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d, pad mode %d",
-                       dst_offset, ret, pad_zero);
+                GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed at "
+                       "offset %ld, error-code %d, pad mode %d", dst_offset, ret, pad_zero);
+                REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, "
+                                  "error-code %d, pad mode %d", dst_offset, ret, pad_zero);
                 return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
               }
             }

ge/common/formats/format_transfers/format_transfer_fractal_nz.cc

@@ -60,7 +60,7 @@ bool CheckShape(Format format, const ShapeVector &shape) {
     default:
       std::string error = "Trans format between " + FmtToStr(TypeUtils::FormatToSerialString(format)) +
           " and FORMAT_FRACTAL_NZ is not supported.";
-      GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, error.c_str());
+      GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_FORMAT_INVALID, error.c_str());
       return false;
   }
 }
@@ -87,7 +87,10 @@ Status TransShapeToFracNz(const ShapeVector &src_shape, DataType data_type, Shap
       hw_shape.push_back(DIM_DEFAULT_VALUE);
       hw_shape.push_back(src_shape[kNdDimIndexN]);
       if (!IsShapeValid(dst_shape)) {
-        GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s", ShapeToString(dst_shape).c_str());
+        GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][DSTShape]Failed, dst shape %s",
+               ShapeToString(dst_shape).c_str());
+        REPORT_CALL_ERROR("E19999", "Failed to check dst shape %s",
+                          ShapeToString(dst_shape).c_str());
         return ACL_ERROR_GE_SHAPE_INVALID;
       }
       return SUCCESS;
@@ -106,7 +109,10 @@ Status TransShapeToFracNz(const ShapeVector &src_shape, DataType data_type, Shap
       hw_shape.push_back(src_shape[size - kNdDimCountBackwardsWH]);
       hw_shape.push_back(src_shape[size - kNdDimCountBackwardsW]);
       if (!IsShapeValid(dst_shape)) {
-        GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s", ShapeToString(dst_shape).c_str());
+        GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][DSTShape]Failed, dst shape %s",
+               ShapeToString(dst_shape).c_str());
+        REPORT_CALL_ERROR("E19999", "Failed to check dst shape %s",
+                          ShapeToString(dst_shape).c_str());
         return ACL_ERROR_GE_SHAPE_INVALID;
       }
       return SUCCESS;
@@ -117,10 +123,12 @@ Status CheckShapeRelation(const TransArgs &args, ShapeVector &hw_shape) {
   ShapeVector expect_src_shape;
   auto ret = TransShapeToFracNz(args.dst_shape, args.src_data_type, expect_src_shape, hw_shape);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Trans shape from %s to %s, shape %s to %s, data type %s failed",
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(),
-           TypeUtils::FormatToSerialString(args.src_format).c_str(), ShapeToString(args.dst_shape).c_str(),
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    GELOGE(ret, "[Transfer][ShapeToFracNz]Failed, shape from %s to %s, shape %s to %s, "
+           "data type %s, error_code:%u", TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+           TypeUtils::FormatToSerialString(args.src_format).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(), ret);
     return ret;
   }
   if (!IsTransShapeSrcCorrect(args, expect_src_shape)) {
@@ -139,9 +147,14 @@ Status TransFormatFromNdToFracNz(const TransArgs &args, TransResult &result, con
 
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size](), std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-           TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to allocate memory "
+           "for dst buf %ld when trans format from %s to %s",
+           dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to allocate memory for dst buf %ld "
+                      "trans format from %s to %s",
+                      dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -175,7 +188,10 @@ Status TransFormatFromNdToFracNz(const TransArgs &args, TransResult &result, con
         auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                             static_cast<size_t>(size * w0));
         if (ret != EOK) {
-          GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d", dst_offset, ret);
+          GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,"[Operate][DSTMemory]Failed at offset %ld, "
+                 "error-code %d", dst_offset, ret);
+          REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, error-code %d",
+                            dst_offset, ret);
           return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
         }
       }
@@ -189,7 +205,10 @@ Status TransFormatFromNdToFracNz(const TransArgs &args, TransResult &result, con
         auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                             static_cast<size_t>(size));
         if (ret != EOK) {
-          GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d", dst_offset, ret);
+          GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,"[Operate][DSTMemory]Failed at offset %ld, "
+                 "error-code %d", dst_offset, ret);
+          REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, error-code %d",
+                            dst_offset, ret);
           return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
         }
       }
@@ -210,9 +229,14 @@ Status TransFormatFromFracNzToNd(const TransArgs &args, TransResult &result, con
 
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to trans format "
+           "from %s to %s, memory for dst buf %ld",
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
            TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+    REPORT_CALL_ERROR("E19999", "Failed to trans format from %s to %s and allocate memory "
+                      "for dst buf %ld",
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -246,7 +270,11 @@ Status TransFormatFromFracNzToNd(const TransArgs &args, TransResult &result, con
         ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                        static_cast<size_t>(size * w0));
         if (ret != EOK) {
-          GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d", dst_offset, ret);
+          GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed at offset %ld, "
+                 "error-code %d",
+                 dst_offset, ret);
+          REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, error-code %d",
+                            dst_offset, ret);
           return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
         }
       }
@@ -260,7 +288,11 @@ Status TransFormatFromFracNzToNd(const TransArgs &args, TransResult &result, con
         ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                        static_cast<size_t>(size));
         if (ret != EOK) {
-          GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d", dst_offset, ret);
+          GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed at offset %ld, "
+                 "error-code %d",
+                 dst_offset, ret);
+          REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, error-code %d",
+                            dst_offset, ret);
           return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
         }
       }
@@ -274,17 +306,39 @@ Status TransFormatFromFracNzToNd(const TransArgs &args, TransResult &result, con
 
 Status FormatTransferFractalNz::TransFormat(const TransArgs &args, TransResult &result) {
   if (!IsDataTypeSupport(args.src_data_type)) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Trans format from %s to %s, src shape %s, dst shape %s, data type %s is not supported",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID,
+           "[Check][Datatype]Failed, trans format from %s to %s, src shape %s, dst shape %s, "
+           "data type %s is not supported",
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), ShapeToString(args.src_shape).c_str(),
-           ShapeToString(args.dst_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Check datatype failed, trans format from %s to %s, src shape %s, "
+                       "dst shape %s, data type %s is not supported",
+                       TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                       TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+                       ShapeToString(args.src_shape).c_str(),
+                       ShapeToString(args.dst_shape).c_str(),
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShape(args.src_format, args.src_shape) || !IsShapeValid(args.dst_shape)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Trans format from %s to %s, src shape %s, dst shape %s, data type %s is not supported",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
+           "[Check][Shape]Failed, trans format from %s to %s, "
+           "src shape %s, dst shape %s, data type %s is not supported",
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), ShapeToString(args.src_shape).c_str(),
-           ShapeToString(args.dst_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Check shape failed, trans format from %s to %s, "
+                       "src shape %s, dst shape %s, data type %s is not supported",
+                       TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                       TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+                       ShapeToString(args.src_shape).c_str(),
+                       ShapeToString(args.dst_shape).c_str(),
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   GELOGD("Begin to trans format from %s to %s, src shape %s, dst shape %s, data type %s",
@@ -307,16 +361,34 @@ Status FormatTransferFractalNz::TransShape(Format src_format, const ShapeVector
                                            Format dst_format, ShapeVector &dst_shape) {
   if (!IsDataTypeSupport(data_type)) {
     GELOGE(ACL_ERROR_GE_DATATYPE_INVALID,
-           "Trans format from %s to %s, src shape %s, data type %s is not supported",
-           TypeUtils::FormatToSerialString(src_format).c_str(), TypeUtils::FormatToSerialString(dst_format).c_str(),
-           ShapeToString(src_shape).c_str(), TypeUtils::DataTypeToSerialString(data_type).c_str());
+           "[Check][Datatype]Failed, trans format from %s to %s, src shape %s, "
+           "data type %s is not supported",
+           TypeUtils::FormatToSerialString(src_format).c_str(),
+           TypeUtils::FormatToSerialString(dst_format).c_str(),
+           ShapeToString(src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Check datatype failed, trans format from %s to %s, src shape %s, "
+                       "data type %s is not supported",
+                       TypeUtils::FormatToSerialString(src_format).c_str(),
+                       TypeUtils::FormatToSerialString(dst_format).c_str(),
+                       ShapeToString(src_shape).c_str(),
+                       TypeUtils::DataTypeToSerialString(data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShape(src_format, src_shape)) {
     GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
-           "Trans format from %s to %s, src shape %s, data type %s is not supported",
-           TypeUtils::FormatToSerialString(src_format).c_str(), TypeUtils::FormatToSerialString(dst_format).c_str(),
-           ShapeToString(src_shape).c_str(), TypeUtils::DataTypeToSerialString(data_type).c_str());
+           "[Check][Shape]Failed, trans format from %s to %s, src shape %s, "
+           "data type %s is not supported",
+           TypeUtils::FormatToSerialString(src_format).c_str(),
+           TypeUtils::FormatToSerialString(dst_format).c_str(),
+           ShapeToString(src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Check shape failed, trans format from %s to %s, src shape %s, "
+                       "data type %s is not supported",
+                       TypeUtils::FormatToSerialString(src_format).c_str(),
+                       TypeUtils::FormatToSerialString(dst_format).c_str(),
+                       ShapeToString(src_shape).c_str(),
+                       TypeUtils::DataTypeToSerialString(data_type).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   ShapeVector hw_shape;
@@ -325,18 +397,40 @@ Status FormatTransferFractalNz::TransShape(Format src_format, const ShapeVector
 
 Status FormatTransferFractalNzND::TransFormat(const TransArgs &args, TransResult &result) {
   if (!IsDataTypeSupport(args.src_data_type)) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Trans format from %s to %s, src shape %s, dst shape %s, data type %s is not supported",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID,
+           "[Check][Datatype]Failed, trans format from %s to %s, src shape %s, dst shape %s, "
+           "data type %s is not supported",
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), ShapeToString(args.src_shape).c_str(),
-           ShapeToString(args.dst_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Check datatype failed, trans format from %s to %s, src shape %s, "
+                       "dst shape %s, data type %s is not supported",
+                       TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                       TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+                       ShapeToString(args.src_shape).c_str(),
+                       ShapeToString(args.dst_shape).c_str(),
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
 
   if (!IsShapeValid(args.src_shape) || !CheckShape(args.dst_format, args.dst_shape)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Trans format from %s to %s, src shape %s, dst shape %s, data type %s is not supported",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
+           "[Check][Shape]Failed, trans format from %s to %s, src shape %s, dst shape %s, "
+           "data type %s is not supported",
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), ShapeToString(args.src_shape).c_str(),
-           ShapeToString(args.dst_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Check shape failed, trans format from %s to %s, src shape %s, "
+                       "dst shape %s, data type %s is not supported",
+                       TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                       TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+                       ShapeToString(args.src_shape).c_str(),
+                       ShapeToString(args.dst_shape).c_str(),
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   GELOGD("Begin to trans format from %s to %s, src shape %s, dst shape %s, data type %s",

ge/common/formats/format_transfers/format_transfer_fractal_z.cc

@@ -29,6 +29,25 @@
 namespace ge {
 namespace formats {
 namespace {
+constexpr int64_t kDim = 1;
+static int64_t Measure(int64_t x, int64_t y) {
+  int64_t z = y;
+  while (x % y != 0) {
+    z = x % y;
+    x = y;
+    y = z;
+  }
+  return z;
+}
+// least common multiple
+static int64_t Lcm(int64_t a, int64_t b) {
+  if (b == 0) {
+    return -1;
+  }
+  int64_t temp = (a * b) / (Measure(a, b));
+  return temp;
+}
+
 Status CheckDataTypeSupport(DataType data_type) { return GetSizeByDataType(data_type) > 0 ? SUCCESS : UNSUPPORTED; }
 
 /**
@@ -54,8 +73,39 @@ Status TransShapeToFz(int64_t n, int64_t c, int64_t h, int64_t w, DataType data_
   dst_shape.push_back(kNiSize);
   dst_shape.push_back(c0);
   if (!IsShapeValid(dst_shape)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Failed, dst shape %s",
+           ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to check dst shape %s", ShapeToString(dst_shape).c_str());
+    return ACL_ERROR_GE_SHAPE_INVALID;
+  }
+  return SUCCESS;
+}
+
+Status TransShapeToFzWithGroups(int64_t n, int64_t c, int64_t h, int64_t w, DataType data_type, std::vector<int64_t> &dst_shape,
+  int64_t groups) {
+  auto c0 = GetCubeSizeByDataType(data_type);
+  if (c0 < 0) {
+    return ACL_ERROR_GE_DATATYPE_INVALID;
+  }
+  int64_t cin_ori = c;
+  int64_t cout_ori = n / groups;
+  int64_t cube_k = GetCubeSizeByDataType(data_type);
+  int64_t e_mult = std::min(
+      Lcm(Lcm(cin_ori, cube_k) / (cin_ori), Lcm(cout_ori, static_cast<int64_t>(kCubeSize)) / (cout_ori)),
+      groups);
+  int64_t cin_opt = Ceil(e_mult * cin_ori, cube_k) * cube_k;
+  int64_t c1_dim = cin_opt / cube_k;
+  int64_t g_dim = Ceil(groups, e_mult);
+  auto n1 = Ceil(cout_ori * e_mult, static_cast<int64_t>(kCubeSize));
+  dst_shape.clear();
+  dst_shape.push_back(g_dim * c1_dim * h * w);
+  dst_shape.push_back(n1);
+  dst_shape.push_back(16);
+  dst_shape.push_back(cube_k);
+  if (!IsShapeValid(dst_shape)) {
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Failed, dst shape %s",
            ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to check dst shape %s", ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   return SUCCESS;
@@ -86,6 +136,21 @@ Status TransShapeHwcnToFz(const std::vector<int64_t> &src_shape, DataType data_t
   return TransShapeToFz(n, c, h, w, data_type, dst_shape);
 }
 
+Status TransShapeHwcnToFzWithGroups(const std::vector<int64_t> &src_shape, DataType data_type, std::vector<int64_t> &dst_shape
+, int64_t groups){
+ if (!CheckShapeValid(src_shape, kHwcnDimsNum)) {
+    return ACL_ERROR_GE_SHAPE_INVALID;
+  }
+
+  auto h = src_shape.at(kHwcnH);
+  auto w = src_shape.at(kHwcnW);
+  auto c = src_shape.at(kHwcnC);
+  auto n = src_shape.at(kHwcnN);
+
+  return TransShapeToFzWithGroups(n, c, h, w, data_type, dst_shape, groups);
+}
+
+
 Status TransShapeNhwcToFz(const std::vector<int64_t> &src_shape, DataType data_type, std::vector<int64_t> &dst_shape) {
   if (!CheckShapeValid(src_shape, kNhwcDimsNum)) {
     return ACL_ERROR_GE_SHAPE_INVALID;
@@ -127,9 +192,14 @@ Status TransFormatFromNchwToFz(const TransArgs &args, TransResult &result) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
   GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
       dst == nullptr,
-      GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-             TypeUtils::FormatToSerialString(args.src_format).c_str(),
-             TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+      GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to allcoate memory "
+             "for dst buf %ld when trans format from %s to %s",
+             dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+             TypeUtils::FormatToSerialString(args.dst_format).c_str());
+      REPORT_CALL_ERROR("E19999", "Failed to allcoate memory for dst buf %ld "
+                        "when trans format from %s to %s",
+                        dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                        TypeUtils::FormatToSerialString(args.dst_format).c_str());
       return ACL_ERROR_GE_MEMORY_ALLOCATION;);
 
   for (int64_t vfi = 0; vfi < vf_cnt; vfi++) {
@@ -173,8 +243,12 @@ Status TransFormatFromNchwToFz(const TransArgs &args, TransResult &result) {
             }
           }
           if (ret != EOK) {
-            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d pad mode %d", offset,
-                   ret, need_pad_zero);
+            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,"[Operate][DSTMemory]Failed at offset %ld, "
+                   "error-code %d pad mode %d",
+                   offset, ret, need_pad_zero);
+            REPORT_CALL_ERROR("E19999","Failed to operate dst memory at offset %ld, "
+                              "error-code %d pad mode %d",
+                              offset, ret, need_pad_zero);
             return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
@@ -187,6 +261,89 @@ Status TransFormatFromNchwToFz(const TransArgs &args, TransResult &result) {
   return SUCCESS;
 }
 
+Status TransFormatHwcnToFzWithGroups(const TransArgs &args, TransResult &result, int64_t groups){
+  int64_t h_dim = args.src_shape[kHwcnH];
+  int64_t w_dim = args.src_shape[kHwcnW];
+  int64_t c_dim = args.src_shape[kHwcnC];
+  int64_t n_dim = args.src_shape[kHwcnN];
+  int64_t cin_ori = c_dim;
+  int64_t cout_ori = n_dim / groups;
+  if (cin_ori == 0 || cout_ori == 0) {
+    GELOGE(GRAPH_FAILED, "[Check][Param]Failed, cin_ori, cout_ori must not be equal 0, "
+           "and current cin_ori, cout_ori, groups are %ld %ld %ld", cin_ori, cout_ori, groups);
+    REPORT_CALL_ERROR("E19999", "Check graph param failed, cin_ori, cout_ori must not be equal 0,"
+                      "and current cin_ori, cout_ori, groups are %ld %ld %ld",
+                      cin_ori, cout_ori, groups);
+    return GRAPH_FAILED;
+  }
+  const int64_t cube_k = GetCubeSizeByDataType(args.src_data_type);
+  int64_t e_mult = std::min(
+      Lcm(Lcm(cin_ori, cube_k) / (cin_ori), Lcm(cout_ori, static_cast<int64_t>(kCubeSize)) / (cout_ori)),
+      groups);
+  int64_t cin_opt = Ceil(e_mult * cin_ori, cube_k) * cube_k;
+  int64_t cout_opt = Ceil(e_mult * cout_ori, static_cast<int64_t>(kCubeSize)) * static_cast<int64_t>(kCubeSize);
+  int64_t c1_dim = cin_opt / cube_k;
+  int64_t g_dim = Ceil(groups, e_mult);
+  int64_t dim_cin = cin_opt / cube_k;
+  int64_t data_size = GetSizeByDataType(args.src_data_type);
+  int64_t size_output_data = g_dim * kDim * dim_cin * h_dim * w_dim * cout_opt * cube_k * data_size;
+  if (size_output_data == 0) {
+      result.length = static_cast<size_t>(size_output_data);
+      return SUCCESS;
+  }
+  errno_t ret = EOK;
+  std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[size_output_data], std::default_delete<uint8_t[]>());
+  if (dst == nullptr) {
+      GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to allcoate memory "
+             "for dst buf %ld when trans format from %s to %s",
+             size_output_data, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+             TypeUtils::FormatToSerialString(args.dst_format).c_str());
+      REPORT_CALL_ERROR("E19999", "Failed to allcoate memory for dst buf %ld "
+                        "when trans format from %s to %s",
+                        size_output_data, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                        TypeUtils::FormatToSerialString(args.dst_format).c_str());
+      return ACL_ERROR_GE_MEMORY_ALLOCATION;
+  }
+  ret = memset_s(dst.get(), static_cast<size_t>(size_output_data), 0, static_cast<size_t>(size_output_data));
+  if (ret != EOK) {
+      GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed, ret is %d", ret);
+      REPORT_CALL_ERROR("E19999", "Failed to operate dst memory, ret is %d", ret);
+      return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
+  }
+  for (int64_t g = 0; g < groups; g++) {
+    for (int64_t d = 0; d < kDim; d++) {
+      for (int64_t c = 0; c < c_dim; c++) {
+        for (int64_t h = 0; h < h_dim; h++) {
+          for (int64_t w = 0; w < w_dim; w++) {
+            for (int64_t n = 0; n < cout_ori; n++) {
+              int64_t e_val = g % e_mult;
+              int64_t dst_ci = e_val * cin_ori + c;
+              int64_t dst_co = e_val * cout_ori + n;
+              int64_t src_co = g * cout_ori + n;
+              int64_t tempory = dst_ci % cube_k;
+              int64_t srx_inx = 0;
+              int64_t dst_inx = (g / e_mult) * kDim * c1_dim * h_dim * w_dim * cout_opt * cube_k +
+                                d * c1_dim * h_dim * w_dim * cout_opt * cube_k +
+                                (dst_ci / cube_k) * h_dim * w_dim * cout_opt * cube_k +
+                                h * w_dim * cout_opt * cube_k + w * cout_opt * cube_k +
+                                dst_co * cube_k + tempory;
+              srx_inx = d * h_dim * w_dim * c_dim * n_dim + h * w_dim * c_dim * n_dim +
+                        w * c_dim * n_dim + c * n_dim + src_co;
+              char *dst_data = reinterpret_cast<char *>(dst.get() + dst_inx * data_size);
+              const char *src_data = reinterpret_cast<const char *>(args.data + srx_inx * data_size);
+              for (int64_t index = 0; index < data_size; index++) {
+                *dst_data++ = *src_data++;
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  result.data = dst;
+  result.length = static_cast<size_t>(size_output_data);
+  return SUCCESS;
+}
 Status TransFormatHwcnToFz(const TransArgs &args, TransResult &result) {
   int64_t h = args.src_shape[kHwcnH];
   int64_t w = args.src_shape[kHwcnW];
@@ -213,9 +370,14 @@ Status TransFormatHwcnToFz(const TransArgs &args, TransResult &result) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
   GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
       dst == nullptr,
-      GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-             TypeUtils::FormatToSerialString(args.src_format).c_str(),
-             TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+      GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to allcoate memory "
+             "for dst buf %ld when trans format from %s to %s",
+             dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+             TypeUtils::FormatToSerialString(args.dst_format).c_str());
+      REPORT_CALL_ERROR("E19999", "Failed to allcoate memory for dst buf %ld "
+                        "when trans format from %s to %s",
+                        dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                        TypeUtils::FormatToSerialString(args.dst_format).c_str());
       return ACL_ERROR_GE_MEMORY_ALLOCATION;);
 
   for (int64_t c1i = 0; c1i < c1; c1i++) {
@@ -235,7 +397,8 @@ Status TransFormatHwcnToFz(const TransArgs &args, TransResult &result) {
                              static_cast<size_t>(data_size));
             } else {
               if (protected_size < data_size) {
-                GELOGE(ACL_ERROR_GE_PARAM_INVALID, "Failed to operate the dst memory, protected_size is %ld and size is %ld",
+                GELOGE(ACL_ERROR_GE_PARAM_INVALID,"[Operate][DSTMemory]Failed, protected_size "
+                       "is %ld and size is %ld",
                        protected_size, data_size);
                 return ACL_ERROR_GE_PARAM_INVALID;
               }
@@ -247,8 +410,11 @@ Status TransFormatHwcnToFz(const TransArgs &args, TransResult &result) {
               }
             }
             if (ret != EOK) {
-              GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d, pad mode %d",
-                     dst_offset, ret, pad_zero);
+              GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed, "
+                     "at offset %ld, error-code %d, pad mode %d", dst_offset, ret, pad_zero);
+              REPORT_CALL_ERROR("E19999", "Failed to operate dst memoery at offset %ld, "
+                                "error-code %d, pad mode %d",
+                                dst_offset, ret, pad_zero);
               return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
             }
           }
@@ -288,9 +454,14 @@ Status TransFormatNhwcToFz(const TransArgs &args, TransResult &result) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
   GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
       dst == nullptr,
-      GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-             TypeUtils::FormatToSerialString(args.src_format).c_str(),
-             TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+      GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to allcoate memory "
+             "for dst buf %ld when trans format from %s to %s",
+             dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+             TypeUtils::FormatToSerialString(args.dst_format).c_str());
+      REPORT_CALL_ERROR("E19999", "Failed to allcoate memory for dst buf %ld "
+                        "when trans format from %s to %s",
+                        dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                        TypeUtils::FormatToSerialString(args.dst_format).c_str());
       return ACL_ERROR_GE_MEMORY_ALLOCATION;);
 
   for (int64_t c1i = 0; c1i < c1; c1i++) {
@@ -310,7 +481,8 @@ Status TransFormatNhwcToFz(const TransArgs &args, TransResult &result) {
                              static_cast<size_t>(data_size));
             } else {
               if (protected_size < data_size) {
-                GELOGE(ACL_ERROR_GE_PARAM_INVALID, "Failed to operate the dst memory, protected_size is %ld and size is %ld",
+                GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Operate][DSTMemory]Failed, protected_size "
+                       "is %ld and size is %ld",
                        protected_size, data_size);
                 return ACL_ERROR_GE_PARAM_INVALID;
               }
@@ -322,8 +494,11 @@ Status TransFormatNhwcToFz(const TransArgs &args, TransResult &result) {
               }
             }
             if (ret != EOK) {
-              GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d, pad mode %d",
-                     dst_offset, ret, pad_zero);
+              GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed at offset %ld,"
+                     " error-code %d, pad mode %d", dst_offset, ret, pad_zero);
+              REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, "
+                                "error-code %d, pad mode %d",
+                                dst_offset, ret, pad_zero);
               return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
             }
           }
@@ -355,15 +530,16 @@ Status FormatTransferFractalZ::TransFormat(const TransArgs &args, TransResult &r
   if (args.src_format == FORMAT_NHWC && args.dst_format == FORMAT_FRACTAL_Z) {
     return TransFormatNhwcToFz(args, result);
   }
-
-  if (args.src_format == FORMAT_HWCN && args.dst_format == FORMAT_FRACTAL_Z) {
+  if ((args.src_format == FORMAT_HWCN) && (GetPrimaryFormat(args.dst_format) == FORMAT_FRACTAL_Z)) {
+    if (GetSubFormat(args.dst_format) > 1) {
+      return TransFormatHwcnToFzWithGroups(args, result, GetSubFormat(args.dst_format));
+    }
     return TransFormatHwcnToFz(args, result);
   }
 
   if (args.src_format == FORMAT_NCHW && args.dst_format == FORMAT_FRACTAL_Z) {
     return TransFormatFromNchwToFz(args, result);
   }
-
   return ACL_ERROR_GE_FORMAT_INVALID;
 }
 
@@ -376,7 +552,10 @@ Status FormatTransferFractalZ::TransShape(Format src_format, const std::vector<i
   if (src_format == FORMAT_NHWC && dst_format == FORMAT_FRACTAL_Z) {
     return TransShapeNhwcToFz(src_shape, data_type, dst_shape);
   }
-  if (src_format == FORMAT_HWCN && dst_format == FORMAT_FRACTAL_Z) {
+  if ((src_format == FORMAT_HWCN) && (GetPrimaryFormat(dst_format) == FORMAT_FRACTAL_Z)) {
+    if (GetSubFormat(dst_format) > 1) {
+        return TransShapeHwcnToFzWithGroups(src_shape, data_type, dst_shape, GetSubFormat(dst_format));
+     }
     return TransShapeHwcnToFz(src_shape, data_type, dst_shape);
   }
   if (src_format == FORMAT_NCHW && dst_format == FORMAT_FRACTAL_Z) {

ge/common/formats/format_transfers/format_transfer_fractal_zz.cc

@@ -59,7 +59,7 @@ bool CheckShape(Format format, const ShapeVector &shape) {
     default:
       std::string error = "Trans format between " + FmtToStr(TypeUtils::FormatToSerialString(format)) +
                           " and FORMAT_FRACTAL_ZZ is not supported.";
-      GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, error.c_str());
+      GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_FORMAT_INVALID, error.c_str());
       return false;
   }
 }
@@ -86,8 +86,10 @@ Status TransShapeToFracZz(const ShapeVector &src_shape, DataType data_type, Shap
       hw_shape.push_back(DIM_DEFAULT_VALUE);
       hw_shape.push_back(src_shape[kNdDimIndexN]);
       if (!IsShapeValid(dst_shape)) {
-        GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s",
+        GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][DSTShape]Failed, dst shape %s",
                ShapeToString(dst_shape).c_str());
+        REPORT_CALL_ERROR("E19999", "Failed to check dst shape %s",
+                          ShapeToString(dst_shape).c_str());
         return ACL_ERROR_GE_SHAPE_INVALID;
       }
       return SUCCESS;
@@ -106,8 +108,10 @@ Status TransShapeToFracZz(const ShapeVector &src_shape, DataType data_type, Shap
       hw_shape.push_back(src_shape[size - kNdDimCountBackwardsWH]);
       hw_shape.push_back(src_shape[size - kNdDimCountBackwardsW]);
       if (!IsShapeValid(dst_shape)) {
-        GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s",
+        GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][DSTShape]Failed, dst shape %s",
                ShapeToString(dst_shape).c_str());
+        REPORT_CALL_ERROR("E19999", "Failed to check dst shape %s",
+                          ShapeToString(dst_shape).c_str());
         return ACL_ERROR_GE_SHAPE_INVALID;
       }
       return SUCCESS;
@@ -118,10 +122,18 @@ Status CheckShapeRelation(const TransArgs &args, ShapeVector &hw_shape) {
   ShapeVector expect_src_shape;
   auto ret = TransShapeToFracZz(args.dst_shape, args.src_data_type, expect_src_shape, hw_shape);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Trans shape from %s to %s, shape %s to %s, data type %s failed",
+    GELOGE(ret, "[Trans][ShapeToFracZz] Failed from %s to %s, shape %s to %s, data type %s",
            TypeUtils::FormatToSerialString(args.dst_format).c_str(),
-           TypeUtils::FormatToSerialString(args.src_format).c_str(), ShapeToString(args.dst_shape).c_str(),
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+           TypeUtils::FormatToSerialString(args.src_format).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to trans shape from %s to %s, shape %s to %s, data type %s",
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      ShapeToString(args.dst_shape).c_str(),
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ret;
   }
   if (!IsTransShapeSrcCorrect(args, expect_src_shape)) {
@@ -140,9 +152,14 @@ Status TransFormatFromNdToFracZz(const TransArgs &args, TransResult &result, con
 
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size](), std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-           TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to allcoate memory "
+           "for dst buf %ld when trans format from %s to %s",
+           dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to allcoate memory for dst buf %ld "
+                      "when trans format from %s to %s",
+                      dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
   // The src&dst_shape can be written as times*H*W & times*H1*W1*H0*W0, respectively. dst_shape_size >= kDimNum4D
@@ -179,7 +196,11 @@ Status TransFormatFromNdToFracZz(const TransArgs &args, TransResult &result, con
           auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                               static_cast<size_t>(size * w0));
           if (ret != EOK) {
-            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d", dst_offset, ret);
+            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed at offset %ld, "
+                   "error-code %d",
+                   dst_offset, ret);
+            REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, error-code %d",
+                              dst_offset, ret);
             return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
@@ -195,7 +216,11 @@ Status TransFormatFromNdToFracZz(const TransArgs &args, TransResult &result, con
           auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                               static_cast<size_t>(size));
           if (ret != EOK) {
-            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d", dst_offset, ret);
+            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed at offset %ld, "
+                   "error-code %d",
+                   dst_offset, ret);
+            REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, error-code %d",
+                              dst_offset, ret);
             return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
@@ -217,9 +242,14 @@ Status TransFormatFromFracZzToNd(const TransArgs &args, TransResult &result, con
 
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size](), std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-           TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed to allcoate memory "
+           "for dst buf %ld when trans format from %s to %s",
+           dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to allcoate memory for dst buf %ld "
+                      "when trans format from %s to %s",
+                      dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -257,7 +287,11 @@ Status TransFormatFromFracZzToNd(const TransArgs &args, TransResult &result, con
           auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                               static_cast<size_t>(size * w0));
           if (ret != EOK) {
-            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d", dst_offset, ret);
+            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed at offset %ld, "
+                   "error-code %d",
+                   dst_offset, ret);
+            REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, error-code %d",
+                              dst_offset, ret);
             return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
@@ -273,7 +307,11 @@ Status TransFormatFromFracZzToNd(const TransArgs &args, TransResult &result, con
           auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                               static_cast<size_t>(size));
           if (ret != EOK) {
-            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "Failed to operate the dst memory at offset %ld, error-code %d", dst_offset, ret);
+            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Operate][DSTMemory]Failed at offset %ld, "
+                   "error-code %d",
+                   dst_offset, ret);
+            REPORT_CALL_ERROR("E19999", "Failed to operate dst memory at offset %ld, error-code %d",
+                              dst_offset, ret);
             return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
@@ -288,17 +326,39 @@ Status TransFormatFromFracZzToNd(const TransArgs &args, TransResult &result, con
 
 Status FormatTransferFractalZz::TransFormat(const TransArgs &args, TransResult &result) {
   if (!IsDataTypeSupport(args.src_data_type)) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Not support trans format from %s to %s, src shape %s, dst shape %s, data type %s",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID,
+           "[Check][Datatype]Failed, not support trans format from %s to %s, "
+           "src shape %s, dst shape %s, data type %s",
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), ShapeToString(args.src_shape).c_str(),
-           ShapeToString(args.dst_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Check datatype failed, not support trans format "
+                       "from %s to %s, src shape %s, dst shape %s, data type %s",
+                       TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                       TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+                       ShapeToString(args.src_shape).c_str(),
+                       ShapeToString(args.dst_shape).c_str(),
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShape(args.src_format, args.src_shape) || !IsShapeValid(args.dst_shape)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Not support trans format from %s to %s, src shape %s, dst shape %s, data type %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
+           "[Check][Shape]Failed, not support trans format from %s to %s, "
+           "src shape %s, dst shape %s, data type %s",
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), ShapeToString(args.src_shape).c_str(),
-           ShapeToString(args.dst_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_CALL_ERROR("E19999",  "Check shape failed, not support trans format from %s to %s, "
+                      "src shape %s, dst shape %s, data type %s",
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+                      ShapeToString(args.src_shape).c_str(),
+                      ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   GELOGD("Begin to trans format from %s to %s, src shape %s, dst shape %s, data type %s",
@@ -321,16 +381,34 @@ Status FormatTransferFractalZz::TransShape(Format src_format, const ShapeVector
                                            Format dst_format, ShapeVector &dst_shape) {
   if (!IsDataTypeSupport(data_type)) {
     GELOGE(ACL_ERROR_GE_DATATYPE_INVALID,
-           "Not support trans format from %s to %s, src shape %s, data type %s",
-           TypeUtils::FormatToSerialString(src_format).c_str(), TypeUtils::FormatToSerialString(dst_format).c_str(),
-           ShapeToString(src_shape).c_str(), TypeUtils::DataTypeToSerialString(data_type).c_str());
+           "[Check][Datatype]Failed, not support trans format from %s to %s, "
+           "src shape %s, data type %s",
+           TypeUtils::FormatToSerialString(src_format).c_str(),
+           TypeUtils::FormatToSerialString(dst_format).c_str(),
+           ShapeToString(src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Check datatype failed, not support trans format from %s to %s, "
+                       "src shape %s, data type %s",
+                       TypeUtils::FormatToSerialString(src_format).c_str(),
+                       TypeUtils::FormatToSerialString(dst_format).c_str(),
+                       ShapeToString(src_shape).c_str(),
+                       TypeUtils::DataTypeToSerialString(data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShape(src_format, src_shape)) {
     GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
-           "Not support trans format from %s to %s, src shape %s, data type %s",
-           TypeUtils::FormatToSerialString(src_format).c_str(), TypeUtils::FormatToSerialString(dst_format).c_str(),
-           ShapeToString(src_shape).c_str(), TypeUtils::DataTypeToSerialString(data_type).c_str());
+           "[Check][Shape]Failed, not support trans format from %s to %s, "
+           "src shape %s, data type %s",
+           TypeUtils::FormatToSerialString(src_format).c_str(),
+           TypeUtils::FormatToSerialString(dst_format).c_str(),
+           ShapeToString(src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Check shape failed, not support trans format from %s to %s, "
+                      "src shape %s, data type %s",
+                      TypeUtils::FormatToSerialString(src_format).c_str(),
+                      TypeUtils::FormatToSerialString(dst_format).c_str(),
+                      ShapeToString(src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(data_type).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   ShapeVector hw_shape;
@@ -339,18 +417,39 @@ Status FormatTransferFractalZz::TransShape(Format src_format, const ShapeVector
 
 Status FormatTransferFractalZzND::TransFormat(const TransArgs &args, TransResult &result) {
   if (!IsDataTypeSupport(args.src_data_type)) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Not support trans format from %s to %s, src shape %s, dst shape %s, data type %s",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID,
+           "[Check][Datatype]Failed, not support trans format from %s to %s, "
+           "src shape %s, dst shape %s, data type %s",
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), ShapeToString(args.src_shape).c_str(),
-           ShapeToString(args.dst_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Check datatype Failed, not support trans format from %s to %s, "
+                       "src shape %s, dst shape %s, data type %s",
+                       TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                       TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+                       ShapeToString(args.src_shape).c_str(),
+                       ShapeToString(args.dst_shape).c_str(),
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
 
   if (!IsShapeValid(args.src_shape) || !CheckShape(args.dst_format, args.dst_shape)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Not support trans format from %s to %s, src shape %s, dst shape %s, data type %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Failed, not support trans format "
+           "from %s to %s, src shape %s, dst shape %s, data type %s",
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), ShapeToString(args.src_shape).c_str(),
-           ShapeToString(args.dst_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+           ShapeToString(args.src_shape).c_str(),
+           ShapeToString(args.dst_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Check shape failed, not support trans format from %s to %s, "
+                      "src shape %s, dst shape %s, data type %s",
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str(),
+                      ShapeToString(args.src_shape).c_str(),
+                      ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   GELOGD("Begin to trans format from %s to %s, src shape %s, dst shape %s, data type %s",

ge/common/formats/format_transfers/format_transfer_fracz_hwcn.cc

@@ -41,16 +41,26 @@ Status CheckArgsForFracZToHwcn(const TransArgs &args) {
     return ACL_ERROR_GE_FORMAT_INVALID;
   }
   if (!CheckDataTypeSupported(args.src_data_type)) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Failed to trans shape from FORMAT_FRACTAL_Z to HWCN, invalid data type %s",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Check][DataType]Failed, "
+           "shape from FORMAT_FRACTAL_Z to HWCN, invalid data type %s",
            TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to trans shape from FORMAT_FRACTAL_Z to HWCN, "
+                       "invalid data type %s",
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShapeValid(src_shape, kFracZDimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check src shape %s", ShapeToString(src_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
+           ShapeToString(src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                      ShapeToString(src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   if (!CheckShapeValid(dst_shape, kHwcnDimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s", ShapeToString(dst_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
+           ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid",
+                      ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   int64_t c0 = GetCubeSizeByDataType(args.src_data_type);
@@ -66,7 +76,7 @@ Status CheckArgsForFracZToHwcn(const TransArgs &args) {
         FmtToStr(ShapeToString(dst_shape));
     GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_SHAPE_INVALID, error.c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
-  } 
+  }
 
   return SUCCESS;
 }
@@ -74,9 +84,17 @@ Status CheckArgsForFracZToHwcn(const TransArgs &args) {
 Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const int size, const int64_t total_size) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld, shape %s",
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION,
+           "[Allocate][DSTMemory]Failed, memory for dst buf %ld, shape %s "
+           "when trans format from %s to %s",
+           total_size, ShapeToString(args.dst_shape).c_str(),
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), total_size, ShapeToString(args.dst_shape).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, shape %s "
+                      "when trans format from %s to %s",
+                      total_size, ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -114,9 +132,12 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
                               static_cast<size_t>(size));
           if (ret != EOK) {
             GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                   "Failed to copy data from FracZ offset %ld to HWCN[%ld, %ld, %ld, %ld] "
-                   "offset %ld, err-code %d",
+                   "[Operate][Memory]Failed to copy data from FracZ offset %ld to "
+                   "HWCN[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
                    src_offset, h_idx, w_idx, c_idx, n_idx, dst_offset, ret);
+            REPORT_CALL_ERROR("E19999", "Failed to copy data from FracZ offset %ld to "
+                              "HWCN[%ld, %ld, %ld, %ld], offset %ld, err-code %d",
+                              src_offset, h_idx, w_idx, c_idx, n_idx, dst_offset, ret);
             return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
@@ -142,9 +163,12 @@ Status FormatTransferFracZHwcn::TransFormat(const TransArgs &args, TransResult &
       result.length = static_cast<size_t>(total_size);
       return SUCCESS;
     }
-
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Get %ld total size from dst shape %s, src shape %s", total_size,
-        ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Get][ShapeSize]Failed, "
+           "total size %ld from dst shape %s, src shape %s", total_size,
+           ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999",  "Failed to get total size %ld from "
+                      "dst shape %s, src shape %s", total_size,
+                      ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   GELOGD("Begin to trans format from FracZ to HWCN, src shape %s, data type %s, dst shape %s, memory size %ld",
@@ -152,9 +176,16 @@ Status FormatTransferFracZHwcn::TransFormat(const TransArgs &args, TransResult &
          ShapeToString(args.dst_shape).c_str(), total_size);
   ret = GetDstDataAfterTrans(args, result, size, total_size);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
-           ShapeToString(args.dst_shape).c_str(), total_size);
+    GELOGE(ret, "[Get][Data]Failed after trans, src shape %s, "
+           "data type %s, dst shape %s, memory size %ld, error_code %u",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+           ShapeToString(args.dst_shape).c_str(), total_size, ret);
+    REPORT_CALL_ERROR("E19999", "Failed to get data after trans, src shape %s, "
+                      "data type %s, dst shape %s, memory size %ld, error_code %u",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                      ShapeToString(args.dst_shape).c_str(), total_size, ret);
     return ret;
   }
   return SUCCESS;

ge/common/formats/format_transfers/format_transfer_fracz_nchw.cc

@@ -37,20 +37,30 @@ Status CheckArgsForFracZToNchw(const TransArgs &args) {
     std::string error = "Dose not support trans format from " +
         FmtToStr(TypeUtils::FormatToSerialString(args.src_format)) + " to " +
         FmtToStr(TypeUtils::FormatToSerialString(args.dst_format));
-    GE_ERRORLOG_AND_ERRORMSG(UNSUPPORTED, error.c_str());
+    GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_FORMAT_INVALID, error.c_str());
     return ACL_ERROR_GE_FORMAT_INVALID;
   }
   if (!CheckDataTypeSupported(args.src_data_type)) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Failed to trans shape from FORMAT_FRACTAL_Z to NCHW, invalid data type %s",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Check][DataType]Failed, "
+           "shape from FORMAT_FRACTAL_Z to NCHW, invalid data type %s",
            TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to trans shape from FORMAT_FRACTAL_Z to NCHW, "
+                       "invalid data type %s",
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShapeValid(src_shape, kFracZDimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check src shape %s", ShapeToString(src_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
+           ShapeToString(src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                      ShapeToString(src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   if (!CheckShapeValid(dst_shape, kNchwDimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s", ShapeToString(dst_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
+           ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid",
+                      ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   int64_t c0 = GetCubeSizeByDataType(args.src_data_type);
@@ -59,10 +69,15 @@ Status CheckArgsForFracZToNchw(const TransArgs &args) {
   }
   int64_t c1 = Ceil(dst_shape.at(kNchwC), c0);
   int64_t n0 = Ceil(dst_shape.at(kNchwN), static_cast<int64_t>(kNiSize));
-  if (src_shape.at(kFracZHWC1) != dst_shape.at(kNchwH) * dst_shape.at(kNchwW) * c1 || src_shape.at(kFracZC0) != c0 ||
-      src_shape.at(kFracZNi) != kNiSize || src_shape.at(kFracZN0) != n0) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check relationship between src and dst shape, src shape %s, dst shape %s",
+  if (src_shape.at(kFracZHWC1) != dst_shape.at(kNchwH) * dst_shape.at(kNchwW) * c1 ||
+      src_shape.at(kFracZC0) != c0 || src_shape.at(kFracZNi) != kNiSize || src_shape.at(kFracZN0) != n0) {
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
+           "[Check][Shape]Failed to check relationship between src and dst shape, "
+           "src shape %s, dst shape %s",
            ShapeToString(src_shape).c_str(), ShapeToString(dst_shape).c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to check relationship between src and dst shape, "
+                       "src shape %s, dst shape %s",
+                       ShapeToString(src_shape).c_str(), ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
 
@@ -72,9 +87,17 @@ Status CheckArgsForFracZToNchw(const TransArgs &args) {
 Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const int size, const int64_t total_size) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld, shape %s",
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION,
+           "[Allocate][DSTMemory]Failed, memory for dst buf %ld, shape %s "
+           "when trans format from %s to %s",
+           total_size, ShapeToString(args.dst_shape).c_str(),
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), total_size, ShapeToString(args.dst_shape).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, shape %s "
+                      "when trans format from %s to %s",
+                      total_size, ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -112,9 +135,12 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
                               static_cast<size_t>(size));
           if (ret != EOK) {
             GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                   "Failed to copy data from FracZ offset %ld to NCHW[%ld, %ld, %ld, %ld] offset %ld, "
-                   "err-code %d",
+                   "[Operate][Memory]Failed to copy data from FracZ offset %ld to "
+                   "NCHW[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
                    src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret);
+            REPORT_CALL_ERROR("E19999","Failed to copy data from FracZ offset %ld to "
+                              "NCHW[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                              src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret );
             return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
@@ -141,8 +167,12 @@ Status FormatTransferFracZNchw::TransFormat(const TransArgs &args, TransResult &
       return SUCCESS;
     }
 
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Get %ld total size from dst shape %s, src shape %s", total_size,
-        ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Get][ShapeSize]Failed, total size %ld from dst shape %s, "
+           "src shape %s", total_size,
+           ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to get total size %ld from dst shape %s, src shape %s",
+                      total_size,
+                      ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   GELOGD("Begin to trans format from FracZ to NCHW, src shape %s, data type %s, dst shape %s, memory size %ld",
@@ -151,9 +181,16 @@ Status FormatTransferFracZNchw::TransFormat(const TransArgs &args, TransResult &
 
   ret = GetDstDataAfterTrans(args, result, size, total_size);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+    GELOGE(ret, "[Get][Data]Failed, after trans, src shape %s, data type %s, "
+           "dst shape %s, memory size %ld",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
            ShapeToString(args.dst_shape).c_str(), total_size);
+    REPORT_CALL_ERROR("E19999", "Failed to get data after trans, src shape %s, "
+                      "data type %s, dst shape %s, memory size %ld",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                      ShapeToString(args.dst_shape).c_str(), total_size);
     return ret;
   }
   return SUCCESS;

ge/common/formats/format_transfers/format_transfer_fracz_nhwc.cc

@@ -37,33 +37,48 @@ Status CheckArgsForFracZToNhwc(const TransArgs &args) {
     std::string error = "Dose not support trans format from " +
         FmtToStr(TypeUtils::FormatToSerialString(args.src_format)) + " to " +
         FmtToStr(TypeUtils::FormatToSerialString(args.dst_format));
-    GE_ERRORLOG_AND_ERRORMSG(UNSUPPORTED, error.c_str());
-    return UNSUPPORTED;
+    GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_FORMAT_INVALID, error.c_str());
+    return ACL_ERROR_GE_FORMAT_INVALID;
   }
   if (!CheckDataTypeSupported(args.src_data_type)) {
-    GELOGE(UNSUPPORTED, "Failed to trans shape from FORMAT_FRACTAL_Z to NHWC, invalid data type %s",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Check][DataType]Failed, "
+           "shape from FORMAT_FRACTAL_Z to NCHW, invalid data type %s",
            TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
-    return UNSUPPORTED;
+    REPORT_INNER_ERROR("E19999", "Failed to trans shape from FORMAT_FRACTAL_Z to NCHW, "
+                       "invalid data type %s",
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShapeValid(src_shape, kFracZDimsNum)) {
-    GELOGE(PARAM_INVALID, "Failed to check src shape %s", ShapeToString(src_shape).c_str());
-    return PARAM_INVALID;
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
+           ShapeToString(src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                      ShapeToString(src_shape).c_str());
+    return ACL_ERROR_GE_SHAPE_INVALID;
   }
   if (!CheckShapeValid(dst_shape, kNhwcDimsNum)) {
-    GELOGE(PARAM_INVALID, "Failed to check dst shape %s", ShapeToString(dst_shape).c_str());
-    return PARAM_INVALID;
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
+           ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid",
+                      ShapeToString(dst_shape).c_str());
+    return ACL_ERROR_GE_SHAPE_INVALID;
   }
   int64_t c0 = GetCubeSizeByDataType(args.src_data_type);
   if (c0 < 0) {
-    return PARAM_INVALID;
+    return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   int64_t c1 = Ceil(dst_shape.at(kNhwcC), c0);
   int64_t n0 = Ceil(dst_shape.at(kNhwcN), static_cast<int64_t>(kNiSize));
-  if (src_shape.at(kFracZHWC1) != dst_shape.at(kNhwcH) * dst_shape.at(kNhwcW) * c1 || src_shape.at(kFracZC0) != c0 ||
-      src_shape.at(kFracZNi) != kNiSize || src_shape.at(kFracZN0) != n0) {
-    GELOGE(PARAM_INVALID, "Failed to check relationship between src and dst shape, src shape %s, dst shape %s",
+  if (src_shape.at(kFracZHWC1) != dst_shape.at(kNhwcH) * dst_shape.at(kNhwcW) * c1 ||
+      src_shape.at(kFracZC0) != c0 || src_shape.at(kFracZNi) != kNiSize || src_shape.at(kFracZN0) != n0) {
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
+           "[Check][Shape]Failed to check relationship between src and dst shape, "
+           "src shape %s, dst shape %s",
            ShapeToString(src_shape).c_str(), ShapeToString(dst_shape).c_str());
-    return PARAM_INVALID;
+    REPORT_INNER_ERROR("E19999", "Failed to check relationship between src and dst shape, "
+                       "src shape %s, dst shape %s",
+                       ShapeToString(src_shape).c_str(), ShapeToString(dst_shape).c_str());
+    return ACL_ERROR_GE_SHAPE_INVALID;
   }
 
   return SUCCESS;
@@ -72,10 +87,18 @@ Status CheckArgsForFracZToNhwc(const TransArgs &args) {
 Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, int size, int64_t total_size) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld, shape %s",
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION,
+           "[Allocate][DSTMemory]Failed, memory for dst buf %ld, "
+           "shape %s when trans format from %s to %s",
+           total_size, ShapeToString(args.dst_shape).c_str(),
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), total_size, ShapeToString(args.dst_shape).c_str());
-    return OUT_OF_MEMORY;
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, "
+                      "shape %s when trans format from %s to %s",
+                      total_size, ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
   auto n0 = args.src_shape.at(kFracZN0);
@@ -111,10 +134,14 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, int size
           auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                               static_cast<size_t>(size));
           if (ret != EOK) {
-            GELOGE(INTERNAL_ERROR,
-                   "Failed to copy data from FracZ offset %ld to HHWC[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
-                   src_offset, n_idx, h_idx, w_idx, c_idx, dst_offset, ret);
-            return INTERNAL_ERROR;
+            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
+                   "[Operate][Memory]Failed to copy data from FracZ offset %ld to "
+                   "NCHW[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                   src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret);
+            REPORT_CALL_ERROR("E19999","Failed to copy data from FracZ offset %ld to "
+                              "NCHW[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                              src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret);
+            return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
       }
@@ -127,8 +154,9 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, int size
 }  // namespace
 
 Status FormatTransferFracZNhwc::TransFormat(const TransArgs &args, TransResult &result) {
-  if (CheckArgsForFracZToNhwc(args) != SUCCESS) {
-    return PARAM_INVALID;
+  Status ret = CheckArgsForFracZToNhwc(args);
+  if (ret != SUCCESS) {
+    return ret;
   }
   int size = GetSizeByDataType(args.src_data_type);
   auto total_size = GetItemNumByShape(args.dst_shape) * size;
@@ -139,18 +167,30 @@ Status FormatTransferFracZNhwc::TransFormat(const TransArgs &args, TransResult &
       return SUCCESS;
     }
 
-    GELOGE(INTERNAL_ERROR, "Get %ld total size from dst shape %s, src shape %s", total_size,
-        ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
-    return PARAM_INVALID;
+    GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Get][ShapeSize]Failed, total size %ld from dst shape %s, "
+           "src shape %s", total_size,
+           ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to get total size %ld from dst shape %s, src shape %s",
+                      total_size,
+                      ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    return ACL_ERROR_GE_PARAM_INVALID;
   }
   GELOGD("Begin to trans format from FracZ to NHWC, src shape %s, data type %s, dst shape %s, memory size %ld",
          ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
          ShapeToString(args.dst_shape).c_str(), total_size);
-  if (GetDstDataAfterTrans(args, result, size, total_size) != SUCCESS) {
-    GELOGE(INTERNAL_ERROR, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
-           ShapeToString(args.dst_shape).c_str(), total_size);
-    return INTERNAL_ERROR;
+  ret = GetDstDataAfterTrans(args, result, size, total_size);
+  if (ret != SUCCESS) {
+    GELOGE(ret, "[Get][Data]Failed, after trans, src shape %s, data type %s, "
+           "dst shape %s, memory size %ld, error_code %u",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+           ShapeToString(args.dst_shape).c_str(), total_size, ret);
+    REPORT_CALL_ERROR("E19999","Failed to get data after trans, src shape %s, data type %s, "
+                      "dst shape %s, memory size %ld, error_code %u",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                      ShapeToString(args.dst_shape).c_str(), total_size, ret);
+    return ret;
   }
   return SUCCESS;
 }
@@ -158,7 +198,7 @@ Status FormatTransferFracZNhwc::TransFormat(const TransArgs &args, TransResult &
 Status FormatTransferFracZNhwc::TransShape(Format src_format, const std::vector<int64_t> &src_shape, DataType data_type,
                                            Format dst_format, std::vector<int64_t> &dst_shape) {
   GELOGD("The shape derivation from FracZ to NHWC is not unique. Trans shape in this direction is not supported");
-  return UNSUPPORTED;
+  return ACL_ERROR_GE_FORMAT_INVALID;
 }
 
 REGISTER_FORMAT_TRANSFER(FormatTransferFracZNhwc, FORMAT_FRACTAL_Z, FORMAT_NHWC)

ge/common/formats/format_transfers/format_transfer_hwcn_c1hwncoc0.cc

@@ -43,8 +43,10 @@ Status TransShapeHwcnToC1hwncoc0(const DataType &data_type, const std::vector<in
   dst_shape.push_back(cube_size);
   dst_shape.push_back(cube_size);
   if (!CheckShapeValid(dst_shape, kC1hwncoc0DimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
            ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid",
+                      ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   return SUCCESS;
@@ -59,16 +61,26 @@ Status CheckArgsForHwcnToC1hwncoc0(const TransArgs &args) {
     return ACL_ERROR_GE_FORMAT_INVALID;
   }
   if (!CheckDataTypeSupported(args.src_data_type)) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Failed to trans shape from HWCN to C1HWNCoC0, invalid data type %s",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Trans][Shape]Failed, "
+           "shape from HWCN to C1HWNCoC0, invalid data type %s",
            TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to trans shape from HWCN to C1HWNCoC0, "
+                       "invalid data type %s",
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShapeValid(args.src_shape, kHwcnDimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check src shape %s", ShapeToString(args.src_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
+           ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                      ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   if (!CheckShapeValid(args.dst_shape, kC1hwncoc0DimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s", ShapeToString(args.dst_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
+           ShapeToString(args.dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid",
+                      ShapeToString(args.dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   std::vector<int64_t> expect_dst_shape;
@@ -78,10 +90,14 @@ Status CheckArgsForHwcnToC1hwncoc0(const TransArgs &args) {
   }
   if (args.dst_shape != expect_dst_shape) {
     GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
-           "Failed to trans format, src and dst shape are not compatible. src shape %s, dst shape %s, "
+           "[Trans][Shape]Failed, src shape %s and dst shape %s are not compatible. "
            "expect dst shape %s",
            ShapeToString(args.src_shape).c_str(), ShapeToString(args.dst_shape).c_str(),
            ShapeToString(expect_dst_shape).c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to trans format, src shape %s and dst shape %s "
+                       "are not compatible. expect dst shape %s",
+                       ShapeToString(args.src_shape).c_str(), ShapeToString(args.dst_shape).c_str(),
+                       ShapeToString(expect_dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
 
@@ -91,9 +107,16 @@ Status CheckArgsForHwcnToC1hwncoc0(const TransArgs &args) {
 Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const int size, const int64_t total_size) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld, shape %s",
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed, "
+           "memory for dst buf %ld, shape %s when trans format from %s to %s",
+           total_size, ShapeToString(args.dst_shape).c_str(),
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), total_size, ShapeToString(args.dst_shape).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, "
+                       "shape %s when trans format from %s to %s",
+                       total_size, ShapeToString(args.dst_shape).c_str(),
+                       TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                       TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -135,11 +158,17 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
                 auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                                     static_cast<size_t>(size));
                 if (ret != EOK) {
-                  GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                         "Failed to copy data from HWCN[%ld, %ld, %ld, %ld] offset %ld to "
+                  GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Copy][Data]Failed, "
+                         "data from HWCN[%ld, %ld, %ld, %ld] offset %ld to "
                          "C1HWNCoC0[%ld, %ld, %ld, %ld, %ld, %ld] offset %ld, err-code %d",
-                         h_idx, w_idx, c_idx, n_idx, src_offset, c1_idx, h_idx, w_idx, n_idx, co_idx, c0_idx,
-                         dst_offset, ret);
+                         h_idx, w_idx, c_idx, n_idx, src_offset, c1_idx, h_idx, w_idx,
+                         n_idx, co_idx, c0_idx, dst_offset, ret);
+                  REPORT_CALL_ERROR("E19999", "Failed to copy data from "
+                                    "HWCN[%ld, %ld, %ld, %ld] offset %ld "
+                                    "to, C1HWNCoC0[%ld, %ld, %ld, %ld, %ld, %ld] "
+                                    "offset %ld, err-code %d",
+                                    h_idx, w_idx, c_idx, n_idx, src_offset, c1_idx, h_idx, w_idx,
+                                    n_idx, co_idx, c0_idx, dst_offset, ret);
                   return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
                 }
               } else {
@@ -147,14 +176,18 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
                     memset_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), 0, static_cast<size_t>(size));
                 if (ret != EOK) {
                   GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                         "Failed to set to 0 to C1HWNCoC0[%ld, %ld, %ld, %ld, %ld, %ld] offset %ld, "
-                         "err-code %d",
+                         "[Operate][Memory]Failed to set to 0 to "
+                         "C1HWNCoC0[%ld, %ld, %ld, %ld, %ld, %ld] offset %ld, err-code %d",
                          c1_idx, h_idx, w_idx, n_idx, co_idx, c0_idx, dst_offset, ret);
+                  REPORT_CALL_ERROR("E19999",  "Failed to set to 0 to "
+                                    "C1HWNCoC0[%ld, %ld, %ld, %ld, %ld, %ld] offset %ld, "
+                                    "err-code %d",
+                                    c1_idx, h_idx, w_idx, n_idx, co_idx, c0_idx, dst_offset, ret);
                   return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
                 }
               }
             }
-          }
+	  }
         }
       }
     }
@@ -179,8 +212,12 @@ Status FormatTransferHwcnC1hwncoc0::TransFormat(const TransArgs &args, TransResu
       return SUCCESS;
     }
 
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Get %ld total size from dst shape %s, src shape %s", total_size,
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Get][ShapeSize]Failed, total size %ld from dst shape %s, "
+           "src shape %s", total_size,
            ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999",  "Failed to get total size %ld from dst shape %s, src shape %s",
+                      total_size,
+                      ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   GELOGD("Begin to trans format from HWCN to C1HWNCoC0, src shape %s, data type %s, dst shape %s, memory size %ld",
@@ -189,9 +226,16 @@ Status FormatTransferHwcnC1hwncoc0::TransFormat(const TransArgs &args, TransResu
 
   ret = GetDstDataAfterTrans(args, result, size, total_size);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
-           ShapeToString(args.dst_shape).c_str(), total_size);
+    GELOGE(ret, "[Get][Data]Failed, after trans, src shape %s, data type %s, "
+           "dst shape %s, memory size %ld, error_code %u",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+           ShapeToString(args.dst_shape).c_str(), total_size, ret);
+    REPORT_CALL_ERROR("E19999", "Failed to get data after trans, src shape %s, data type %s, "
+                      "dst shape %s, memory size %ld, error_code %u",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                      ShapeToString(args.dst_shape).c_str(), total_size, ret);
     return ret;
   }
   return SUCCESS;
@@ -201,8 +245,10 @@ Status FormatTransferHwcnC1hwncoc0::TransShape(Format src_format, const std::vec
                                                DataType data_type, Format dst_format, std::vector<int64_t> &dst_shape) {
   if (src_format == FORMAT_HWCN && CheckDataTypeSupported(data_type)) {
     if (!CheckShapeValid(src_shape, kHwcnDimsNum)) {
-      GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check src shape %s",
+      GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
              ShapeToString(src_shape).c_str());
+      REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                        ShapeToString(src_shape).c_str());
       return ACL_ERROR_GE_SHAPE_INVALID;
     }
     return TransShapeHwcnToC1hwncoc0(data_type, src_shape, dst_shape);

ge/common/formats/format_transfers/format_transfer_nc1hwc0_nchw.cc

@@ -37,33 +37,49 @@ Status CheckArgsForNc1hwc0ToNchw(const TransArgs &args) {
     std::string error = "Dose not support trans format from " +
         FmtToStr(TypeUtils::FormatToSerialString(args.src_format)) + " to " +
         FmtToStr(TypeUtils::FormatToSerialString(args.dst_format));
-    GE_ERRORLOG_AND_ERRORMSG(UNSUPPORTED, error.c_str());
-    return UNSUPPORTED;
+    GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_FORMAT_INVALID, error.c_str());
+    return ACL_ERROR_GE_FORMAT_INVALID;
   }
   if (!CheckDataTypeSupported(args.src_data_type)) {
-    GELOGE(UNSUPPORTED, "Failed to trans shape from NC1HWC0 to NCHW, invalid data type %s",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Check][DataType]Failed, shape from NC1HWC0 to NCHW, "
+           "invalid data type %s",
            TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
-    return UNSUPPORTED;
+    REPORT_INNER_ERROR("E19999", "Failed to trans shape from NC1HWC0 to NCHW, invalid data type %s",
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShapeValid(args.src_shape, kNc1hwc0DimsNum)) {
-    GELOGE(PARAM_INVALID, "Failed to check src shape %s", ShapeToString(args.src_shape).c_str());
-    return PARAM_INVALID;
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
+           ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                      ShapeToString(args.src_shape).c_str());
+    return ACL_ERROR_GE_SHAPE_INVALID;
   }
   if (!CheckShapeValid(args.dst_shape, kNchwDimsNum)) {
-    GELOGE(PARAM_INVALID, "Failed to check dst shape %s", ShapeToString(args.dst_shape).c_str());
-    return PARAM_INVALID;
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
+           ShapeToString(args.dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid",
+                      ShapeToString(args.dst_shape).c_str());
+    return ACL_ERROR_GE_SHAPE_INVALID;
   }
   int64_t c0 = GetCubeSizeByDataType(args.src_data_type);
   if (c0 <= 0) {
-    GELOGE(PARAM_INVALID, "Failed to get cube size, the data type is invalid");
-    return PARAM_INVALID;
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Get][Cube]Failed, the data type %s is invalid",
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to get cube size, the data tyep %s is invalid",
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (src_shape.at(kNc1hwc0H) != dst_shape.at(kNchwH) || src_shape.at(kNc1hwc0W) != dst_shape.at(kNchwW) ||
       src_shape.at(kNc1hwc0N) != dst_shape.at(kNchwN) || src_shape.at(kNc1hwc0C0) != c0 ||
       src_shape.at(kNc1hwc0C1) != (Ceil(dst_shape.at(kNchwC), c0))) {
-    GELOGE(PARAM_INVALID, "Failed to check relationship between src and dst shape, src shape %s, dst shape %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Failed to check relationship between "
+           "src shape %s and dst shape %s",
            ShapeToString(src_shape).c_str(), ShapeToString(dst_shape).c_str());
-    return PARAM_INVALID;
+    REPORT_INNER_ERROR("E19999",  "Failed to check relationship between src shape %s "
+                       "and dst shape %s",
+                       ShapeToString(src_shape).c_str(), ShapeToString(dst_shape).c_str());
+    return ACL_ERROR_GE_SHAPE_INVALID;
   }
 
   return SUCCESS;
@@ -72,10 +88,17 @@ Status CheckArgsForNc1hwc0ToNchw(const TransArgs &args) {
 Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const int size, const int64_t total_size) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld, shape %s",
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed, "
+           "memory for dst buf %ld, shape %s when trans format from %s to %s",
+           total_size, ShapeToString(args.dst_shape).c_str(),
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), total_size, ShapeToString(args.dst_shape).c_str());
-    return OUT_OF_MEMORY;
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, "
+                      "shape %s when trans format from %s to %s",
+                      total_size, ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
   auto h = args.src_shape.at(kNc1hwc0H);
@@ -109,11 +132,17 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
           auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
                               static_cast<size_t>(size));
           if (ret != EOK) {
-            GELOGE(INTERNAL_ERROR,
-                   "Failed to copy data from NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld to NCHW[%ld, %ld, %ld, %ld]"
-                   " offset %ld, err-code %d",
-                   n_idx, c1_idx, h_idx, w_idx, c0_idx, src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret);
-            return INTERNAL_ERROR;
+            GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Copy][Data]Failed, data from "
+                   "NC1HWC0[%ld, %ld, %ld, %ld, %ld] "
+                   "src offset %ld to NCHW[%ld, %ld, %ld, %ld], dst offset %ld, err-code %d",
+                   n_idx, c1_idx, h_idx, w_idx, c0_idx, src_offset, n_idx,
+                   c_idx, h_idx, w_idx, dst_offset, ret);
+            REPORT_CALL_ERROR("E19999", "Failed to copy data from NC1HWC0[%ld, %ld, %ld, %ld, %ld] "
+                              "src offset %ld to NCHW[%ld, %ld, %ld, %ld], dst offset %ld, "
+                              "err-code %d",
+                              n_idx, c1_idx, h_idx, w_idx, c0_idx, src_offset, n_idx,
+                              c_idx, h_idx, w_idx, dst_offset, ret);
+            return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
       }
@@ -126,8 +155,9 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
 }  // namespace
 
 Status FormatTransferNc1hwc0Nchw::TransFormat(const TransArgs &args, TransResult &result) {
-  if (CheckArgsForNc1hwc0ToNchw(args) != SUCCESS) {
-    return PARAM_INVALID;
+  Status ret = CheckArgsForNc1hwc0ToNchw(args);
+  if (ret != SUCCESS) {
+    return ret;
   }
   int size = GetSizeByDataType(args.src_data_type);
   auto total_size = GetItemNumByShape(args.dst_shape) * size;
@@ -138,18 +168,30 @@ Status FormatTransferNc1hwc0Nchw::TransFormat(const TransArgs &args, TransResult
       return SUCCESS;
     }
 
-    GELOGE(INTERNAL_ERROR, "Get %ld total size from dst shape %s, src shape %s", total_size,
+    GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Get][ShapeSize]Failed, total size %ld from dst shape %s, "
+           "src shape %s", total_size,
            ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
-    return PARAM_INVALID;
+    REPORT_CALL_ERROR("E19999", "Failed to get total size %ld from dst shape %s, src shape %s",
+                      total_size,
+                      ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    return ACL_ERROR_GE_PARAM_INVALID;
   }
   GELOGD("Begin to trans format from NC1HWC0 to NCHW, src shape %s, data type %s, dst shape %s, memory size %ld",
          ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
          ShapeToString(args.dst_shape).c_str(), total_size);
-  if (GetDstDataAfterTrans(args, result, size, total_size) != SUCCESS) {
-    GELOGE(INTERNAL_ERROR, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+  ret = GetDstDataAfterTrans(args, result, size, total_size);
+  if (ret != SUCCESS) {
+    GELOGE(ret, "[Get][Data]Failed, after trans, src shape %s, data type %s, "
+           "dst shape %s, memory size %ld",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
            ShapeToString(args.dst_shape).c_str(), total_size);
-    return INTERNAL_ERROR;
+    REPORT_CALL_ERROR("E19999", "Failed to get data after trans, src shape %s, data type %s, "
+                      "dst shape %s, memory size %ld",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                      ShapeToString(args.dst_shape).c_str(), total_size);
+    return ret;
   }
   return SUCCESS;
 }
@@ -157,7 +199,7 @@ Status FormatTransferNc1hwc0Nchw::TransFormat(const TransArgs &args, TransResult
 Status FormatTransferNc1hwc0Nchw::TransShape(Format src_format, const std::vector<int64_t> &src_shape,
                                              DataType data_type, Format dst_format, std::vector<int64_t> &dst_shape) {
   GELOGD("The shape derivation from NC1HWC0 to NCHW is not unique. Trans shape in this direction is not supported");
-  return UNSUPPORTED;
+  return ACL_ERROR_GE_FORMAT_INVALID;
 }
 
 REGISTER_FORMAT_TRANSFER(FormatTransferNc1hwc0Nchw, FORMAT_NC1HWC0, FORMAT_NCHW)

ge/common/formats/format_transfers/format_transfer_nc1hwc0_nhwc.cc

@@ -41,28 +41,44 @@ Status CheckArgsForNc1hwc0ToNhwc(const TransArgs &args) {
     return ACL_ERROR_GE_FORMAT_INVALID;
   }
   if (!CheckDataTypeSupported(args.src_data_type)) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Failed to trans shape from NC1HWC0 to NHWC, invalid data type %s",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Check][DataType]Failed, shape from NC1HWC0 to NHWC, "
+           "invalid data type %s",
            TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to trans shape from NC1HWC0 to NHWC, invalid data type %s",
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShapeValid(args.src_shape, kNc1hwc0DimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check src shape %s", ShapeToString(args.src_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
+           ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                      ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   if (!CheckShapeValid(args.dst_shape, kNhwcDimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s", ShapeToString(args.dst_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
+           ShapeToString(args.dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid",
+                      ShapeToString(args.dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   int64_t c0 = GetCubeSizeByDataType(args.src_data_type);
   if (c0 <= 0) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Failed to get cube size, the data type is invalid");
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Get][Cube]Failed, the data type %s is invalid",
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to get cube size, the data type %s is invalid",
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (src_shape.at(kNc1hwc0H) != dst_shape.at(kNhwcH) || src_shape.at(kNc1hwc0W) != dst_shape.at(kNhwcW) ||
       src_shape.at(kNc1hwc0N) != dst_shape.at(kNhwcN) || src_shape.at(kNc1hwc0C0) != c0 ||
       src_shape.at(kNc1hwc0C1) != (Ceil(dst_shape.at(kNhwcC), c0))) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check relationship between src and dst shape, src shape %s, dst shape %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Failed to check relationship between "
+           "src shape %s and dst shape %s",
            ShapeToString(src_shape).c_str(), ShapeToString(dst_shape).c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to check relationship between src shape %s "
+                       "and dst shape %s",
+                       ShapeToString(src_shape).c_str(), ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
 
@@ -72,9 +88,16 @@ Status CheckArgsForNc1hwc0ToNhwc(const TransArgs &args) {
 Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const int size, const int64_t total_size) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld, shape %s",
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allocate][DSTMemory]Failed, memory for dst buf %ld, "
+           "shape %s when trans format from %s to %s",
+           total_size, ShapeToString(args.dst_shape).c_str(),
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), total_size, ShapeToString(args.dst_shape).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, "
+                      "shape %s when trans format from %s to %s",
+                      total_size, ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -110,9 +133,14 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
                               static_cast<size_t>(size));
           if (ret != EOK) {
             GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                   "Failed to copy data from NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld to NHWC[%ld, %ld, %ld, %ld]"
-                   " offset %ld, err-code %d",
-                   n_idx, c1_idx, h_idx, w_idx, c0_idx, src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret);
+                   "[Copy][Data]Failed, data from NC1HWC0[%ld, %ld, %ld, %ld, %ld] "
+                   "offset %ld to NHWC[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                   n_idx, c1_idx, h_idx, w_idx, c0_idx, src_offset, n_idx, c_idx,
+                   h_idx, w_idx, dst_offset, ret);
+            REPORT_CALL_ERROR("E19999", "Failed to copy data from NC1HWC0[%ld, %ld, %ld, %ld, %ld] "
+                              "offset %ld to NHWC[%ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                              n_idx, c1_idx, h_idx, w_idx, c0_idx, src_offset, n_idx, c_idx,
+                              h_idx, w_idx, dst_offset, ret);
             return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
           }
         }
@@ -139,19 +167,31 @@ Status FormatTransferNc1hwc0Nhwc::TransFormat(const TransArgs &args, TransResult
       return SUCCESS;
     }
 
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Get %ld total size from dst shape %s, src shape %s", total_size,
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Get][ShapeSize]Failed, total size %ld from dst shape %s, "
+           "src shape %s", total_size,
            ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to get total size %ld from dst shape %s, src shape %s",
+                      total_size,
+                      ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
-  GELOGD("Begin to trans format from NC1HWC0 to NCHW, src shape %s, data type %s, dst shape %s, memory size %ld",
+  GELOGD("[Trans][Format]Begin to trans format from NC1HWC0 to NCHW, "
+         "src shape %s, data type %s, dst shape %s, memory size %ld",
          ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
          ShapeToString(args.dst_shape).c_str(), total_size);
 
   ret = GetDstDataAfterTrans(args, result, size, total_size);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
-           ShapeToString(args.dst_shape).c_str(), total_size);
+    GELOGE(ret, "[Get][Data]Failed, after trans, src shape %s, data type %s, "
+           "dst shape %s, memory size %ld, error_code %u",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+           ShapeToString(args.dst_shape).c_str(), total_size, ret);
+    REPORT_CALL_ERROR("E19999", "[Get][Data]Failed, after trans, src shape %s, "
+                      "data type %s, dst shape %s, memory size %ld, error_code %u",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                      ShapeToString(args.dst_shape).c_str(), total_size, ret);
     return ret;
   }
   return SUCCESS;

ge/common/formats/format_transfers/format_transfer_nchw_fz_c04.cc

@@ -59,8 +59,9 @@ Status TransShape(int64_t n, int64_t c, int64_t h, int64_t w, DataType data_type
   dst_shape.push_back(c0);
 
   if (!IsShapeValid(dst_shape)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
            ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid", ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   return SUCCESS;
@@ -94,7 +95,13 @@ Status TransFormatFromNchwToFzC04(const TransArgs &args, TransResult &result) {
   std::vector<int64_t> expect_shape = {n, h, w, c};
   auto ret = ge::formats::Transpose(data, args.src_shape, args.src_data_type, perm_arg_1, trans_result_1);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Failed to Transpose from NCHW to HWCN");
+    GELOGE(ret, "[Trans][Formats]Failed from NCHW to HWCN, src_shape %s, src_data_type %s",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Failede to trans formats from NCHW to HWCN, src_shape %s, "
+                      "src_data_type %s",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ret;
   }
 
@@ -104,7 +111,9 @@ Status TransFormatFromNchwToFzC04(const TransArgs &args, TransResult &result) {
   // check size it should be same with original
   size_t expect_size = n * c * h * w * size;  // before has do check about mul
   if (trans_result_1.length != expect_size) {
-    GELOGE(ACL_ERROR_GE_PARAM_INVALID, "size is not match after transpose!");
+    GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Shape]size %zu is not match expect size %zu "
+           "after transpose",
+           trans_result_1.length, expect_size);
     return ACL_ERROR_GE_PARAM_INVALID;
   }
 
@@ -118,19 +127,32 @@ Status TransFormatFromNchwToFzC04(const TransArgs &args, TransResult &result) {
 
   // data overflow check totally
   GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(h_o, w_o),
-                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "int64 mul overflow.A[%ld], B[%ld]", h_o, w_o);
+                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][Shape]Failed, "
+                         "int64 mul overflow.A[%ld], B[%ld]", h_o, w_o);
+                  REPORT_CALL_ERROR("E19999", "Check shape failed, int64 mul overflow.A[%ld], "
+                                    "B[%ld]", h_o, w_o);
                   return ACL_ERROR_GE_INTERNAL_ERROR);
   GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(n_o, c_o),
-                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "int64 mul overflow.A[%ld], B[%ld]", n_o, c_o);
+                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][Shape]Failed, "
+                         "int64 mul overflow.A[%ld], B[%ld]", n_o, c_o);
+                  REPORT_CALL_ERROR("E19999", "Check shape failed, int64 mul overflow.A[%ld], "
+                                    "B[%ld]", n_o, c_o);
                   return ACL_ERROR_GE_INTERNAL_ERROR);
   auto t1 = h_o * w_o;
   auto t2 = n_o * c_o;
-  GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(t1, t2), GELOGE(INTERNAL_ERROR, "int64 mul overflow.A[%ld], B[%ld]", t1, t2);
+  GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(t1, t2),
+                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][Shape]Failed, "
+                         "int64 mul overflow.A[%ld], B[%ld]", t1, t2);
+                  REPORT_CALL_ERROR("E19999", "Check shape failed, "
+                                    "int64 mul overflow.A[%ld], B[%ld]", t1, t2);
                   return ACL_ERROR_GE_INTERNAL_ERROR);
 
   int64_t total_ele_cnt = n_o * c_o * h_o * w_o;
   GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(total_ele_cnt, size),
-                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "int64 mul overflow.A[%ld], B[%d]", total_ele_cnt, size);
+                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][Shape]Failed, "
+                  "int64 mul overflow.A[%ld], B[%d]", total_ele_cnt, size);
+                  REPORT_CALL_ERROR("E19999", "Check shape failed, int64 mul overflow.A[%ld], "
+                  "B[%d]", total_ele_cnt, size);
                   return ACL_ERROR_GE_INTERNAL_ERROR);
   int64_t dst_size = total_ele_cnt * size;
   if (dst_size == 0) {
@@ -140,14 +162,21 @@ Status TransFormatFromNchwToFzC04(const TransArgs &args, TransResult &result) {
 
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-           TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to alloc the memory for dst buf %ld "
+           "when trans format from %s to %s",
+           dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999",  "Failed to alloc the memory for dst buf %ld "
+                      "when trans format from %s to %s",
+                      dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
   auto retMem = memset_s(dst.get(), dst_size, 0, dst_size);
   if (retMem != EOK) {
-    GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "memst failed!");
+    GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Set][Memory]Failed, dst buf %ld, error_code %d",
+           dst_size, retMem);
+    REPORT_CALL_ERROR("E19999", "Set memory failed, dst buf %ld, error_code %d", dst_size, retMem);
     return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
   }
   // copy data
@@ -159,7 +188,10 @@ Status TransFormatFromNchwToFzC04(const TransArgs &args, TransResult &result) {
   for (auto k = 0; k < n; k++) {
     ret = memcpy_s(p_d + k * stride, protectSize, p_s + k * block, block);
     if (ret != EOK) {
-      GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "memcpy_s failed!");
+      GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Set][Memcpy]Failed, block %zu, stride %zu, "
+             "protect_size %ld, error_code %d", block, stride, protectSize, ret);
+      REPORT_CALL_ERROR("E19999", "[Set][Memcpy]Failed, block %zu, stride %zu, "
+                        "protect_size %ld, error_code %d", block, stride, protectSize, ret);
       return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
     }
     protectSize = protectSize - block;
@@ -169,7 +201,8 @@ Status TransFormatFromNchwToFzC04(const TransArgs &args, TransResult &result) {
   std::vector<int64_t> perm_arg_2 = {2, 0, 1, 3};
   ret = ge::formats::Transpose(dst.get(), shape_o, args.src_data_type, perm_arg_2, result);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Failed to Transpose from NCHW to HWCN");
+    GELOGE(ret, "[Trans][Formats]Failed from NCHW to HWCN, error_code %u", ret);
+    REPORT_CALL_ERROR("E19999", "Failed to trans formats from NCHW to HWCN, error_code %u", ret);
     return ret;
   }
 
@@ -190,7 +223,8 @@ Status PaddingNC(const TransArgs &args, TransArgs &args_tmp, std::shared_ptr<uin
   auto w = src_shape.at(kNchwW);
 
   if (c > kMaxDimsNumC) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Invalie dim c num[%lu].It should be in (0,4]", c);
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Invalid dim c num[%lu]. "
+           "It should be in (0,4]", c);
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
 
@@ -205,20 +239,33 @@ Status PaddingNC(const TransArgs &args, TransArgs &args_tmp, std::shared_ptr<uin
 
   // data overflow check
   GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(h_o, w_o),
-                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "int64 mul overflow.A[%ld], B[%ld]", h_o, w_o);
+                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][Shape]Failed, "
+                         "int64 mul overflow.A[%ld], B[%ld]", h_o, w_o);
+                  REPORT_CALL_ERROR("E19999", "Check shape failed, int64 mul overflow.A[%ld], "
+                                    "B[%ld]", h_o, w_o);
                   return ACL_ERROR_GE_INTERNAL_ERROR);
   GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(n_o, c_o),
-                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "int64 mul overflow.A[%ld], B[%ld]", n_o, c_o);
+                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][Shape]Failed, "
+                         "int64 mul overflow.A[%ld], B[%ld]", n_o, c_o);
+                  REPORT_CALL_ERROR("E19999", "Check shape failed, int64 mul overflow.A[%ld], "
+                                    "B[%ld]", n_o, c_o);
                   return ACL_ERROR_GE_INTERNAL_ERROR);
   auto t1 = h_o * w_o;
   auto t2 = n_o * c_o;
-  GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(t1, t2), GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "int64 mul overflow.A[%ld], B[%ld]", t1, t2);
+  GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(t1, t2),
+                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][Shape]Failed, "
+                         "int64 mul overflow.A[%ld], B[%ld]", t1, t2);
+                  REPORT_CALL_ERROR("E19999", "Check shape failed, int64 mul overflow.A[%ld], "
+                                    "B[%ld]", t1, t2);
                   return ACL_ERROR_GE_INTERNAL_ERROR);
 
   int64_t total_ele_cnt = n_o * c_o * h_o * w_o;
   int size = GetSizeByDataType(args.src_data_type);
   GE_IF_BOOL_EXEC(!CheckInt64MulOverflow(total_ele_cnt, size),
-                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "int64 mul overflow.A[%ld], B[%d]", total_ele_cnt, size);
+                  GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][Shape]Failed, "
+                         "int64 mul overflow.A[%ld], B[%d]", total_ele_cnt, size);
+                  REPORT_CALL_ERROR("E19999", "Check shape failed, int64 mul overflow.A[%ld], "
+                                    "B[%d]", total_ele_cnt, size);
                   return ACL_ERROR_GE_INTERNAL_ERROR);
 
   int64_t dst_size = total_ele_cnt * size;
@@ -228,14 +275,21 @@ Status PaddingNC(const TransArgs &args, TransArgs &args_tmp, std::shared_ptr<uin
 
   dst.reset(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
-           TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to alloc the memory for dst buf %ld when "
+           "trans format from %s to %s",
+           dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld when "
+                      "trans format from %s to %s",
+                      dst_size, TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
   auto ret = memset_s(dst.get(), dst_size, 0, dst_size);
   if (ret != EOK) {
-    GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "memst failed!");
+    GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Set][Memory]Failed, dst buf %ld, error_code %d",
+           dst_size, ret);
+    REPORT_CALL_ERROR("E19999", "Set memory failed, dst buf %ld, error_code %d", dst_size, ret);
     return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
   }
 
@@ -249,7 +303,10 @@ Status PaddingNC(const TransArgs &args, TransArgs &args_tmp, std::shared_ptr<uin
       ret = memcpy_s(p_d + (i * c_o * h_o * w_o + j * h_o * w_o) * size, protectSize,
                      p_s + (i * c * h * w + j * h * w) * size, block);
       if (ret != EOK) {
-        GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "memcpy_s failed!");
+        GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED, "[Set][Memcpy]Failed, block %zu, "
+               "protect_size %ld, error_code %d", block, protectSize, ret);
+        REPORT_CALL_ERROR("E19999", "[Set][Memcpy]Failed, block %zu, protect_size %ld, "
+                          "error_code %d", block, protectSize, ret);
         return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
       }
       protectSize = protectSize - block;
@@ -270,12 +327,14 @@ Status FormatTransferNchwToFZC04::TransFormat(const TransArgs &args, TransResult
   std::shared_ptr<uint8_t> dst = nullptr;
   auto ret = PaddingNC(args, args_tmp, dst);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Padding in NC axis failed!");
+    GELOGE(ret, "[Padding][NCAxis]Failed, error_code %u", ret);
+    REPORT_CALL_ERROR("E19999", "Padding in NC axis failed, error_code %u", ret);
     return ret;
   }
 
   std::vector<int64_t> expect_shape;
-  ret = TransShape(args_tmp.src_format, args_tmp.src_shape, args_tmp.src_data_type, args_tmp.dst_format, expect_shape);
+  ret = TransShape(args_tmp.src_format, args_tmp.src_shape, args_tmp.src_data_type,
+                   args_tmp.dst_format, expect_shape);
   if (ret != SUCCESS) {
     return ret;
   }

ge/common/formats/format_transfers/format_transfer_nchw_nc1hwc0.cc

@@ -32,12 +32,17 @@ Status TransShapeNchwToNc1hwc0(const std::vector<int64_t> &src_shape, DataType d
                                std::vector<int64_t> &dst_shape) {
   int64_t c0 = GetCubeSizeByDataType(data_type);
   if (c0 <= 0) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Failed to get cube size, the data type is invalid");
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Get][Cube]Failed, the data type %s is invalid",
+           TypeUtils::DataTypeToSerialString(data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to get cube size, the data type %s is invalid",
+                      TypeUtils::DataTypeToSerialString(data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShapeValid(src_shape, kNchwDimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check src shape %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
            ShapeToString(src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                      ShapeToString(src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   dst_shape.clear();
@@ -47,8 +52,10 @@ Status TransShapeNchwToNc1hwc0(const std::vector<int64_t> &src_shape, DataType d
   dst_shape.push_back(src_shape.at(kNchwW));
   dst_shape.push_back(c0);
   if (!CheckShapeValid(dst_shape, kNc1hwc0DimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
            ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid",
+                      ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   return SUCCESS;
@@ -69,10 +76,17 @@ Status CheckArgsForNchwToNc1hwc0(const TransArgs &args) {
   }
   if (expect_5d_shape != args.dst_shape) {
     GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
-           "Failed to trans format, the src and dst shape are not compatible. data"
-           " type %s, src shape %s, dst shape %s, expect dst shape %s",
-           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(), ShapeToString(args.src_shape).c_str(),
+           "[Trans][Format]Failed, the src and dst shape are not compatible. "
+           "data type %s, src shape %s, dst shape %s, expect dst shape %s",
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+           ShapeToString(args.src_shape).c_str(),
            ShapeToString(args.dst_shape).c_str(), ShapeToString(expect_5d_shape).c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to trans formats, the src and dst shape are not "
+                       "compatible. data type %s, src shape %s, dst shape %s, expect dst shape %s",
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                       ShapeToString(args.src_shape).c_str(),
+                       ShapeToString(args.dst_shape).c_str(),
+                       ShapeToString(expect_5d_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
 
@@ -83,10 +97,16 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
     GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION,
-           "Failed to trans format from %s to %s, can not alloc the memory for"
-           " dst buf %ld, shape %s",
+           "[Allcoate][Memory]Failed to alloc the memory for dst buf %ld, "
+           "shape %s when trans format from %s to %s",
+           total_size, ShapeToString(args.dst_shape).c_str(),
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), total_size, ShapeToString(args.dst_shape).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, "
+                      "shape %s when trans format from %s to %s",
+                      total_size, ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -97,7 +117,10 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
 
   int64_t c0 = GetCubeSizeByDataType(args.src_data_type);
   if (c0 <= 0) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "The c0 is invalid %ld", c0);
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Check][Shape]The c0 is invalid %ld, data_type %s",
+           c0, TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Check shape failed, the c0 is invalid %ld, data_type %s",
+                      c0, TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   int64_t c1 = (c - 1) / c0 + 1;
@@ -130,9 +153,13 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
                                   static_cast<size_t>(size));
               if (ret != EOK) {
                 GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                       "Failed to copy data from NCHW[%ld] offset %ld to "
-                       "NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                       "[Operate][Memory]Failed to copy data from NCHW[%ld] offset %ld "
+                       "to NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld, err-code %d",
                        srcIdx, src_offset, n_idx, c1_idx, h_idx, w_idx, c0_idx, dst_offset, ret);
+                REPORT_CALL_ERROR("E19999", "Failed to copy data from NCHW[%ld] offset %ld "
+                                  "to NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                                  srcIdx, src_offset, n_idx, c1_idx, h_idx, w_idx, c0_idx,
+                                  dst_offset, ret);
                 return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
               }
             } else {
@@ -140,9 +167,12 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
                   memset_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), 0, static_cast<size_t>(size));
               if (ret != EOK) {
                 GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                       "Failed to set to 0 to "
-                       "NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                       "[Operate][Memory]Failed to set to 0 to NC1HWC0[%ld, %ld, %ld, %ld, %ld] "
+                       "offset %ld, err-code %d",
                        n_idx, c1_idx, h_idx, w_idx, c0_idx, dst_offset, ret);
+                REPORT_CALL_ERROR("E19999", "Failed to set to 0 to "
+                                  "NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld, err-code %d",
+                                  n_idx, c1_idx, h_idx, w_idx, c0_idx, dst_offset, ret);
                 return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
               }
             }
@@ -173,8 +203,12 @@ Status FormatTransferNchwNc1hwc0::TransFormat(const TransArgs &args, TransResult
       return SUCCESS;
     }
 
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Get %ld total size from dst shape %s, src shape %s", total_size,
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Get][Shape]Failed, total size %ld from dst shape %s, "
+           "src shape %s", total_size,
            ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to get total size %ld from dst shape %s, src shape %s",
+                      total_size,
+                      ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   GELOGD(
@@ -184,9 +218,16 @@ Status FormatTransferNchwNc1hwc0::TransFormat(const TransArgs &args, TransResult
       ShapeToString(args.dst_shape).c_str(), total_size);
   ret = GetDstDataAfterTrans(args, result, size, total_size);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
-           ShapeToString(args.dst_shape).c_str(), total_size);
+    GELOGE(ret, "[Get][Data]Failed, after trans, src shape %s, data type %s, "
+           "dst shape %s, memory size %ld, error_code %u",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+           ShapeToString(args.dst_shape).c_str(), total_size, ret);
+    REPORT_CALL_ERROR("E19999", "Failed to get data after trans, src shape %s, data type %s, "
+                      "dst shape %s, memory size %ld, error_code %u",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                      ShapeToString(args.dst_shape).c_str(), total_size, ret);
     return ret;
   }
   return SUCCESS;

ge/common/formats/format_transfers/format_transfer_nhwc_nc1hwc0.cc

@@ -34,7 +34,10 @@ Status TransShapeNhwcToNc1hwc0(const std::vector<int64_t> &src_shape, DataType d
                                std::vector<int64_t> &dst_shape) {
   int64_t c0 = GetCubeSizeByDataType(data_type);
   if (c0 <= 0) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Failed to get cube size, the data type is invalid");
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Get][Cube]Failed, the data type %s is invalid",
+           TypeUtils::DataTypeToSerialString(data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to get cube size, the data type %s is invalid",
+                      TypeUtils::DataTypeToSerialString(data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   dst_shape.clear();
@@ -44,8 +47,10 @@ Status TransShapeNhwcToNc1hwc0(const std::vector<int64_t> &src_shape, DataType d
   dst_shape.push_back(src_shape.at(kNhwcW));
   dst_shape.push_back(c0);
   if (!CheckShapeValid(dst_shape, kNc1hwc0DimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s",
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
            ShapeToString(dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid",
+                      ShapeToString(dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   return SUCCESS;
@@ -60,16 +65,25 @@ Status CheckArgsForNhwcToNc1hwc0(const TransArgs &args) {
     return ACL_ERROR_GE_FORMAT_INVALID;
   }
   if (!CheckDataTypeSupported(args.src_data_type)) {
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Failed to trans shape from NHWC to NC1HWC0, invalid data type %s",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Check][DataType]Failed from NHWC to NC1HWC0, "
+           "invalid data type %s",
            TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to trans shape from NHWC to NC1HWC0, invalid data type %s",
+                       TypeUtils::DataTypeToSerialString(args.src_data_type).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   if (!CheckShapeValid(args.src_shape, kNhwcDimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check src shape %s", ShapeToString(args.src_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
+           ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                      ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   if (!CheckShapeValid(args.dst_shape, kNc1hwc0DimsNum)) {
-    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check dst shape %s", ShapeToString(args.dst_shape).c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s",
+           ShapeToString(args.dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "Dst shape %s check valid",
+                      ShapeToString(args.dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
   std::vector<int64_t> expect_dst_shape;
@@ -79,10 +93,14 @@ Status CheckArgsForNhwcToNc1hwc0(const TransArgs &args) {
   }
   if (args.dst_shape != expect_dst_shape) {
     GELOGE(ACL_ERROR_GE_SHAPE_INVALID,
-           "Failed to trans format, the src and dst shape are not compatible. src shape %s, dst shape %s, "
+           "[Trans][Format]Failed , the src shape %s and dst shape %s are not compatible. "
            "expect dst shape %s",
            ShapeToString(args.src_shape).c_str(), ShapeToString(args.dst_shape).c_str(),
            ShapeToString(expect_dst_shape).c_str());
+    REPORT_CALL_ERROR("E19999",  "Failed to trans format, the src shape %s and "
+                      "dst shape %s are not compatible. expect dst shape %s",
+                      ShapeToString(args.src_shape).c_str(), ShapeToString(args.dst_shape).c_str(),
+                      ShapeToString(expect_dst_shape).c_str());
     return ACL_ERROR_GE_SHAPE_INVALID;
   }
 
@@ -92,9 +110,16 @@ Status CheckArgsForNhwcToNc1hwc0(const TransArgs &args) {
 Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const int size, const int64_t total_size) {
   std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[total_size], std::default_delete<uint8_t[]>());
   if (dst == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld, shape %s",
+    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Allcoate][Memory]Failed, memory for dst buf %ld, "
+           "shape %s when trans format from %s to %s",
+           total_size, ShapeToString(args.dst_shape).c_str(),
            TypeUtils::FormatToSerialString(args.src_format).c_str(),
-           TypeUtils::FormatToSerialString(args.dst_format).c_str(), total_size, ShapeToString(args.dst_shape).c_str());
+           TypeUtils::FormatToSerialString(args.dst_format).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, "
+                      "shape %s when trans format from %s to %s",
+                      total_size, ShapeToString(args.dst_shape).c_str(),
+                      TypeUtils::FormatToSerialString(args.src_format).c_str(),
+                      TypeUtils::FormatToSerialString(args.dst_format).c_str());
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
 
@@ -132,17 +157,27 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
               auto ret = memcpy_s(dst.get() + dst_offset, protected_size, args.data + src_offset, size);
               if (ret != EOK) {
                 GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                       "Failed to copy data from NHWC[%ld, %ld, %ld, %ld] offset %ld to "
-                       "NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld err-code %d",
-                       n_idx, h_idx, w_idx, c_idx, src_offset, n_idx, c1_idx, h_idx, w_idx, c0_idx, dst_offset, ret);
+                       "[Operate][Memory]Failed to copy data from NHWC[%ld, %ld, %ld, %ld] "
+                       "offset %ld to NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld err-code %d",
+                       n_idx, h_idx, w_idx, c_idx, src_offset,
+                       n_idx, c1_idx, h_idx, w_idx, c0_idx, dst_offset, ret);
+                REPORT_CALL_ERROR("E19999", "Failed to copy data from NHWC[%ld, %ld, %ld, %ld] "
+                                  "offset %ld to "
+                                  "NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld err-code %d",
+                                  n_idx, h_idx, w_idx, c_idx, src_offset,
+                                  n_idx, c1_idx, h_idx, w_idx, c0_idx, dst_offset, ret);
                 return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
               }
             } else {
               auto ret = memset_s(dst.get() + dst_offset, protected_size, 0, size);
               if (ret != EOK) {
                 GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-                       "Failed to set 0 to NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld base err-code %d", n_idx, c1_idx,
-                       h_idx, w_idx, c0_idx, dst_offset, ret);
+                       "[Operate][Memory]Failed to set 0 to "
+                       "NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld base err-code %d",
+                       n_idx, c1_idx, h_idx, w_idx, c0_idx, dst_offset, ret);
+                REPORT_CALL_ERROR("E19999", "Failed to set 0 to "
+                                  "NC1HWC0[%ld, %ld, %ld, %ld, %ld] offset %ld base err-code %d",
+                                  n_idx, c1_idx, h_idx, w_idx, c0_idx, dst_offset, ret);
                 return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
               }
             }
@@ -171,8 +206,12 @@ Status FormatTransferNhwcNc1hwc0::TransFormat(const TransArgs &args, TransResult
       return SUCCESS;
     }
 
-    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "Get %ld total size from dst shape %s, src shape %s", total_size,
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Get][ShapeSize]Failed, "
+           "total size %ld from dst shape %s, src shape %s", total_size,
            ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999", "[Get][Shape]Failed, total size %ld from "
+                      "dst shape %s, src shape %s", total_size,
+                      ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_DATATYPE_INVALID;
   }
   GELOGD("Begin to trans format from NHWC to NC1HWC0, src shape %s, data type %s, dst shape %s, memory size %ld",
@@ -181,9 +220,16 @@ Status FormatTransferNhwcNc1hwc0::TransFormat(const TransArgs &args, TransResult
 
   ret = GetDstDataAfterTrans(args, result, size, total_size);
   if (ret != SUCCESS) {
-    GELOGE(ret, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
-           ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
-           ShapeToString(args.dst_shape).c_str(), total_size);
+    GELOGE(ret, "[Get][Data]Failed, after trans, src shape %s, data type %s, "
+           "dst shape %s, memory size %ld, error_code %u",
+           ShapeToString(args.src_shape).c_str(),
+           TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+           ShapeToString(args.dst_shape).c_str(), total_size, ret);
+    REPORT_CALL_ERROR("E19999", "Failed to get data after trans, src shape %s, data type %s, "
+                      "dst shape %s, memory size %ld, error_code %u",
+                      ShapeToString(args.src_shape).c_str(),
+                      TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
+                      ShapeToString(args.dst_shape).c_str(), total_size, ret);
     return ret;
   }
   return SUCCESS;
@@ -193,8 +239,10 @@ Status FormatTransferNhwcNc1hwc0::TransShape(Format src_format, const std::vecto
                                              DataType data_type, Format dst_format, std::vector<int64_t> &dst_shape) {
   if (src_format == FORMAT_NHWC && CheckDataTypeSupported(data_type)) {
     if (!CheckShapeValid(src_shape, kNhwcDimsNum)) {
-      GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "Failed to check src shape %s",
+      GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s",
              ShapeToString(src_shape).c_str());
+      REPORT_CALL_ERROR("E19999", "Src shape %s check invalid",
+                        ShapeToString(src_shape).c_str());
       return ACL_ERROR_GE_SHAPE_INVALID;
     }
     return TransShapeNhwcToNc1hwc0(src_shape, data_type, dst_shape);

ge/common/formats/format_transfers/format_transfer_transpose.cc

@@ -50,21 +50,21 @@ std::map<Format, std::map<Format, std::vector<int64_t>>> perm_args{
 bool IsShapeArgValid(const std::vector<int64_t> &src_shape, const std::vector<int64_t> &perm_arg) {
   if (src_shape.empty()) {
     std::string error = "Failed to transpose, empty src shape";
-    GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, error.c_str());
-    GELOGE(PARAM_INVALID, "Failed to transpose, empty src shape");
+    GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_SHAPE_INVALID, error.c_str());
+    GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Trans][Shape]Failed, empty src shape");
     return false;
   }
   for (auto dim : src_shape) {
     if (dim < 0) {
       std::string error = "Failed to transpose, negative dim in src shape " + FmtToStr(ShapeToString(src_shape));
-      GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, error.c_str());
+      GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_SHAPE_INVALID, error.c_str());
       return false;
     }
   }
   if (perm_arg.size() != src_shape.size()) {
     std::string error = "Failed to transpose, the size of src shape" + FmtToStr(src_shape.size()) +
         " and perm arg" +  FmtToStr(perm_arg.size()) + " are different";
-    GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, error.c_str());
+    GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_SHAPE_INVALID, error.c_str());
     return false;
   }
 
@@ -73,7 +73,7 @@ bool IsShapeArgValid(const std::vector<int64_t> &src_shape, const std::vector<in
     if (perm < 0 || static_cast<size_t>(perm) >= perm_arg.size() || ++exists[perm] > 1) {
       std::string error = "Failed to transpose, duplicated perm arg " + FmtToStr(perm) +
         ", perm arg " +  FmtToStr(JoinToString(perm_arg));
-      GE_ERRORLOG_AND_ERRORMSG(PARAM_INVALID, error.c_str());
+      GE_ERRORLOG_AND_ERRORMSG(ACL_ERROR_GE_PARAM_INVALID, error.c_str());
       return false;
     }
   }
@@ -82,12 +82,14 @@ bool IsShapeArgValid(const std::vector<int64_t> &src_shape, const std::vector<in
 bool IsTransposeArgValid(const uint8_t *src, const std::vector<int64_t> &src_shape, DataType src_data_type,
                          const std::vector<int64_t> &perm_arg) {
   if (src == nullptr) {
-    GELOGE(PARAM_INVALID, "Failed to transpose, the src is null");
+    GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Trans][Param]Failed, the src is null");
     return false;
   }
   if (GetSizeByDataType(src_data_type) < 0) {
-    GELOGE(UNSUPPORTED, "Failed to transpose, the data type %s is not support",
+    GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Trans][Param]Failed, the data type %s is not support",
            TypeUtils::DataTypeToSerialString(src_data_type).c_str());
+    REPORT_CALL_ERROR("E19999", "Failed to transpose, the data type %s is not support",
+                      TypeUtils::DataTypeToSerialString(src_data_type).c_str());
     return false;
   }
   return IsShapeArgValid(src_shape, perm_arg);
@@ -173,10 +175,15 @@ Status Transpose(const uint8_t *src, const std::vector<int64_t> &src_shape, Data
                         static_cast<size_t>(data_size));
     if (ret != EOK) {
       GELOGE(ACL_ERROR_GE_MEMORY_OPERATE_FAILED,
-             "Failed to transpose, src shape %s, perm arg %s, dst shape %s, "
+             "[Operate][Memory]Failed to transpose, src shape %s, perm arg %s, dst shape %s, "
              "failed to write to dst offset %ld, current dim offset %s",
              ShapeToString(src_shape).c_str(), ShapeToString(perm_arg).c_str(), ShapeToString(dst_shape).c_str(),
              dst_offset_bytes, ShapeToString(dst_indexes).c_str());
+      REPORT_CALL_ERROR("E19999", "Failed to transpose, src shape %s, perm arg %s, dst shape %s, "
+                        "failed to write to dst offset %ld, current dim offset %s",
+                        ShapeToString(src_shape).c_str(), ShapeToString(perm_arg).c_str(),
+                        ShapeToString(dst_shape).c_str(),
+                        dst_offset_bytes, ShapeToString(dst_indexes).c_str());
       return ACL_ERROR_GE_MEMORY_OPERATE_FAILED;
     }
     AddOne(dst_shape, dst_indexes);

ge/common/formats/formats.cc

@@ -44,7 +44,12 @@ GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY Status TransFormat(const TransArg
 
   auto src_shape_size = GetItemNumByShape(args.src_shape);
   if (args.data == nullptr && src_shape_size != 0) {
-    GELOGE(ACL_ERROR_GE_PARAM_INVALID, "Invalid input null data");
+    GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Shape]Failed, input data is null "
+           "or shape size not euqal to 0, src_shape %s",
+           ShapeToString(args.src_shape).c_str());
+    REPORT_CALL_ERROR("E19999","Failed to check shape, input data is null "
+                      "or shape size not equal to 0, src_shape %s",
+                      ShapeToString(args.src_shape).c_str());
     return ACL_ERROR_GE_PARAM_INVALID;
   }
 
@@ -82,7 +87,8 @@ GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY Status TransDataType(const CastAr
   }
 
   if (args.data == nullptr && args.src_data_size != 0) {
-    GELOGE(ACL_ERROR_GE_PARAM_INVALID, "Invalid input null data");
+    GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Param]Failed, input data is null "
+           "or data size not equal to 0, src_data_size %ld", args.src_data_size);
     return ACL_ERROR_GE_PARAM_INVALID;
   }
 

ge/common/helper/model_helper.cc

@@ -87,12 +87,13 @@ Status ModelHelper::SaveSizeToModelDef(const GeModelPtr &ge_model) {
 
   std::shared_ptr<ModelTaskDef> model_task_def = ge_model->GetModelTaskDefPtr();
   if (model_task_def == nullptr) {
-    GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Create model task def ptr failed");
-    return ACL_ERROR_GE_MEMORY_ALLOCATION;
+    GELOGD("SaveSizeToModelDef task_info_size is 0.");
+    om_info.push_back(0);
+  } else {
+    size_t partition_task_size = model_task_def->ByteSizeLong();
+    GELOGD("SaveSizeToModelDef task_info_size is %zu", partition_task_size);
+    om_info.push_back(partition_task_size);
   }
-  size_t partition_task_size = model_task_def->ByteSizeLong();
-  GELOGD("SaveSizeToModelDef task_info_size is %zu", partition_task_size);
-  om_info.push_back(partition_task_size);
 
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(*(ge_model.get()), "om_info_list", om_info),
                    GELOGE(FAILED, "SetListInt of om_info_list failed.");
@@ -598,6 +599,7 @@ Status ModelHelper::GenerateGeRootModel(OmFileLoadHelper &om_load_helper) {
       is_first_model = false;
       root_model_->SetRootGraph(GraphUtils::GetComputeGraph(cur_model->GetGraph()));
       root_model_->SetModelId(cur_model->GetModelId());
+      root_model_->SetModelName(cur_model->GetName());
       model_ = cur_model;
       continue;
     }

ge/common/helper/om_file_helper.cc

@@ -416,8 +416,7 @@ Status OmFileSaveHelper::SaveRootModel(const SaveParam &save_param, const char *
   if (is_offline) {
     ret = FileSaver::SaveToFile(output_file, model_header_, model_partition_tabels, all_model_partitions);
   } else {
-    GELOGW("do not support save ge root model to buff now");
-    return FAILED;
+    ret = FileSaver::SaveToBuffWithFileHeader(model_header_, model_partition_tabels, all_model_partitions, model);
   }
   if (ret == SUCCESS) {
     GELOGD("Save model success without encrypt.");

ge/common/profiling/profiling_manager.cc

@@ -24,6 +24,7 @@
 #include "graph/types.h"
 #include "runtime/base.h"
 #include "graph/load/model_manager/davinci_model.h"
+#include "mmpa/mmpa_api.h"
 
 namespace {
 const char *const kTrainingTrace = "training_trace";
@@ -31,7 +32,6 @@ const char *const kFpPoint = "fp_point";
 const char *const kBpPoint = "bp_point";
 
 #ifdef DAVINCI_SUPPORT_PROFILING
-const size_t kReportMaxLen = 2048;
 const int32_t kMaxDeviceNum = 256;
 const uint32_t kInteval = 2;
 const std::string kConfigNumsdev = "devNums";
@@ -47,6 +47,10 @@ const std::string kOptype = "op_type";
 const std::string kBlockDim = "block_dims";
 const std::string kTaskId = "task_id";
 const std::string kStreamId = "stream_id";
+const std::string kThreadId = "thread_id";
+const std::string kIndexId = "index_id";
+const std::string kTimeStamp = "time_stamp";
+const std::string kTagId = "tag_id";
 const std::string kShapeType = "shape_type";
 const std::string kCurIterNum = "cur_iter_num";
 const std::string kTaskType = "task_type";
@@ -287,27 +291,80 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin
 #endif
 }
 
+FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfileStepInfo(
+  uint64_t index_id, uint64_t model_id, uint16_t tag_id, rtStream_t stream, int32_t device_id) {
+#ifdef DAVINCI_SUPPORT_PROFILING
+  rtError_t rt_ret = RT_ERROR_NONE;
+#ifndef ONLY_COMPILE_OPEN_SRC
+  GELOGD("Profiling Step Info TraceTask execute async start, index_id = %lu, model_id = %lu, tag_id = %u",
+         index_id, model_id, tag_id);
+  rt_ret = rtProfilerTraceEx(index_id, model_id, tag_id, stream);
+  if (rt_ret != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "[Call][rtProfilerTraceEx] failed, ret: 0x%X", rt_ret);
+    return RT_ERROR_TO_GE_STATUS(rt_ret);
+  }
+  GELOGD("Profiling Step Info TraceTask execute async success, index_id = %lu, model_id = %lu, tag_id = %u",
+         index_id, model_id, tag_id);
+#endif
+
+  mmTimespec timespec = mmGetTickCount();
+  // 1000 ^ 3 converts second to nanosecond
+  int64_t time = timespec.tv_sec * 1000 * 1000 * 1000 + timespec.tv_nsec;
+  uint32_t task_id = 0;
+  uint32_t stream_id = 0;
+  rt_ret = rtGetTaskIdAndStreamID(&task_id, &stream_id);
+  if (rt_ret != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "[Get][RtsInfo] task_id and stream_id failed, ret: 0x%X.", rt_ret);
+    return RT_ERROR_TO_GE_STATUS(rt_ret);
+  }
+  GELOGD("Get profiling args, task_id[%u], stream_id[%u]", task_id, stream_id);
+
+  Json step_info;
+  step_info[kIndexId] = index_id;
+  step_info[kModelId] = model_id;
+  step_info[kTimeStamp] = time;
+  step_info[kTagId] = tag_id;
+  step_info[kTaskId] = task_id;
+  step_info[kStreamId] = stream_id;
+  step_info[kThreadId] = mmGetTid();
+
+  std::string reported_data;
+  try {
+    reported_data = step_info.dump(kInteval, ' ', false, Json::error_handler_t::ignore);
+  } catch (std::exception &e) {
+    GELOGE(FAILED, "Failed to convert JSON to string, reason: %s.", e.what());
+  } catch (...) {
+    GELOGE(FAILED, "Failed to convert JSON to string.");
+  }
+  reported_data.append(",")
+               .append("\n");
+  ReportData(device_id, reported_data, "step_info");
+#endif
+  return SUCCESS;
+}
+
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportData(
     const int32_t &device_id, const string &data, const string &tag_name) {
 #ifdef DAVINCI_SUPPORT_PROFILING
   ReporterData reporter_data{};
   int ret = -1;
   int32_t cb_ret = -1;
-  size_t index = data.size() / kReportMaxLen;
+  size_t report_max_len = reporter_max_len_;
+  size_t index = data.size() / report_max_len;
   if (index >= 1) {
     reporter_data.deviceId = device_id;
     ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, tag_name.c_str(), tag_name.size());
     GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag [%s] memcpy error!", tag_name.c_str()); return;);
     for (size_t i = 0; i < index; ++i) {
-      reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * i;
-      reporter_data.dataLen = kReportMaxLen;
+      reporter_data.data = (unsigned char *)data.c_str() + report_max_len * i;
+      reporter_data.dataLen = report_max_len;
       cb_ret = CallMsprofReport(reporter_data);
       GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data [%s] failed, ret:%d", tag_name.c_str(), cb_ret);
                       return;);
     }
-    reporter_data.dataLen = data.size() - kReportMaxLen * index;
+    reporter_data.dataLen = data.size() - report_max_len * index;
     if (reporter_data.dataLen != 0) {
-      reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * index;
+      reporter_data.data = (unsigned char *)data.c_str() + report_max_len * index;
       cb_ret = CallMsprofReport(reporter_data);
       GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data [%s] failed, ret:%d", tag_name.c_str(), cb_ret);
                       return;);
@@ -745,15 +802,32 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ProfilingManager::Profilin
   return  execute_model_prof_on;
 }
 
-FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::PluginInit() const {
+FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::PluginInit() {
   if (prof_cb_.msprofReporterCallback == nullptr) {
     GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr.");
     return ge::PARAM_INVALID;
   }
-  return prof_cb_.msprofReporterCallback(
+  int32_t cb_ret = prof_cb_.msprofReporterCallback(
       static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK),
       static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_INIT),
       nullptr, 0);
+  if (cb_ret != MSPROF_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Profiling reporter init failed, ret = %d.", cb_ret);
+    GELOGE(INTERNAL_ERROR, "[Init][ProfilingReporter] profiling init failed, ret = %d.", cb_ret);
+    return INTERNAL_ERROR;
+  }
+
+  cb_ret = prof_cb_.msprofReporterCallback(
+      static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK),
+      static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_DATA_MAX_LEN),
+      &reporter_max_len_, sizeof(uint32_t));
+  if (cb_ret != MSPROF_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Get profiling reporter data max len failed, ret = %d.", cb_ret);
+    GELOGE(INTERNAL_ERROR, "[Init][ProfilingReporter] Get profiling reporter data max len failed, ret = %d.", cb_ret);
+    return INTERNAL_ERROR;
+  }
+
+ return SUCCESS;
 }
 
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit() const {

ge/common/profiling/profiling_manager.h

@@ -27,6 +27,7 @@
 #include "framework/common/ge_types.h"
 #include "external/register/register_types.h"
 #include "toolchain/prof_callback.h"
+#include "runtime/stream.h"
 
 using std::map;
 using std::string;
@@ -88,7 +89,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager {
   void ProfilingTaskDescInfo(uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info,
                              const int32_t &device_id);
   void ProfilingOpInputOutInfo(const TaskDescInfo &task, Json &task_json);
-  Status PluginInit() const;
+  Status PluginInit();
   void PluginUnInit() const;
   Status CallMsprofReport(ReporterData &reporter_data) const;
   struct MsprofCallback &GetMsprofCallback() { return prof_cb_; }
@@ -97,6 +98,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager {
   void GetFpBpPoint(std::string &fp_point, std::string &bp_point);
   void GetOpInputOutputInfo(const OpDescPtr &op, TaskDescInfo &task_desc_info) const;
   void ReportData(const int32_t &device_id, const std::string &data, const std::string &tag_name);
+  Status ProfileStepInfo(uint64_t index_id, uint64_t model_id, uint16_t tag_id, rtStream_t stream, int32_t device_id);
  private:
   Status InitFromOptions(const Options &options, MsprofGeOptions &prof_conf);
   Status ParseOptions(const std::string &options);
@@ -119,6 +121,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager {
   MsprofCallback prof_cb_;
   std::string fp_point_;
   std::string bp_point_;
+  uint32_t reporter_max_len_ = 0;
 };
 }  // namespace ge
 #endif  // GE_COMMON_PROFILING_PROFILING_MANAGER_H_

ge/common/proto/insert_op.proto

@@ -88,6 +88,7 @@ message AippOpParams {
 	int32 right_padding_size = 69;
 	int32 top_padding_size = 70;
 	int32 bottom_padding_size = 71;
+	float padding_value = 72;
 
 	int32 mean_chn_0 = 10;
 	int32 mean_chn_1 = 11;

ge/common/tbe_kernel_store.cc

@@ -15,6 +15,8 @@
  */
 
 #include "common/tbe_kernel_store.h"
+#include "graph/utils/attr_utils.h"
+#include "graph/debug/ge_attr_define.h"
 
 namespace ge {
 
@@ -31,6 +33,15 @@ void TBEKernelStore::LoadTBEKernelBinToOpDesc(const std::shared_ptr<ge::OpDesc>
       GE_IF_BOOL_EXEC(!op_desc->SetExtAttr(ge::OP_EXTATTR_NAME_TBE_KERNEL, kernel_bin),
                       GELOGW("LoadKernelTBEBinToOpDesc: SetExtAttr for kernel_bin failed");)
       GELOGI("Load tbe kernel:%s, %zu", kernel_bin->GetName().c_str(), kernel_bin->GetBinDataSize());
+
+      std::string atomic_kernel_name;
+      (void) AttrUtils::GetStr(op_desc, ATOMIC_ATTR_TBE_KERNEL_NAME, atomic_kernel_name);
+      if (!atomic_kernel_name.empty()) {
+        GELOGI("Get atomic kernel name is %s.", atomic_kernel_name.c_str());
+        auto atomic_kernel_bin = FindKernel(atomic_kernel_name);
+        GE_IF_BOOL_EXEC(!op_desc->SetExtAttr(EXT_ATTR_ATOMIC_TBE_KERNEL, atomic_kernel_bin),
+                        GELOGW("LoadKernelTBEBinToOpDesc: SetExtAttr for atomic kernel_bin failed");)
+      }
     }
   }
 }

ge/common/types.cc

@@ -90,6 +90,8 @@ REGISTER_OPTYPE_DEFINE(DEPCONVOLUTION, "ConvolutionDepthwise");
 REGISTER_OPTYPE_DEFINE(DROPOUT, "Dropout");
 REGISTER_OPTYPE_DEFINE(DROPOUTGENMASK, "DropOutGenMask");
 REGISTER_OPTYPE_DEFINE(DROPOUTDOMASK, "DropOutDoMask");
+REGISTER_OPTYPE_DEFINE(DROPOUTDOMASKV3, "DropOutDoMaskV3");
+REGISTER_OPTYPE_DEFINE(DROPOUTDOMASKV3D, "DropOutDoMaskV3D");
 REGISTER_OPTYPE_DEFINE(CONCAT, "Concat");
 REGISTER_OPTYPE_DEFINE(ROIPOOLING, "ROIPooling");
 REGISTER_OPTYPE_DEFINE(PROPOSAL, "Proposal");

ge/executor/CMakeLists.txt

@@ -8,6 +8,7 @@ set(PROTO_LIST
 )
 
 protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
+protobuf_generate(ge_static PROTO_STATIC_SRCS PROTO_STATIC_HDRS ${PROTO_LIST})
 
 set(SRC_LIST
     "ge_executor.cc"
@@ -15,6 +16,7 @@ set(SRC_LIST
     "../common/ge/plugin_manager.cc"
     "../common/ge/op_tiling_manager.cc"
     "../common/dump/dump_properties.cc"
+    "../common/dump/exception_dumper.cc"
     "../common/dump/dump_manager.cc"
     "../common/dump/dump_op.cc"
     "../common/dump/opdebug_register.cc"
@@ -162,7 +164,7 @@ set(SRC_LIST
 )
 
 ######## libge_executor.a ########
-add_library(ge_executor STATIC ${SRC_LIST} ${PROTO_HDRS})
+add_library(ge_executor STATIC ${SRC_LIST} ${PROTO_STATIC_HDRS})
 
 target_compile_options(ge_executor PRIVATE
     $<$<OR:$<STREQUAL:${TARGET_SYSTEM_NAME},Linux>,$<STREQUAL:${TARGET_SYSTEM_NAME},Android>>:-fvisibility=hidden -O2 -Werror -Wno-deprecated-declarations -fno-common>
@@ -178,6 +180,7 @@ target_compile_definitions(ge_executor PRIVATE
     google=ascend_private
     $<IF:$<STREQUAL:${TARGET_SYSTEM_NAME},Windows>,OS_TYPE=WIN,OS_TYPE=0>
     $<$<STREQUAL:${TARGET_SYSTEM_NAME},Windows>:SECUREC_USING_STD_SECURE_LIB=0 NOMINMAX>
+    $<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
     LOG_CPP
 )
 
@@ -191,7 +194,7 @@ target_include_directories(ge_executor SYSTEM PRIVATE
     ${METADEF_DIR}/inc/external/graph
     ${METADEF_DIR}/inc/graph
     ${CMAKE_BINARY_DIR}
-    ${CMAKE_BINARY_DIR}/proto/ge
+    ${CMAKE_BINARY_DIR}/proto/ge_static
     #### yellow zone ####
     ${GE_CODE_DIR}/../inc
     ${GE_CODE_DIR}/../inc/cce
@@ -212,6 +215,7 @@ target_link_libraries(ge_executor PRIVATE
 add_library(ge_executor_shared SHARED ${SRC_LIST} ${PROTO_HDRS})
 
 target_compile_options(ge_executor_shared PRIVATE
+    -fno-common
     -Werror
     -O2
     -Wno-deprecated-declarations
@@ -223,6 +227,7 @@ target_compile_definitions(ge_executor_shared PRIVATE
     DAVINCI_SUPPORT_PROFILING
     google=ascend_private
     FUNC_VISIBILITY
+    $<$<STREQUAL:${ENABLE_OPEN_SRC},True>:ONLY_COMPILE_OPEN_SRC>
 )
 
 target_include_directories(ge_executor_shared PRIVATE

ge/executor/ge_executor.cc

@@ -30,6 +30,8 @@
 #include "single_op/single_op_manager.h"
 #include "graph/load/model_manager/davinci_model.h"
 #include "opskernel_manager/ops_kernel_builder_manager.h"
+#include "graph/opsproto_manager.h"
+#include "ge_local_engine/engine/host_cpu_engine.h"
 
 using std::string;
 using std::vector;
@@ -199,6 +201,33 @@ bool IsDynmaicDimsSizeMatchModel(const vector<uint64_t> cur_dynamic_dims,
 namespace ge {
 bool GeExecutor::isInit_ = false;
 
+static void InitOpsProtoManager() {
+  string opsproto_path;
+  const char *path_env = std::getenv("ASCEND_OPP_PATH");
+  if (path_env != nullptr) {
+    string path = path_env;
+    string file_path = RealPath(path.c_str());
+    if (file_path.empty()) {
+      GELOGE(FAILED, "[Check][EnvPath]ASCEND_OPP_PATH path [%s] is invalid.", path.c_str());
+      REPORT_INPUT_ERROR("E68016", {"ASCEND_OPP_PATH", path}); 
+      return;
+    }
+    opsproto_path = (path + "/op_proto/custom/" + ":") + (path + "/op_proto/built-in/");
+    GELOGI("Get opsproto so path from env : %s", path.c_str());
+  } else {
+    string path_base = PluginManager::GetPath();
+    GELOGI("path_base is %s", path_base.c_str());
+    path_base = path_base.substr(0, path_base.rfind('/'));
+    path_base = path_base.substr(0, path_base.rfind('/') + 1);
+    opsproto_path = (path_base + "ops/op_proto/custom/" + ":") + (path_base + "ops/op_proto/built-in/");
+  }
+  GELOGI("Get opsproto path is %s", opsproto_path.c_str());
+  OpsProtoManager *manager = OpsProtoManager::Instance();
+  map<string, string> option_tmp;
+  option_tmp.emplace(std::pair<string, string>(string("ge.opsProtoLibPath"), opsproto_path));
+  (void)manager->Initialize(option_tmp);
+}
+
 GeExecutor::GeExecutor() {}
 
 Status GeExecutor::Initialize() {
@@ -208,6 +237,16 @@ Status GeExecutor::Initialize() {
     return ge::SUCCESS;
   }
 
+  OpTilingManager::GetInstance().LoadSo();
+
+  Status init_hostcpu_engine_status = HostCpuEngine::GetInstance().Initialize();
+  if (init_hostcpu_engine_status != SUCCESS) {
+    GELOGE(init_hostcpu_engine_status, "Failed to initialize HostCpuEngine");
+    return init_hostcpu_engine_status;
+  }
+
+  InitOpsProtoManager();
+
   std::vector<rtMemType_t> mem_type(1, RT_MEMORY_HBM);
   mem_type.push_back(RT_MEMORY_P2P_DDR);
   auto ret = MemManager::Instance().Initialize(mem_type);

ge/executor/proto/insert_op.proto

@@ -88,6 +88,7 @@ message AippOpParams {
 	int32 right_padding_size = 69;
 	int32 top_padding_size = 70;
 	int32 bottom_padding_size = 71;
+	float padding_value = 72;
 
 	int32 mean_chn_0 = 10;
 	int32 mean_chn_1 = 11;

ge/ge_inference.mk

@@ -114,6 +114,7 @@ OMG_HOST_SRC_FILES := \
     graph/passes/atomic_addr_clean_pass.cc \
     graph/passes/mark_same_addr_pass.cc \
     graph/passes/mark_graph_unknown_status_pass.cc \
+    graph/passes/mark_node_unknown_shape_pass.cc \
     graph/passes/mark_agnostic_pass.cc \
     graph/common/omg_util.cc \
     graph/common/bcast.cc \
@@ -222,6 +223,7 @@ OMG_HOST_SRC_FILES := \
     graph/passes/hccl_group_pass.cc \
     graph/passes/memcpy_addr_async_pass.cc \
     graph/passes/set_input_output_offset_pass.cc \
+    graph/passes/buffer_pool_memory_pass.cc \
 
 OMG_DEVICE_SRC_FILES := $(OMG_HOST_SRC_FILES)
 

ge/ge_local_engine/CMakeLists.txt

@@ -20,6 +20,8 @@ set(OPS_KERNEL_SRC_LIST
 )
 
 protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
+protobuf_generate(ge_ops_shared PROTO_OPS_SHARED_SRCS PROTO_OPS_SHARED_HDRS ${PROTO_LIST})
+protobuf_generate(ge_ops_static PROTO_OPS_STATIC_SRCS PROTO_OPS_STATIC_HDRS ${PROTO_LIST})
 
 ############ libge_local_engine.so ############
 add_library(ge_local_engine SHARED ${SRC_LIST} ${PROTO_HDRS})
@@ -119,7 +121,7 @@ set_target_properties(atc_ge_local_engine PROPERTIES
 )
 
 ############ libge_local_opskernel_builder.so ############
-add_library(ge_local_opskernel_builder SHARED ${OPS_KERNEL_SRC_LIST} ${PROTO_HDRS})
+add_library(ge_local_opskernel_builder SHARED ${OPS_KERNEL_SRC_LIST} ${PROTO_OPS_SHARED_HDRS})
 
 target_compile_options(ge_local_opskernel_builder PRIVATE
     -Werror
@@ -143,7 +145,7 @@ target_include_directories(ge_local_opskernel_builder PRIVATE
     ${METADEF_DIR}/inc/external/graph
     ${METADEF_DIR}/inc/graph
     ${CMAKE_BINARY_DIR}
-    ${CMAKE_BINARY_DIR}/proto/ge
+    ${CMAKE_BINARY_DIR}/proto/ge_ops_shared
     #### yellow zone ####
     ${GE_CODE_DIR}/../inc
     #### blue zone ####
@@ -166,7 +168,7 @@ target_link_libraries(ge_local_opskernel_builder PRIVATE
 )
 
 ############ atclib/libge_local_opskernel_builder.so ############
-add_library(atc_ge_local_opskernel_builder SHARED ${OPS_KERNEL_SRC_LIST} ${PROTO_HDRS})
+add_library(atc_ge_local_opskernel_builder SHARED ${OPS_KERNEL_SRC_LIST} ${PROTO_OPS_SHARED_HDRS})
 
 target_compile_options(atc_ge_local_opskernel_builder PRIVATE
     -Werror
@@ -190,7 +192,7 @@ target_include_directories(atc_ge_local_opskernel_builder PRIVATE
     ${METADEF_DIR}/inc/external/graph
     ${METADEF_DIR}/inc/graph
     ${CMAKE_BINARY_DIR}
-    ${CMAKE_BINARY_DIR}/proto/ge
+    ${CMAKE_BINARY_DIR}/proto/ge_ops_shared
     #### yellow zone ####
     ${GE_CODE_DIR}/../inc
     #### blue zone ####
@@ -218,7 +220,7 @@ set_target_properties(atc_ge_local_opskernel_builder PROPERTIES
 )
 
 ############ libge_local_opskernel_builder.a ############
-add_library(ge_local_opskernel_builder_static STATIC ${OPS_KERNEL_SRC_LIST} ${PROTO_HDRS})
+add_library(ge_local_opskernel_builder_static STATIC ${OPS_KERNEL_SRC_LIST} ${PROTO_OPS_STATIC_HDRS})
 
 target_compile_options(ge_local_opskernel_builder_static PRIVATE
     -Werror
@@ -243,7 +245,7 @@ target_include_directories(ge_local_opskernel_builder_static PRIVATE
     ${METADEF_DIR}/inc/external/graph
     ${METADEF_DIR}/inc/graph
     ${CMAKE_BINARY_DIR}
-    ${CMAKE_BINARY_DIR}/proto/ge
+    ${CMAKE_BINARY_DIR}/proto/ge_ops_static
     #### yellow zone ####
     ${GE_CODE_DIR}/../inc
     #### blue zone ####

ge/ge_local_engine/ops_kernel_store/op/ge_deleted_op.cc

@@ -38,6 +38,7 @@ REGISTER_OP_CREATOR(ExpandDims, GeDeletedOp);
 REGISTER_OP_CREATOR(Reshape, GeDeletedOp);
 REGISTER_OP_CREATOR(ReFormat, GeDeletedOp);
 REGISTER_OP_CREATOR(Squeeze, GeDeletedOp);
+REGISTER_OP_CREATOR(Unsqueeze, GeDeletedOp);
 REGISTER_OP_CREATOR(Size, GeDeletedOp);
 REGISTER_OP_CREATOR(Shape, GeDeletedOp);
 REGISTER_OP_CREATOR(ShapeN, GeDeletedOp);

ge/ge_runner.mk

@@ -114,6 +114,7 @@ LIBGE_LOCAL_SRC_FILES := \
     graph/passes/atomic_addr_clean_pass.cc \
     graph/passes/mark_same_addr_pass.cc \
     graph/passes/mark_graph_unknown_status_pass.cc \
+    graph/passes/mark_node_unknown_shape_pass.cc \
     graph/passes/mark_agnostic_pass.cc \
     graph/partition/dynamic_shape_partition.cc \
     graph/partition/stage_partition.cc \
@@ -246,6 +247,7 @@ LIBGE_LOCAL_SRC_FILES := \
     graph/passes/end_of_sequence_add_control_pass.cc \
     graph/passes/memcpy_addr_async_pass.cc \
     graph/passes/set_input_output_offset_pass.cc \
+    graph/passes/buffer_pool_memory_pass.cc \
     graph/preprocess/graph_preprocess.cc \
     graph/preprocess/insert_op/ge_aipp_op.cc \
     graph/preprocess/insert_op/util_insert_aipp_op.cc \

ge/ge_runtime/task/label_goto_task.cc

@@ -16,14 +16,12 @@
 
 #include "ge_runtime/task/label_goto_task.h"
 #include "ge_runtime/task/task_factory.h"
+#include "framework/common/util.h"
 
 namespace ge {
 namespace model_runner {
 LabelGotoTask::LabelGotoTask(const ModelContext &model_context, const std::shared_ptr<LabelGotoTaskInfo> &task_info)
-    : TaskRepeater<LabelGotoTaskInfo>(model_context, task_info),
-      task_info_(task_info),
-      stream_(nullptr),
-      label_(nullptr) {
+    : TaskRepeater<LabelGotoTaskInfo>(model_context, task_info), task_info_(task_info) {
   if (task_info_ == nullptr) {
     GELOGW("task_info_ is null!");
     return;
@@ -42,29 +40,78 @@ LabelGotoTask::LabelGotoTask(const ModelContext &model_context, const std::share
   label_ = label_list[label_id];
 }
 
-LabelGotoTask::~LabelGotoTask() {}
+LabelGotoTask::~LabelGotoTask() {
+  GE_FREE_RT_LOG(label_info_);
+  GE_FREE_RT_LOG(index_value_);
+}
 
 bool LabelGotoTask::Distribute() {
   GELOGI("LabelGotoTask Distribute start.");
+  if (!CheckParamValid()) {
+    return false;
+  }
+
+  const std::vector<void *> label_list = { label_ };
+  rtError_t rt_ret = rtMalloc(&index_value_, sizeof(uint64_t), RT_MEMORY_HBM);
+  if (rt_ret != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt api failed, ret: %#x", rt_ret);
+    return false;
+  }
+
+  uint64_t branch_index = 0;
+  rt_ret = rtMemcpy(index_value_, sizeof(uint64_t), &branch_index, sizeof(uint64_t), RT_MEMCPY_HOST_TO_DEVICE);
+  if (rt_ret != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt api failed, ret: %#x", rt_ret);
+    return false;
+  }
+
+  uint32_t label_info_size = sizeof(rtLabelDevInfo) * label_list.size();
+  rt_ret = rtMalloc(&label_info_, label_info_size, RT_MEMORY_HBM);
+  if (rt_ret != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt api failed, ret: %#x", rt_ret);
+    return false;
+  }
+
+  rt_ret = rtLabelListCpy(label_list.data(), label_list.size(), label_info_, label_info_size);
+  if (rt_ret != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt api failed, ret: %#x", rt_ret);
+    return false;
+  }
+
+  rt_ret = rtLabelSwitchByIndex(index_value_, label_list.size(), label_info_, stream_);
+  if (rt_ret != RT_ERROR_NONE) {
+    GELOGE(RT_FAILED, "Call rt api failed, ret: %#x", rt_ret);
+    return false;
+  }
+
+  GELOGI("DistributeTask end.");
+  return true;
+}
+
+bool LabelGotoTask::CheckParamValid() {
   if (stream_ == nullptr) {
     GELOGE(PARAM_INVALID, "stream is null!");
     return false;
   }
+
   if (label_ == nullptr) {
     GELOGE(PARAM_INVALID, "label is null!");
     return false;
   }
-  rtError_t rt_ret = rtLabelGotoEx(label_, stream_);
-  if (rt_ret != RT_ERROR_NONE) {
-    GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
+
+  if (label_info_ != nullptr) {
+    GELOGE(PARAM_INVALID, "label_info_ has dirty data.");
+    return false;
+  }
+
+  if (index_value_ != nullptr) {
+    GELOGE(PARAM_INVALID, "index_value_ has dirty data.");
     return false;
   }
 
-  GELOGI("DistributeTask end.");
   return true;
 }
 
 REGISTER_TASK(TaskInfoType::LABEL_GOTO, LabelGotoTask, LabelGotoTaskInfo);
-
 }  // namespace model_runner
 }  // namespace ge

ge/ge_runtime/task/label_goto_task.h

@@ -31,9 +31,13 @@ class LabelGotoTask : public TaskRepeater<LabelGotoTaskInfo> {
   bool Distribute() override;
 
  private:
+  bool CheckParamValid();
+
   std::shared_ptr<LabelGotoTaskInfo> task_info_;
-  void *stream_;
-  void *label_;
+  void *stream_{nullptr};
+  void *label_{nullptr};
+  void *label_info_{nullptr};
+  void *index_value_{nullptr};
 };
 }  // namespace model_runner
 }  // namespace ge

ge/generator/ge_generator.cc

@@ -36,6 +36,7 @@
 #include "graph/utils/type_utils.h"
 #include "init/gelib.h"
 #include "model/ge_model.h"
+#include "analyzer/analyzer.h"
 
 using std::map;
 using std::string;
@@ -50,9 +51,14 @@ const char *const kFileNameSuffix = "online";
 const char *const kAicpuAllshape = "_AllShape";
 constexpr char const *kAttrSupportDynamicShape = "support_dynamicshape";
 const int64_t kDynamicDimValue = -2;
+const int kDefaultDeviceId = 0;
+const int kDefaultJobId = 0;
+const int32_t kFuzzBuildPattern = 1;
 
 std::map<ge::OpEngineType, std::string> engine_type_map{
-    {ge::ENGINE_SYS, kEngineNameDefault}, {ge::ENGINE_AICORE, kAIcoreEngine}, {ge::ENGINE_VECTOR, kVectorEngine}};
+    {ge::ENGINE_SYS, kEngineNameDefault},
+    {ge::ENGINE_AICORE, kAIcoreEngine},
+    {ge::ENGINE_VECTOR, kVectorEngine}};
 
 bool ContainsDynamicInpus(const ge::OpDesc &op_desc) {
   for (auto &tensor_desc : op_desc.GetAllInputsDescPtr()) {
@@ -63,6 +69,10 @@ bool ContainsDynamicInpus(const ge::OpDesc &op_desc) {
   }
   return false;
 }
+// if optional in/out, format is format_reserved and dtype is dt_undefined
+bool IsOptional(const ge::GeTensorDesc &tensor_desc) {
+  return tensor_desc.GetFormat() == ge::FORMAT_RESERVED && tensor_desc.GetDataType() == ge::DT_UNDEFINED;
+}
 }  // namespace
 
 namespace ge {
@@ -83,8 +93,9 @@ static Status CheckEngineTypeSupport(const NodePtr &node, OpEngineType engine_ty
   } else {
     ErrorManager::GetInstance().ATCReportErrMessage("E14001", {"opname", "optype", "value", "reason"},
         {op_desc->GetName(), op_desc->GetType(), "engine type",
-        "it only support kEngineNameDefault/kAIcoreEngine/kVectorEngine"});
-    GELOGE(FAILED, "CheckEngineType: engine type: %d not support", static_cast<int>(engine_type));
+        "it only support default/AIcoreEngine/VectorEngine"});
+    GELOGE(FAILED, "[Check][EngineType]value:%d not support, "
+           "only support default/AIcoreEngine/VectorEngine now", static_cast<int>(engine_type));
     return FAILED;
   }
 
@@ -149,7 +160,7 @@ static Status CheckEngineTypeSupport(const NodePtr &node, OpEngineType engine_ty
 }
 
 static Status AddInputs(const ComputeGraphPtr &graph, const NodePtr &node, const GeTensorDesc &tensor, int32_t index,
-                        bool attr) {
+                        bool attr, int32_t &data_index) {
   GE_CHECK_NOTNULL_EXEC(graph, return PARAM_INVALID);
   GE_CHECK_NOTNULL_EXEC(node, return PARAM_INVALID);
 
@@ -188,17 +199,21 @@ static Status AddInputs(const ComputeGraphPtr &graph, const NodePtr &node, const
 
   (void)AttrUtils::SetBool(data_op, "_is_single_op", true);
 
-  GE_CHK_BOOL_EXEC(data_op->AddInputDesc(tensor) == GRAPH_SUCCESS, return FAILED, "Add input desc fail.");
-  GE_CHK_BOOL_EXEC(data_op->AddOutputDesc(tensor) == GRAPH_SUCCESS, return FAILED, "Add output desc fail.");
-  if (attr) {
-    GE_CHK_BOOL_EXEC(AttrUtils::SetInt(data_op, ATTR_NAME_INDEX, index), return FAILED, "Set index fail.");
+  GE_CHK_BOOL_EXEC(data_op->AddInputDesc(tensor) == GRAPH_SUCCESS, return FAILED,
+                   "[Add][InputDesc]fail for node:%s", data_op->GetName().c_str());
+  GE_CHK_BOOL_EXEC(data_op->AddOutputDesc(tensor) == GRAPH_SUCCESS, return FAILED,
+                   "[Add][OutputDesc]fail for node:%s", data_op->GetName().c_str());
+  if (attr && !is_const) {
+    GE_CHK_BOOL_EXEC(AttrUtils::SetInt(data_op, ATTR_NAME_INDEX, data_index), return FAILED,
+                     "[Set][Attr:%s]fail for node:%s", ATTR_NAME_INDEX.c_str(), data_op->GetName().c_str());
+    ++data_index;
   }
 
   ge::NodePtr arg_node = graph->AddNode(data_op);
-  GE_CHK_BOOL_EXEC(arg_node != nullptr, return FAILED, "Insert Data node fail.");
+  GE_CHK_BOOL_EXEC(arg_node != nullptr, return FAILED, "Insert Data node fail");
 
   GE_CHK_STATUS(GraphUtils::AddEdge(arg_node->GetOutDataAnchor(0), node->GetInDataAnchor(index)),
-                "Add edge[%s->%s] fail.", data_op->GetName().c_str(), node->GetName().c_str());
+                "[Add][Edge]fail from node:%s to node:%s", data_op->GetName().c_str(), node->GetName().c_str());
 
   return SUCCESS;
 }
@@ -213,20 +228,23 @@ static Status AddOutputs(const ComputeGraphPtr &graph, const NodePtr &node, cons
   for (const auto &out_desc : outputs) {
     GeTensorDesc tensor = out_desc.GetTensorDesc();
     TensorUtils::SetInputTensor(tensor, true);
-    GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(tensor) == GRAPH_SUCCESS, return FAILED, "Add input desc fail");
+    GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(tensor) == GRAPH_SUCCESS, return FAILED,
+                     "[Add][InputDesc]fail for node:%s", op_desc->GetName().c_str());
 
     TensorUtils::SetInputTensor(tensor, false);
     TensorUtils::SetOutputTensor(tensor, true);
-    GE_CHK_BOOL_EXEC(op_desc->AddOutputDesc(tensor) == GRAPH_SUCCESS, return FAILED, "Add output desc fail");
+    GE_CHK_BOOL_EXEC(op_desc->AddOutputDesc(tensor) == GRAPH_SUCCESS, return FAILED,
+                     "[Add][OutputDesc]fail for node:%s", op_desc->GetName().c_str());
     count++;
   }
   GE_CHECK_NOTNULL_EXEC(graph, return PARAM_INVALID);
   ge::NodePtr out_node = graph->AddNode(op_desc);
-  GE_CHK_BOOL_EXEC(out_node != nullptr, return FAILED, "Insert Output node fail.");
+  GE_CHK_BOOL_EXEC(out_node != nullptr, return FAILED,
+                   "[Add][Node:%s]fail in graph:%u", op_desc->GetName().c_str(), graph->GetGraphID());
   GE_CHECK_NOTNULL_EXEC(node, return PARAM_INVALID);
   for (int32_t i = 0; i < count; ++i) {
     GE_CHK_STATUS(GraphUtils::AddEdge(node->GetOutDataAnchor(i), out_node->GetInDataAnchor(i)),
-                  "Add edge[%s->%s] fail.", node->GetName().c_str(), out_node->GetName().c_str());
+                  "[Add][Edge]fail from node:%s to node:%s", node->GetName().c_str(), out_node->GetName().c_str());
   }
 
   return SUCCESS;
@@ -283,13 +301,44 @@ static Status ResetTensorVecShape(const vector<GeTensor> &inputs, vector<GeTenso
   return SUCCESS;
 }
 
+static Status GetFuzzBuildAttrs(const OpDescPtr &op_desc, const GeRootModelPtr &ge_root_model,
+                                GeAttrValue::LIST_NAMED_ATTRS &fuzz_build_attrs) {
+  GELOGD("Start get fuzz build attrs of %s.", op_desc->GetName().c_str());
+  GE_CHECK_NOTNULL(ge_root_model->GetRootGraph());
+  for (const auto &node : ge_root_model->GetRootGraph()->GetAllNodes()) {
+    GE_CHECK_NOTNULL(node);
+    GE_CHECK_NOTNULL(node->GetOpDesc());
+    GELOGD("Delete fuzz build attr of %s after build.", node->GetName().c_str());
+    node->GetOpDesc()->DelAttr(ATTR_NAME_FUZZ_BUILD);
+  }
+  (void)AttrUtils::GetListNamedAttrs(op_desc, ATTR_NAME_FUZZ_BUILD_RES_ATTRS, fuzz_build_attrs);
+  if (!fuzz_build_attrs.empty()) {
+    GELOGD("%s has split, get ATTR_NAME_FUZZ_BUILD_RES_ATTRS directly.", op_desc->GetName().c_str());
+    return SUCCESS;
+  } else {
+    GELOGW("%s build with fuzz build pattern, but not set ATTR_NAME_FUZZ_BUILD_RES_ATTRS.", op_desc->GetName().c_str());
+  }
+  return SUCCESS;
+}
+
+static bool HasShapeRange(const vector<GeTensor> &inputs) {
+  for (const auto &input : inputs) {
+    vector<pair<int64_t, int64_t>> shape_range;
+    (void)input.GetTensorDesc().GetShapeRange(shape_range);
+    if (!shape_range.empty()) {
+      GELOGD("Has set shape range.");
+      return true;
+    }
+  }
+  return false;
+}
+
 class GeGenerator::Impl {
  public:
   Impl(OmgContext &omg_context) : omg_context_(omg_context) {}
   ~Impl() = default;
 
   Status BuildModel(const Graph &graph, const vector<GeTensor> &inputs, GeRootModelPtr &ge_models);
-
   Status SaveModel(const string &file_name_prefix, GeModelPtr &models, ModelBufferData &model);
 
   Status SaveRootModel(const string &file_name_prefix, GeRootModelPtr &model, ModelBufferData &model_buff);
@@ -554,6 +603,42 @@ bool GeGenerator::Impl::SetOmSystemInfo(AttrHolder &obj) {
   return true;
 }
 
+Status GeGenerator::SetModelNameForDump(const GeRootModelPtr &ge_root_model) {
+  bool is_unknown_shape = false;
+  Status ret = ge_root_model->CheckIsUnknownShape(is_unknown_shape);
+  if (ret != SUCCESS) {
+    GELOGE(FAILED, "[Check][IsUnknownShape]Check root model is unknown shape failed, model id:%u",
+           ge_root_model->GetModelId());
+    REPORT_CALL_ERROR("E19999", "Check root model is unknown shape failed, model id:%u",
+                      ge_root_model->GetModelId());
+    return FAILED;
+  }
+  GeModelPtr model_root = nullptr;
+  if (is_unknown_shape) {
+    model_root = MakeShared<GeModel>();
+    GE_CHECK_NOTNULL(model_root);
+    model_root->SetGraph(GraphUtils::CreateGraphFromComputeGraph(ge_root_model->GetRootGraph()));
+    ge_root_model->SetSubgraphInstanceNameToModel(ge_root_model->GetRootGraph()->GetName(), model_root);
+  }
+
+  ModelHelper model_helper;
+  string model_name;
+  GE_CHECK_NOTNULL(ge_root_model->GetRootGraph());
+  Status name_ret = model_helper.GetModelNameFromMergedGraphName(ge_root_model->GetRootGraph()->GetName(),
+                                                                 model_name);
+  if (name_ret != SUCCESS) {
+    ErrorManager::GetInstance().ATCReportErrMessage("E10000", {"parameter"}, {"output"});
+    GELOGE(FAILED, "[Check][GetModelNameStep]Get model_name failed. Param --output is invalid, root graph name: %s",
+           ge_root_model->GetRootGraph()->GetName().c_str());
+    return PARAM_INVALID;
+  }
+  map<string, GeModelPtr> name_to_ge_model = ge_root_model->GetSubgraphInstanceNameToModel();
+  GeModelPtr &ge_model = name_to_ge_model[ge_root_model->GetRootGraph()->GetName()];
+  GE_CHECK_NOTNULL(ge_model);
+  ge_model->SetName(model_name);
+  return SUCCESS;
+}
+
 Status GeGenerator::GenerateModel(const Graph &graph, const string &file_name_prefix, const vector<GeTensor> &inputs,
                                   ModelBufferData &model, bool is_offline) {
   rtContext_t ctx = nullptr;
@@ -588,20 +673,10 @@ Status GeGenerator::GenerateModel(const Graph &graph, const string &file_name_pr
   }
 
   GE_CHECK_NOTNULL(ge_root_model);
-  GE_CHECK_NOTNULL(ge_root_model->GetRootGraph());
-  ModelHelper model_helper;
-  string model_name = "";
-  Status name_ret = model_helper.GetModelNameFromMergedGraphName(ge_root_model->GetRootGraph()->GetName(),
-                                                                 model_name);
-  if (name_ret != SUCCESS) {
-    ErrorManager::GetInstance().ATCReportErrMessage("E10000", {"parameter"}, {"output"});
-    GELOGE(FAILED, "Get model_name failed. Param --output is invalid.");
-    return PARAM_INVALID;
+  ret = SetModelNameForDump(ge_root_model);
+  if (ret != SUCCESS) {
+    return ret;
   }
-  map<string, GeModelPtr> name_to_ge_model = ge_root_model->GetSubgraphInstanceNameToModel();
-  GeModelPtr &ge_model = name_to_ge_model[ge_root_model->GetRootGraph()->GetName()];
-  GE_RETURN_WITH_LOG_IF_FALSE(ge_model != nullptr, "ge_model cannot be null");
-  ge_model->SetName(model_name);
   ret = impl_->SaveRootModel(file_name_prefix, ge_root_model, model);
   if (ret != SUCCESS) {
     GELOGE(ret, "Save model failed");
@@ -654,6 +729,34 @@ namespace {
   }
 }
 
+bool GeGenerator::CheckNoAicore(const ComputeGraphPtr &graph) {
+  for (const auto &node : graph->GetDirectNode()) {
+    if (node == nullptr) {
+      continue;
+    }
+    auto op_desc = node->GetOpDesc();
+    if (op_desc == nullptr) {
+      continue;
+    }
+    if (op_desc->GetOpEngineName() == kAIcoreEngine) {
+      return false;
+    }
+  }
+  return true;
+}
+
+void GeGenerator::RemoveConst(const vector<GeTensor> &inputs, vector<GeTensor> &outputs) {
+  for (auto &input : inputs) {
+    GeTensorDesc input_desc = input.GetTensorDesc();
+    bool is_const = false;
+    (void)AttrUtils::GetBool(input_desc, CONST_ATTR_NAME_INPUT, is_const);
+    bool is_optional = IsOptional(input_desc);
+    if (!is_optional && !is_const) {
+      outputs.emplace_back(input);
+    }
+  }
+}
+
 Status GeGenerator::CheckForSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &inputs,
                                      const vector<GeTensor> &outputs) {
   GE_CHECK_NOTNULL_EXEC(op_desc, return PARAM_INVALID);
@@ -676,7 +779,8 @@ Status GeGenerator::CheckForSingleOp(OpDescPtr &op_desc, const vector<GeTensor>
 
 Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &inputs, const vector<GeTensor> &outputs,
                                   const string &model_file_name, OpEngineType engine_type, ModelBufferData &model_buff,
-                                  bool is_offline) {
+                                  bool is_offline, int32_t compile_flag) {
+  GELOGD("Inputs size is %zu, outputs size is %zu.", inputs.size(), outputs.size());
   GE_CHECK_NOTNULL_EXEC(impl_, return PARAM_INVALID);
   impl_->is_offline_ = is_offline;
   if (!is_offline) {
@@ -698,6 +802,16 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
   OpDescPtr op_desc_tmp = AttrUtils::CloneOpDesc(op_desc);
   GE_CHECK_NOTNULL(op_desc_tmp);
 
+  bool fuzz_compile_flag = false;
+  if (!HasShapeRange(inputs) && compile_flag == kFuzzBuildPattern) {
+    fuzz_compile_flag = true;
+  }
+  if (!AttrUtils::SetBool(op_desc, ATTR_NAME_FUZZ_BUILD, fuzz_compile_flag)) {
+    GELOGE(FAILED, "[Set][ATTR_NAME_FUZZ_BUILD] Failed to set attr for %s.", op_desc->GetName().c_str());
+    return FAILED;
+  }
+  impl_->omg_context_.fuzz_compile_flag = fuzz_compile_flag;
+
   // 1. Create ComputeGraph.
   string name = ge::CurrentTimeInStr() + "_" + model_file_name;
   Graph graph;
@@ -710,7 +824,7 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
     auto node = comp_graph->FindNode(op_desc->GetName());
     Status ret = CheckEngineTypeSupport(node, engine_type);
     if (ret != SUCCESS) {
-      GELOGE(ret, "check engine type failed.");
+      GELOGE(ret, "[Check][EngineType]value:%d for node:%s not support", engine_type, node->GetName().c_str());
       return ret;
     }
   }
@@ -718,7 +832,9 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
   GELOGI("ATC parser success in single op build.");
 
   GeRootModelPtr ge_root_model = nullptr;
-  GE_CHK_STATUS_RET_NOLOG(impl_->BuildModel(graph, inputs, ge_root_model));
+  vector<GeTensor> data_inputs;
+  RemoveConst(inputs, data_inputs);
+  GE_CHK_STATUS_RET_NOLOG(impl_->BuildModel(graph, data_inputs, ge_root_model));
   map<string, GeAttrValue> op_attrs = op_desc_tmp->GetAllAttrs();
   GE_CHECK_NOTNULL(ge_root_model);
   GE_CHECK_NOTNULL(ge_root_model->GetRootGraph());
@@ -734,7 +850,7 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
 
   bool all_shape = false;
   (void)AttrUtils::GetBool(op_desc, kAicpuAllshape, all_shape);
-  if (all_shape) {
+  if (all_shape && CheckNoAicore(root_graph)) {
     GELOGD("Get aicpu all_shape kernel!");
     vector<GeTensor> inputs_dynamic;
     vector<GeTensor> outputs_dynamic;
@@ -742,6 +858,19 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
     GE_CHK_STATUS_RET_NOLOG(ResetTensorVecShape(outputs, outputs_dynamic));
     GE_CHK_STATUS_RET_NOLOG(
       impl_->SaveParams(ge_model, op_desc_tmp->GetType(), op_attrs, inputs_dynamic, outputs_dynamic));
+  } else if (fuzz_compile_flag) {
+    GELOGD("Get fuzz build result of %s.", op_desc->GetName().c_str());
+    (void)AttrUtils::SetInt(ge_model, ATTR_NAME_BUILD_MODE, fuzz_compile_flag);
+    GeAttrValue::LIST_NAMED_ATTRS fuzz_build_attrs;
+    if (GetFuzzBuildAttrs(op_desc, ge_root_model, fuzz_build_attrs) != SUCCESS) {
+      GELOGE(FAILED, "[Get][FuzzRet]Failed to get fuzz build result of %s.", op_desc->GetName().c_str());
+      return FAILED;
+    }
+    if (!fuzz_build_attrs.empty()) {
+      GE_CHK_BOOL_EXEC(AttrUtils::SetListNamedAttrs(ge_model, ATTR_NAME_FUZZ_BUILD_RES_ATTRS, fuzz_build_attrs),
+                       return FAILED, "Set ATTR_NAME_FUZZ_BUILD_RES_ATTRS failed.");
+    }
+    GE_CHK_STATUS_RET_NOLOG(impl_->SaveParams(ge_model, op_desc_tmp->GetType(), op_attrs, inputs, outputs));
   } else {
     GE_CHK_STATUS_RET_NOLOG(impl_->SaveParams(ge_model, op_desc_tmp->GetType(), op_attrs, inputs, outputs));
   }
@@ -757,15 +886,17 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector<GeTensor> &in
  * @param [in] vector<GeTensor> &inputs: Operator input data description information.
  * @param [in] vector<GeTensor> &outputs: Operator output data description information.
  * @param [in] const string &model_file_name: Offline model filename.
+ * @param [in] compile_flag: op build flag from atc
  * @return SUCCESS handle successfully / others handle failed
  */
 Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor> &inputs,
-                                       const vector<GeTensor> &outputs, const string &model_file_name) {
+                                       const vector<GeTensor> &outputs, const string &model_file_name,
+                                       int32_t compile_flag) {
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
   GELOGI("Start to build single op offline model, input size: %zu, output size: %zu", inputs.size(), outputs.size());
   ModelBufferData model_buff;
   OpEngineType engine_type = ENGINE_SYS;
-  Status status = BuildSingleOp(op_desc, inputs, outputs, model_file_name, engine_type, model_buff, true);
+  Status status = BuildSingleOp(op_desc, inputs, outputs, model_file_name, engine_type, model_buff, true, compile_flag);
   GELOGI("Finish build single offline model, status: %u", status);
   return status;
 }
@@ -777,9 +908,11 @@ Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor
  * @param [in] vector<GeTensor> &inputs: Operator input data description information.
  * @param [in] vector<GeTensor> &outputs: Operator output data description information.
  * @param [in] engine_type: specific engine.
+ * @param [in] compile_flag: op build flag, compile flag by acl
  * @param [out] ModelBufferData &Model_buff: Model_buff: model buffer of the op.
  * @return SUCCESS handle successfully / others handle failed
  */
+
 Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor> &inputs,
                                        const vector<GeTensor> &outputs, OpEngineType engine_type,
                                        ModelBufferData &model_buff) {
@@ -790,6 +923,17 @@ Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor
   return status;
 }
 
+Status GeGenerator::BuildSingleOpModel(OpDescPtr &op_desc, const vector<GeTensor> &inputs,
+                                       const vector<GeTensor> &outputs, OpEngineType engine_type, int32_t compile_flag,
+                                       ModelBufferData &model_buff) {
+  ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
+  GELOGI("Start to build single op online, input size: %zu, output size: %zu", inputs.size(), outputs.size());
+  Status status = BuildSingleOp(op_desc, inputs, outputs, kFileNameSuffix, engine_type, model_buff, false,
+                                compile_flag);
+  GELOGI("Finish build single online model, status: %u", status);
+  return status;
+}
+
 Status GeGenerator::BuildSingleOpGraph(OpDescPtr &op_desc, const vector<GeTensor> &inputs,
                                        const vector<GeTensor> &outputs, std::string graph_name, Graph &graph) {
   ge::ComputeGraphPtr compute_graph = MakeShared<ComputeGraph>(graph_name);
@@ -801,18 +945,19 @@ Status GeGenerator::BuildSingleOpGraph(OpDescPtr &op_desc, const vector<GeTensor
 
   // 2. Create InputData node.
   int32_t arg_index = 0;
+  int32_t data_index = 0;
   if (inputs.empty()) {
     for (const auto &input_desc : op_desc->GetAllInputsDescPtr()) {
       GE_CHECK_NOTNULL_EXEC(input_desc, return INTERNAL_ERROR);
       if (!IsNeedConnectInputOpForSingleOp(*input_desc)) {
         continue;
       }
-      GE_CHK_STATUS_RET_NOLOG(AddInputs(compute_graph, op_node, *input_desc, arg_index, false));
+      GE_CHK_STATUS_RET_NOLOG(AddInputs(compute_graph, op_node, *input_desc, arg_index, false, data_index));
       arg_index++;
     }
   } else {
     for (const auto &in_desc : inputs) {
-      GE_CHK_STATUS_RET_NOLOG(AddInputs(compute_graph, op_node, in_desc.GetTensorDesc(), arg_index, true));
+      GE_CHK_STATUS_RET_NOLOG(AddInputs(compute_graph, op_node, in_desc.GetTensorDesc(), arg_index, true, data_index));
       arg_index++;
     }
   }
@@ -871,13 +1016,12 @@ Status GeGenerator::Impl::SaveRootModel(const string &file_name_prefix, GeRootMo
                    "ge root model has no sub model")
   GeModelPtr model_root = nullptr;
   if (is_unknown_shape) {
-    model_root = make_shared<GeModel>();
-    model_root->SetGraph(GraphUtils::CreateGraphFromComputeGraph(ge_root_model->GetRootGraph()));
-    ge_root_model->SetSubgraphInstanceNameToModel(ge_root_model->GetRootGraph()->GetName(), model_root);
-    model_root->SetName(ge_root_model->GetRootGraph()->GetName());
+    auto name_to_ge_model = ge_root_model->GetSubgraphInstanceNameToModel();
+    model_root = name_to_ge_model[ge_root_model->GetRootGraph()->GetName()];
   } else {
     model_root = ge_root_model->GetSubgraphInstanceNameToModel().begin()->second;
   }
+  GE_CHECK_NOTNULL(model_root);
   // set atc version
   if (!SetAtcVersionInfo(*(model_root.get()))) {
     GELOGW("SetPackageVersionInfo of atc failed!");
@@ -915,6 +1059,13 @@ Status GeGenerator::Impl::BuildModel(const Graph &graph, const vector<GeTensor>
 
   static std::atomic<uint64_t> atomic_session_id(0);
   auto session_id = atomic_session_id.fetch_add(1);
+  // This is a temporary add for graph with variable
+  auto version = static_cast<int32_t>(SessionVersion::ClOUD_VERSION);
+  ret = VarManager::Instance(session_id)->Init(version, session_id, kDefaultDeviceId, kDefaultJobId);
+  GELOGI("Start init var instance, session_id %lu", session_id);
+  if (ret != SUCCESS) {
+    GELOGW("Failed init var instance, session_id %lu", session_id);
+  }
   if (is_singleop_unregistered_) {
     ret = graph_manager_.BuildGraphForUnregisteredOp(graph_id, inputs, ge_root_model, session_id);
   } else {
@@ -924,13 +1075,13 @@ Status GeGenerator::Impl::BuildModel(const Graph &graph, const vector<GeTensor>
   ErrorManager::GetInstance().SetStage(ErrorMessage::kModelCompile, ErrorMessage::kOther);
   if (ret != SUCCESS) {
     GELOGE(GE_GENERATOR_GRAPH_MANAGER_BUILD_GRAPH_FAILED, "GraphManager build graph fail, graph id: %u", graph_id);
-    VarManagerPool::Instance().RemoveVarManager(session_id);
-    return GE_GENERATOR_GRAPH_MANAGER_BUILD_GRAPH_FAILED;
+    ret = GE_GENERATOR_GRAPH_MANAGER_BUILD_GRAPH_FAILED;
   }
 
+  RtContextUtil::GetInstance().DestroyRtContexts(session_id);
+  Analyzer::GetInstance()->DestroySessionJsonObject(session_id);
   VarManagerPool::Instance().RemoveVarManager(session_id);
-
-  return SUCCESS;
+  return ret;
 }
 
 Status GeGenerator::Impl::GenerateInfershapeGraph(const Graph &graph) {

ge/graph/build/graph_builder.cc

@@ -77,6 +77,8 @@ Status HandleSubgraphNode(NodePtr &src_node, OutDataAnchorPtr &src_out_anchor) {
 Status HandleSubgraphDataNode(NodePtr &src_node, OutDataAnchorPtr &src_out_anchor) {
   uint32_t index = 0;
   if (!AttrUtils::GetInt(src_node->GetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, index)) {
+    REPORT_INNER_ERROR("E19999", "get attr:%s failed from node:%s",
+                       ATTR_NAME_PARENT_NODE_INDEX.c_str(), src_node->GetName().c_str());
     GELOGE(FAILED, "Get attr ATTR_NAME_PARENT_NODE_INDEX failed, node:%s.", src_node->GetName().c_str());
     return FAILED;
   }
@@ -109,6 +111,8 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) {
   GE_CHECK_NOTNULL(graph);
   auto instance_ptr = ge::GELib::GetInstance();
   if (instance_ptr == nullptr || !instance_ptr->InitFlag()) {
+    REPORT_INNER_ERROR("E19999", "check gelib instance null, graph:%s",
+                       graph->GetName().c_str());
     GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GraphBuilder: GE is not initialized");
     return GE_CLI_GE_NOT_INITIALIZED;
   }
@@ -121,6 +125,8 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) {
       (void)instance_ptr->DNNEngineManagerObj().GetDNNEngineName(node_ptr);
       kernel_lib_name = node_ptr->GetOpDesc()->GetOpKernelLibName();
       if (kernel_lib_name.empty()) {
+        REPORT_INNER_ERROR("E19999", "op kernel lib is empty in node:%s(%s)",
+                           node_ptr->GetName().c_str(), node_ptr->GetType().c_str());
         GELOGE(INTERNAL_ERROR, "Get node:%s(%s) kernel lib failed.", node_ptr->GetName().c_str(),
                node_ptr->GetType().c_str());
         return INTERNAL_ERROR;
@@ -129,12 +135,16 @@ Status GraphBuilder::CalcOpParam(const ge::ComputeGraphPtr &graph) {
 
     auto ret = SetInputSize(node_ptr);
     if (ret != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Set node:%s(%s) inputDesc size failed",
+                        node_ptr->GetName().c_str(), node_ptr->GetType().c_str());
       GELOGE(ret, "Set node inputDesc size failed, node name is %s", node_ptr->GetName().c_str());
       return ret;
     }
 
     ret = OpsKernelBuilderManager::Instance().CalcOpRunningParam(*node_ptr);
     if (ret != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Call Calculate op:%s(%s) running param failed",
+                        node_ptr->GetName().c_str(), node_ptr->GetType().c_str());
       GELOGE(ret, "Calculate op running param failed, node name is %s", node_ptr->GetName().c_str());
       return ret;
     }
@@ -191,6 +201,7 @@ Status GraphBuilder::UpdateParentNodeOutputSize(const ge::ComputeGraphPtr &graph
 
 Status GraphBuilder::Build(ComputeGraphPtr &comp_graph, GeRootModelPtr &ge_root_model_ptr, uint64_t session_id) {
   if (comp_graph == nullptr) {
+    REPORT_INNER_ERROR("E19999", "check compute_graph nullptr, session_id:%lu", session_id);
     GELOGE(GE_GRAPH_PARAM_NULLPTR, "Graph build comp_graph is null.");
     return GE_GRAPH_PARAM_NULLPTR;
   }
@@ -302,6 +313,8 @@ Status GraphBuilder::SetConstantInputOffset(ComputeGraphPtr &comp_graph) {
 
       std::vector<GeTensorPtr> weights = OpDescUtils::MutableWeights(peer_node);
       if (weights.empty()) {
+        REPORT_INNER_ERROR("E19999", "check weights size of node %s(%s) is empty",
+                           node->GetName().c_str(), node->GetType().c_str());
         GELOGE(FAILED, "weights size of node %s is empty", node->GetName().c_str());
         return FAILED;
       }
@@ -382,54 +395,6 @@ Status GraphBuilder::BuildForHostCpuGraph(ComputeGraphPtr &comp_graph, GeModelPt
   return BuildForUnknownShapeGraph(comp_graph, ge_model_ptr, session_id);
 }
 
-static Status InsertMemcpyNode(const ComputeGraphPtr &graph, const OutDataAnchorPtr &out_anchor,
-                               const std::vector<InDataAnchorPtr> &in_anchors, const std::string &name) {
-  GE_CHECK_NOTNULL(out_anchor);
-  NodePtr in_node = out_anchor->GetOwnerNode();
-  GE_CHECK_NOTNULL(in_node);
-  OpDescBuilder op_desc_builder(name, MEMCPYADDRASYNC);
-  OpDescPtr op_desc = op_desc_builder.AddInput("x", in_node->GetOpDesc()->GetOutputDesc(0))
-                                     .AddOutput("y", in_node->GetOpDesc()->GetOutputDesc(0))
-                                     .Build();
-  (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false);
-  if (GraphUtils::InsertNodeAfter(out_anchor, in_anchors, graph->AddNode(op_desc)) != GRAPH_SUCCESS) {
-    GELOGE(FAILED, "Insert IDENTITY node %s after %s failed.", name.c_str(), in_node->GetName().c_str());
-    return FAILED;
-  }
-  return SUCCESS;
-}
-
-static Status GenerateTaskForConstant(const std::shared_ptr<ComputeGraph> &graph) {
-  for (auto &node : graph->GetDirectNode()) {
-    // CONSTANT not generate task, so insert IDENTITY between CONSTANT and NETOUTPUT
-    auto op_desc = node->GetOpDesc();
-    if (op_desc == nullptr) {
-      continue;
-    }
-    auto op_type = op_desc->GetType();
-    if (op_type == NETOUTPUT) {
-      for (InDataAnchorPtr &in_data_anchor : node->GetAllInDataAnchors()) {
-        const OutDataAnchorPtr &peer_out_anchor = in_data_anchor->GetPeerOutAnchor();
-        GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue);
-        NodePtr in_node = peer_out_anchor->GetOwnerNode();
-        GE_CHECK_NOTNULL(in_node);
-
-        std::string in_node_op_type = in_node->GetType();
-        if (in_node_op_type == CONSTANT) {
-          GELOGD("Insert MemcpyAsync node between %s and %s.", in_node->GetName().c_str(), node->GetName().c_str());
-          std::string name = node->GetName() + "_input_" + std::to_string(in_data_anchor->GetIdx()) + "_Memcpy";
-          if (InsertMemcpyNode(graph, peer_out_anchor, {in_data_anchor}, name) != SUCCESS) {
-            GELOGE(FAILED, "Insert memcpy between %s and %s failed.",
-                   in_node->GetName().c_str(), node->GetName().c_str());
-            return FAILED;
-          }
-        }
-      }
-    }
-  }
-  return SUCCESS;
-}
-
 Status GraphBuilder::MarkFpBpProfilingTaskAttr(ComputeGraphPtr &com_graph) {
   bool original_unknown_shape_flag = com_graph->GetGraphUnknownFlag();
   com_graph->SetGraphUnknownFlag(false);
@@ -466,6 +431,8 @@ Status GraphBuilder::MarkFpBpProfilingTaskAttr(ComputeGraphPtr &com_graph) {
         GELOGI("The all reduce node of dynamic graph is %s, idx %u", op_desc->GetName().c_str(), node_index);
         (void)ge::AttrUtils::SetBool(op_desc, ATTR_NAME_INSERT_BP_PROFILILNG_TASK, true);
         GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep),
+                        REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id "
+                                           "for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str());
                         GELOGE(FAILED, "Multiply result is out of range.");
                           return FAILED);
         int64_t log_id = i * kProfilingArStep + kProfilingArStartLogid;
@@ -512,9 +479,6 @@ Status GraphBuilder::BuildForDynamicShapeGraph(ComputeGraphPtr &comp_graph,
         !sub_graph->GetParentGraph()->GetGraphUnknownFlag()) {
       continue;
     }
-
-    GE_CHK_STATUS_RET(GenerateTaskForConstant(sub_graph), "Generate task For constant node in subgraph failed.");
-
     if (sub_graph->GetGraphUnknownFlag()) {
       // unknown shape build flow
       GE_CHK_STATUS_RET(BuildForUnknownShapeGraph(sub_graph, ge_model_ptr, session_id),
@@ -545,16 +509,19 @@ Status GraphBuilder::GetTaskInfo(const ge::ModelBuilder &builder, const ModelPtr
 
   int64_t memory_size = 0;
   if (!AttrUtils::GetInt(model_ptr, ATTR_MODEL_MEMORY_SIZE, memory_size)) {
+    REPORT_INNER_ERROR("E19999", "Get Attr:%s fail in model", ATTR_MODEL_MEMORY_SIZE.c_str());
     GELOGE(INTERNAL_ERROR, "Get memory size fail.");
     return INTERNAL_ERROR;
   }
   int64_t p2p_memory_size = 0;
   if (!AttrUtils::GetInt(model_ptr, ATTR_MODEL_P2P_MEMORY_SIZE, p2p_memory_size)) {
+    REPORT_INNER_ERROR("E19999", "Get Attr:%s fail in model", ATTR_MODEL_P2P_MEMORY_SIZE.c_str());
     GELOGE(INTERNAL_ERROR, "Get p2p memory size fail.");
     return INTERNAL_ERROR;
   }
   int64_t weight_size = 0;
   if (!AttrUtils::GetInt(model_ptr, ATTR_MODEL_WEIGHT_SIZE, weight_size)) {
+    REPORT_INNER_ERROR("E19999", "Get Attr:%s fail in model", ATTR_MODEL_WEIGHT_SIZE.c_str());
     GELOGE(INTERNAL_ERROR, "Get weight memory size fail.");
     return INTERNAL_ERROR;
   }
@@ -664,6 +631,7 @@ Status GraphBuilder::SetInputSize(const ge::NodePtr &node_ptr) {
 Status GraphBuilder::UpdateDataInputSize(const ge::NodePtr &node_ptr) {
   const auto &op_desc = node_ptr->GetOpDesc();
   if (op_desc == nullptr) {
+    REPORT_INNER_ERROR("E19999", "check op_desc is nullptr");
     GELOGE(FAILED, "Op desc is nullptr.");
     return FAILED;
   }
@@ -681,6 +649,8 @@ Status GraphBuilder::UpdateDataInputSize(const ge::NodePtr &node_ptr) {
     int64_t real_dim_size = 0;
     ge::graphStatus graph_status = TensorUtils::GetTensorSizeInBytes(output_desc, real_dim_size);
     if (graph_status != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Get tensor size in bytes failed for op:%s(%s) index:0",
+                        op_desc->GetName().c_str(), op_desc->GetType().c_str());
       GELOGE(FAILED, "Get tensor size in bytes failed.");
       return FAILED;
     }
@@ -688,6 +658,8 @@ Status GraphBuilder::UpdateDataInputSize(const ge::NodePtr &node_ptr) {
     ge::GeTensorDesc input_desc = op_desc->GetInputDesc(0);
     ge::TensorUtils::SetSize(input_desc, real_dim_size);
     if (op_desc->UpdateInputDesc(0, input_desc) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Update input desc size failed for op:%s(%s) index:0",
+                        op_desc->GetName().c_str(), op_desc->GetType().c_str());
       GELOGE(FAILED, "Update input desc size failed.");
       return FAILED;
     }
@@ -716,6 +688,8 @@ Status GraphBuilder::CalcDynShapeRootGraphDataSize(const ge::OpDescPtr &op_desc)
     int64_t real_dim_size = 0;
     ge::graphStatus graph_status = TensorUtils::GetTensorSizeInBytes(output_desc, real_dim_size);
     if (graph_status != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Get tensor size in bytes failed for op:%s(%s) index:0 ",
+                        op_desc->GetName().c_str(), op_desc->GetType().c_str());
       GELOGE(FAILED, "Get tensor size in bytes failed.");
       return FAILED;
     }
@@ -723,6 +697,8 @@ Status GraphBuilder::CalcDynShapeRootGraphDataSize(const ge::OpDescPtr &op_desc)
     ge::TensorUtils::SetSize(output_desc, real_dim_size);
     GELOGI("Update dynamic shape graph data output size to [%ld].", real_dim_size);
     if (op_desc->UpdateOutputDesc(0, output_desc) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Update output desc size failed for op:%s(%s) index:0 ",
+                        op_desc->GetName().c_str(), op_desc->GetType().c_str());
       GELOGE(FAILED, "Update dynamic shape graph data output desc size failed.");
       return FAILED;
     }
@@ -740,6 +716,8 @@ Status GraphBuilder::SecondPartition(ge::ComputeGraphPtr &comp_graph) {
   GE_CHK_STATUS_RET(ret, "Graph partition Failed.");
   const auto &graph_2_subgraphlist = graph_partitioner_.GetSubGraphMap();
   if (graph_2_subgraphlist.find(comp_graph) == graph_2_subgraphlist.end()) {
+    REPORT_INNER_ERROR("E19999", "find subgraphlis in graph:%s failed",
+                       comp_graph->GetName().c_str());
     GELOGE(FAILED, "Find subgraph failed.");
     return FAILED;
   }
@@ -768,6 +746,9 @@ Status GraphBuilder::AddOutputMemTypeForNode(const NodePtr &node) {
              mem_type);
       if (!AttrUtils::SetInt(src_desc->MutableOutputDesc(src_out_anchor->GetIdx()), ATTR_OUTPUT_MEMORY_TYPE,
                              mem_type)) {
+        REPORT_INNER_ERROR("E19999", "Set Attr:%s for node:%s(%s) out_index:%u failed",
+                           ATTR_OUTPUT_MEMORY_TYPE.c_str(), src_desc->GetName().c_str(), src_desc->GetType().c_str(),
+                           src_out_anchor->GetIdx());
         GELOGE(INTERNAL_ERROR, "Set out_memory_type attr for [%s:%d] failed.", src_desc->GetName().c_str(),
                src_out_anchor->GetIdx());
         return INTERNAL_ERROR;

ge/graph/build/label_allocator.cc

@@ -28,6 +28,7 @@ LabelAllocator::LabelAllocator(const ComputeGraphPtr &graph) : compute_graph_(gr
 
 Status LabelAllocator::AssignFunctionalLabels() {
   if (compute_graph_ == nullptr) {
+    REPORT_INNER_ERROR("E19999", "check param compute_graph nullptr");
     GELOGE(INTERNAL_ERROR, "ComputeGraph not set, Assign labels failed.");
     return INTERNAL_ERROR;
   }
@@ -46,11 +47,15 @@ Status LabelAllocator::AssignFunctionalLabels() {
   for (auto node : functional_nodes) {
     LabelMakerPtr maker = LabelMakerFactory::Instance().Create(node->GetType(), compute_graph_, node);
     if (maker == nullptr) {
+      REPORT_CALL_ERROR("E19999", "Check Node:%s(%s) label maker not registed",
+                        node->GetName().c_str(), node->GetType().c_str());
       GELOGE(INTERNAL_ERROR, "Node: %s label maker not registed.", node->GetType().c_str());
       return INTERNAL_ERROR;
     }
 
     if (maker->Run(label_index) != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Node:%s(%s) run label maker failed",
+                        node->GetName().c_str(), node->GetType().c_str());
       GELOGE(INTERNAL_ERROR, "Node: %s run label maker failed.", node->GetType().c_str());
       return INTERNAL_ERROR;
     }
@@ -63,6 +68,7 @@ Status LabelAllocator::AssignFunctionalLabels() {
 
 bool LabelAllocator::CollectFunctionalNode(ComputeGraphPtr &graph, std::set<NodePtr> &functional_nodes) {
   if (graph == nullptr) {
+    REPORT_INNER_ERROR("E19999", "check param compute_graph nullptr");
     GELOGE(INTERNAL_ERROR, "Sub ComputeGraph is null.");
     return false;
   }
@@ -74,12 +80,16 @@ bool LabelAllocator::CollectFunctionalNode(ComputeGraphPtr &graph, std::set<Node
 
   NodePtr func_node = graph->GetParentNode();
   if (func_node == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Parent node not set in node:%s(%s), graph:%s",
+                       func_node->GetName().c_str(), func_node->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Parent functional node not set: %s.", graph->GetName().c_str());
     return false;
   }
 
   ComputeGraphPtr owner_graph = func_node->GetOwnerComputeGraph();
   if (owner_graph == nullptr) {
+    REPORT_INNER_ERROR("E19999", "ComputeGraph owner not set in node:%s(%s), graph:%s",
+                       func_node->GetName().c_str(), func_node->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "ComputeGraph owner not set: %s.", func_node->GetName().c_str());
     return false;
   }

ge/graph/build/logical_stream_allocator.cc

@@ -33,13 +33,21 @@ using std::queue;
 namespace ge {
 LogicalStreamPass::LogicalStreamPass(const string &name) : name_(name) {}
 
-const string &LogicalStreamPass::GetName() const { return name_; }
+const string &LogicalStreamPass::GetName() const {
+  return name_;
+}
 
-bool LogicalStreamPass::IsEngineSkip(const Subgraph &subgraph) const { return subgraph.engine_conf.skip_assign_stream; }
+bool LogicalStreamPass::IsEngineSkip(const Subgraph &subgraph) const {
+  return subgraph.engine_conf.skip_assign_stream;
+}
 
-bool LogicalStreamPass::IsEngineAttach(const Subgraph &subgraph) const { return subgraph.engine_conf.attach; }
+bool LogicalStreamPass::IsEngineAttach(const Subgraph &subgraph) const {
+  return subgraph.engine_conf.attach;
+}
 
-bool LogicalStreamPass::IsEngineIndependent(const Subgraph &subgraph) const { return subgraph.engine_conf.independent; }
+bool LogicalStreamPass::IsEngineIndependent(const Subgraph &subgraph) const {
+  return subgraph.engine_conf.independent;
+}
 
 bool LogicalStreamPass::HasStreamLabel(const Subgraph &subgraph) const {
   return !subgraph.subgraph_info.GetStreamLabel().empty();
@@ -60,14 +68,14 @@ Status AssignByLabelPass::Run(ComputeGraphPtr graph, const vector<SubgraphPtr> &
       // Subgraphs of the same stream_label are assigned to the same stream,
       // and different stream_labels are assigned new streams.
       auto iter = label_streams.find(stream_label);
-      if (iter != label_streams.end()) {
-        subgraph->stream_id = iter->second;
-      } else {
+      if (iter == label_streams.end()) {
         subgraph->stream_id = next_stream;
-        GELOGI("Assign new stream %ld for label %s.", next_stream, stream_label.c_str());
+        GELOGI("[Assign][NewStreamId] %ld for label %s.", next_stream, stream_label.c_str());
 
         label_streams.emplace(stream_label, next_stream);
-        ++next_stream;
+        next_stream++;
+      } else {
+        subgraph->stream_id = iter->second;
       }
       changed = true;
     }
@@ -92,15 +100,15 @@ Status IndependentStreamPass::Run(ComputeGraphPtr graph, const vector<SubgraphPt
     const string &stream_label = subgraph->subgraph_info.GetStreamLabel();
     auto &label_streams = engine_streams[engine];
     auto iter = label_streams.find(stream_label);
-    if (iter != label_streams.end()) {
-      subgraph->stream_id = iter->second;
-    } else {
+    if (iter == label_streams.end()) {
       subgraph->stream_id = next_stream;
-      GELOGI("Assign new independent stream %ld for engine %s (label: %s).", next_stream, engine.c_str(),
+      GELOGI("[Assign][NewStreamId:independent] %ld for engine %s (label: %s).", next_stream, engine.c_str(),
              stream_label.c_str());
 
       label_streams.emplace(stream_label, next_stream);
-      ++next_stream;
+      next_stream++;
+    } else {
+      subgraph->stream_id = iter->second;
     }
     changed = true;
   }
@@ -121,14 +129,16 @@ Status AssignByDependencyPass::Run(ComputeGraphPtr graph, const vector<SubgraphP
     }
 
     SubgraphPtr reusable_subgraph = GetReusableSubgraph(subgraph, end_subgraph_map, pld_subgraph_map);
-    if (reusable_subgraph != nullptr) {
+    if (reusable_subgraph == nullptr) {
+      (void)AssignNewStream(subgraph);
+    } else {
       if (HasAssignedStream(*reusable_subgraph)) {
         subgraph->stream_id = reusable_subgraph->stream_id;
       } else {
         int64_t stream_id = AssignNewStream(reusable_subgraph);
         subgraph->stream_id = stream_id;
-        GELOGI("Reusable subgraph %s has not been assigned a stream, now assign new stream %ld.",
-               reusable_subgraph->name.c_str(), stream_id);
+        GELOGI("[Assign][NewStreamId] %ld for Reusable subgraph %s cause has not been assigned before.",
+               stream_id, reusable_subgraph->name.c_str());
       }
 
       if (reusable_subgraph->reused_subgraph != nullptr) {
@@ -137,11 +147,10 @@ Status AssignByDependencyPass::Run(ComputeGraphPtr graph, const vector<SubgraphP
 
       subgraph->reused_subgraph = reusable_subgraph;
       reused_subgraphs_.emplace_back(subgraph, reusable_subgraph);
-      GELOGI("Subgraph %s of engine %s reuses stream of subgraph %s of engine %s.", subgraph->name.c_str(),
+      GELOGI("[Reuse][Stream]Subgraph %s of engine %s reuses stream of subgraph %s of engine %s.",
+             subgraph->name.c_str(),
              subgraph->engine_conf.id.c_str(), reusable_subgraph->name.c_str(),
              reusable_subgraph->engine_conf.id.c_str());
-    } else {
-      (void)AssignNewStream(subgraph);
     }
     changed = true;
   }
@@ -191,13 +200,15 @@ bool AssignByDependencyPass::CouldReuse(const SubgraphPtr &subgraph, const Subgr
     auto iter = pld_subgraph_map.find(end_pld_pair.second);
     if (iter != pld_subgraph_map.end()) {
       const SubgraphPtr &pred_subgraph_succ = iter->second;
-      if (pred_subgraph_succ != subgraph && pred_subgraph_succ->engine_conf.id == pred_subgraph->engine_conf.id) {
+      if ((pred_subgraph_succ != subgraph) &&
+          (pred_subgraph_succ->engine_conf.id == pred_subgraph->engine_conf.id)) {
         return false;
       }
     }
   }
 
-  if ((subgraph->engine_conf.id == pred_subgraph->engine_conf.id) || IsEngineAttach(*subgraph)) {
+  if ((subgraph->engine_conf.id == pred_subgraph->engine_conf.id) ||
+      IsEngineAttach(*subgraph)) {
     return true;
   }
 
@@ -249,7 +260,7 @@ int64_t AssignByDependencyPass::AssignNewStream(SubgraphPtr subgraph) {
     engine_stream_num_[engine_name] = stream_id + 1;
   }
 
-  GELOGI("Subgraph %s assigns new temp stream %ld (engine: %s).", subgraph->name.c_str(), stream_id,
+  GELOGI("[Assign][NewStreamId:temp]id:%ld for Subgraph %s (engine: %s).", stream_id, subgraph->name.c_str(),
          engine_name.c_str());
 
   return stream_id;
@@ -282,7 +293,7 @@ void AssignByDependencyPass::UpdateAssignedSubgraphs(Context &context) {
       GELOGI("Subgraph %s of engine %s reuses default stream %ld.", subgraph->name.c_str(),
              subgraph->engine_conf.id.c_str(), context.default_stream);
     } else {
-      GELOGI("Stream of subgraph %s has been updated to %ld.", subgraph->name.c_str(), subgraph->stream_id);
+      GELOGI("[Update][StreamId]id:%ld for subgraph %s.", subgraph->stream_id, subgraph->name.c_str());
     }
   }
 }
@@ -293,7 +304,7 @@ void AssignByDependencyPass::UpdateReusedSubgraphs() {
     auto &cur_subgraph = item.first;
     auto &reused_graph = item.second;
     cur_subgraph->stream_id = reused_graph->stream_id;
-    GELOGI("Stream of subgraph %s has been updated to %ld.", cur_subgraph->name.c_str(), cur_subgraph->stream_id);
+    GELOGI("[Update][StreamId]id:%ld for subgraph %s.", cur_subgraph->stream_id, cur_subgraph->name.c_str());
   }
 }
 
@@ -309,6 +320,8 @@ Status SingleStreamPass::Run(ComputeGraphPtr graph, const vector<SubgraphPtr> &s
     if (!HasAssignedStream(*subgraph)) {
       const string &stream_label = subgraph->subgraph_info.GetStreamLabel();
       if (!stream_label.empty()) {
+        REPORT_INNER_ERROR("E19999", "Stream labels are not supported in SingleStream mode "
+                           "(subgraph: %s, stream label: %s)", subgraph->name.c_str(), stream_label.c_str());
         GELOGE(INTERNAL_ERROR, "Stream labels are not supported (subgraph: %s, stream label: %s).",
                subgraph->name.c_str(), stream_label.c_str());
         return INTERNAL_ERROR;
@@ -326,11 +339,13 @@ Status NodeStreamUpdatePass::Run(ComputeGraphPtr graph, const vector<SubgraphPtr
     const string &engine_name = subgraph->engine_conf.id;
 
     if (!IsEngineSkip(*subgraph) && !HasAssignedStream(*subgraph)) {
+      REPORT_INNER_ERROR("E19999", "Subgraph %s has not yet been assigned a stream (engine: %s)",
+                         subgraph->name.c_str(), engine_name.c_str());
       GELOGE(INTERNAL_ERROR, "Subgraph %s has not yet been assigned a stream (engine: %s).", subgraph->name.c_str(),
              engine_name.c_str());
       return INTERNAL_ERROR;
     } else {
-      GELOGI("Subgraph %s is assigned stream %ld (engine: %s).", subgraph->name.c_str(), subgraph->stream_id,
+      GELOGI("[Assign][StreamId] %ld for Subgraph %s (engine: %s).", subgraph->stream_id, subgraph->name.c_str(),
              engine_name.c_str());
     }
   }
@@ -353,12 +368,12 @@ Status NodeStreamUpdatePass::Run(ComputeGraphPtr graph, const vector<SubgraphPtr
         GELOGD("Node %s of type %s in subgraph %s is assigned parent stream %ld (engine: %s).", node->GetName().c_str(),
                node->GetType().c_str(), subgraph->name.c_str(), context.default_stream, engine_name.c_str());
       } else if (IsEngineSkip(*subgraph) && node->GetInNodes().empty()) {
-        GELOGD("Node %s of type %s in subgraph %s doesn't need to assign a stream (engine: %s).",
+        GELOGD("[Skip][StreamIdAssign]Node %s of type %s in subgraph %s doesn't need (engine: %s).",
                node->GetName().c_str(), node->GetType().c_str(), subgraph->name.c_str(), engine_name.c_str());
       } else {
         node->GetOpDesc()->SetStreamId(stream_id);
-        GELOGD("Node %s of type %s in subgraph %s is assigned stream %ld (engine: %s).", node->GetName().c_str(),
-               node->GetType().c_str(), subgraph->name.c_str(), stream_id, engine_name.c_str());
+        GELOGD("[Assign][StreamId]id:%ld for Node %s of type %s in subgraph %s (engine: %s).", stream_id,
+               node->GetName().c_str(), node->GetType().c_str(), subgraph->name.c_str(), engine_name.c_str());
       }
     }
   }
@@ -366,6 +381,48 @@ Status NodeStreamUpdatePass::Run(ComputeGraphPtr graph, const vector<SubgraphPtr
   return SUCCESS;
 }
 
+Status UpdateForParallelGroupPass::Run(ComputeGraphPtr graph, const vector<SubgraphPtr> &subgraphs, Context &context) {
+  std::map<int, vector<OpDescPtr>> stream_op_map;
+  for (const SubgraphPtr &subgraph : subgraphs) {
+    auto compute_graph = subgraph->subgraph_info.GetSubGraph();
+    for (const NodePtr &node : compute_graph->GetDirectNode()) {
+      OpDescPtr op_desc = node->GetOpDesc();
+      GE_CHECK_NOTNULL(op_desc);
+      if (op_desc->HasAttr(ATTR_NAME_PARALLEL_GROUP)) {
+        int64_t op_desc_stream_id = op_desc->GetStreamId();
+        stream_op_map[op_desc_stream_id].push_back(op_desc);
+      }
+    }
+  }
+  for (const auto &itr : stream_op_map) {
+    if (itr.first == kInvalidStream) {
+      continue;
+    }
+    std::map<std::string, int64_t> group_2_stream_id;
+    for (const auto &op_desc : itr.second) {
+      std::string group_name;
+      if (!AttrUtils::GetStr(op_desc, ATTR_NAME_PARALLEL_GROUP, group_name)) {
+        GELOGE(FAILED, "[GetAttr][OpDesc]Get node %s ATTR_NAME_PARALLEL_GROUP failed.", op_desc->GetName().c_str());
+        REPORT_INNER_ERROR("E19999", "Get node %s ATTR_NAME_PARALLEL_GROUP failed.", op_desc->GetName().c_str());
+        return FAILED;
+      }
+      const auto &itr = group_2_stream_id.find(group_name);
+      int64_t new_stream_id = kInvalidStream;
+      int64_t old_stream_id = op_desc->GetStreamId();
+      if (itr != group_2_stream_id.end()) {
+        new_stream_id = itr->second;
+      } else {
+        new_stream_id = context.next_stream++;
+        group_2_stream_id[group_name] = new_stream_id;
+      }
+      op_desc->SetStreamId(new_stream_id);
+      GELOGD("Node %s assigned stream %ld from stream %ld.",
+             op_desc->GetName().c_str(), new_stream_id, old_stream_id);
+    }
+  }
+  return SUCCESS;
+}
+
 int64_t UpdateForSkippedEnginePass::GetSingleInoutStream(const NodePtr &node) const {
   set<int64_t> stream_ids;
 
@@ -387,8 +444,8 @@ int64_t UpdateForSkippedEnginePass::GetSingleInoutStream(const NodePtr &node) co
 
   if (stream_ids.size() == 1) {
     int64_t stream_id = *(stream_ids.begin());
-    GELOGI("The stream of all input and output nodes of node %s (type: %s) is %ld.", node->GetName().c_str(),
-           node->GetType().c_str(), stream_id);
+    GELOGI("[Get][SingleStreamId]The stream of all input and output nodes of node %s (type: %s) is %ld.",
+           node->GetName().c_str(), node->GetType().c_str(), stream_id);
     return stream_id;
   }
 
@@ -406,7 +463,7 @@ Status UpdateForSkippedEnginePass::Run(ComputeGraphPtr graph, const vector<Subgr
         auto op_desc = node->GetOpDesc();
         GE_CHECK_NOTNULL(op_desc);
         auto stream_id = op_desc->GetStreamId();
-        if (stream_id != kInvalidStream && !HasStreamLabel(*subgraph)) {
+        if ((stream_id != kInvalidStream) && !HasStreamLabel(*subgraph)) {
           ops_without_label.emplace(op_desc);
         }
       }
@@ -427,8 +484,8 @@ Status UpdateForSkippedEnginePass::Run(ComputeGraphPtr graph, const vector<Subgr
         int64_t inout_stream = GetSingleInoutStream(node);
         if (inout_stream != kInvalidStream) {
           op_desc->SetStreamId(inout_stream);
-          GELOGI("Node %s of type %s reassign to stream %ld from stream %ld.", node->GetName().c_str(),
-                 node->GetType().c_str(), inout_stream, stream_id);
+          GELOGI("[Reassign][StreamId]%ld for Node %s of type %s from stream %ld.",
+                 inout_stream, node->GetName().c_str(), node->GetType().c_str(), stream_id);
         }
       }
     }
@@ -455,7 +512,7 @@ Status AllReduceParallelPass::Run(ComputeGraphPtr graph, const vector<SubgraphPt
     return NOT_CHANGED;
   }
 
-  GELOGI("AllReduceParallelPass is enabled.");
+  GELOGI("[Run][AllReduceParallelPass] start");
   GE_DUMP(graph, "BeforeAllReduceParallel");
 
   // All successors of HcomAllReduce.
@@ -463,7 +520,7 @@ Status AllReduceParallelPass::Run(ComputeGraphPtr graph, const vector<SubgraphPt
 
   for (const NodePtr &node : graph->GetDirectNode()) {
     if (!IsHcomNode(node->GetType()) ||
-        node->GetInDataNodes().size() <= 1) {
+        (node->GetInDataNodes().size() <= 1)) {
       continue;
     }
 
@@ -565,7 +622,7 @@ Status LogicalStreamAllocator::Assign(const ComputeGraphPtr &root_graph, const G
   RefreshContinuousStreams(root_graph);
 
   stream_num = context_.next_stream;
-  GELOGI("Assigned logical stream num: %ld.", stream_num);
+  GELOGI("[Assign][LogicalStream] At last, stream num: %ld.", stream_num);
 
   return SUCCESS;
 }
@@ -575,7 +632,7 @@ Status LogicalStreamAllocator::DoAssign(const ComputeGraphPtr &graph, const Grap
   GE_CHECK_NOTNULL(graph);
 
   NodePtr parent_node = graph->GetParentNode();
-  if (parent_node == nullptr || parent_node->GetOpDesc() == nullptr) {
+  if ((parent_node == nullptr) || (parent_node->GetOpDesc() == nullptr)) {
     context_.default_stream = kInvalidStream;
   } else {
     context_.default_stream = parent_node->GetOpDesc()->GetStreamId();
@@ -583,6 +640,8 @@ Status LogicalStreamAllocator::DoAssign(const ComputeGraphPtr &graph, const Grap
 
   auto iter = subgraph_map.find(graph);
   if (iter == subgraph_map.end()) {
+    REPORT_INNER_ERROR("E19999", "Graph %s not found in subgraph_map when do logical stream assign ",
+                       graph->GetName().c_str());
     GELOGE(FAILED, "Graph %s not found.", graph->GetName().c_str());
     return FAILED;
   }
@@ -597,7 +656,7 @@ Status LogicalStreamAllocator::DoAssign(const ComputeGraphPtr &graph, const Grap
     return status;
   }
 
-  GELOGD("Subgraphs of graph %s:", graph->GetName().c_str());
+  GELOGD("[Show][Subgraphs] in graph %s", graph->GetName().c_str());
   for (const auto &subgraph : subgraphs) {
     if (subgraph != nullptr) {
       GELOGD("subgraph: %s", subgraph->name.c_str());
@@ -622,6 +681,8 @@ Status LogicalStreamAllocator::ConvertSubgraphs(const vector<SubGraphInfoPtr> &s
     const string &engine_name = subgraph_info->GetEngineName();
     auto engine_conf_iter = engine_confs.find(engine_name);
     if ((engine_conf_iter == engine_confs.end()) || (engine_conf_iter->second == nullptr)) {
+      REPORT_INNER_ERROR("E19999", "Engine conf of subgraph %s not found (engine name: %s)",
+                         subgraph_name.c_str(), engine_name.c_str());
       GELOGE(INTERNAL_ERROR, "Engine conf of subgraph %s not found (engine name: %s).", subgraph_name.c_str(),
              engine_name.c_str());
 
@@ -655,6 +716,7 @@ Status LogicalStreamAllocator::RunPasses(const ComputeGraphPtr &graph, const vec
     passes.emplace_back(MakeShared<IndependentStreamPass>());
     passes.emplace_back(MakeShared<AssignByDependencyPass>());
     passes.emplace_back(MakeShared<NodeStreamUpdatePass>());
+    passes.emplace_back(MakeShared<UpdateForParallelGroupPass>());
     passes.emplace_back(MakeShared<AllReduceParallelPass>());
     passes.emplace_back(MakeShared<UpdateForSkippedEnginePass>());
   }
@@ -664,10 +726,11 @@ Status LogicalStreamAllocator::RunPasses(const ComputeGraphPtr &graph, const vec
 
     Status status = pass->Run(graph, subgraphs, context_);
     if (status == SUCCESS) {
-      GELOGD("Stream pass %s return SUCCESS.", pass->GetName().c_str());
+      GELOGD("[Show][Status]Stream pass %s return SUCCESS.", pass->GetName().c_str());
     } else if (status == NOT_CHANGED) {
-      GELOGD("Stream pass %s return NOT_CHANGED.", pass->GetName().c_str());
+      GELOGD("[Show][Status]Stream pass %s return NOT_CHANGED.", pass->GetName().c_str());
     } else {
+      REPORT_CALL_ERROR("E19999", "Stream pass %s run failed.", pass->GetName().c_str());
       GELOGE(status, "Stream pass %s failed.", pass->GetName().c_str());
       return status;
     }
@@ -686,7 +749,7 @@ void LogicalStreamAllocator::RefreshContinuousStreams(const ComputeGraphPtr &gra
       auto op_desc = node->GetOpDesc();
       if (op_desc != nullptr) {
         int64_t stream_id = op_desc->GetStreamId();
-        if (stream_id != kInvalidStream && stream_id < stream_num) {
+        if ((stream_id != kInvalidStream) && (stream_id < stream_num)) {
           stream_has_node[stream_id] = true;
         }
       }
@@ -695,10 +758,10 @@ void LogicalStreamAllocator::RefreshContinuousStreams(const ComputeGraphPtr &gra
 
   context_.next_stream = 0;
   vector<int64_t> old_to_new_streams(stream_num, kInvalidStream);
-  for (size_t old_stream = 0; old_stream < stream_has_node.size(); ++old_stream) {
+  for (size_t old_stream = 0; old_stream < stream_has_node.size(); old_stream++) {
     if (stream_has_node[old_stream]) {
       old_to_new_streams[old_stream] = context_.next_stream;
-      ++context_.next_stream;
+      context_.next_stream++;
     }
   }
 
@@ -706,7 +769,7 @@ void LogicalStreamAllocator::RefreshContinuousStreams(const ComputeGraphPtr &gra
     auto op_desc = node->GetOpDesc();
     if (op_desc != nullptr) {
       int64_t stream_id = op_desc->GetStreamId();
-      if (stream_id != kInvalidStream && stream_id < stream_num) {
+      if ((stream_id != kInvalidStream) && (stream_id < stream_num)) {
         op_desc->SetStreamId(old_to_new_streams[stream_id]);
       }
     }

ge/graph/build/logical_stream_allocator.h

@@ -149,6 +149,13 @@ class NodeStreamUpdatePass : public LogicalStreamPass {
   Status Run(ComputeGraphPtr graph, const std::vector<SubgraphPtr> &subgraphs, Context &context) override;
 };
 
+// assign stream by parallel group
+class UpdateForParallelGroupPass : public LogicalStreamPass {
+ public:
+  STREAM_PASS_DEFAULT_FUNC(UpdateForParallelGroupPass);
+  Status Run(ComputeGraphPtr graph, const std::vector<SubgraphPtr> &subgraphs, Context &context) override;
+};
+
 // Update the stream of subgraphs to nodes.
 class UpdateForSkippedEnginePass : public LogicalStreamPass {
  public:

ge/graph/build/memory/binary_block_mem_assigner.cc

@@ -70,7 +70,10 @@ Status BinaryBlockMemAssigner::GetMemoryRanges(vector<int64_t> &range_ceils) {
     return SUCCESS;
   }
   if ((all_memory_size.front() <= 0) || (log(kLogBase) == 0)) {
-    GELOGE(FAILED, "Memory size:%ld is invalid.", all_memory_size.front());
+    GELOGE(FAILED, "[Check][MemRangeStep]first mem_range_step:%ld less than 0,invalid,"
+          "maybe has dynamic shape in graph", all_memory_size.front());
+    REPORT_INNER_ERROR("E19999", "first mem_range_step:%ld less than 0,invalid,"
+          "maybe has dynamic shape in graph", all_memory_size.front());
     return FAILED;
   }
   // Memory size is 512 aligned, so it is not necessary to take less than 512
@@ -81,12 +84,18 @@ Status BinaryBlockMemAssigner::GetMemoryRanges(vector<int64_t> &range_ceils) {
   GELOGD("Range number: %zu", range_number);
 
   vector<vector<int64_t>> ranges(range_number);
-  GE_CHK_BOOL_EXEC((range_number != 0), return PARAM_INVALID, "range_number can't be 0.");
+  GE_CHK_BOOL_EXEC((range_number != 0),
+    REPORT_INNER_ERROR("E19999", "inner data[range_number] is 0, judge invalid");
+    return PARAM_INVALID,
+    "[Check][RangeNumber]inner data is 0, judge invalid.");
   size_t range_number_limit = all_memory_size.size() / range_number;
   int64_t range_ceil = min_memory_size;
   for (size_t i = 1; i <= range_number; i++) {
     GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(static_cast<uint64_t>(range_ceil), kRangeCeilInterval),
-                    GELOGE(FAILED, "Multiply result is out of range.");
+                    GELOGE(FAILED, "[Check][MemRangeCeil]Multiply result is out of range,"
+                      "range_ceil:%ld, interval:%u", range_ceil, kRangeCeilInterval);
+                    REPORT_INNER_ERROR("E19999", "process mem_range_ceil,multiply result out of range,"
+                      "range_ceil:%ld, interval:%u", range_ceil, kRangeCeilInterval);
                     return FAILED);
     range_ceil *= kRangeCeilInterval;  // The block size of each interval is doubled every time.
     for (auto iter = all_memory_size.begin(); iter != all_memory_size.end();) {

ge/graph/build/memory/block_mem_assigner.cc

@@ -30,6 +30,7 @@
 #include "graph/utils/node_utils.h"
 #include "graph/utils/op_desc_utils.h"
 #include "graph/utils/tensor_utils.h"
+#include "graph/utils/type_utils.h"
 
 #include "graph/debug/ge_attr_define.h"
 
@@ -429,17 +430,14 @@ void SetLastUsedInputMemAttr(NodePtr &node, int input_index) {
   }
   auto node_op_desc = node->GetOpDesc();
   if (node_op_desc != nullptr) {
-    auto input_desc = node_op_desc->GetInputDesc(input_index);
-    if (!ge::AttrUtils::SetInt(input_desc, ATTR_NAME_IS_END_OF_INPUTMEM_LIFECYCLE, true)) {
+    auto input_desc = node_op_desc->MutableInputDesc(input_index);
+    if (!ge::AttrUtils::SetInt(*input_desc, ATTR_NAME_IS_END_OF_INPUTMEM_LIFECYCLE, true)) {
       GELOGW("Set %s input[%d] ATTR_NAME_IS_END_OF_INPUTMEM_LIFECYCLE to true failed.", node_op_desc->GetName().c_str(),
              input_index);
       return;
     }
     GELOGD("Set %s input[%d] ATTR_NAME_IS_END_OF_INPUTMEM_LIFECYCLE to true success.", node_op_desc->GetName().c_str(),
            input_index);
-    if (node_op_desc->UpdateInputDesc(input_index, input_desc) != GRAPH_SUCCESS) {
-      GELOGW("Update %s input[%d] desc failed.", node_op_desc->GetName().c_str(), input_index);
-    }
   }
 }
 
@@ -457,7 +455,16 @@ Status GetNoAlignSize(const ge::OpDesc &desc, uint32_t index, size_t &size) {
   DataType data_type = output_op_desc->GetDataType();
   graphStatus graph_status = TensorUtils::CalcTensorMemSize(shape, format, data_type, tensor_size);
   if (graph_status != GRAPH_SUCCESS) {
-    GELOGE(graph_status, "CalcTensorMemSize failed!");
+    GELOGE(graph_status, "[Calculate][TensorSize]shape:%s, format:%s, data_type:%s, op:%s, out_index:%u",
+           shape.ToString().c_str(),
+           TypeUtils::FormatToSerialString(format).c_str(),
+           TypeUtils::DataTypeToSerialString(data_type).c_str(),
+           desc.GetName().c_str(), index);
+    REPORT_CALL_ERROR("E19999", "CalcTensorMemSize fail, shape:%s, format:%s, data_type:%s, op:%s, out_index:%u",
+                      shape.ToString().c_str(),
+                      TypeUtils::FormatToSerialString(format).c_str(),
+                      TypeUtils::DataTypeToSerialString(data_type).c_str(),
+                      desc.GetName().c_str(), index);
     return FAILED;
   }
   size = static_cast<size_t>(tensor_size);
@@ -498,7 +505,7 @@ BlockMemAssigner::BlockMemAssigner(ComputeGraphPtr compute_graph, const map<stri
       symbol_to_anchors_(symbol_to_anchors), anchor_to_symbol_(anchor_to_symbol), life_time_(0) {}
 
 BlockMemAssigner::~BlockMemAssigner() {
-  GELOGD("blocks_store_ size : %lu", blocks_store_.size());
+  GELOGD("[Destruct][BlockMemAssigner]blocks_store_ size : %lu", blocks_store_.size());
   for (MemoryBlock *memory_block : blocks_store_) {
     GE_DELETE_NEW_SINGLE(memory_block);
   }
@@ -583,11 +590,16 @@ void BlockMemAssigner::GetOutAndWorkSpaceMem(vector<int64_t> &all_memory_size) {
     }
 
     for (auto &out_anchor : n->GetAllOutDataAnchors()) {
-      GeTensorDesc output_desc = node_op_desc->GetOutputDesc(out_anchor->GetIdx());
+      auto output_desc = node_op_desc->GetOutputDescPtr(out_anchor->GetIdx());
       int64_t size = 0;
-      GE_IF_BOOL_EXEC(ge::TensorUtils::GetSize(output_desc, size) != SUCCESS, GELOGI("Get size failed"));
-      GE_IF_BOOL_EXEC(size < 0, GELOGE(FAILED, "Node:%s size:%ld is invalid, maybe it is unknown shape node.",
-                                       node_op_desc->GetName().c_str(), size);
+      GE_IF_BOOL_EXEC(ge::TensorUtils::GetSize(*output_desc, size) != SUCCESS, GELOGI("Get size failed"));
+      GE_IF_BOOL_EXEC(size < 0,
+                      GELOGE(FAILED, "[Check][TensorSize]tensor_size:%ld is invalid, "
+                             "maybe it is unknown shape node, Node_name:%s",
+                             size, node_op_desc->GetName().c_str());
+                      REPORT_INNER_ERROR("E19999", "tensor_size:%ld is invalid, "
+                                         "maybe it is unknown shape node, Node_name:%s",
+                                         size, node_op_desc->GetName().c_str());
                       return;);
       batch_all_memory_size[batch_label].emplace_back(size);
       if (batch_total_size.find(batch_label) == batch_total_size.end()) {
@@ -678,22 +690,34 @@ bool BlockMemAssigner::IsOutNodeSetContinuousInput(const NodePtr &n, uint32_t ou
   if (static_cast<size_t>(out_index) < n->GetAllOutDataAnchors().size()) {
     auto out_anchor = n->GetOutDataAnchor(out_index);
     GE_IF_BOOL_EXEC(out_anchor == nullptr,
-                    GELOGE(FAILED, "Node[%s] output[%u] anchor is null.", n->GetName().c_str(), out_index);
+                    GELOGE(FAILED, "[Check][Anchor]Node[%s] output[%u] anchor is null.",
+                           n->GetName().c_str(), out_index);
+                    REPORT_INNER_ERROR("E19999", "output anchor is null, node_name: %s output_index: %u.",
+                                       n->GetName().c_str(), out_index);
                     return false;);
     for (auto const &peer_in_anchor : out_anchor->GetPeerInDataAnchors()) {
       GE_IF_BOOL_EXEC(peer_in_anchor == nullptr,
-                      GELOGE(FAILED, "Node[%s] output[%u] peer_in_anchor 0 is null.", n->GetName().c_str(), out_index);
+                      GELOGE(FAILED, "[Check][Anchor]Node[%s] output[%u] peer_in_anchor 0 is null.",
+                             n->GetName().c_str(), out_index);
+                      REPORT_INNER_ERROR("E19999", "output anchor peer is null, node_name: %s output_index: %u.",
+                                         n->GetName().c_str(), out_index);
                       return false;);
       auto peer_node = peer_in_anchor->GetOwnerNode();
       GE_IF_BOOL_EXEC(peer_node == nullptr,
-                      GELOGE(FAILED, "Node[%s] output[%u] node is null.", n->GetName().c_str(), out_index);
+                      GELOGE(FAILED, "[Check][Node]Node[%s] output[%u] peer node is null.",
+                             n->GetName().c_str(), out_index);
+                      REPORT_INNER_ERROR("E19999", "output anchor peer node is null, node_name: %s output_index: %u.",
+                                         n->GetName().c_str(), out_index);
                       return false;);
 
       // Get the continuous input type of the node, default is false
       bool is_input_continuous = false;
       auto peer_in_node_desc = peer_node->GetOpDesc();
       GE_IF_BOOL_EXEC(peer_in_node_desc == nullptr,
-                      GELOGE(FAILED, "Node[%s] output[%u] nodedesc is null.", n->GetName().c_str(), out_index);
+                      GELOGE(FAILED, "[Check][OpDesc]Node[%s] output[%u] nodedesc is null.",
+                             n->GetName().c_str(), out_index);
+                      REPORT_INNER_ERROR("E19999", "output anchor peer op_desc is null, node_name:%s output_index:%u.",
+                                         n->GetName().c_str(), out_index);
                       return false;);
 
       // If GetBool fail, is_input_continuous is false.
@@ -793,7 +817,10 @@ bool BlockMemAssigner::IsContinuousMemoryReuse(const NodePtr &n, const NodePtr &
     if ((in_anchor == nullptr) || (in_anchor->GetPeerOutAnchor() == nullptr) ||
         (in_anchor->GetPeerOutAnchor()->GetOwnerNode() == nullptr) ||
         (in_anchor->GetPeerOutAnchor()->GetOwnerNode()->GetOpDesc() == nullptr)) {
-      GELOGE(FAILED, "Node[%s] output[%u] peer input node desc is null.", n->GetName().c_str(), out_index);
+      GELOGE(FAILED, "[Check][OpDesc]Node[%s] output[%u] peer input node desc is null.",
+             n->GetName().c_str(), out_index);
+      REPORT_INNER_ERROR("E19999", "get output anchor peer op_desc fail, node_name: %s output_index: %u.",
+                         n->GetName().c_str(), out_index);
       return false;
     }
     auto peer_out_node_desc = in_anchor->GetPeerOutAnchor()->GetOwnerNode()->GetOpDesc();
@@ -1077,7 +1104,10 @@ MemoryBlock *BlockMemAssigner::ApplyMemory(size_t block_size, size_t real_size,
                                            OpMemoryType mem_type, const NodePtr &n, uint32_t out_index,
                                            const vector<bool> &workspace_reuse_flag, const bool is_op_reuse_mem,
                                            const bool continuous, int64_t memory_type) {
-  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(n == nullptr, return nullptr, "Input parameter n is null.");
+  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
+      n == nullptr,
+      REPORT_INNER_ERROR("E19999", "Input parameter n(type:node_ptr) is null, apply memory failed");
+      return nullptr, "[Check][Param]Input parameter n(type:node_ptr) is null.");
   auto node_op_desc = n->GetOpDesc();
   GE_IF_BOOL_EXEC(node_op_desc == nullptr, return nullptr);
   std::string batch_label;
@@ -1129,7 +1159,12 @@ MemoryBlock *BlockMemAssigner::ApplyMemory(size_t block_size, size_t real_size,
   }
 
   auto block = new (std::nothrow) MemoryBlock(block_size, node_op_desc->GetStreamId(), is_reuse_memory, memory_type);
-  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(block == nullptr, return nullptr, "new an object failed.");
+  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
+      block == nullptr,
+      REPORT_INNER_ERROR("E19999", "new a memoryblock object failed. node_name:%s out_index:%u",
+                         n->GetName().c_str(), out_index);
+      return nullptr,
+      "[New][Object]new MemoryBlock failed, node_name:%s out_index:%u", n->GetName().c_str(), out_index);
 
   // Data and netoutput need zero copy block
   block->is_zero_copy_ = IsZeroCopyBlock(n, continuous);
@@ -1188,9 +1223,15 @@ void BlockMemAssigner::ContinuousOutRefCheck(bool &isAllOutputRef, bool &isOutpu
 
 Status BlockMemAssigner::ApplyContinuousMemory(const NodePtr &n, const vector<int64_t> &ranges,
                                                const bool is_op_reuse_mem) {
-  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(n == nullptr, return INTERNAL_ERROR, "input node is null.");
+  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
+      n == nullptr,
+      REPORT_INNER_ERROR("E19999", "Input parameter n(type:node_ptr) is null");
+      return INTERNAL_ERROR, "[check][param]Input parameter n(type:NodePtr) is null.");
   auto node_op_desc = n->GetOpDesc();
-  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(node_op_desc == nullptr, return INTERNAL_ERROR, "node_op_desc is null.");
+  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
+      node_op_desc == nullptr,
+      REPORT_INNER_ERROR("E19999", "Input parameter n(type:OpDescPtr) is null");
+      return INTERNAL_ERROR, "[Check][Param]Input parameter n(type:OpDescPtr) is null");
 
   // continuous output support ref only when all output ref input
   bool isAllOutputRef = true;
@@ -1204,7 +1245,9 @@ Status BlockMemAssigner::ApplyContinuousMemory(const NodePtr &n, const vector<in
   }
 
   if (!isAllOutputRef && isOutputHasRef) {
-    GELOGE(INTERNAL_ERROR, "continuous output node ref part input, not support this situation, node_name:%s",
+    REPORT_INNER_ERROR("E19999", "continuous output node ref part input, not support now. node_name:%s",
+                       n->GetName().c_str());
+    GELOGE(INTERNAL_ERROR, "[Check][OutRefStatus]continuous output node ref part input, not support, node_name:%s",
            n->GetName().c_str());
     return INTERNAL_ERROR;
   }
@@ -1215,7 +1258,9 @@ Status BlockMemAssigner::ApplyContinuousMemory(const NodePtr &n, const vector<in
   for (uint32_t index = 0; index < static_cast<uint32_t>(node_op_desc->GetOutputsSize()); index++) {
     auto output_op_desc = node_op_desc->GetOutputDescPtr(index);
     if (output_op_desc == nullptr) {
-      GELOGE(INTERNAL_ERROR, "Get output desc failed, node_name:%s, output_index:%u", n->GetName().c_str(), index);
+      REPORT_INNER_ERROR("E19999", "get output_desc failed, node_name:%s, output_index:%u",
+                         n->GetName().c_str(), index);
+      GELOGE(INTERNAL_ERROR, "[Get][OutputDesc]node_name:%s, output_index:%u", n->GetName().c_str(), index);
       return INTERNAL_ERROR;
     }
 
@@ -1226,7 +1271,9 @@ Status BlockMemAssigner::ApplyContinuousMemory(const NodePtr &n, const vector<in
 
     int64_t size = 0;
     if (ge::TensorUtils::GetSize(*output_op_desc, size) != SUCCESS) {
-      GELOGE(INTERNAL_ERROR, "Get size failed, node_name:%s, output_index:%u", n->GetName().c_str(), index);
+      REPORT_CALL_ERROR("E19999", "get tensor_size failed, node_name:%s, output_index:%u",
+                        n->GetName().c_str(), index);
+      GELOGE(INTERNAL_ERROR, "[Get][TensorSize]node_name:%s, output_index:%u", n->GetName().c_str(), index);
       return INTERNAL_ERROR;
     }
     size_t align_size = static_cast<size_t>(size);
@@ -1266,7 +1313,9 @@ Status BlockMemAssigner::ApplyContinuousMemory(const NodePtr &n, const vector<in
     block->last_continuous_block_ = true;
     ++(block->ref_count_);
   } else {
-    GELOGE(INTERNAL_ERROR, "node apply continuous output memory failed. node_name:%s", n->GetName().c_str());
+    REPORT_CALL_ERROR("E19999", "apply continuousMemory failed, node_name:%s, total_size:%ld",
+                      n->GetName().c_str(), total_size);
+    GELOGE(INTERNAL_ERROR, "[Apply][ContinuousMemory]node_name:%s, total_size:%ld", n->GetName().c_str(), total_size);
     return INTERNAL_ERROR;
   }
   return SUCCESS;
@@ -1274,25 +1323,44 @@ Status BlockMemAssigner::ApplyContinuousMemory(const NodePtr &n, const vector<in
 
 MemoryBlock *BlockMemAssigner::ApplyOutMemory(const NodePtr &n, uint32_t index, const vector<int64_t> &ranges,
                                               const bool is_op_reuse_mem, const bool continuous) {
-  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(n == nullptr, return nullptr, "input node is null.");
+  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
+      n == nullptr,
+      REPORT_INNER_ERROR("E19999", "Input parameter n(type:NodePtr) is null");
+      return nullptr, "[Check][Param]Input parameter n(type:NodePtr) is null");
   auto node_op_desc = n->GetOpDesc();
-  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(node_op_desc == nullptr, return nullptr, "node_op_desc is null.");
+  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
+      node_op_desc == nullptr,
+      REPORT_INNER_ERROR("E19999", "Input parameter n(type:OpDescPtr) is null");
+      return nullptr, "[Check][Param]Input parameter n(type:OpDescPtr) is null");
   MemoryBlock *block = nullptr;
   NodeIndexIO node_index_io(n, index, kOut);
   int64_t size = 0;
   auto output_op_desc = node_op_desc->GetOutputDescPtr(index);
-  GE_IF_BOOL_EXEC(output_op_desc == nullptr, return nullptr);
+  GE_IF_BOOL_EXEC(
+      output_op_desc == nullptr,
+      REPORT_INNER_ERROR("E19999", "get output_desc failed, node_name:%s, output_index:%u",
+                         n->GetName().c_str(), index);
+      GELOGE(FAILED, "[Get][OutputDesc]node_name:%s, output_index:%u", n->GetName().c_str(), index);
+      return nullptr);
   GE_IF_BOOL_EXEC(ge::TensorUtils::GetSize(*output_op_desc, size) != SUCCESS, GELOGI("Get size failed"));
   size_t no_align_size = 0;
-  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(GetNoAlignSize(*node_op_desc, index, no_align_size) != SUCCESS,
-                                 return nullptr, "Get no align size failed");
+  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
+      GetNoAlignSize(*node_op_desc, index, no_align_size) != SUCCESS,
+      REPORT_CALL_ERROR("E19999", "Get no align size failed, node_name:%s, output_index:%u",
+                        n->GetName().c_str(), index);
+      return nullptr,
+      "[Get][TensorSize]Get no align size, node_name:%s, output_index:%u", n->GetName().c_str(), index);
 
   std::string symbol;
   bool reuse_input = false;
   if (IsSymbolExist(node_index_io, symbol)) {
     block = symbol_blocks_[symbol];
-    GE_IF_BOOL_EXEC(block == nullptr, GELOGE(FAILED, "Node %s ref block is nullptr.", node_op_desc->GetName().c_str());
-        return nullptr);
+    GE_IF_BOOL_EXEC(block == nullptr,
+      REPORT_INNER_ERROR("E19999", "get ref block failed, node_name:%s, symbol:%s",
+                         node_op_desc->GetName().c_str(), node_index_io.ToString().c_str());
+      GELOGE(FAILED, "[Get][RefBlock]node_name:%s, symbol:%s",
+             node_op_desc->GetName().c_str(), node_index_io.ToString().c_str());
+      return nullptr);
     // reduce old size
     size_t align_size = block->Size();
     AlignMemOffset(align_size);
@@ -1335,12 +1403,28 @@ MemoryBlock *BlockMemAssigner::ApplyOutMemory(const NodePtr &n, uint32_t index,
     vector<bool> workspace_reuse_flag;
     block = ApplyMemory(block_size, size, no_align_size, kOutput, n, index,
                         workspace_reuse_flag, is_op_reuse_mem, continuous, memory_type);
+    GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
+        block == nullptr,
+        REPORT_CALL_ERROR("E19999", "apply out Memory failed, node_name:%s, block_size:%ld, out_index:%u",
+                          n->GetName().c_str(), block_size, index);
+        return nullptr,
+        "[Apply][Memory]node_name:%s, block_size:%ld, out_index:%u",
+        n->GetName().c_str(), block_size, index);
   }
-  GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(block == nullptr, return nullptr, "Block is nullptr.");
   int out_count = 0;
-  GE_IF_BOOL_EXEC(index >= n->GetAllOutDataAnchors().size(), GELOGE(FAILED, "index is out of range."); return nullptr);
+  GE_IF_BOOL_EXEC(
+      index >= n->GetAllOutDataAnchors().size(),
+      REPORT_INNER_ERROR("E19999", "out index:%u exceed out_size:%lu, node_name:%s",
+                         index, n->GetAllOutDataAnchors().size(), n->GetName().c_str());
+      GELOGE(FAILED, "[Check][OutIndex]index:%u exceed out_size:%lu, node_name:%s",
+             index, n->GetAllOutDataAnchors().size(), n->GetName().c_str());
+      return nullptr);
   auto out_data_anchor = n->GetOutDataAnchor(index);
-  GE_IF_BOOL_EXEC(out_data_anchor == nullptr, GELOGE(FAILED, "Out data anchor is nullptr."); return nullptr);
+  GE_IF_BOOL_EXEC(
+      out_data_anchor == nullptr,
+      REPORT_INNER_ERROR("E19999", "out anchor is null, index:%u, node_name:%s", index, n->GetName().c_str());
+      GELOGE(FAILED, "[Check][OutAnchor]is null, index:%u, node_name:%s", index, n->GetName().c_str());
+      return nullptr);
   for (const auto &in_anchor : out_data_anchor->GetPeerInDataAnchors()) {
     auto owner_node = in_anchor->GetOwnerNode();
     auto op_desc = owner_node->GetOpDesc();
@@ -1546,8 +1630,14 @@ Status BlockMemAssigner::AssignOutputMemoryWithReuse(const NodePtr &node, vector
   GELOGD("Assign memory node[%s], output size[%zu], output memory type size[%zu]", op_desc->GetName().c_str(),
          op_desc->GetOutputsSize(), memorys_type.size());
   if (has_mem_type_attr && (memorys_type.size() != op_desc->GetOutputsSize())) {
-    GELOGE(INTERNAL_ERROR, "fusion: node[%s], output memory size err[outputsize:%zu, memorysize:%zu]",
-           op_desc->GetName().c_str(), op_desc->GetOutputsSize(), memorys_type.size());
+    REPORT_INNER_ERROR("E19999", "Attr[%s] size:%zu not equal to node output size:%zu, node_name:%s",
+                       ATTR_NAME_OUTPUT_MEM_TYPE_LIST.c_str(), memorys_type.size(),
+                       op_desc->GetOutputsSize(), op_desc->GetName().c_str());
+    GELOGE(
+        INTERNAL_ERROR,
+        "[Check][MemTypeAttr]Attr %s size:%zu not equal to node output size:%zu, node_name:%s",
+        ATTR_NAME_OUTPUT_MEM_TYPE_LIST.c_str(), memorys_type.size(),
+        op_desc->GetOutputsSize(), op_desc->GetName().c_str());
     return INTERNAL_ERROR;
   }
 
@@ -1565,12 +1655,15 @@ Status BlockMemAssigner::AssignOutputMemoryWithReuse(const NodePtr &node, vector
   bool is_atomic = false;
   // If GetBool fail, is_atomic is false.
   (void)ge::AttrUtils::GetBool(op_desc, ATOMIC_ATTR_IS_ATOMIC_NODE, is_atomic);
+  bool is_buffer_pool_mem_supported = (op_desc->HasAttr(ATTR_NAME_BUFFER_POOL_ID)) &&
+                                      (op_desc->HasAttr(ATTR_NAME_BUFFER_POOL_SIZE)) && (!root_unknown_shape_flag_);
   // Allocate memory for the current node and release node memory of the same size in the workspace
   GE_IF_BOOL_EXEC(ge_disable_reuse_mem_env_ != "1",
                   for (auto iter = stream_workspace_blocks_.begin(); iter != stream_workspace_blocks_.end();
                        ++iter) { ReleaseMemorys(iter->second[stream_id], reusable_blocks_[iter->first][stream_id]);
                                  iter->second[stream_id].clear();});
-  if (IsContinuousOutput(node)) {
+  bool need_apply_continuous_memory = IsContinuousOutput(node) && (!is_buffer_pool_mem_supported);
+  if (need_apply_continuous_memory) {
     return ApplyContinuousMemory(node, ranges, is_op_reuse_mem_);
   }
   for (uint32_t i = 0; i < static_cast<uint32_t>(op_desc->GetOutputsSize()); i++) {
@@ -1604,7 +1697,7 @@ Status BlockMemAssigner::AssignOutputMemoryWithReuse(const NodePtr &node, vector
       GE_IF_BOOL_EXEC(!no_need_assign_memory,
           no_need_assign_memory = IsAtomicOutputMemory(node, i, is_atomic, out_node_set_continuous_input););
     }
-    no_need_assign_memory = (no_need_assign_memory || IsKnownSubgraphData(node));
+    no_need_assign_memory = (no_need_assign_memory || IsKnownSubgraphData(node) || is_buffer_pool_mem_supported);
     if (no_need_assign_memory) {
       zero_memory_list_.emplace_back(node, kOutput, i, false);
       continue;
@@ -1645,11 +1738,18 @@ Status BlockMemAssigner::AssignOutputMemoryWithReuse(const NodePtr &node, vector
 ///
 void BlockMemAssigner::AssignMemoryWithReuse(vector<int64_t> &ranges) {
   (void)ge::GetContext().GetOption(OPTION_EXEC_DISABLE_REUSED_MEMORY, ge_disable_reuse_mem_env_);
-  GELOGD("Reuse memory %s", ge_disable_reuse_mem_env_ == "1" ? "close" : "open");
+  GEEVENT("Reuse memory %s", ge_disable_reuse_mem_env_ == "1" ? "close" : "open");
   string op_no_reuse_mem_str;
   const char *op_no_reuse_mem = std::getenv(OP_NO_REUSE_MEM);
   GE_IF_BOOL_EXEC(op_no_reuse_mem != nullptr, op_no_reuse_mem_str = string(op_no_reuse_mem);
                   CheckAndGetOpReuseEnv(op_no_reuse_mem_str, op_no_reuse_mem_vec_, op_reuse_env_valid_););
+  auto root_graph = GraphUtils::FindRootGraph(compute_graph_);
+  if (root_graph == nullptr) {
+    GELOGE(INTERNAL_ERROR, "[Check][RootGraph]Root graph is nullptr, graph:%s.", compute_graph_->GetName().c_str());
+    REPORT_INNER_ERROR("E19999", "Root graph is nullptr, graph:%s.", compute_graph_->GetName().c_str());
+    return;
+  }
+  root_unknown_shape_flag_ = root_graph->GetGraphUnknownFlag();
 
   for (NodePtr &n : compute_graph_->GetAllNodes()) {
     auto node_op_desc = n->GetOpDesc();
@@ -1673,8 +1773,12 @@ void BlockMemAssigner::AssignMemoryWithReuse(vector<int64_t> &ranges) {
            temp.size(), tvm_workspace_memory_type.size());
 
     if (has_tvm_workspace_mem_type_attr && (temp.size() != tvm_workspace_memory_type.size())) {
-      GELOGE(INTERNAL_ERROR, "fusion: node[%s], tvm workspace memory size error![v_temp:%zu, workspace:%zu]",
-             n->GetName().c_str(), temp.size(), tvm_workspace_memory_type.size());
+      REPORT_INNER_ERROR("E19999", "Attr[%s]size:%zu is not equal to workspace size:%zu, node_name:%s",
+                         TVM_ATTR_NAME_WORKSPACE_TYPE.c_str(), tvm_workspace_memory_type.size(),
+                         temp.size(), n->GetName().c_str());
+      GELOGE(INTERNAL_ERROR, "[Check][Attr]Attr %s size:%zu is not equal to workspace size:%zu, node_name:%s",
+             TVM_ATTR_NAME_WORKSPACE_TYPE.c_str(), tvm_workspace_memory_type.size(),
+             temp.size(), n->GetName().c_str());
       return;
     }
     for (size_t i = 0; i < temp.size(); i++) {
@@ -2031,7 +2135,7 @@ void SetBlockOpMemOffset(MemoryBlock *block, int32_t child_block_level) {
 
   child_block_level++;
   for (MemoryBlock *child_block : block->ChildBlockList()) {
-      SetBlockOpMemOffset(child_block, child_block_level);
+    SetBlockOpMemOffset(child_block, child_block_level);
   }
 }
 
@@ -2059,7 +2163,7 @@ void BlockMemAssigner::SetOpMemOffset(bool is_zero_copy) {
 Status BlockMemAssigner::Assign() {
   vector<int64_t> ranges;
   if (GetMemoryRanges(ranges) != SUCCESS) {
-    GELOGE(FAILED, "GetMemoryRanges Fail!");
+    GELOGE(FAILED, "[Get][MemoryRanges] Fail!");
     return FAILED;
   }
   GE_IF_BOOL_EXEC(ranges.empty(), return SUCCESS);
@@ -2083,8 +2187,12 @@ bool BlockMemAssigner::GetWorkSpaceMemoryType(const NodePtr &node, size_t index,
   bool has_workspace_mem_type_attr =
       ge::AttrUtils::GetListInt(op_desc, TVM_ATTR_NAME_WORKSPACE_TYPE, workspace_memory_type);
   if (has_workspace_mem_type_attr && (workspace_memory_type.size() <= index)) {
-    GELOGE(INTERNAL_ERROR, "node[%s], workspace_memory size error![index:%zu, workspace:%zu]",
-           node->GetName().c_str(), index, workspace_memory_type.size());
+    REPORT_INNER_ERROR("E19999", "get workspace mem_type failed, "
+                       "index %zu invalid, bigger than attr %s size:%zu, node_name:%s",
+                       index, TVM_ATTR_NAME_WORKSPACE_TYPE.c_str(),
+                       workspace_memory_type.size(), node->GetName().c_str());
+    GELOGE(INTERNAL_ERROR, "[Get][WorkspaceMemType]index %zu invalid, bigger than attr %s size:%zu, node_name:%s",
+           index, TVM_ATTR_NAME_WORKSPACE_TYPE.c_str(), workspace_memory_type.size(), node->GetName().c_str());
     return false;
   }
   memory_type = has_workspace_mem_type_attr ? workspace_memory_type[index] : RT_MEMORY_HBM;

ge/graph/build/memory/block_mem_assigner.h

@@ -494,6 +494,8 @@ class BlockMemAssigner : public MemAssigner {
   /// @          [stream2][nodeid]
   ///
   DependStreamLife total_node_depend_stream_life_;
+
+  bool root_unknown_shape_flag_ = false;
 };
 }  // namespace ge
 #endif  // GE_GRAPH_BUILD_MEMORY_BLOCK_MEM_ASSIGNER_H_

ge/graph/build/memory/buffer_pool_mem_assigner.cc

@@ -0,0 +1,234 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "graph/build/memory/buffer_pool_mem_assigner.h"
+#include "graph/common/omg_util.h"
+#include "graph/utils/tensor_utils.h"
+#include "framework/common/util.h"
+#include "graph/compute_graph.h"
+#include "graph/debug/ge_attr_define.h"
+#include "common/math/math_util.h"
+#include "common/util/error_manager/error_manager.h"
+
+namespace ge {
+namespace {
+const size_t kBufferPoolNodeMemInfoLength = 2;
+const uint32_t kBufferPoolNodeOutputSizeIndex = 0;
+const uint32_t kBufferPoolNodeOutputOffsetIndex = 1;
+} // namespace
+
+Status BufferPoolMemAssigner::Assign() {
+  if (compute_graph_ == nullptr) {
+    GELOGE(PARAM_INVALID, "[Check][Graph]Graph is nullptr");
+    REPORT_INNER_ERROR("E19999", "Input graph is nullptr");
+    return PARAM_INVALID;
+  }
+  Status ret = InitAssigner(compute_graph_);
+  if (ret != SUCCESS) {
+    GELOGE(FAILED, "[Init][Assigner]Graph:%s.", compute_graph_->GetName().c_str());
+    return FAILED;
+  }
+  ret = AssignOutput();
+  if (ret != SUCCESS) {
+    GELOGE(FAILED, "[Assign][Output]Graph:%s.", compute_graph_->GetName().c_str());
+    return FAILED;
+  }
+  return SUCCESS;
+}
+
+Status BufferPoolMemAssigner::GetOutputMemoryType(const NodePtr &node, size_t idx, int64_t &memory_type) {
+  GE_CHECK_NOTNULL(node->GetOpDesc());
+  memory_type = RT_MEMORY_HBM;
+  std::vector<int64_t> type_list;
+  bool has_mem_type = ge::AttrUtils::GetListInt(node->GetOpDesc(), ATTR_NAME_OUTPUT_MEM_TYPE_LIST, type_list);
+  if (has_mem_type && (type_list.size() != node->GetOpDesc()->GetOutputsSize() || idx >= type_list.size())) {
+    GELOGE(PARAM_INVALID, "[Check][OutputParam]Output param invalid, output size:%zu, mem type size:%zu, index:%zu.",
+           node->GetOpDesc()->GetOutputsSize(), type_list.size(), idx);
+    REPORT_INNER_ERROR("E19999", "Output param invalid, output size:%zu, mem type size:%zu, index:%zu.",
+                       node->GetOpDesc()->GetOutputsSize(), type_list.size(), idx);
+    return PARAM_INVALID;
+  }
+  memory_type = has_mem_type ? type_list[idx] : RT_MEMORY_HBM;
+  return SUCCESS;
+}
+
+Status BufferPoolMemAssigner::InitAssigner(const ComputeGraphPtr &graph) {
+  for (const NodePtr &node : graph->GetAllNodes()) {
+    int64_t buffer_pool_id = 0;
+    int64_t buffer_pool_size = 0;
+    bool get_attr = AttrUtils::GetInt(node->GetOpDesc(), ATTR_NAME_BUFFER_POOL_ID, buffer_pool_id);
+    get_attr = get_attr && (AttrUtils::GetInt(node->GetOpDesc(), ATTR_NAME_BUFFER_POOL_SIZE, buffer_pool_size));
+    if (get_attr) {
+      std::string batch_label;
+      (void) AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_BATCH_LABEL, batch_label);
+      buffer_pool_nodes_[batch_label][buffer_pool_id].emplace_back(node);
+      auto iter = buffer_pool_size_[batch_label].find(buffer_pool_id);
+      if (iter == buffer_pool_size_[batch_label].end()) {
+        buffer_pool_size_[batch_label][buffer_pool_id] = buffer_pool_size;
+      }
+      Status ret = InitMemOffsetBase(node);
+      if (ret != SUCCESS) {
+        GELOGE(ret, "[Init][MemOffsetBase]Batch label:%s.", batch_label.c_str());
+        REPORT_INNER_ERROR("E19999", "Failed to init offset base, batch label:%s.", batch_label.c_str());
+        return ret;
+      }
+    }
+  }
+
+  int64_t max_size = 0;
+  for (const auto &iter : buffer_pool_size_) {
+    std::string batch_label = iter.first;
+    int64_t batch_offset = mem_offset_base_;
+    for (const auto &buffer_pool : iter.second) {
+      int64_t buffer_pool_id = buffer_pool.first;
+      int64_t buffer_pool_size = buffer_pool.second;
+      buffer_pool_offset_base_[batch_label][buffer_pool_id] = batch_offset;
+      FMK_INT64_ADDCHECK(buffer_pool_size, kBufferPoolMemAlignSize);
+      AlignMemSize(buffer_pool_size, kBufferPoolMemAlignSize);
+      FMK_INT64_ADDCHECK(batch_offset, (buffer_pool_size + kBufferPoolMemAlignSize));
+      batch_offset += (buffer_pool_size + kBufferPoolMemAlignSize);
+    }
+    int64_t batch_mem_size = batch_offset - mem_offset_base_;
+    GELOGI("[Init][Assigner]Get batch mem size, batch label:%s, mem size:%ld.", batch_label.c_str(), batch_mem_size);
+    if (max_size < batch_mem_size) {
+      max_size = batch_mem_size;
+    }
+  }
+  FMK_INT64_ADDCHECK(mem_offset_base_, max_size);
+  mem_offset_ = static_cast<size_t>(mem_offset_base_ + max_size);
+  GELOGI("[Init][Assigner]Init buffer pool mem assigner successfully, "
+         "mem type:%ld, mem offset base:%ld, mem offset:%zu.", mem_type_, mem_offset_base_, mem_offset_);
+  return SUCCESS;
+}
+
+Status BufferPoolMemAssigner::InitMemOffsetBase(const NodePtr &node) {
+  int64_t mem_type;
+  Status ret = GetOutputMemoryType(node, static_cast<size_t>(kBufferPoolNodeOutIndex), mem_type);
+  if (ret != SUCCESS) {
+    GELOGE(ret, "[Get][MemType]Node:%s, index:%u.", node->GetName().c_str(), kBufferPoolNodeOutIndex);
+    REPORT_INNER_ERROR("E19999", "Failed to get output memory type, node:%s, index:%u.",
+                       node->GetName().c_str(), kBufferPoolNodeOutIndex);
+    return ret;
+  }
+  if (mem_type_ != mem_type && init_offset_base_) {
+    GELOGE(PARAM_INVALID, "[Check][MemType]The memory type of all buffer pool nodes must be the same, node:%s, "
+           "required:%ld, actually: %ld", node->GetName().c_str(), mem_type_, mem_type);
+    REPORT_INNER_ERROR("E19999", "The memory type of all buffer pool nodes must be the same, node:%s, "
+                                 "required:%ld, actually: %ld", node->GetName().c_str(), mem_type_, mem_type);
+    return PARAM_INVALID;
+  }
+  if (!init_offset_base_) {
+    auto iter = mem_type_to_offset_.find(mem_type);
+    if (iter == mem_type_to_offset_.end()) {
+      GELOGE(PARAM_INVALID, "[Check][MemType]Memory type is not supported, node:%s, mem type:%ld.",
+             node->GetName().c_str(), mem_type);
+      REPORT_INNER_ERROR("E19999", "Memory type is not supported, node:%s, mem type:%ld.",
+                         node->GetName().c_str(), mem_type);
+      return PARAM_INVALID;
+    }
+    mem_offset_base_ = static_cast<int64_t>(iter->second);
+    FMK_INT64_ADDCHECK(mem_offset_base_, (kBufferPoolMemAlignSize + kBufferPoolMemAlignSize));
+    AlignMemSize(mem_offset_base_, kBufferPoolMemAlignSize);
+    // The HCOM nodes may access the previous 512 bytes.
+    mem_offset_base_ += kBufferPoolMemAlignSize;
+    mem_type_ = mem_type;
+    init_offset_base_ = true;
+    GELOGI("[Init][MemOffsetBase]Init offset base:%ld, memory type:%ld", mem_offset_base_, mem_type);
+  }
+  return SUCCESS;
+}
+
+Status BufferPoolMemAssigner::AssignOutput() {
+  for (auto &batch_pool_nodes_map : buffer_pool_nodes_) {
+    std::string batch_label = batch_pool_nodes_map.first;
+    for (auto &pool_nodes_map : batch_pool_nodes_map.second) {
+      int64_t buffer_pool_id = pool_nodes_map.first;
+      auto iter_buffer_id_size = buffer_pool_size_[batch_label].find(buffer_pool_id);
+      if (iter_buffer_id_size == buffer_pool_size_[batch_label].end()) {
+        GELOGE(INTERNAL_ERROR, "[Get][BufferPoolSize]Pool id:%ld.", buffer_pool_id);
+        REPORT_INNER_ERROR("E19999", "Failed to get buffer pool size, pool id:%ld.", buffer_pool_id);
+        return INTERNAL_ERROR;
+      }
+      auto iter_buffer_id_offset = buffer_pool_offset_base_[batch_label].find(buffer_pool_id);
+      if (iter_buffer_id_offset == buffer_pool_offset_base_[batch_label].end()) {
+        GELOGE(INTERNAL_ERROR, "[Get][BufferPoolBaseOffset]Pool id:%ld.", buffer_pool_id);
+        REPORT_INNER_ERROR("E19999", "Failed to get buffer pool base offset, pool id:%ld.", buffer_pool_id);
+        return INTERNAL_ERROR;
+      }
+      int64_t buffer_pool_size = iter_buffer_id_size->second;
+      int64_t output_offset_base = iter_buffer_id_offset->second;
+      Status ret = AssignOutputInOneBufferPool(batch_label, output_offset_base, pool_nodes_map.second);
+      if (ret != SUCCESS) {
+        GELOGE(ret, "[Assign][OneBufferPool]Batch label:%s, pool id:%ld, pool size:%ld, offset base:%ld.",
+               batch_label.c_str(), buffer_pool_id, buffer_pool_size, output_offset_base);
+        REPORT_INNER_ERROR("E19999", "Failed to assign output memory, batch label:%s, "
+                           "pool id:%ld, pool size:%ld, offset base:%ld.",
+                           batch_label.c_str(), buffer_pool_id, buffer_pool_size, output_offset_base);
+        return ret;
+      }
+      GELOGI("[Assign][Output]Assign output successfully, batch label:%s, pool id:%ld, pool size:%ld, offset base:%ld.",
+             batch_label.c_str(), buffer_pool_id, buffer_pool_size, output_offset_base);
+    }
+  }
+  return SUCCESS;
+}
+
+Status BufferPoolMemAssigner::AssignOutputInOneBufferPool(const std::string &batch_label,
+                                                          int64_t output_offset_base,
+                                                          const std::vector<NodePtr> &buffer_pool_nodes) {
+  for (const NodePtr &node : buffer_pool_nodes) {
+    int64_t output_size = 0;
+    Status ret = GetMemorySize(node, output_size);
+    if (ret != SUCCESS) {
+      GELOGE(ret, "[Get][MemSize]Node:%s.", node->GetName().c_str());
+      REPORT_INNER_ERROR("E19999", "Failed to get output size, node:%s.", node->GetName().c_str());
+      return ret;
+    }
+    OpDescPtr op_desc = node->GetOpDesc();
+    GE_CHECK_NOTNULL(op_desc);
+    vector<int64_t> memory_size_and_offset;
+    bool get_attr = AttrUtils::GetListInt(op_desc, ATTR_NAME_BUFFER_POOL_NODE_SIZE_AND_OFFSET, memory_size_and_offset);
+    if (!get_attr || memory_size_and_offset.size() != kBufferPoolNodeMemInfoLength) {
+      GELOGE(PARAM_INVALID, "[Get][Attr]Node:%s, mem info size:%zu, required size:%zu.",
+             node->GetName().c_str(), memory_size_and_offset.size(), kBufferPoolNodeMemInfoLength);
+      REPORT_INNER_ERROR("E19999", "Failed to get pool node memory info, node:%s, info size:%zu, required size:%zu.",
+                         node->GetName().c_str(), memory_size_and_offset.size(), kBufferPoolNodeMemInfoLength);
+      return PARAM_INVALID;
+    }
+    if (output_size != memory_size_and_offset[kBufferPoolNodeOutputSizeIndex]) {
+      GELOGE(PARAM_INVALID, "[Check][MemSize]Something wrong with memory size, pre size:%ld, curr size:%ld, node:%s.",
+             memory_size_and_offset[kBufferPoolNodeOutputSizeIndex], output_size, node->GetName().c_str());
+      REPORT_INNER_ERROR("E19999", "Something wrong with memory size, pre size:%ld, curr size:%ld, node:%s.",
+                         memory_size_and_offset[kBufferPoolNodeOutputSizeIndex], output_size, node->GetName().c_str());
+      return PARAM_INVALID;
+    }
+
+    int64_t logical_offset = memory_size_and_offset[kBufferPoolNodeOutputOffsetIndex];
+    vector<int64_t> output_list = {(output_offset_base + logical_offset)};
+    op_desc->SetOutputOffset(output_list);
+    // log for IMAS tools
+    GELOGI("[IMAS]Set %s name[%s] optype[%s] %s[%u] offset to [%ld] streamid[%ld] memtype[%ld] "
+           "size[%zu] realsize[%zu] noalignsize[%zu] life time begin[%d] life time end[%d] "
+           "child[%d:%d:%d:%d:%d] isref[%d] batch[%s]",
+           compute_graph_->GetName().c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(),
+           "output", kBufferPoolNodeOutIndex, output_list[kBufferPoolNodeOutIndex], op_desc->GetStreamId(), mem_type_,
+           static_cast<size_t>(output_size), static_cast<size_t>(output_size), static_cast<size_t>(output_size),
+           0, 0, 0, 0, 0, 0, 0, 0, batch_label.c_str());
+  }
+  return SUCCESS;
+}
+
+}  // namespace ge

ge/graph/build/memory/buffer_pool_mem_assigner.h

@@ -0,0 +1,83 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef GE_GRAPH_BUILD_MEMORY_BUFFER_POOL_MEM_ASSIGNER_H_
+#define GE_GRAPH_BUILD_MEMORY_BUFFER_POOL_MEM_ASSIGNER_H_
+
+#include <vector>
+#include <map>
+#include <unordered_map>
+#include "graph/build/memory/mem_assigner.h"
+#include "runtime/mem.h"
+
+namespace ge {
+class BufferPoolMemAssigner : public MemAssigner {
+ public:
+  BufferPoolMemAssigner(ComputeGraphPtr compute_graph, const std::map<int64_t, size_t> &mem_type_to_offset)
+      : MemAssigner(), compute_graph_(compute_graph),
+        mem_type_(0),
+        mem_offset_(0),
+        mem_offset_base_(0),
+        init_offset_base_(false),
+        mem_type_to_offset_(mem_type_to_offset) {}
+
+  BufferPoolMemAssigner(const BufferPoolMemAssigner &) = delete;
+
+  BufferPoolMemAssigner &operator=(const BufferPoolMemAssigner &) = delete;
+
+  ~BufferPoolMemAssigner() override = default;
+
+  Status Assign() override;
+
+  size_t GetMemOffset() const { return mem_offset_; }
+
+  int64_t GetMemType() const { return mem_type_; }
+
+ private:
+  static Status GetOutputMemoryType(const NodePtr &node, size_t idx,  int64_t &memory_type);
+
+  Status InitAssigner(const ComputeGraphPtr &graph);
+
+  Status InitMemOffsetBase(const NodePtr &node);
+
+  Status AssignOutput();
+
+  Status AssignOutputInOneBufferPool(const std::string &batch_label,
+                                     int64_t output_offset_base,
+                                     const std::vector<NodePtr> &buffer_pool_nodes);
+
+  ComputeGraphPtr compute_graph_;
+
+  int64_t mem_type_;
+
+  size_t mem_offset_;
+
+  int64_t mem_offset_base_;
+
+  bool init_offset_base_;
+
+  std::map<int64_t, size_t> mem_type_to_offset_;
+
+  // Use map to ensure that each visit is in the order of pool id
+  std::unordered_map<std::string, std::map<int64_t, std::vector<NodePtr>>> buffer_pool_nodes_;
+
+  // Use map to ensure that each visit is in the order of pool id
+  std::unordered_map<std::string, std::map<int64_t, int64_t>> buffer_pool_size_;
+
+  std::unordered_map<std::string, std::unordered_map<int64_t, int64_t>> buffer_pool_offset_base_;
+};
+}  // namespace ge
+#endif  // GE_GRAPH_BUILD_MEMORY_BUFFER_POOL_MEM_ASSIGNER_H_

ge/graph/build/memory/graph_mem_assigner.h

@@ -110,8 +110,11 @@ class GraphMemoryAssigner {
   ge::Status SetInputOffset();
 
   ge::Status UpdateOpInputOffset(const NodePtr &node) const;
+  ge::Status UpdateRefOpOutputOffset(const NodePtr &node, const std::map<int32_t, int32_t> &out2ins, const int ref_in,
+                                     const int64_t input_offset) const;
 
   ge::Status CheckOffset();
+  ge::Status CheckRefNodeOffset(const NodePtr &node);
 
   ge::Status AssignReferenceMemory();
 
@@ -125,7 +128,7 @@ class GraphMemoryAssigner {
 
   ge::Status ReAssignAtomicMemory(bool is_loop_graph);
 
-  ge::Status GetAllRef(const NodePtr &node, std::map<int32_t, int32_t> &out2ins);
+  ge::Status TryGetNodeRefIndexes(const NodePtr &node, std::map<int32_t, int32_t> &out2ins) const;
 
   bool AssignContinuousInputMemoryWithAtomicProcessDirectly(const NodePtr &input_continuous_node,
                                                             std::map<NodePtr, uint32_t> &node_2_continuous_type);
@@ -188,6 +191,8 @@ class GraphMemoryAssigner {
 
   void PrintMemoryOffset();
 
+  Status AssignBufferPoolMemory();
+
   MemoryOffsetMap memory_offset_;
   ge::ComputeGraphPtr compute_graph_;
   HybridMemAssignerPtr mem_assigner_;

ge/graph/build/memory/hybrid_mem_assigner.cc

@@ -42,6 +42,7 @@ Status HybridMemAssigner::AssignMemory(std::unique_ptr<BlockMemAssigner> &block_
 
 Status HybridMemAssigner::Assign() {
   if (GraphUtils::GetRefMapping(compute_graph_, symbol_to_anchors_, anchor_to_symbol_) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Get ref-mapping for graph %s failed", compute_graph_->GetName().c_str());
     GELOGE(FAILED, "Get ref-mapping for graph %s failed.", compute_graph_->GetName().c_str());
     return FAILED;
   }

ge/graph/build/memory/module.mk

@@ -8,6 +8,7 @@ local_lib_src_files :=  memory_assigner.cc \
                         hybrid_mem_assigner.cc \
                         max_block_mem_assigner.cc \
                         var_mem_assign_util.cc \
+                        buffer_pool_mem_assigner.cc \
 
 local_lib_inc_path :=   ${LOCAL_PATH} \
                         ${TOPDIR}inc \

ge/graph/build/memory/var_mem_assign_util.cc

@@ -53,6 +53,8 @@ Status VarMemAssignUtil::AssignStaticMemory2Node(ge::ComputeGraphPtr &compute_gr
     GE_IF_BOOL_EXEC(ge::AttrUtils::GetStr(n->GetOpDesc(), REF_VAR_SRC_VAR_NAME, ref_var_src_var_name), continue);
     string node_name = n->GetName();
     GE_IF_BOOL_EXEC(n->GetOpDesc()->GetAllOutputsDesc().empty(),
+                    REPORT_INNER_ERROR("E19999", "check node:%s has no OutputDesc",
+                                       n->GetName().c_str());
                     GELOGE(FAILED, "node:%s has no OutputDesc.", n->GetName().c_str());
                     return FAILED);
     ge::ConstGeTensorDescPtr tensor_desc = n->GetOpDesc()->GetOutputDescPtr(0);
@@ -116,6 +118,8 @@ Status VarMemAssignUtil::SetOutVariableAttr(const ge::NodePtr &node, const ge::N
   GE_CHECK_NOTNULL(node->GetOpDesc());
   output_list = node->GetOpDesc()->GetOutputOffset();
   if (output_list.empty()) {
+    REPORT_INNER_ERROR("E19999", "check node:%s output_offset_list is empty",
+                       node->GetName().c_str());
     GELOGE(PARAM_INVALID, "Output_list is empty");
     return PARAM_INVALID;
   }
@@ -126,7 +130,12 @@ Status VarMemAssignUtil::SetOutVariableAttr(const ge::NodePtr &node, const ge::N
       VarManager::Instance(session_id)->GetVarAddr(var_node->GetName(), var_tensor_desc, &dev_ptr, memory_type));
 
   int out_list_size = static_cast<int>(output_list.size());
-  GE_CHK_BOOL_RET_STATUS(index < out_list_size, FAILED, "index %d >= output_list.size() %d", index, out_list_size);
+  if (index >= out_list_size) {
+    REPORT_INNER_ERROR("E19999", "param index:%d >= output_list.size() %d in node %s, check invalid",
+                       index, out_list_size, node->GetName().c_str());
+    GELOGE(FAILED, "index %d >= output_list.size() %d", index, out_list_size);
+    return FAILED;
+  }
 
   output_list[index] = static_cast<int64_t>(reinterpret_cast<intptr_t>(dev_ptr));
   GELOGI("Assign node outputOffset[index] is: %ld", output_list[index]);
@@ -168,9 +177,13 @@ Status VarMemAssignUtil::DealBroadCastNode(uint32_t graph_id, const ge::NodePtr
 
   auto broad_cast_index = static_cast<size_t>(broad_cast_info.idx);
   auto input_tensor_desc_ptr_vistor = op_desc->GetAllInputsDescPtr();
-  GE_CHK_BOOL_RET_STATUS(input_tensor_desc_ptr_vistor.size() > broad_cast_index, FAILED,
-                         "Get broadcast op %s input tensor desc size [%zu] < idx [%d]", node->GetName().c_str(),
-                         input_tensor_desc_ptr_vistor.size(), broad_cast_info.idx);
+  if (input_tensor_desc_ptr_vistor.size() <= broad_cast_index) {
+    REPORT_INNER_ERROR("E19999", "Get broadcast op %s input tensor desc size [%zu] < idx [%d]",
+                       node->GetName().c_str(), input_tensor_desc_ptr_vistor.size(), broad_cast_info.idx);
+    GELOGE(FAILED, "Get broadcast op %s input tensor desc size [%zu] < idx [%d]", node->GetName().c_str(),
+           input_tensor_desc_ptr_vistor.size(), broad_cast_info.idx);
+    return FAILED;
+  }
   const ge::GeTensorDescPtr input_tensor_desc =
       input_tensor_desc_ptr_vistor.at(static_cast<size_t>(broad_cast_info.idx));
   int64_t input_size = 0;
@@ -298,6 +311,7 @@ Status VarMemAssignUtil::SetOutTransNodeToAssign(const ge::NodePtr &node, const
 }
 
 Status VarMemAssignUtil::AssignMemory2HasRefAttrNode(ge::ComputeGraphPtr &compute_graph) {
+  GraphToNodeMap graph_to_node;
   for (const ge::NodePtr &n : compute_graph->GetAllNodes()) {
     string ref_var_src_var_name;
     auto op_desc = n->GetOpDesc();
@@ -305,7 +319,8 @@ Status VarMemAssignUtil::AssignMemory2HasRefAttrNode(ge::ComputeGraphPtr &comput
     for (uint32_t idx = 0; idx < op_desc->GetOutputsSize(); idx += 1) {
       const auto out_desc = op_desc->MutableOutputDesc(idx);
       if (ge::AttrUtils::GetStr(out_desc, REF_VAR_SRC_VAR_NAME, ref_var_src_var_name)) {
-        GE_CHK_STATUS_RET(AssignData2VarRef(n, ref_var_src_var_name, compute_graph->GetSessionID(), idx));
+        GE_CHK_STATUS_RET(
+          AssignData2VarRef(n, ref_var_src_var_name, compute_graph->GetSessionID(), idx, graph_to_node));
       }
     }
   }
@@ -313,16 +328,37 @@ Status VarMemAssignUtil::AssignMemory2HasRefAttrNode(ge::ComputeGraphPtr &comput
 }
 
 Status VarMemAssignUtil::AssignData2VarRef(const ge::NodePtr &has_ref_attr_node, const string &src_var_name,
-                                           uint64_t session_id, uint32_t out_index) {
+                                           uint64_t session_id, uint32_t out_index,
+                                           GraphToNodeMap &graph_to_node) {
   // Get ref_var_src_var address
   auto root_graph = GraphUtils::FindRootGraph(has_ref_attr_node->GetOwnerComputeGraph());
   GE_CHECK_NOTNULL(root_graph);
-  ge::NodePtr var_ref_src_var = root_graph->FindNode(src_var_name);
-  if (var_ref_src_var == nullptr) {
+  // Cache mapping (name to nodeptr) simproves query performance
+  auto &name_to_node = graph_to_node[root_graph];
+  if (name_to_node.empty()) {
+    for (const ge::NodePtr &n : root_graph->GetDirectNode()) {
+      name_to_node.emplace(n->GetName(), n);
+    }
+    for (auto sub_graph : root_graph->GetAllSubgraphs()) {
+      auto &name_to_node_sub = graph_to_node[sub_graph];
+      if (name_to_node_sub.empty()) {
+        for (const ge::NodePtr &n : sub_graph->GetDirectNode()) {
+          name_to_node_sub.emplace(n->GetName(), n);
+        }
+      }
+    }
+  }
+
+  ge::NodePtr var_ref_src_var = nullptr;
+  auto it = name_to_node.find(src_var_name);
+  if ((it != name_to_node.end()) && (it->second != nullptr)) {
+    var_ref_src_var = it->second;
+  } else {
     for (auto sub_graph : root_graph->GetAllSubgraphs()) {
-        auto node_ptr = sub_graph->FindNode(src_var_name);
-        if (node_ptr != nullptr) {
-          var_ref_src_var = node_ptr;
+      auto &name_to_node_sub = graph_to_node[sub_graph];
+      it = name_to_node_sub.find(src_var_name);
+      if ((it != name_to_node_sub.end()) && (it->second != nullptr)) {
+          var_ref_src_var = it->second;
           break;
         }
     }

ge/graph/build/memory/var_mem_assign_util.h

@@ -22,6 +22,8 @@
 #include "graph/utils/node_utils.h"
 
 namespace ge {
+using GraphToNodeMap = std::map<ge::ComputeGraphPtr, std::map<std::string, ge::NodePtr>>;
+
 class VarMemAssignUtil {
  public:
   static Status AssignVarMemory(ge::ComputeGraphPtr &compute_graph);
@@ -47,7 +49,7 @@ class VarMemAssignUtil {
   static Status DealTransNode(const ge::NodePtr &final_trans_node);
   static Status DealExportTransNode(const ge::NodePtr &node, const ge::NodePtr &final_trans_node);
   static Status AssignData2VarRef(const ge::NodePtr &variable_ref, const std::string &src_var_name, uint64_t session_id,
-                                  uint32_t out_index);
+                                  uint32_t out_index, GraphToNodeMap &graph_to_node);
 
   static Status SetOutTransNodeToAssign(const ge::NodePtr &node, const ge::NodePtr &final_trans_node, size_t index);
 };

ge/graph/build/model_builder.cc

@@ -116,11 +116,15 @@ Status ModelBuilder::CalcOutputSize(const ge::NodePtr &n) {
     int64_t size_temp = 0;
     graphStatus graph_status = TensorUtils::GetTensorMemorySizeInBytes(desc_temp, size_temp);
     if (graph_status != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Get tensor size in bytes failed for op:%s(%s) index:%u",
+                        node_op_desc->GetName().c_str(), node_op_desc->GetType().c_str(), index);
       GELOGE(graph_status, "GetTensorMemorySizeInBytes failed!");
       return FAILED;
     }
     TensorUtils::SetSize(desc_temp, size_temp);
     if (node_op_desc->UpdateOutputDesc(index, desc_temp) != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Update Output desc size failed for op:%s(%s) index:%u",
+                        node_op_desc->GetName().c_str(), node_op_desc->GetType().c_str(), index);
       GELOGE(FAILED, "UpdateOutputDesc failed.");
       return FAILED;
     }
@@ -197,8 +201,7 @@ void ModelBuilder::SetInputIsConst(const ge::NodePtr &n) {
     }
   }
 
-  std::string input_const_info = ToString(is_input_const);
-  GELOGD("update opdesc:%s InputConst:%s", node_op_desc->GetName().c_str(), input_const_info.c_str());
+  GELOGD("update opdesc:%s InputConst:%s", node_op_desc->GetName().c_str(), ToString(is_input_const).c_str());
   node_op_desc->SetIsInputConst(is_input_const);
 }
 
@@ -207,11 +210,15 @@ Status ModelBuilder::AdjustConstWeightSize(const ge::NodePtr &node, size_t &mem_
   if (node->GetType() == CONSTANT) {
     vector<GeTensorPtr> weights = OpDescUtils::MutableWeights(node);
     if (weights.empty()) {
+      REPORT_INNER_ERROR("E19999", "Check weights size of node %s(%s) is empty",
+                         node->GetName().c_str(), node->GetType().c_str());
       GELOGE(FAILED, "weights size of node %s is empty", node->GetName().c_str());
       return FAILED;
     }
     GeTensorPtr weight = weights[0];
     if (weight == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Check weight of node %s(%s) is nullptr",
+                         node->GetName().c_str(), node->GetType().c_str());
       GELOGE(FAILED, "weights[0] is null.");
       return FAILED;
     }
@@ -353,6 +360,9 @@ Status ModelBuilder::AdjustInputTensorFlag() {
           auto input_desc = owner_node_op_desc->GetInputDesc(in_anchors->GetIdx());
           ge::TensorUtils::SetInputTensor(input_desc, true);
           if (owner_node_op_desc->UpdateInputDesc(in_anchors->GetIdx(), input_desc) != SUCCESS) {
+            REPORT_CALL_ERROR("E19999", "Update Input desc size failed for op:%s(%s) index:%u",
+                              owner_node_op_desc->GetName().c_str(), owner_node_op_desc->GetType().c_str(),
+                              in_anchors->GetIdx());
             GELOGE(FAILED, "UpdateOutputDesc failed.");
             return FAILED;
           }
@@ -381,33 +391,51 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) {
 
   max_mem_offset_ = mem_type_to_mem_offset_[RT_MEMORY_HBM];
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_MEMORY_SIZE, max_mem_offset_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_MODEL_MEMORY_SIZE.c_str());
                    GELOGE(FAILED, "SetInt of ATTR_MODEL_MEMORY_SIZE failed.");
                    return FAILED);
   if (mem_type_to_mem_offset_.find(RT_MEMORY_P2P_DDR) != mem_type_to_mem_offset_.end()) {
     p2p_mem_offset_ = mem_type_to_mem_offset_[RT_MEMORY_P2P_DDR];
   }
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_P2P_MEMORY_SIZE, p2p_mem_offset_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_MODEL_P2P_MEMORY_SIZE.c_str());
                    GELOGE(FAILED, "SetInt of ATTR_MODEL_P2P_MEMORY_SIZE failed.");
                        return FAILED);
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_WEIGHT_SIZE, weight_offset_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_MODEL_WEIGHT_SIZE.c_str());
                    GELOGE(FAILED, "SetInt of ATTR_MODEL_WEIGHT_SIZE failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_STREAM_NUM, stream_num_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_MODEL_STREAM_NUM.c_str());
                    GELOGE(FAILED, "SetInt of ATTR_MODEL_STREAM_NUM failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_EVENT_NUM, event_num_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_MODEL_EVENT_NUM.c_str());
                    GELOGE(FAILED, "SetInt of ATTR_MODEL_EVENT_NUM failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListInt(&model, ATTR_MODEL_HUGE_STREAM_LIST, huge_streams_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_MODEL_HUGE_STREAM_LIST.c_str());
                    GELOGE(FAILED, "SetInt of ATTR_MODEL_HUGE_STREAM_LIST failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_LABEL_NUM, label_num_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_MODEL_LABEL_NUM.c_str());
                    GELOGE(FAILED, "SetInt of ATTR_MODEL_LABEL_NUM failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetInt(&model, ATTR_MODEL_ZERO_COPY_MEMORY_SIZE, zero_copy_mem_size_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_MODEL_ZERO_COPY_MEMORY_SIZE.c_str());
                    GELOGE(FAILED, "SetInt of ATTR_MODEL_ZERO_COPY_MEMORY_SIZE failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(&model, ATTR_MODEL_OUT_NODES_NAME, GetLocalOmgContext().net_out_nodes),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_MODEL_OUT_NODES_NAME.c_str());
                    GELOGE(FAILED, "SetListStr of ATTR_MODEL_OUT_NODES_NAME failed.");
                    return FAILED);
   GELOGI("For model, max_mem_offset_: %zu, p2p_mem_size: %zu, zero_copy_mem_size_: %zu", max_mem_offset_,
@@ -415,6 +443,8 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) {
   string fp_ceiling_mode;
   if (ge::GetContext().GetOption("ge.fpCeilingMode", fp_ceiling_mode) == SUCCESS) {
     if (!ge::AttrUtils::SetStr(&model, ATTR_FP_CEILING_MODE, fp_ceiling_mode)) {
+      REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                         ATTR_FP_CEILING_MODE.c_str());
       GELOGE(FAILED, "Failed to set attr ATTR_FP_CEILING_MODE");
       return FAILED;
     }
@@ -429,22 +459,30 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) {
   int64_t core_type = (ge_core_type == kVectorCore) ? 1 : 0;
   GELOGI("core_type: %ld", core_type);
   if (!ge::AttrUtils::SetInt(&model, ATTR_MODEL_CORE_TYPE, core_type)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                       ATTR_MODEL_CORE_TYPE.c_str());
     GELOGE(FAILED, "SetInt of ATTR_CORE_TYPE failed.");
   }
   InitL1FusionOption();
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(&model, ATTR_NAME_SWITCH_FOR_L1_FUSION, is_l1_fusion_enable_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                                      ATTR_NAME_SWITCH_FOR_L1_FUSION.c_str());
                    GELOGE(FAILED, "SetBool of ATTR_NAME_SWITCH_FOR_L1_FUSION failed.");
                    return FAILED);
   const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(session_id_);
   bool is_op_debug = dump_properties.IsOpDebugOpen();
   if (is_op_debug) {
     if (!ge::AttrUtils::SetBool(&model, ATTR_OP_DEBUG_FLAG, is_op_debug)) {
+      REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                         ATTR_OP_DEBUG_FLAG.c_str());
       GELOGE(FAILED, "SetBool of ATTR_OP_DEBUG_FLAG failed.");
       return FAILED;
     }
     uint32_t op_debug_mode = dump_properties.GetOpDebugMode();
     GELOGI("Get op debug mode:%d", op_debug_mode);
     if (!ge::AttrUtils::SetInt(&model, ATTR_OP_DEBUG_MODE, op_debug_mode)) {
+      REPORT_INNER_ERROR("E19999", "Set Attr:%s in model failed",
+                         ATTR_OP_DEBUG_MODE.c_str());
       GELOGE(FAILED, "SetBool of ATTR_OP_DEBUG_MODE failed.");
       return FAILED;
     }
@@ -516,6 +554,8 @@ Status ModelBuilder::MergeWeights() {
     // If MutableTensor failed, weight is nullptr.
     (void)ge::AttrUtils::MutableTensor(op_desc, ATTR_NAME_WEIGHTS, weight);
     if (weight == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Can't get const weight in op:%s(%s)",
+                         op_desc->GetName().c_str(), op_desc->GetType().c_str());
       GELOGE(FAILED, "Can't get const op weight, name: %s", node->GetName().c_str());
       return FAILED;
     }
@@ -538,8 +578,15 @@ Status ModelBuilder::MergeWeights() {
       continue;
     }
     if (weight_data.data() != nullptr) {
-      GE_IF_BOOL_EXEC(base_addr == nullptr, GELOGE(FAILED, "Base addr is nullptr."); return FAILED);
+      GE_IF_BOOL_EXEC(base_addr == nullptr,
+                      REPORT_INNER_ERROR("E19999", "Check weight in op:%s(%s) is nullptr",
+                                         op_desc->GetName().c_str(), op_desc->GetType().c_str());
+                      GELOGE(FAILED, "Base addr is nullptr.");
+                      return FAILED);
       if (weight_offset_ - offset < weight_data.size()) {
+        REPORT_INNER_ERROR("E19999", "left weight size not enough for op:%s(%s) left_size:%zu, weight_size:%zu",
+                           op_desc->GetName().c_str(), op_desc->GetType().c_str(),
+                           weight_offset_ - offset, weight_data.size());
         GELOGE(FAILED, "left weight size not enough. left_size:%lu, weight_size:%lu",
                weight_offset_ - offset, weight_data.size());
         return FAILED;
@@ -551,6 +598,9 @@ Status ModelBuilder::MergeWeights() {
         auto err = memcpy_s(reinterpret_cast<void *>(dst_ptr), SECUREC_MEM_MAX_LEN, reinterpret_cast<void *>(src_ptr),
                             SECUREC_MEM_MAX_LEN);
         if (err != EOK) {
+          REPORT_CALL_ERROR("E19999", "mem copy failed. errret:%u, "
+                            "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu,",
+                            err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN);
           GELOGE(FAILED, "mem copy failed. errret:%u, "
                  "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu",
                  err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN);
@@ -562,6 +612,9 @@ Status ModelBuilder::MergeWeights() {
       }
       auto err = memcpy_s(reinterpret_cast<void *>(dst_ptr), left_size, reinterpret_cast<void *>(src_ptr), left_size);
       if (err != EOK) {
+        REPORT_CALL_ERROR("E19999", "mem copy failed. errret:%u, "
+                          "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu,",
+                          err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN);
         GELOGE(FAILED, "mem copy failed. errret:%u, "
                "dst_ptr:%lx, dst_size:%lu, src_ptr:%lx, src_size:%lu",
                err, dst_ptr, SECUREC_MEM_MAX_LEN, src_ptr, SECUREC_MEM_MAX_LEN);
@@ -574,6 +627,50 @@ Status ModelBuilder::MergeWeights() {
   return SUCCESS;
 }
 
+Status ModelBuilder::SaveAtomicTBEKernel(const OpDescPtr &op_desc) {
+  ge::NodePtr atomic_clean_node = nullptr;
+  atomic_clean_node = op_desc->TryGetExtAttr("atomic_clean_node_ptr", atomic_clean_node);
+  if (atomic_clean_node == nullptr) {
+    return SUCCESS;
+  }
+
+  ge::OpDescPtr atomic_op_desc = atomic_clean_node->GetOpDesc();
+  GE_CHECK_NOTNULL(atomic_op_desc);
+  TBEKernelPtr tbe_kernel = atomic_op_desc->TryGetExtAttr(ge::OP_EXTATTR_NAME_TBE_KERNEL, TBEKernelPtr());
+  if (tbe_kernel == nullptr) {
+    std::string kernel_name;
+    GeAttrValue::BYTES kernel_buffer;
+    (void) AttrUtils::GetStr(atomic_op_desc, ATTR_NAME_TBE_KERNEL_NAME, kernel_name);
+    (void) AttrUtils::GetBytes(atomic_op_desc, ATTR_NAME_TBE_KERNEL_BUFFER, kernel_buffer);
+    if (!kernel_name.empty() && (kernel_buffer.GetSize() > 0)) {
+      GE_CHECK_NOTNULL(kernel_buffer.GetData());
+      std::vector<char> data(kernel_buffer.GetData(), kernel_buffer.GetData() + kernel_buffer.GetSize());
+      tbe_kernel = MakeShared<OpKernelBin>(kernel_name, std::move(data));
+      GE_CHECK_NOTNULL(tbe_kernel);
+    }
+  }
+  if (tbe_kernel == nullptr) {
+    GELOGD("Atomic_clean_node doesn't have tbe_kernel.");
+    return SUCCESS;
+  }
+  tbe_kernel_store_.AddTBEKernel(tbe_kernel);
+  GELOGD("Atomic_clean_node tbe_kernel_name %s!", tbe_kernel->GetName().c_str());
+  (void) AttrUtils::SetStr(op_desc, ATOMIC_ATTR_TBE_KERNEL_NAME, tbe_kernel->GetName());
+
+  std::string kernel_name;
+  (void) AttrUtils::GetStr(atomic_op_desc, atomic_op_desc->GetName() + "_kernelname", kernel_name);
+  (void) AttrUtils::SetStr(op_desc, op_desc->GetName() + "_atomic_kernelname", kernel_name);
+
+  std::string meta_data;
+  (void) AttrUtils::GetStr(atomic_op_desc, TVM_ATTR_NAME_METADATA, meta_data);
+  (void) AttrUtils::SetStr(op_desc, ATOMIC_ATTR_TVM_METADATA, meta_data);
+
+  std::string json_string;
+  (void) AttrUtils::GetStr(atomic_op_desc, TVM_ATTR_NAME_MAGIC, json_string);
+  (void) AttrUtils::SetStr(op_desc, ATOMIC_ATTR_TVM_MAGIC, json_string);
+  return SUCCESS;
+}
+
 Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) {
   // Add weight
   ge_model.SetWeight(weight_buffer_);
@@ -602,11 +699,15 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) {
     }
     GE_IF_BOOL_EXEC(tbe_kernel == nullptr, continue);
     if (tbe_name_set.count(tbe_kernel->GetName()) > 0) {
+      REPORT_INNER_ERROR("E19999", "tbe_kernel name %s can't be the same, judge for op:%s(%s),",
+                         tbe_kernel->GetName().c_str(), n->GetName().c_str(), n->GetType().c_str());
       GELOGE(FAILED, "tbe_kernel name %s can't be the same", tbe_kernel->GetName().c_str());
       return FAILED;
     }
     tbe_name_set.insert(tbe_kernel->GetName());
     tbe_kernel_store_.AddTBEKernel(tbe_kernel);
+
+    GE_CHK_STATUS_RET(SaveAtomicTBEKernel(node_op_desc), "[Save][TBEKernel] save atomic tbekernel failed!");
   }
 
   SetModelCheckAicpuAttr(model, aicpu_op_types, aicpu_tf_op_types);
@@ -618,6 +719,8 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) {
         node_op_desc->TryGetExtAttr(ge::OP_EXTATTR_CUSTAICPU_KERNEL, CustAICPUKernelPtr());
     GE_IF_BOOL_EXEC(cust_aicpu_kernel == nullptr, continue);
     if (aicpu_name_set.count(cust_aicpu_kernel->GetName()) > 0) {
+      REPORT_INNER_ERROR("E19999", "aicpu_kernel name %s can't be the same, judge for op:%s(%s),",
+                         cust_aicpu_kernel->GetName().c_str(), n->GetName().c_str(), n->GetType().c_str());
       GELOGE(FAILED, "aicpu_kernel name %s can't be the same", cust_aicpu_kernel->GetName().c_str());
       return FAILED;
     }
@@ -640,6 +743,7 @@ Status ModelBuilder::SaveDataToModel(ge::Model &model, ge::GeModel &ge_model) {
   // Add task
   GeAttrValue::BYTES task_def_bytes;
   if (!AttrUtils::GetZeroCopyBytes(model, MODEL_ATTR_TASKS, task_def_bytes)) {
+    REPORT_CALL_ERROR("E19999", "Get attr:%s in model failed", MODEL_ATTR_TASKS.c_str());
     GELOGE(INTERNAL_ERROR, "Get zero copy bytes fail.");
     return INTERNAL_ERROR;
   }
@@ -675,6 +779,7 @@ void ModelBuilder::SetModelVersion(ge::Model &model) {
 
 Status ModelBuilder::PreBuildModel() {
   if ((compute_graph_ == nullptr) || !(compute_graph_->IsValid())) {
+    REPORT_INNER_ERROR("E19999", "Check compute_graph no valid");
     GELOGE(FAILED, "Graph_ is not valid.");
     return FAILED;
   }
@@ -754,6 +859,7 @@ Status ModelBuilder::CompileSingleOp() {
   // Create ge instance
   std::shared_ptr<GELib> instance = ge::GELib::GetInstance();
   if ((instance == nullptr) || !instance->InitFlag()) {
+    REPORT_INNER_ERROR("E19999", "Check GELib instance not init before");
     GELOGE(ge::GE_CLI_GE_NOT_INITIALIZED, "CompileSingleOp failed.");
     return ge::GE_CLI_GE_NOT_INITIALIZED;
   }
@@ -775,6 +881,8 @@ Status ModelBuilder::CompileSingleOp() {
         (void)instance->DNNEngineManagerObj().GetDNNEngineName(node);
         kernel_lib_name = op_desc->GetOpKernelLibName();
         if (kernel_lib_name.empty()) {
+          REPORT_INNER_ERROR("E19999", "Check kernel lib name empty of op:%s(%s)",
+                             node->GetName().c_str(), node->GetType().c_str());
           GELOGE(ge::INTERNAL_ERROR, "Get node:%s(%s) kernel lib failed.", node->GetName().c_str(),
                  node->GetType().c_str());
           return ge::INTERNAL_ERROR;
@@ -785,6 +893,8 @@ Status ModelBuilder::CompileSingleOp() {
       if (kernel_info != nullptr) {
         node_vector_map[kernel_lib_name].emplace_back(node);
       } else {
+        REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s,",
+                           node->GetName().c_str(), node->GetType().c_str(), kernel_lib_name.c_str());
         GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "Get op %s ops kernel info store failed", node->GetName().c_str());
         return ge::GE_GRAPH_PARAM_NULLPTR;
       }
@@ -800,6 +910,8 @@ Status ModelBuilder::CompileSingleOp() {
     GELOGI("[GEPERFTRACE] The node size of compile op of %s is %zu", kernel_lib_name.c_str(), node_vector.size());
     GE_TIMESTAMP_ADD(BatchCompileOp);
     if (ret != ge::SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Batch compile op failed, kernel lib name, node size:%zu,",
+                        node_vector.size());
       GELOGE(ret, "Compile op failed, kernel lib name is %s", kernel_lib_name.c_str());
       return ret;
     }

ge/graph/build/model_builder.h

@@ -89,6 +89,8 @@ class ModelBuilder {
   void SetModelCheckAicpuAttr(ge::Model &model, std::set<std::string> &aicpu_op_types,
                                 std::set<std::string> &aicpu_tf_op_types);
 
+  Status SaveAtomicTBEKernel(const OpDescPtr &op_desc);
+
   uint64_t session_id_;
 
   map<int64_t, size_t> mem_type_to_mem_offset_;

ge/graph/build/run_context.cc

@@ -18,6 +18,7 @@
 #include "common/util.h"
 #include "framework/common/debug/ge_log.h"
 #include "graph/debug/ge_attr_define.h"
+#include "graph/common/omg_util.h"
 
 namespace ge {
 RunContextUtil::~RunContextUtil() { DestroyRtModelResources(); }
@@ -27,15 +28,21 @@ Status RunContextUtil::InitMemInfo(uint8_t *data_mem_base, uint64_t data_mem_siz
                                    std::map<int64_t, uint64_t> mem_type_to_data_mem_size, uint8_t *weight_mem_base,
                                    uint64_t weight_mem_size) {
   if ((data_mem_size > 0) && (data_mem_base == nullptr)) {
+    REPORT_INNER_ERROR("E19999", "InitMemInfo param data_mem_base is null but data_mem_size = %lu", data_mem_size);
     GELOGE(PARAM_INVALID, "InitMemInfo param data_mem_base is null but data_mem_size = %lu.", data_mem_size);
     return PARAM_INVALID;
   }
   if ((weight_mem_size > 0) && (weight_mem_base == nullptr)) {
+    REPORT_INNER_ERROR("E19999", "InitMemInfo param weight_mem_base is null but weight_mem_size = %lu",
+                       weight_mem_size);
     GELOGE(PARAM_INVALID, "InitMemInfo param weight_mem_base is null but weight_mem_size = %lu.", weight_mem_size);
     return PARAM_INVALID;
   }
   if (mem_type_to_data_mem_base.empty() || mem_type_to_data_mem_size.empty() ||
       mem_type_to_data_mem_base.size() != mem_type_to_data_mem_size.size()) {
+    REPORT_INNER_ERROR("E19999", "InitMemInfo param mem_type_to_data_mem_base size[%zu] "
+                       "is not equal to the size of mem_type_to_data_mem_size[%zu].",
+                       mem_type_to_data_mem_base.size(), mem_type_to_data_mem_size.size());
     GELOGE(PARAM_INVALID,
            "InitMemInfo param mem_type_to_data_mem_base size[%zu] is not equal to the size of "
            "mem_type_to_data_mem_size[%zu].",
@@ -55,6 +62,7 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even
   // Create rt model
   rtError_t rt_ret = rtModelCreate(&rt_model_, 0);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "call rtModelCreate failed, ret:%d,", static_cast<int>(rt_ret));
     GELOGE(RT_FAILED, "rtModelCreate failed. rt_ret = %d", static_cast<int>(rt_ret));
     return RT_FAILED;
   }
@@ -64,6 +72,8 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even
     rtStream_t stream = nullptr;
     rt_ret = rtStreamCreate(&stream, 0);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "call rtStreamCreate failed, ret:%d, index:%u,",
+                        static_cast<int>(rt_ret), i);
       GELOGE(RT_FAILED, "rtStreamCreate failed. rt_ret = %d, index = %u", static_cast<int>(rt_ret), i);
       return RT_FAILED;
     }
@@ -71,16 +81,22 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even
 
     rt_ret = rtModelBindStream(rt_model_, stream, 0);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "call rtModelBindStream failed, ret:%d, index:%u,",
+                        static_cast<int>(rt_ret), i);
       GELOGE(RT_FAILED, "Bind stream and model failed. rt_ret = %d, index = %u", static_cast<int>(rt_ret), i);
       return RT_FAILED;
     }
   }
 
   // Create rt event
+  uint32_t create_flag = static_cast<uint32_t>((event_num > kEventReuseThreshold) ? RT_EVENT_WITH_FLAG :
+                                                                                    RT_EVENT_DEFAULT);
   for (uint32_t i = 0; i < event_num; ++i) {
     rtEvent_t event = nullptr;
-    rt_ret = rtEventCreate(&event);
+    rt_ret = rtEventCreateWithFlag(&event, create_flag);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "call rtEventCreate failed, ret:%d, index:%u,",
+                        static_cast<int>(rt_ret), i);
       GELOGE(RT_FAILED, "rtEventCreate failed. rt_ret = %d, index = %u", static_cast<int>(rt_ret), i);
       return RT_FAILED;
     }
@@ -92,6 +108,8 @@ Status RunContextUtil::CreateRtModelResources(uint32_t stream_num, uint32_t even
     rtLabel_t label = nullptr;
     rt_ret = rtLabelCreateV2(&label, rt_model_);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "call rtLabelCreateV2 failed, ret:%d, index:%u,",
+                        static_cast<int>(rt_ret), i);
       GELOGE(RT_FAILED, "rtLabelCreate failed. rt_ret = %d, index = %u", static_cast<int>(rt_ret), i);
       return RT_FAILED;
     }
@@ -143,12 +161,15 @@ Status RunContextUtil::CreateRunContext(Model &model, const ComputeGraphPtr &gra
   GELOGD("Begin to Create RunContext, session_id = %lu", session_id);
   // check params
   if (graph == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param graph nullptr, session_id:%lu,", session_id);
     GELOGE(PARAM_INVALID, "CreateRunContext param graph is null. session_id=%lu", session_id);
     return PARAM_INVALID;
   }
 
   uint32_t stream_num = 0;
   if (!AttrUtils::GetInt(&model, ATTR_MODEL_STREAM_NUM, stream_num)) {
+    REPORT_INNER_ERROR("E19999", "Get Attr:%s failed for model, session_id:%lu,",
+                       ATTR_MODEL_STREAM_NUM.c_str(), session_id);
     GELOGE(INTERNAL_ERROR, "Get stream_num attr from model_def failed. session_id=%lu", session_id);
     return INTERNAL_ERROR;
   }
@@ -156,6 +177,8 @@ Status RunContextUtil::CreateRunContext(Model &model, const ComputeGraphPtr &gra
 
   uint32_t event_num = 0;
   if (!AttrUtils::GetInt(&model, ATTR_MODEL_EVENT_NUM, event_num)) {
+    REPORT_INNER_ERROR("E19999", "Get Attr:%s failed for model, session_id:%lu,",
+                       ATTR_MODEL_EVENT_NUM.c_str(), session_id);
     GELOGE(INTERNAL_ERROR, "Get event_num attr from model failed. session_id=%lu", session_id);
     return INTERNAL_ERROR;
   }
@@ -163,6 +186,8 @@ Status RunContextUtil::CreateRunContext(Model &model, const ComputeGraphPtr &gra
 
   uint32_t label_num = 0;
   if (!AttrUtils::GetInt(&model, ATTR_MODEL_LABEL_NUM, label_num)) {
+    REPORT_INNER_ERROR("E19999", "Get Attr:%s failed for model, session_id:%lu,",
+                       ATTR_MODEL_LABEL_NUM.c_str(), session_id);
     GELOGE(INTERNAL_ERROR, "Get label_num attr from model failed. session_id=%lu", session_id);
     return INTERNAL_ERROR;
   }

ge/graph/build/stream_allocator.cc

@@ -27,6 +27,8 @@
 #include "graph/ge_context.h"
 #include "graph/utils/graph_utils.h"
 #include "init/gelib.h"
+#include "common/string_util.h"
+#include "common/util/error_manager/error_manager.h"
 
 using std::map;
 using std::set;
@@ -38,6 +40,13 @@ const int64_t kTaskNumPerNormalNode = 3;
 const int64_t kTaskNumPerHcclNode = 245;
 const char *const kTrueStr = "true";
 const char *const kFalseStr = "false";
+const size_t kEventMultiplexingItemCount = 3;
+const size_t kKeyWordIndex = 0;
+const size_t kNodeNameIndex = 1;
+const size_t kEventIdIndex = 2;
+const char *const kSend = "SendTo";
+const char *const kRecv = "RecvFrom";
+const char kDelim = ';';
 
 inline bool HasContinuousStreamLabel(const ge::OpDescPtr &op_desc, std::string &continuous_stream_label) {
   if (ge::AttrUtils::GetStr(op_desc, ge::ATTR_NAME_CONTINUOUS_STREAM_LABEL, continuous_stream_label)) {
@@ -52,6 +61,97 @@ bool IsHcclOp(const string &op_type) {
                                    ge::HCOMALLREDUCE, ge::HCOMREDUCESCATTER, ge::HCOMREDUCE});
   return hccl_op_types.find(op_type) != hccl_op_types.end();
 }
+
+ge::Status ParseNodeEventMultiplexing(const ge::NodePtr &node,
+    const std::vector<std::string> &raw_event_multiplexing,
+    std::unordered_map<ge::NodePtr, std::vector<std::pair<std::string, uint32_t>>> &node_to_send,
+    std::unordered_map<ge::NodePtr, std::vector<std::pair<std::string, uint32_t>>> &node_to_recv) {
+  GE_CHECK_NOTNULL(node);
+  for (const auto &str : raw_event_multiplexing) {
+    std::vector<std::string> ele = ge::StringUtils::Split(str, kDelim);
+    if (ele.size() != kEventMultiplexingItemCount) {
+      GELOGE(ge::PARAM_INVALID, "[Check][RawMultiplexing]Size error, node:%s, require size:%zu, actually:%zu.",
+             node->GetName().c_str(), kEventMultiplexingItemCount, ele.size());
+      REPORT_INNER_ERROR("E19999", "Raw event multiplexing is invalid, node:%s, require size:%zu, actually:%zu.",
+                         node->GetName().c_str(), kEventMultiplexingItemCount, ele.size());
+      return ge::PARAM_INVALID;
+    }
+    int value;
+    try {
+      value = std::stoi(ele[kEventIdIndex]);
+    } catch (std::invalid_argument &) {
+      GELOGE(ge::PARAM_INVALID, "[Throw][Exception]Event id is invalid, node:%s, raw:%s.",
+             node->GetName().c_str(), ele[kEventIdIndex].c_str());
+      REPORT_INNER_ERROR("E19999", "Event id is invalid, node:%s, raw:%s.",
+                         node->GetName().c_str(), ele[kEventIdIndex].c_str());
+      return ge::PARAM_INVALID;
+    } catch (std::out_of_range &) {
+      GELOGE(ge::PARAM_INVALID, "[Throw][Exception]Event id is out of range, node:%s, raw:%s.",
+             node->GetName().c_str(), ele[kEventIdIndex].c_str());
+      REPORT_INNER_ERROR("E19999", "Event id is out of range, node:%s, raw:%s.",
+                         node->GetName().c_str(), ele[kEventIdIndex].c_str());
+      return ge::PARAM_INVALID;
+    }
+    if (value < 0) {
+      GELOGE(ge::PARAM_INVALID, "[Check][EventId]Event id is out of range, node:%s, raw:%s, value:%d.",
+             node->GetName().c_str(), ele[kEventIdIndex].c_str(), value);
+      REPORT_INNER_ERROR("E19999", "Event id is out of range, node:%s, raw:%s, value:%d.",
+                         node->GetName().c_str(), ele[kEventIdIndex].c_str(), value);
+      return ge::PARAM_INVALID;
+    }
+    if (ele[kKeyWordIndex] == kSend) {
+      node_to_send[node].emplace_back(std::make_pair(ele[kNodeNameIndex], static_cast<uint32_t>(value)));
+    } else if (ele[kKeyWordIndex] == kRecv) {
+      node_to_recv[node].emplace_back(std::make_pair(ele[kNodeNameIndex], static_cast<uint32_t>(value)));
+    } else {
+      GELOGE(ge::PARAM_INVALID, "[Check][KeyWord]Key word is not supported, node:%s, key:%s.",
+             node->GetName().c_str(), ele[kEventIdIndex].c_str());
+      REPORT_INNER_ERROR("E19999", "Key word is not supported, node:%s, key:%s.",
+                         node->GetName().c_str(), ele[kEventIdIndex].c_str());
+      return ge::PARAM_INVALID;
+    }
+  }
+  return ge::SUCCESS;
+}
+
+ge::Status ParseAllNodeEventMultiplexing(const ge::ComputeGraphPtr &graph,
+    std::unordered_map<std::string, ge::NodePtr> &name_to_node_map,
+    std::unordered_map<ge::NodePtr, std::vector<std::pair<std::string, uint32_t>>> &node_to_send,
+    std::unordered_map<ge::NodePtr, std::vector<std::pair<std::string, uint32_t>>> &node_to_recv) {
+  for (const auto &node : graph->GetNodes(graph->GetGraphUnknownFlag())) {
+    ge::OpDescPtr op_desc = node->GetOpDesc();
+    GE_CHECK_NOTNULL(op_desc);
+    name_to_node_map.insert({node->GetName(), node});
+    std::vector<std::string> raw_event_multiplexing;
+    if (!(op_desc->HasAttr(ge::ATTR_NAME_EVENT_MULTIPLEXING))) {
+      continue;
+    }
+    bool get_attr = ge::AttrUtils::GetListStr(op_desc, ge::ATTR_NAME_EVENT_MULTIPLEXING, raw_event_multiplexing);
+    if (!get_attr) {
+      GELOGE(ge::PARAM_INVALID, "[Get][Attr]Node:%s.", node->GetName().c_str());
+      REPORT_INNER_ERROR("E19999", "Failed to get raw event multiplexing, node:%s.", node->GetName().c_str());
+      return ge::PARAM_INVALID;
+    }
+    auto parse_ret = ParseNodeEventMultiplexing(node, raw_event_multiplexing, node_to_send, node_to_recv);
+    if (parse_ret != ge::SUCCESS) {
+      GELOGE(parse_ret, "[Parse][Eventmultiplexing]Node:%s.", node->GetName().c_str());
+      REPORT_INNER_ERROR("E19999", "Failed to parse node event multiplexing, node:%s.", node->GetName().c_str());
+      return parse_ret;
+    }
+  }
+  return ge::SUCCESS;
+}
+
+std::vector<uint32_t> GetIntersection(std::vector<uint32_t> &a, std::vector<uint32_t> &b) {
+  std::unordered_set<uint32_t> ele_of_a(a.begin(), a.end());
+  std::vector<uint32_t> res;
+  for (auto &ele : b) {
+    if (ele_of_a.count(ele) > 0) {
+      res.emplace_back(ele);
+    }
+  }
+  return res;
+}
 }  // namespace
 
 namespace ge {
@@ -76,6 +176,7 @@ Status StreamAllocator::AssignLogicalStreams(const std::map<std::string, int> &m
 
   auto gelib = GELib::GetInstance();
   if (gelib == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check GELib instance nullptr");
     GELOGE(FAILED, "Get GELib instance failed.");
     return FAILED;
   }
@@ -149,6 +250,12 @@ Status StreamAllocator::RefreshRealStream(int64_t &stream_num, int64_t &event_nu
     return status;
   }
 
+  status = RefreshEventsWithReuse();
+  if (status != SUCCESS) {
+    GELOGE(status, "[Refresh][Events]RefreshEventsWithReuse failed!");
+    return status;
+  }
+
   status = InsertSyncEventNodes();
   if (status != SUCCESS) {
     GELOGE(status, "InsertSyncEventNode failed!");
@@ -184,6 +291,8 @@ Status StreamAllocator::AssignSingleStream() {
   }
 
   if (stream_num_ > 1) {
+    REPORT_INNER_ERROR("E19999", "The number of ts streams is %ld, only one is supported",
+                       stream_num_);
     GELOGE(FAILED, "The number of ts streams is %ld, only one is supported.", stream_num_);
     return FAILED;
   }
@@ -257,6 +366,9 @@ Status StreamAllocator::SetActiveStreamsByLabel() {
       }
     }
     GE_CHK_BOOL_EXEC(AttrUtils::SetListInt(node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, activated_stream_list),
+                     REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed",
+                                        ATTR_NAME_ACTIVE_STREAM_LIST.c_str(),
+                                        node->GetName().c_str(), node->GetType().c_str());
                      GELOGE(FAILED, "SetListInt failed.");
                      return FAILED);
   }
@@ -307,6 +419,9 @@ Status StreamAllocator::SetActiveStreamsForSubgraphs() {
     }
 
     if (!AttrUtils::SetListInt(first_active_node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) {
+      REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed",
+                         ATTR_NAME_ACTIVE_STREAM_LIST.c_str(),
+                         first_active_node->GetName().c_str(), first_active_node->GetType().c_str());
       GELOGE(FAILED, "Set active streams for node %s failed.", first_active_node->GetName().c_str());
       return FAILED;
     }
@@ -376,6 +491,8 @@ Status StreamAllocator::InsertOneEventInTwoNodes(const NodePtr &cur_node, const
   }
 
   if (next_stream_id == kInvalidStream) {
+    REPORT_INNER_ERROR("E19999", "Stream id of next_node %s(%s) should not be %ld",
+                       next_node->GetName().c_str(), next_node->GetType().c_str(), kInvalidStream);
     GELOGE(FAILED, "Stream id of next_node %s should not be %ld", next_node->GetName().c_str(), kInvalidStream);
     return FAILED;
   }
@@ -589,8 +706,14 @@ Status StreamAllocator::OptimizeByStreamActivate() {
 // -> stream(streamSwitch) -> stream(streamActivate) -> stream(stream true or false)
 // No need to insert an event between node in stream(normal) and node in stream(stream true or false)
 bool StreamAllocator::IsRecvNodeActivatedBySendNode(const NodePtr &send_node_ptr, const NodePtr &recv_node_ptr) const {
-  GE_CHECK_NOTNULL_EXEC(send_node_ptr->GetOpDesc(), GELOGE(FAILED, "op desc is nullptr"); return false);
-  GE_CHECK_NOTNULL_EXEC(recv_node_ptr->GetOpDesc(), GELOGE(FAILED, "op desc is nullptr"); return false);
+  GE_CHECK_NOTNULL_EXEC(send_node_ptr->GetOpDesc(),
+                        REPORT_INNER_ERROR("E19999", "Check param send_node_ptr nullptr");
+                        GELOGE(FAILED, "op desc is nullptr");
+                        return false);
+  GE_CHECK_NOTNULL_EXEC(recv_node_ptr->GetOpDesc(),
+                        REPORT_INNER_ERROR("E19999", "Check param recv_node_ptr nullptr");
+                        GELOGE(FAILED, "op desc is nullptr");
+                        return false);
   auto cur_stream_id = send_node_ptr->GetOpDesc()->GetStreamId();
   if (AttrUtils::HasAttr(recv_node_ptr->GetOpDesc(), ATTR_NAME_STREAM_LABEL)) {
     // find streamActivate node
@@ -714,6 +837,8 @@ Status StreamAllocator::SplitStreams(vector<set<int64_t>> &split_streams) {
       continue;
     }
     if (stream_id > last_stream_id) {
+      REPORT_INNER_ERROR("E19999", "streamid(%ld) > last_stream_id(%ld), check invalid",
+                         stream_id, last_stream_id);
       GELOGE(FAILED, "SplitStreams:streamid(%ld) > last_stream_id(%ld)", stream_id, last_stream_id);
       return FAILED;
     }
@@ -727,6 +852,8 @@ Status StreamAllocator::SplitStreams(vector<set<int64_t>> &split_streams) {
       stream_continuous_2_node_num_map[continuous_stream_label]++;
       // return error
       if (stream_continuous_2_node_num_map[continuous_stream_label] > max_node_num_one_stream) {
+        REPORT_INNER_ERROR("E19999", "Check node[%s] stream_id[%ld] continuous stream label[%s] unsatisfied",
+                           op_desc->GetName().c_str(), stream_id, continuous_stream_label.c_str());
         GELOGE(FAILED, "SplitStreams:node[%s] stream_id[%ld] continuous stream label[%s] unsatisfied ",
                op_desc->GetName().c_str(), stream_id, continuous_stream_label.c_str());
         return FAILED;
@@ -881,6 +1008,8 @@ Status StreamAllocator::UpdateActiveStreamsForSwitchNode(NodePtr &switch_node) {
   GE_CHECK_NOTNULL(op_desc);
 
   if (!AttrUtils::SetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, stream_ids)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(),
+                       op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(FAILED, "SetListInt failed.");
     return FAILED;
   }
@@ -895,6 +1024,8 @@ Status StreamAllocator::InsertActiveNodesAfterSwitch(NodePtr &switch_node, vecto
   vector<string> ori_active_label_list;
   if (!AttrUtils::GetListStr(switch_desc, ATTR_NAME_ACTIVE_LABEL_LIST, ori_active_label_list) ||
       ori_active_label_list.empty()) {
+    REPORT_INNER_ERROR("E19999", "Get Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_LABEL_LIST.c_str(),
+                       switch_node->GetName().c_str(), switch_node->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "Get active label list of switch %s failed.", switch_node->GetName().c_str());
     return INTERNAL_ERROR;
   }
@@ -918,6 +1049,8 @@ Status StreamAllocator::InsertActiveNodesAfterSwitch(NodePtr &switch_node, vecto
   for (auto &active_node : added_active_nodes) {
     GE_CHECK_NOTNULL(switch_node->GetOutControlAnchor());
     if (switch_node->GetOutControlAnchor()->LinkTo(active_node->GetInControlAnchor()) != GRAPH_SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Link from %s to %s failed",
+                        switch_node->GetName().c_str(), active_node->GetName().c_str());
       GELOGE(FAILED, "Link %s to %s failed.", switch_node->GetName().c_str(), active_node->GetName().c_str());
       return FAILED;
     }
@@ -933,6 +1066,8 @@ Status StreamAllocator::UpdateActiveStreamsForActiveNode(const vector<set<int64_
     vector<uint32_t> new_active_streams = active_streams;
     for (uint32_t logical_stream : active_streams) {
       if (static_cast<size_t>(logical_stream) >= split_streams.size()) {
+        REPORT_INNER_ERROR("E19999", "Check logical stream:%u is out of range:%zu",
+                           logical_stream, split_streams.size());
         GELOGE(FAILED, "logical stream is out of range.");
         return FAILED;
       }
@@ -951,6 +1086,8 @@ Status StreamAllocator::UpdateActiveStreamsForActiveNode(const vector<set<int64_
       }
     }
     if (!AttrUtils::SetListInt(node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, new_active_streams)) {
+      REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(),
+                         node->GetName().c_str(), node->GetType().c_str());
       GELOGE(FAILED, "Set active streams for node %s failed.", node->GetName().c_str());
       return FAILED;
     }
@@ -991,6 +1128,8 @@ Status StreamAllocator::UpdateActiveStreamsForSubgraphs() const {
     new_active_streams.emplace(static_cast<uint32_t>(new_split_stream));
     active_streams.assign(new_active_streams.begin(), new_active_streams.end());
     if (!AttrUtils::SetListInt(active_op, ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) {
+      REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(),
+                         active_op->GetName().c_str(), active_op->GetType().c_str());
       GELOGE(FAILED, "Set active streams for node %s failed.", active_node->GetName().c_str());
       return FAILED;
     }
@@ -1059,6 +1198,8 @@ Status StreamAllocator::SetActiveStreamsForLoop() {
 
       NodePtr pre_switch_node = FindSwitchNodeBeforeLoopActiveNode(node);
       if (pre_switch_node == nullptr) {
+        REPORT_INNER_ERROR("E19999", "Find switch node before loop active node %s fail",
+                           node->GetName().c_str());
         GELOGE(FAILED, "find switch node before loop active node %s failed", node->GetName().c_str());
         return FAILED;
       }
@@ -1066,6 +1207,9 @@ Status StreamAllocator::SetActiveStreamsForLoop() {
       if (!AttrUtils::GetListStr(node->GetOpDesc(), ATTR_NAME_ACTIVE_LABEL_LIST, activated_label_list) ||
           activated_label_list.empty()) {
         GE_CHK_BOOL_EXEC(AttrUtils::SetListInt(node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, loop_active_streams),
+                         REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)",
+                                            ATTR_NAME_ACTIVE_STREAM_LIST.c_str(),
+                                            node->GetName().c_str(), node->GetType().c_str());
                          GELOGE(FAILED, "SetListInt failed.");
                          return FAILED);
         for (const auto &stream_id : loop_active_streams) {
@@ -1112,6 +1256,8 @@ Status StreamAllocator::CheckStreamActived() const {
       uint32_t stream_id = static_cast<uint32_t>(node->GetOpDesc()->GetStreamId());
       auto iter = find(active_streams.begin(), active_streams.end(), stream_id);
       if (iter != active_streams.end()) {
+        REPORT_INNER_ERROR("E19999", "Node:%s(%s) cannot active its own stream %u, check invalid ",
+                           node->GetName().c_str(), node->GetType().c_str(), stream_id);
         GELOGE(FAILED, "Node %s cannot active its own stream %u.", node->GetName().c_str(), stream_id);
         return FAILED;
       }
@@ -1121,6 +1267,94 @@ Status StreamAllocator::CheckStreamActived() const {
   return SUCCESS;
 }
 
+Status StreamAllocator::ReuseEvent(bool send_to,
+    const std::unordered_map<std::string, ge::NodePtr> &name_to_node_map,
+    const std::unordered_map<ge::NodePtr, std::vector<std::pair<std::string, uint32_t>>> &node_to_event_id) {
+  for (const auto &node_event_id : node_to_event_id) {
+    ge::NodePtr curr_node = node_event_id.first;
+    NodePtr send_node = send_to ? curr_node : nullptr;
+    NodePtr recv_node = send_to ? nullptr : curr_node;
+    for (const auto &event_pair : node_event_id.second) {
+      auto peer_node_iter = name_to_node_map.find(event_pair.first);
+      if (peer_node_iter == name_to_node_map.end()) {
+        GELOGE(PARAM_INVALID, "[Get][Node]Name:%s.", event_pair.first.c_str());
+        REPORT_INNER_ERROR("E19999", "Failed to find node, name:%s.", event_pair.first.c_str());
+        return PARAM_INVALID;
+      }
+      recv_node = send_to ? peer_node_iter->second : recv_node;
+      send_node = send_to ? send_node : peer_node_iter->second;
+      GE_CHECK_NOTNULL(send_node);
+      GE_CHECK_NOTNULL(recv_node);
+      auto event_id = GetIntersection(node_to_send_events_[send_node], node_to_recv_events_[recv_node]);
+      uint32_t new_event = event_pair.second + event_num_;
+      if (event_id.empty()) {
+        GELOGI("[Check][Optimized]Send:%s, recv:%s.", send_node->GetName().c_str(), recv_node->GetName().c_str());
+        continue;
+      } else if (event_id.size() != 1) {
+        GELOGW("[Check][Event]More than one event are found between %s and %s, event num:%zu.",
+               send_node->GetName().c_str(), recv_node->GetName().c_str(), event_id.size());
+      }
+      uint32_t old_event = event_id[0];
+      auto reuse_event_id = [] (vector<uint32_t> &event_list, uint32_t old_event, uint32_t new_event) -> void {
+        event_list.erase(std::remove(event_list.begin(), event_list.end(), old_event), event_list.end());
+        event_list.push_back(new_event);
+        return;
+      };
+      reuse_event_id(node_to_send_events_[send_node], old_event, new_event);
+      reuse_event_id(node_to_recv_events_[recv_node], old_event, new_event);
+      GELOGI("[Reuse][Event]Replace event successfully, send node:%s, recv node:%s, old id:%u, new id:%u.",
+             send_node->GetName().c_str(), recv_node->GetName().c_str(), old_event, new_event);
+    }
+  }
+  return ge::SUCCESS;
+}
+
+// Refresh events to reuse events
+Status StreamAllocator::RefreshEventsWithReuse() {
+  GELOGI("[Refresh][Events]Refresh events with reuse, stream num:%ld, original event num:%u.", stream_num_, event_num_);
+  if (event_num_ <= kEventReuseThreshold) {
+    GELOGI("[Check][ReuseThreshold]Event used num is %u, less than %u, skip reuse.",
+           event_num_, kEventReuseThreshold);
+    return SUCCESS;
+  }
+  std::unordered_map<std::string, NodePtr> name_to_node_map;
+  std::unordered_map<NodePtr, std::vector<std::pair<std::string, uint32_t>>> node_to_send;
+  std::unordered_map<NodePtr, std::vector<std::pair<std::string, uint32_t>>> node_to_recv;
+  Status ret = ParseAllNodeEventMultiplexing(whole_graph_, name_to_node_map, node_to_send, node_to_recv);
+  if (ret != SUCCESS) {
+    GELOGE(ret, "[Parse][AllNodeEventMultiplexing]Graph:%s.", whole_graph_->GetName().c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to parse all node event multiplexing, graph:%s.",
+                       whole_graph_->GetName().c_str());
+    return ret;
+  }
+  if (node_to_send.empty() && node_to_recv.empty()) {
+    return SUCCESS;
+  }
+
+  ret = ReuseEvent(true, name_to_node_map, node_to_send);
+  if (ret != SUCCESS) {
+    GELOGE(ret, "[Reuse][Event]Phase:Send, graph:%s.", whole_graph_->GetName().c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to reuse event, phase:Send, graph:%s.", whole_graph_->GetName().c_str());
+    return ret;
+  }
+
+  ret = ReuseEvent(false, name_to_node_map, node_to_recv);
+  if (ret != SUCCESS) {
+    GELOGE(ret, "[Reuse][Event]Phase:Recv, graph:%s.", whole_graph_->GetName().c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to reuse event, phase:Recv, graph:%s.", whole_graph_->GetName().c_str());
+    return ret;
+  }
+
+  Status status = RefreshContinuousEvents();
+  if (status != SUCCESS) {
+    GELOGE(status, "[Refresh][ContinuousEvents]Graph:%s.", whole_graph_->GetName().c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to refresh continuous events, graph:%s.", whole_graph_->GetName().c_str());
+    return status;
+  }
+  GELOGI("[Refresh][Events]RefreshEventsWithReuse successfully, event num:%u.", event_num_);
+  return SUCCESS;
+}
+
 // Refresh events to continuous events
 Status StreamAllocator::RefreshContinuousEvents() {
   // Establish a mapping relationship from old to new event id
@@ -1128,8 +1362,10 @@ Status StreamAllocator::RefreshContinuousEvents() {
   uint32_t new_event_id = 0;
   for (const auto &one_pair : node_to_send_events_) {
     for (const auto &event_id : one_pair.second) {
-      old_to_new_events[event_id] = new_event_id;
-      new_event_id++;
+      if (old_to_new_events.find(event_id) == old_to_new_events.end()) {
+        old_to_new_events[event_id] = new_event_id;
+        new_event_id++;
+      }
     }
   }
 
@@ -1139,6 +1375,7 @@ Status StreamAllocator::RefreshContinuousEvents() {
     for (size_t i = 0; i < send_events.size(); i++) {
       auto find_it = old_to_new_events.find(send_events[i]);
       if (find_it == old_to_new_events.end()) {
+        REPORT_INNER_ERROR("E19999", "Check invalid send event %u", send_events[i]);
         GELOGE(FAILED, "RefreshContinuousEvents: invalid send event %u", send_events[i]);
         return FAILED;
       }
@@ -1152,6 +1389,7 @@ Status StreamAllocator::RefreshContinuousEvents() {
     for (size_t i = 0; i < recv_events.size(); i++) {
       auto find_it = old_to_new_events.find(recv_events[i]);
       if (find_it == old_to_new_events.end()) {
+        REPORT_INNER_ERROR("E19999", "Check invalid recv event %u", recv_events[i]);
         GELOGE(FAILED, "RefreshContinuousEvents: invalid recv event %u", recv_events[i]);
         return FAILED;
       }
@@ -1166,6 +1404,7 @@ Status StreamAllocator::RefreshContinuousEvents() {
 
 // Insert the real send/recv node in the graph
 Status StreamAllocator::InsertSyncEventNodes() {
+  unordered_map<string, uint32_t> sync_event_name;
   for (const auto &node : whole_graph_->GetNodes(whole_graph_->GetGraphUnknownFlag())) {
     // Add the node corresponding to the recv event
     vector<uint32_t> recv_event_id_list;
@@ -1175,12 +1414,23 @@ Status StreamAllocator::InsertSyncEventNodes() {
     GE_CHECK_NOTNULL(node->GetOutControlAnchor());
     for (auto &event_id : recv_event_id_list) {
       string recv_node_name = whole_graph_->GetName() + "_Recv_" + to_string(event_id);
+      auto iter = sync_event_name.find(recv_node_name);
+      if (iter == sync_event_name.end()) {
+          sync_event_name[recv_node_name] = 1;
+      } else {
+          recv_node_name = recv_node_name + "_Reuse_" + to_string(iter->second);
+          ++(iter->second);
+      }
       OpDescPtr op_desc_ptr = MakeShared<OpDesc>(recv_node_name, RECV);
       GE_CHECK_NOTNULL(op_desc_ptr);
 
       int64_t temp_stream_id = node->GetOpDesc()->GetStreamId();
       op_desc_ptr->SetStreamId(temp_stream_id);
-      GE_CHK_BOOL_EXEC(AttrUtils::SetInt(op_desc_ptr, RECV_ATTR_EVENT_ID, event_id), GELOGE(FAILED, "SetInt failed.");
+      GE_CHK_BOOL_EXEC(AttrUtils::SetInt(op_desc_ptr, RECV_ATTR_EVENT_ID, event_id),
+                       REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed, event_id:%u,",
+                                          RECV_ATTR_EVENT_ID.c_str(),
+                                          node->GetName().c_str(), node->GetType().c_str(), event_id);
+                       GELOGE(FAILED, "SetInt failed.");
                        return FAILED);
       (void)AttrUtils::SetListStr(op_desc_ptr, ATTR_NAME_DATA_DUMP_ORIGIN_OP_NAMES,
                                   std::move(std::vector<std::string>()));
@@ -1189,6 +1439,8 @@ Status StreamAllocator::InsertSyncEventNodes() {
       GE_CHECK_NOTNULL(recv_node->GetOutControlAnchor());
       Status status = GraphUtils::AddEdge(recv_node->GetOutControlAnchor(), node->GetInControlAnchor());
       if (status != SUCCESS) {
+        REPORT_INNER_ERROR("E19999", "Add edge from node %s to node %s failed",
+                           recv_node->GetName().c_str(), node->GetName().c_str());
         GELOGE(status, "Add edge for node %s and node %s failed.", recv_node->GetName().c_str(),
                node->GetName().c_str());
         return status;
@@ -1203,6 +1455,13 @@ Status StreamAllocator::InsertSyncEventNodes() {
 
     for (auto &event_id : send_event_id_list) {
       string send_node_name = whole_graph_->GetName() + "_Send_" + to_string(event_id);
+      auto iter = sync_event_name.find(send_node_name);
+      if (iter == sync_event_name.end()) {
+          sync_event_name[send_node_name] = 1;
+      } else {
+        send_node_name = send_node_name + "_Reuse_" + to_string(iter->second);
+          ++(iter->second);
+      }
       OpDescPtr op_desc_ptr = MakeShared<OpDesc>(send_node_name, SEND);
       GE_CHECK_NOTNULL(op_desc_ptr);
 
@@ -1217,6 +1476,8 @@ Status StreamAllocator::InsertSyncEventNodes() {
       GE_CHECK_NOTNULL(send_node->GetInControlAnchor());
       Status status = GraphUtils::AddEdge(node->GetOutControlAnchor(), send_node->GetInControlAnchor());
       if (status != SUCCESS) {
+        REPORT_INNER_ERROR("E19999", "Add edge from node %s to node %s failed",
+                           node->GetName().c_str(), send_node->GetName().c_str());
         GELOGE(status, "Add edge for node %s and node %s failed.", node->GetName().c_str(),
                send_node->GetName().c_str());
         return status;
@@ -1228,6 +1489,8 @@ Status StreamAllocator::InsertSyncEventNodes() {
 
   Status status = whole_graph_->InsertGraphEvents();
   if (status != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Insert Graph Events fail, graph:%s,",
+                      whole_graph_->GetName().c_str());
     GELOGE(status, "Graph ReorderEventNodes failed");
     return status;
   }
@@ -1248,12 +1511,16 @@ void StreamAllocator::DumpEvents() {
     GELOGD("After RefreshRealStream: stream %ld.", stream_id);
 
     for (const auto &node : one_pair.second) {
+      if (node == nullptr || node->GetOpDesc() == nullptr) {
+        continue;
+      }
       string send_event_str;
       for (const auto &send_event_id : node_to_send_events_[node]) {
         send_event_str += " " + to_string(send_event_id);
       }
       if (!send_event_str.empty()) {
-        GELOGI("node: %s, send events: %s", node->GetName().c_str(), send_event_str.c_str());
+        GELOGI("node: %s, id: %ld, stream id :%ld, send events: %s.", node->GetName().c_str(),
+               node->GetOpDesc()->GetId(), node->GetOpDesc()->GetStreamId(), send_event_str.c_str());
       }
 
       string recv_event_str;
@@ -1261,7 +1528,8 @@ void StreamAllocator::DumpEvents() {
         recv_event_str += " " + to_string(recv_event_id);
       }
       if (!recv_event_str.empty()) {
-        GELOGI("node: %s, recv events: %s", node->GetName().c_str(), recv_event_str.c_str());
+        GELOGI("node: %s, id: %ld, stream id :%ld, recv events: %s.", node->GetName().c_str(),
+               node->GetOpDesc()->GetId(), node->GetOpDesc()->GetStreamId(), recv_event_str.c_str());
       }
     }
   }
@@ -1274,6 +1542,8 @@ Status StreamAllocator::GetMaxStreamAndTask(bool huge_stream, uint32_t &max_stre
   }
   rtError_t ret = rtGetMaxStreamAndTask(stream_type, &max_stream_count, &max_task_count);
   if (ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "call rtGetMaxStreamAndTask fail, ret:%d, stream_type:%u,",
+                      static_cast<int>(ret), stream_type);
     GELOGE(FAILED, "Get max stream and task count by rts failed.");
     return FAILED;
   }
@@ -1416,6 +1686,7 @@ Status StreamAllocator::AddActiveNodes(NodePtr &switch_node, const vector<string
   for (size_t i = 0; i < label_num; i++) {
     const string &active_label = ori_active_label_list[i];
     if (labeled_streams_.find(active_label) == labeled_streams_.end()) {
+      REPORT_INNER_ERROR("E19999", "can not find stream label:%s", active_label.c_str());
       GELOGE(FAILED, "can not find stream label %s", active_label.c_str());
       return FAILED;
     }
@@ -1442,11 +1713,15 @@ Status StreamAllocator::AddActiveNodes(NodePtr &switch_node, const vector<string
       }
       GE_CHECK_NOTNULL(switch_node->GetOutControlAnchor());
       if (switch_node->GetOutControlAnchor()->Unlink(node->GetInControlAnchor()) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Unlink %s to %s failed",
+                          switch_node->GetName().c_str(), node->GetName().c_str());
         GELOGE(FAILED, "Unlink %s to %s failed.", switch_node->GetName().c_str(), node->GetName().c_str());
         return FAILED;
       }
       GE_CHECK_NOTNULL(active_node->GetOutControlAnchor());
       if (active_node->GetOutControlAnchor()->LinkTo(node->GetInControlAnchor()) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Link %s to %s failed",
+                          active_node->GetName().c_str(), node->GetName().c_str());
         GELOGE(FAILED, "Link %s to %s failed.", active_node->GetName().c_str(), node->GetName().c_str());
         return FAILED;
       }
@@ -1477,12 +1752,15 @@ Status StreamAllocator::AddActiveNodes(NodePtr &switch_node, const vector<string
 
 Status StreamAllocator::SetActiveStreamList(NodePtr &active_node, const string &active_label) {
   if (labeled_streams_.find(active_label) == labeled_streams_.end()) {
+    REPORT_INNER_ERROR("E19999", "Can not find stream label:%s", active_label.c_str());
     GELOGE(FAILED, "Can not find stream label %s.", active_label.c_str());
     return FAILED;
   }
   set<int64_t> &streams = labeled_streams_[active_label];
   vector<int64_t> active_streams(streams.begin(), streams.end());
   if (!AttrUtils::SetListInt(active_node->GetOpDesc(), ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_STREAM_LIST.c_str(),
+                       active_node->GetName().c_str(), active_node->GetType().c_str());
     GELOGE(FAILED, "SetListInt of %s failed.", ATTR_NAME_ACTIVE_STREAM_LIST.c_str());
     return FAILED;
   }

ge/graph/build/stream_allocator.h

@@ -71,6 +71,10 @@ class StreamAllocator {
   Status SetActiveStreamsForLoop();
   Status CheckStreamActived() const;
 
+  Status ReuseEvent(bool send_to,
+    const std::unordered_map<std::string, ge::NodePtr> &name_to_node_map,
+    const std::unordered_map<ge::NodePtr, std::vector<std::pair<std::string, uint32_t>>> &node_to_event_id);
+  Status RefreshEventsWithReuse();
   Status RefreshContinuousEvents();
 
   Status InsertSyncEventNodes();

ge/graph/build/stream_graph_optimizer.cc

@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 #include "stream_graph_optimizer.h"
+
+#include <securec.h>
+
 #include "common/util.h"
 #include "framework/common/debug/ge_log.h"
 #include "graph/utils/node_utils.h"
@@ -122,13 +125,16 @@ Status StreamGraphOptimizer::OptimizeStreamedSubGraph(const ComputeGraphPtr &com
         GE_CHECK_NOTNULL(op_desc);
         int64_t stream_id = op_desc->GetStreamId();
         if (static_cast<size_t>(stream_id) >= run_context.graphStreamList.size()) {
+          REPORT_INNER_ERROR("E19999", "Check stream_id:%ld in op:%s(%s) is bigger than "
+                             "run_context.graphStreamList.size():%zu", stream_id, op_desc->GetName().c_str(),
+                             op_desc->GetType().c_str(), run_context.graphStreamList.size());
           GELOGE(FAILED, "stream_id %ld is bigger than run_context.graphStreamList.size() %zu", stream_id,
                  run_context.graphStreamList.size());
           return FAILED;
         }
         run_context.stream = run_context.graphStreamList[stream_id];
-	std::string batch_label;
-	(void)AttrUtils::GetStr(subgraph, ATTR_NAME_BATCH_LABEL, batch_label);
+        std::string batch_label;
+        (void)AttrUtils::GetStr(subgraph, ATTR_NAME_BATCH_LABEL, batch_label);
         GELOGD("Subgraph has same stream id, subgraph: %s, engine_name: %s, stream_id: %ld, rtstream: %lu, "
 	       "batch_label: %s", subgraph->GetName().c_str(), engine_name.c_str(), stream_id,
                static_cast<uint64_t>(reinterpret_cast<uintptr_t>(run_context.stream)), batch_label.c_str());
@@ -136,6 +142,9 @@ Status StreamGraphOptimizer::OptimizeStreamedSubGraph(const ComputeGraphPtr &com
           GE_CHECK_NOTNULL(*iter);
           Status ret = (*iter)->OptimizeStreamGraph(*subgraph, run_context);
           if (ret != SUCCESS) {
+            REPORT_CALL_ERROR("E19999", "Call optimize streamed subgraph failed, subgraph: %s, engine_name: %s, graph "
+                              "Optimizer num: %zu, ret: %u", subgraph->GetName().c_str(), engine_name.c_str(),
+                              graph_optimizers.size(), ret);
             GELOGE(
               ret,
               "[optimizeStreamedSubGraph]: optimize streamed subgraph failed, subgraph: %s, engine_name: %s, graph "

ge/graph/build/task_generator.cc

@@ -49,6 +49,7 @@ const char *const kIsLastNode = "is_last_node";
 const char *const kIsInputVar = "INPUT_IS_VAR";
 const char *const kIsOutputVar = "OUTPUT_IS_VAR";
 const char *const kProfilingMode = "PROFILING_MODE";
+const char *const kIteratorV2 = "IteratorV2";
 const uint32_t kProfilingArStep = 2;
 const uint64_t kProfilingFpStartLogid = 1;
 const uint64_t kProfilingBpEndLogid = 2;
@@ -57,6 +58,7 @@ const uint64_t kProfilingArEndLogid = 4;
 const uint64_t kProfilingIterEndLogid = 65535;
 const int64_t kHashFactor = 100000;
 const int64_t kInvalidGroupId = -1;
+const std::set<std::string> kFpNodeTypes = {ge::DATA, ge::GETNEXT, kIteratorV2};
 }  // namespace
 namespace ge {
 TaskGenerator::TaskGenerator(uint8_t *var_mem_base, uint64_t var_mem_size) {
@@ -69,6 +71,7 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t
   GELOGD("Begin to Get TaskInfo. session_id=%lu", session_id);
   // Check params
   if (graph == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param graph is null, session_id:%lu", session_id);
     GELOGE(PARAM_INVALID, "GetTaskInfo param graph is null. session_id=%lu", session_id);
     return PARAM_INVALID;
   }
@@ -93,6 +96,8 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t
     op_name.push_back(iter.second);
   }
   GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListStr(model, ATTR_MODEL_TASK_INDEX_OP_NAME, op_name),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s",
+                                      ATTR_MODEL_TASK_INDEX_OP_NAME.c_str(), model.GetName().c_str());
                    GELOGE(FAILED, "SetListStr failed.");
                    return FAILED);
 
@@ -106,6 +111,8 @@ Status TaskGenerator::GetTaskInfo(Model &model, ComputeGraphPtr &graph, uint64_t
   for (const TaskDef &task_def_temp : task_def_list) {
     TaskDef *task_def = model_task_def.add_task();
     if (task_def == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Add task_def in ModelTaskDef fail, session_id:%lu, graph:%s, model:%s",
+                         session_id, graph->GetName().c_str(), model.GetName().c_str());
       GELOGE(FAILED, "task_def is nullptr.");
       return FAILED;
     }
@@ -126,30 +133,44 @@ Status TaskGenerator::AddModelTaskToModel(const ModelTaskDef &model_task_def, ui
                                           RunContext &run_context) {
   GE_CHK_BOOL_EXEC(
       AttrUtils::SetInt(model, MODEL_ATTR_TASK_GEN_BASE_ADDR, reinterpret_cast<uintptr_t>(run_context.dataMemBase)),
+      REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s",
+                         MODEL_ATTR_TASK_GEN_BASE_ADDR.c_str(), model.GetName().c_str());
       GELOGE(FAILED, "SetInt MODEL_ATTR_TASK_GEN_BASE_ADDR failed.");
       return FAILED);
   GE_CHK_BOOL_EXEC(
       AttrUtils::SetInt(model, MODEL_ATTR_TASK_GEN_WEIGHT_ADDR, reinterpret_cast<uintptr_t>(run_context.weightMemBase)),
+      REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s",
+                         MODEL_ATTR_TASK_GEN_WEIGHT_ADDR.c_str(), model.GetName().c_str());
       GELOGE(FAILED, "SetInt MODEL_ATTR_TASK_GEN_WEIGHT_ADDR failed.");
       return FAILED);
   GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, ATTR_MODEL_TASK_GEN_VAR_ADDR, reinterpret_cast<uintptr_t>(var_mem_base_)),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s",
+                                      ATTR_MODEL_TASK_GEN_VAR_ADDR.c_str(), model.GetName().c_str());
                    GELOGE(FAILED, "SetInt ATTR_MODEL_TASK_GEN_VAR_ADDR failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, ATTR_MODEL_VAR_SIZE, var_mem_size_),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for model:%s",
+                                      ATTR_MODEL_VAR_SIZE.c_str(), model.GetName().c_str());
                    GELOGE(FAILED, "SetInt ATTR_MODEL_VAR_SIZE failed.");
                    return FAILED);
   GE_CHK_BOOL_EXEC(AttrUtils::SetInt(model, MODEL_ATTR_SESSION_ID, session_id),
+                   REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for mode:%s",
+                                      MODEL_ATTR_SESSION_ID.c_str(), model.GetName().c_str());
                    GELOGE(FAILED, "SetInt MODEL_ATTR_SESSION_ID failed.");
                    return FAILED);
 
   size_t task_size = model_task_def.ByteSizeLong();
   ge::Buffer serial_buff(task_size);
   if (!model_task_def.SerializePartialToArray(serial_buff.GetData(), static_cast<int>(task_size))) {
+    REPORT_INNER_ERROR("E19999", "model_task_def's serialize failed,  model name = %s, task_size=%zu",
+                       model.GetName().c_str(), task_size);
     GELOGE(FAILED, "model_task_def's serialize failed,  model name = %s, task_size=%zu.", model.GetName().c_str(),
            task_size);
     return FAILED;
   }
   if (!AttrUtils::SetZeroCopyBytes(model, MODEL_ATTR_TASKS, std::move(serial_buff))) {
+    REPORT_INNER_ERROR("E19999", "Set model task to model failed,  model name = %s, task_size=%zu",
+                       model.GetName().c_str(), task_size);
     GELOGE(FAILED, "Set model task to model failed,  model name = %s, task_size=%zu.", model.GetName().c_str(),
            task_size);
     return FAILED;
@@ -167,7 +188,10 @@ Status TaskGenerator::UpdateOpIsVarAttr(const OpDescPtr &op_desc, uint64_t sessi
     for (int64_t input : input_offsets) {
       input_var.push_back(VarManager::Instance(session_id)->IsVarAddr(input));
     }
-    GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsInputVar, input_var), GELOGE(FAILED, "SetListBool failed.");
+    GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsInputVar, input_var),
+                     REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsInputVar,
+                                        op_desc->GetName().c_str(), op_desc->GetType().c_str());
+                     GELOGE(FAILED, "SetListBool failed.");
                      return FAILED);
   }
 
@@ -177,7 +201,10 @@ Status TaskGenerator::UpdateOpIsVarAttr(const OpDescPtr &op_desc, uint64_t sessi
     for (int64_t output : output_offsets) {
       output_var.push_back(VarManager::Instance(session_id)->IsVarAddr(output));
     }
-    GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsOutputVar, output_var), GELOGE(FAILED, "SetListBool failed.");
+    GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsOutputVar, output_var),
+                     REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsOutputVar,
+                                        op_desc->GetName().c_str(), op_desc->GetType().c_str());
+                     GELOGE(FAILED, "SetListBool failed.");
                      return FAILED);
   }
   return SUCCESS;
@@ -252,6 +279,7 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra
   GELOGD("Beign to generate task, graph name is %s.", graph->GetName().c_str());
   std::shared_ptr<GELib> ge_lib = GELib::GetInstance();
   if ((ge_lib == nullptr) || !ge_lib->InitFlag()) {
+    REPORT_INNER_ERROR("E19999", "Check GELib instance not init before");
     GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GenerateTask failed.");
     return GE_CLI_GE_NOT_INITIALIZED;
   }
@@ -319,6 +347,8 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra
     }
     auto kernel_info_store = ops_kernel_manager.GetOpsKernelInfoStore(op_kernel_lib_name);
     if (kernel_info_store == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s",
+                         node->GetName().c_str(), node->GetType().c_str(), op_kernel_lib_name.c_str());
       GELOGE(INTERNAL_ERROR,
              "No ops kernel store or ops kernel builder found. node:%s(%s), op_kernel_lib_name=%s.",
              name.c_str(),
@@ -344,6 +374,8 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra
     auto ret = OpsKernelBuilderManager::Instance().GenerateTask(*node, run_context, task_def_list);
     GE_TIMESTAMP_ADD(GenerateTask);
     if (ret != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Call OpsKernelBuilderManager GenerateTask fail for op:%s(%s)",
+                        node->GetName().c_str(), node->GetType().c_str());
       GELOGE(ret, "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task failed.",
              op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), op_id, stream_id);
       return ret;
@@ -353,6 +385,9 @@ Status TaskGenerator::GenerateTask(RunContext &run_context, ComputeGraphPtr &gra
     size_t task_list_size_after = task_def_list.size();
     // If tasks is reduced
     if (task_list_size_after < task_list_size_before) {
+      REPORT_INNER_ERROR("E19999", "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task "
+                         "but task num from %zu to %zu, check invalid", op_kernel_lib_name.c_str(), name.c_str(),
+                         type.c_str(), op_id, stream_id, task_list_size_before, task_list_size_after);
       GELOGE(FAILED, "Call %s to generate node[name:%s(%s), id:%ld, stream_id:%ld] task. but task num from %zu to %zu.",
              op_kernel_lib_name.c_str(), name.c_str(), type.c_str(), op_id, stream_id, task_list_size_before,
              task_list_size_after);
@@ -417,6 +452,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info
       size_t task_list_size_before = task_def_list.size();
       OpsKernelInfoStorePtr kernel_info_store = ops_kernel_manager.GetOpsKernelInfoStore(op_kernel_lib_name);
       if (kernel_info_store == nullptr) {
+        REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s",
+                           op_desc->GetName().c_str(), op_desc->GetType().c_str(),
+                           op_kernel_lib_name.c_str());
         GELOGE(INTERNAL_ERROR,
                "Fusion: No ops kernel store or ops kernel builder found. fusion_node:%s(%s), op_kernel_lib_name=%s.",
                fusion_node_name.c_str(), fusion_node_type.c_str(), op_kernel_lib_name.c_str());
@@ -433,6 +471,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info
       int64_t op_id = op_desc->GetId();
       int64_t stream_id = op_desc->GetStreamId();
       if (stream_id < 0 || stream_id >= (int64_t)run_context.graphStreamList.size()) {
+        REPORT_INNER_ERROR("E19999", "Fusion: fusion_node[name:%s(%s), id:%ld] stream id is invalid, "
+                           "stream list size=%zu", fusion_node_name.c_str(), fusion_node_type.c_str(),
+                           op_id, run_context.graphStreamList.size());
         GELOGE(INTERNAL_ERROR, "Fusion: fusion_node[name:%s(%s), id:%ld] stream id is invalid, stream list size=%zu",
                fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, run_context.graphStreamList.size());
         return INTERNAL_ERROR;
@@ -444,6 +485,9 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info
              op_kernel_lib_name.c_str(), fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, stream_id);
       ret = OpsKernelBuilderManager::Instance().GenerateTask(*fusion_node, run_context, task_def_list);
       if (ret != SUCCESS) {
+        REPORT_CALL_ERROR("E19999", " Call %s to generate fusion_node:[fusion_node_name:%s(%s), "
+                          "id:%ld, stream_id:%ld] task failed", op_kernel_lib_name.c_str(),
+                          fusion_node_name.c_str(), fusion_node_type.c_str(), op_id, stream_id);
         GELOGE(ret,
                "Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), "
                "id:%ld, stream_id:%ld] task failed.",
@@ -455,6 +499,10 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info
       size_t task_list_size_after = task_def_list.size();
       // if tasks is reduced
       if (task_list_size_after < task_list_size_before) {
+        REPORT_INNER_ERROR("E19999", "InsertProfilingTask for fusion_node:[fusion_node_name:%s(%s), kernel_name:%s"
+                           "id:%ld, stream_id:%ld] task, but task num from %zu to %zu, check invalid",
+                           fusion_node_name.c_str(), fusion_node_type.c_str(), op_kernel_lib_name.c_str(),
+                           op_id, stream_id, task_list_size_before, task_list_size_after);
         GELOGE(FAILED,
                "Fusion: Call %s to generate fusion_node:[fusion_node_name:%s(%s), "
                "id:%ld, stream_id:%ld] task. but task num from %zu to %zu.",
@@ -489,6 +537,8 @@ Status TaskGenerator::GenerateTaskForFusionNode(FusionTaskInfo &fusion_task_info
 
 Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) {
   if (NodeUtils::SetAllAnchorStatus(node) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "SetAllAnchorStatus fail for op:%s(%s)",
+                      node->GetName().c_str(), node->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "NodeUtils::SetAllAnchorStatus failed.");
     return INTERNAL_ERROR;
   }
@@ -496,6 +546,8 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) {
     auto peer_anchor = anchor->GetPeerOutAnchor();
     if (peer_anchor == nullptr) {
       if (AnchorUtils::SetStatus(anchor, ANCHOR_SUSPEND) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Set in peer anchor status fail for op:%s(%s), anchor_index:%d,",
+                          node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx());
         GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed.");
         return INTERNAL_ERROR;
       }
@@ -506,11 +558,15 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) {
     bool is_const = NodeUtils::GetConstOpType(peer_anchor->GetOwnerNode(), const_type);
     if (is_const && (const_type == CONSTANT)) {
       if (AnchorUtils::SetStatus(anchor, ANCHOR_CONST) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Set in anchor CONST status fail for op:%s(%s), anchor_index:%d,",
+                          node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx());
         GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed.");
         return INTERNAL_ERROR;
       }
     } else {
       if (AnchorUtils::SetStatus(anchor, ANCHOR_DATA) != GRAPH_SUCCESS) {
+        REPORT_CALL_ERROR("E19999", "Set in anchor DATA status fail for op:%s(%s), anchor_index:%d,",
+                          node->GetName().c_str(), node->GetType().c_str(), anchor->GetIdx());
         GELOGE(INTERNAL_ERROR, "AnchorUtils::SetStatus failed.");
         return INTERNAL_ERROR;
       }
@@ -523,12 +579,15 @@ Status TaskGenerator::UpdateAnchorStatus(const NodePtr &node) {
 Status TaskGenerator::MarkNodeAndSetIndex(ComputeGraphPtr &graph) {
   auto ge_lib = GELib::GetInstance();
   if ((ge_lib == nullptr) || !ge_lib->InitFlag()) {
+    REPORT_INNER_ERROR("E19999", "Check GELib instance not init before");
     GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized or is finalized.");
     return GE_CLI_GE_NOT_INITIALIZED;
   }
 
   const auto all_nodes = graph->GetNodes(graph->GetGraphUnknownFlag());
   if (all_nodes.empty()) {
+    REPORT_INNER_ERROR("E19999", "Check param all_nodes empty in graph:%s",
+                       graph->GetName().c_str());
     GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Graph's node is empty");
     return GE_GRAPH_GRAPH_NODE_NULL;
   }
@@ -584,6 +643,9 @@ Status TaskGenerator::MarkFirstAndLastOps(const vector<OpDescPtr> &ops, bool is_
     for (auto &op_desc : continuous_ops) {
       string op_kernel_lib_name = op_desc->GetOpKernelLibName();
       if (op_kernel_lib_name.empty()) {
+        REPORT_INNER_ERROR("E19999", "Get ops kernel info store failed for op:%s(%s), op_kernel_name:%s",
+                           op_desc->GetName().c_str(), op_desc->GetType().c_str(),
+                           op_kernel_lib_name.c_str());
         GELOGE(INTERNAL_ERROR, "node:%s(%s) get op kernel lib failed.", op_desc->GetName().c_str(),
                op_desc->GetType().c_str());
         return INTERNAL_ERROR;
@@ -599,9 +661,15 @@ Status TaskGenerator::MarkFirstAndLastOps(const vector<OpDescPtr> &ops, bool is_
 
     for (auto &it : first_and_last_ops) {
       auto &op_pair = it.second;
-      GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.first, kIsFirstNode, true), GELOGE(FAILED, "SetBool failed.");
+      GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.first, kIsFirstNode, true),
+                       REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsFirstNode,
+                                          op_pair.first->GetName().c_str(), op_pair.first->GetType().c_str());
+                       GELOGE(FAILED, "SetBool failed.");
                        return FAILED);
-      GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.second, kIsLastNode, true), GELOGE(FAILED, "SetBool failed.");
+      GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(op_pair.second, kIsLastNode, true),
+                       REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsLastNode,
+                                          op_pair.second->GetName().c_str(), op_pair.second->GetType().c_str());
+                       GELOGE(FAILED, "SetBool failed.");
                        return FAILED);
     }
   }
@@ -621,8 +689,10 @@ Status TaskGenerator::AutoFindFpOpIndex(const ComputeGraphPtr &graph, ProfilingP
     if (op_kernel_lib_name.empty()) {
       continue;
     }
-
-    if (op_desc->GetType() == GETNEXT || op_desc->GetType() == DATA) {
+    auto type = op_desc->GetType();
+    std::string original_type;
+    (void)AttrUtils::GetStr(op_desc, ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE, original_type);
+    if (kFpNodeTypes.find(type) != kFpNodeTypes.end() || kFpNodeTypes.find(original_type) != kFpNodeTypes.end()) {
       auto out_anchor = node->GetOutDataAnchor(0);
       for (auto &peer_in_anchor : out_anchor->GetPeerInDataAnchors()) {
         GE_CHECK_NOTNULL(peer_in_anchor);
@@ -723,7 +793,9 @@ uint32_t TaskGenerator::FindLastBpFromBpNode(const ComputeGraphPtr &graph, const
     GELOGI("bp_op_desc is %s, id is %ld", bp_op_desc->GetName().c_str(), bp_op_desc->GetId());
   }
 
-  GE_CHECK_NOTNULL(bp_op_desc);
+  if (bp_op_desc == nullptr) {
+    return last_bp;
+  }
   uint32_t current_idx = 0;
   for (auto &node : graph->GetNodes(graph->GetGraphUnknownFlag())) {
     OpDescPtr op_desc = node->GetOpDesc();
@@ -906,6 +978,8 @@ Status TaskGenerator::InsertProfilingArTaskBefore(const OpDescPtr &op_desc, std:
     for (size_t i = 0; i < all_reduce_nodes.size(); i++) {
       if (all_reduce_nodes[i] == node_index) {
         GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep),
+                        REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id "
+                                           "for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str());
                         GELOGE(FAILED, "Multiply result is out of range.");
                         return FAILED);
         ar_log_id = i * kProfilingArStep + kProfilingArStartLogid;
@@ -998,6 +1072,8 @@ Status TaskGenerator::InsertProfilingArTaskAfter(const OpDescPtr &op_desc, std::
     for (size_t i = 0; i < all_reduce_nodes.size(); i++) {
       if (all_reduce_nodes[i] == node_index) {
         GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(i, kProfilingArStep),
+                        REPORT_INNER_ERROR("E19999", "Multiply result is out of range when calc profiling ar log id "
+                                           "for node:%s(%s)", op_desc->GetName().c_str(), op_desc->GetType().c_str());
                         GELOGE(FAILED, "Multiply result is out of range.");
                         return FAILED);
         ar_log_id = i * kProfilingArStep + kProfilingArEndLogid;
@@ -1107,6 +1183,7 @@ Status TaskGenerator::SetUnknownShapeStream(RunContext &run_context, rtStream_t
   run_context.stream = stream;
   rtError_t rt_ret = rtModelBindStream(run_context.model, stream, 0);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtModelBindStream failed, ret:0x%X", rt_ret);
     GELOGE(FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     GE_CHK_RT_RET(rtStreamDestroy(stream));
     return FAILED;

ge/graph/common/bcast.cc

@@ -73,6 +73,8 @@ Status BCast::SetShapeDifferentInfo(const kVecInt &x, const kVecInt &y) {
       y_bcast_i = x_i;
       grad_y_reduce_idx_.push_back(n - 1 - i);
     } else {
+      REPORT_INNER_ERROR("E19999", "SetShapeDifferentInfo failed. Two tensor shapes are not compatible "
+                         "according to the broadcasting rule.");
       GELOGE(domi::PARAM_INVALID,
              "SetShapeDifferentInfo failed. Two tensor shapes are not compatible "
              "according to the broadcasting rule.");

ge/graph/common/bcast.h

@@ -111,11 +111,14 @@ class BCast {
                       const std::function<OutT(InT const &, InT const &)> &func) {
     Status ret;
     if (func == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Check param func nullptr");
       GELOGE(domi::PARAM_INVALID, "Param func is null");
       return domi::PARAM_INVALID;
     }
     // Min input num is 2
     if (input.size() < kMinDimNum) {
+      REPORT_INNER_ERROR("E19999", "Param input.size():%zu < %zu, check invalid",
+                         input.size(), kMinDimNum);
       GELOGE(domi::PARAM_INVALID, "Input size is smaller than two.");
       return domi::PARAM_INVALID;
     }
@@ -149,11 +152,14 @@ class BCast {
   Status BCastComputeCheck(const std::vector<ConstGeTensorPtr> &input, std::vector<OutT> &v_output,
                            const std::function<OutT(InT const &, InT const &, DataType &type, Status &)> &func) {
     if (func == nullptr) {
+      REPORT_INNER_ERROR("E19999", "Check param func nullptr");
       GELOGE(PARAM_INVALID, "Param func is null");
       return PARAM_INVALID;
     }
     // Min input num is 2
     if (input.size() < kMinDimNum) {
+      REPORT_INNER_ERROR("E19999", "Param input.size():%zu < %zu, check invalid",
+                         input.size(), kMinDimNum);
       GELOGE(PARAM_INVALID, "Input size is smaller than two.");
       return PARAM_INVALID;
     }
@@ -179,6 +185,7 @@ class BCast {
       auto value = func((*(reinterpret_cast<const InT *>(x1_data) + x_index)),
                         (*(reinterpret_cast<const InT *>(x2_data) + y_index)), data_type, ret);
       if (ret != SUCCESS) {
+        REPORT_INNER_ERROR("E19999", "BCastComputeCheck func execute failed, datatype is %d.", data_type);
         GELOGE(ret, "BCastComputeCheck func execute failed, datatype is %d.", data_type);
         return ret;
       }

ge/graph/common/omg_util.cc

@@ -21,6 +21,8 @@
 #include "framework/common/debug/ge_log.h"
 #include "graph/debug/ge_attr_define.h"
 #include "graph/utils/graph_utils.h"
+#include "graph/utils/tensor_utils.h"
+#include "common/math/math_util.h"
 
 namespace ge {
 ///
@@ -36,6 +38,8 @@ Status GetOriginalType(const ge::NodePtr &node, string &type) {
   GE_CHECK_NOTNULL(node->GetOpDesc());
   bool ret = ge::AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE, type);
   if (!ret) {
+    REPORT_INNER_ERROR("E19999", "Get Attr:%s fail for op:%s(%s)", ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE.c_str(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "Get FrameWorkOp original type [%s]", type.c_str());
     return INTERNAL_ERROR;
   }
@@ -55,6 +59,8 @@ Status SetStreamLabel(const ge::NodePtr &node, const std::string &label) {
   GE_CHECK_NOTNULL(tmp_desc);
 
   if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_STREAM_LABEL, label)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_STREAM_LABEL.c_str(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "Op: %s set ATTR_NAME_STREAM_LABEL failed", node->GetName().c_str());
     return FAILED;
   }
@@ -72,6 +78,8 @@ Status SetCycleEvent(const ge::NodePtr &node) {
   OpDescPtr tmp_desc = node->GetOpDesc();
   GE_CHECK_NOTNULL(tmp_desc);
   if (!AttrUtils::SetBool(tmp_desc, ge::ATTR_NAME_STREAM_CYCLE_EVENT_FLAG, true)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_STREAM_CYCLE_EVENT_FLAG.c_str(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "Op: %s set ATTR_NAME_STREAM_CYCLE_EVENT_FLAG failed", node->GetName().c_str());
     return FAILED;
   }
@@ -90,6 +98,8 @@ Status SetActiveLabelList(const ge::NodePtr &node, const std::vector<std::string
   OpDescPtr tmp_desc = node->GetOpDesc();
   GE_CHECK_NOTNULL(tmp_desc);
   if (!AttrUtils::SetListStr(tmp_desc, ge::ATTR_NAME_ACTIVE_LABEL_LIST, active_label_list)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ACTIVE_LABEL_LIST.c_str(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "Op: %s set ATTR_NAME_ACTIVE_LABEL_LIST failed", node->GetName().c_str());
     return FAILED;
   }
@@ -108,6 +118,8 @@ Status SetSwitchBranchNodeLabel(const ge::NodePtr &node, const std::string &bran
   OpDescPtr tmp_desc = node->GetOpDesc();
   GE_CHECK_NOTNULL(tmp_desc);
   if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_SWITCH_BRANCH_NODE_LABEL, branch_label)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_SWITCH_BRANCH_NODE_LABEL.c_str(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "Op: %s set ATTR_NAME_SWITCH_BRANCH_NODE_LABEL failed", node->GetName().c_str());
     return FAILED;
   }
@@ -126,6 +138,8 @@ Status SetSwitchTrueBranchFlag(const ge::NodePtr &node, bool value) {
   OpDescPtr tmp_desc = node->GetOpDesc();
   GE_CHECK_NOTNULL(tmp_desc);
   if (!AttrUtils::SetBool(tmp_desc, ge::ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG, value)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG.c_str(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "Op: %s set ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG failed", node->GetName().c_str());
     return FAILED;
   }
@@ -144,6 +158,8 @@ Status SetOriginalNodeName(const ge::NodePtr &node, const std::string &orig_name
   OpDescPtr tmp_desc = node->GetOpDesc();
   GE_CHECK_NOTNULL(tmp_desc);
   if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_ORIG_NODE_NAME, orig_name)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_ORIG_NODE_NAME.c_str(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "Op: %s set ATTR_NAME_ORIG_NODE_NAME failed", node->GetName().c_str());
     return FAILED;
   }
@@ -161,6 +177,8 @@ Status SetCyclicDependenceFlag(const ge::NodePtr &node) {
   OpDescPtr tmp_desc = node->GetOpDesc();
   GE_CHECK_NOTNULL(tmp_desc);
   if (!AttrUtils::SetBool(tmp_desc, ge::ATTR_NAME_CYCLIC_DEPENDENCE_FLAG, true)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_CYCLIC_DEPENDENCE_FLAG.c_str(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "Op: %s set ATTR_NAME_CYCLIC_DEPENDENCE_FLAG failed", node->GetName().c_str());
     return FAILED;
   }
@@ -180,10 +198,50 @@ Status SetNextIteration(const ge::NodePtr &node, const std::string &next) {
   GE_CHECK_NOTNULL(tmp_desc);
 
   if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_NEXT_ITERATION, next)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_NEXT_ITERATION.c_str(),
+                       node->GetName().c_str(), node->GetType().c_str());
     GELOGE(FAILED, "Op: %s set ATTR_NAME_NEXT_ITERATION failed", node->GetName().c_str());
     return FAILED;
   }
 
   return SUCCESS;
 }
+
+///
+/// @brief Align the memory
+/// @param [in/out] memory size
+/// @param [in] alinment
+/// @return void
+///
+void AlignMemSize(int64_t &mem_size, int64_t align_size) {
+  if (mem_size <= 0) {
+    return;
+  }
+  mem_size = (mem_size + align_size - 1) / align_size * align_size;
+}
+
+///
+/// @brief Get memory size from tensor desc
+/// @param [in] node
+/// @param [out] memory size
+/// @return Status
+///
+Status GetMemorySize(const NodePtr &node, int64_t &output_size) {
+  GE_CHECK_NOTNULL(node->GetOpDesc());
+  auto output_op_desc = node->GetOpDesc()->GetOutputDescPtr(kBufferPoolNodeOutIndex);
+  GE_CHECK_NOTNULL(output_op_desc);
+  int64_t size = 0;
+  auto ret = ge::TensorUtils::GetSize(*output_op_desc, size);
+  if (ret != ge::GRAPH_SUCCESS) {
+    GELOGE(INTERNAL_ERROR, "[Get][Size]Node:%s.", node->GetName().c_str());
+    REPORT_INNER_ERROR("E19999", "Failed to get output size, node:%s.", node->GetName().c_str());
+    return INTERNAL_ERROR;
+  }
+  FMK_INT64_ADDCHECK(size, kBufferPoolMemAlignSize);
+  AlignMemSize(size, kBufferPoolMemAlignSize);
+  // The HCOM operator requires an additional 512 bytes before and after
+  FMK_INT64_ADDCHECK(size, (kBufferPoolMemAlignSize + kBufferPoolMemAlignSize));
+  output_size = kBufferPoolMemAlignSize + size + kBufferPoolMemAlignSize;
+  return SUCCESS;
+}
 }  // namespace ge

ge/graph/common/omg_util.h

@@ -27,6 +27,11 @@
 #include "graph/node.h"
 
 namespace ge {
+namespace {
+const int64_t kBufferPoolMemAlignSize = 512;
+const uint32_t kBufferPoolNodeOutIndex = 0;
+const uint32_t kEventReuseThreshold = 65500;
+} // namespace
 ///
 /// @brief get the Original Type of FrameworkOp
 /// @param [in] node
@@ -96,6 +101,22 @@ Status SetCyclicDependenceFlag(const ge::NodePtr &node);
 /// @return Status
 ///
 Status SetNextIteration(const ge::NodePtr &node, const std::string &next);
+
+///
+/// @brief Align the memory
+/// @param [in/out] memory size
+/// @param [in] alinment
+/// @return void
+///
+void AlignMemSize(int64_t &mem_size, int64_t align_size);
+
+///
+/// @brief Get memory size from tensor desc
+/// @param [in] node
+/// @param [out] memory size
+/// @return Status
+///
+Status GetMemorySize(const NodePtr &node, int64_t &output_size);
 }  // namespace ge
 
 #endif  // GE_GRAPH_COMMON_OMG_UTIL_H_

ge/graph/execute/graph_execute.cc

@@ -20,9 +20,12 @@
 #include <string>
 
 #include "graph/load/model_manager/model_manager.h"
+#include "graph/load/model_manager/davinci_model.h"
 #include "omm/csa_interact.h"
 
 namespace ge {
+using Uint32Pair = pair<uint32_t, uint32_t>;
+const uint32_t kInvalidModelId = UINT32_MAX;
 GraphExecutor::GraphExecutor()
     : init_flag_(false),
       train_graph_flag_(false),
@@ -40,6 +43,7 @@ GraphExecutor::~GraphExecutor() {
       rtError_t rt_ret;
       rt_ret = rtFreeHost(buffer_addr);
       if (rt_ret != RT_ERROR_NONE) {
+        REPORT_CALL_ERROR("E19999", "Call rtFreeHost failed, ret:0x%X", rt_ret);
         GELOGE(RT_FAILED, "[GraphManager] subgraph free buffer failed, ret: 0x%X", rt_ret);
       }
     }
@@ -51,14 +55,17 @@ GraphExecutor::~GraphExecutor() {
 Status GraphExecutor::SetCondition(std::mutex *mutex, std::condition_variable *cond,
                                    std::shared_ptr<GraphModelListener> listener) {
   if (mutex == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param mutex nullptr");
     GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetCondition] input param mutex is nullptr.");
     return GE_GRAPH_PARAM_NULLPTR;
   }
   if (cond == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param cond nullptr");
     GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetCondition] input param cond is nullptr.");
     return GE_GRAPH_PARAM_NULLPTR;
   }
   if (listener == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param listener nullptr");
     GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetCondition] input param listener is nullptr.");
     return GE_GRAPH_PARAM_NULLPTR;
   }
@@ -75,6 +82,7 @@ Status GraphExecutor::SetCondition(std::mutex *mutex, std::condition_variable *c
 
 Status GraphExecutor::SetGraphContext(GraphContextPtr graph_context_ptr) {
   if (graph_context_ptr == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param graph_context_ptr nullptr");
     GELOGE(GE_GRAPH_PARAM_NULLPTR, "[SetGraphContext] input param graph_context_ptr is nullptr");
     return GE_GRAPH_PARAM_NULLPTR;
   }
@@ -101,6 +109,7 @@ Status GraphExecutor::FreeInOutBuffer() {
       rtError_t rt_ret;
       rt_ret = rtFreeHost(*iter);
       if (rt_ret != RT_ERROR_NONE) {
+        REPORT_CALL_ERROR("E19999", "Call rtFreeHost failed, ret:0x%X", rt_ret);
         GELOGE(RT_FAILED, "[GraphManager] subgraph free buffer failed, ret: 0x%X", rt_ret);
         (void)buffer_addr_.erase(buffer_addr_.begin(), iter);
         return GE_GRAPH_FREE_FAILED;
@@ -146,6 +155,8 @@ Status GraphExecutor::MallocInOutBuffer(const std::vector<uint64_t> &buffer_size
     void *tmp_buf = nullptr;
     rt_ret = rtMallocHost(&tmp_buf, buffer_size[i]);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, size:%lu, ret:0x%X",
+                        buffer_size[i], rt_ret);
       GELOGE(RT_FAILED, "[GraphManager] subgraph malloc buffer failed, ret: 0x%X", rt_ret);
       return GE_GRAPH_MALLOC_FAILED;
     }
@@ -191,6 +202,8 @@ Status GraphExecutor::PrepareInputData(const std::vector<GeTensor> &input_tensor
       rtError_t rt_ret = rtMemcpy(addrVec[i], bufferSizeVec[i], in_tensor->GetData().data(),
                                   in_tensor->GetData().size(), RT_MEMCPY_HOST_TO_HOST);
       if (rt_ret != RT_ERROR_NONE) {
+        REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, dst_size:%lu, src_size:%zu, ret:0x%X",
+                          bufferSizeVec[i], in_tensor->GetData().size(), rt_ret);
         GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
         return RT_FAILED;
       }
@@ -250,6 +263,8 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vector<GeTe
   }
 
   if (graph_run_listener_->ResetResult() != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Call graph_run_listener_.ResetResult fail, model_id:%u",
+                      model_id);
     GELOGE(GE_GRAPH_EXECUTE_FAILED, "Reset result failed");
     return GE_GRAPH_EXECUTE_FAILED;
   }
@@ -273,6 +288,8 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vector<GeTe
     // Run graph return
     uint32_t result_code = graph_run_listener_->GetResultCode();
     if (result_code != SUCCESS && result_code != END_OF_SEQUENCE) {
+      REPORT_CALL_ERROR("E19999", "Graph_run_listener_ run fail, result:%u, model_id:%u",
+                        result_code, model_id);
       GELOGE(GE_GRAPH_EXECUTE_FAILED, "[GraphExecutor] execute model failed, ret=%u, modelId=%u.", result_code,
              model_id);
       return GE_GRAPH_EXECUTE_FAILED;
@@ -281,10 +298,14 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vector<GeTe
   for (size_t i = 0; i < output_data.blobs.size(); ++i) {
     DataBuffer outputDataTmp = output_data.blobs[i];
     CHECK_FALSE_EXEC(outputDataTmp.length != 0,
+                     REPORT_INNER_ERROR("E19999", "Param output_data.length is 0 in model:%u, check invalid",
+                                        model_id);
                      GELOGE(GE_GRAPH_EXECUTE_FAILED, "Failed to allocate memory, length is 0.");
                      return GE_GRAPH_EXECUTE_FAILED);
     std::unique_ptr<uint8_t> outBufTmp(new (std::nothrow) uint8_t[outputDataTmp.length]);
     if (outBufTmp == nullptr) {
+      REPORT_CALL_ERROR("E19999", "New output buffer fail, length:%lu, model:%u",
+                        outputDataTmp.length, model_id);
       GELOGE(FAILED, "Failed to allocate memory.");
       return FAILED;
     }
@@ -292,6 +313,8 @@ Status GraphExecutor::SyncExecuteModel(uint32_t model_id, const std::vector<GeTe
     rtError_t ret_value = rtMemcpy(outBufTmp.get(), outputDataTmp.length, outputDataTmp.data, outputDataTmp.length,
                                    RT_MEMCPY_HOST_TO_HOST);
     CHECK_FALSE_EXEC(ret_value == RT_ERROR_NONE,
+                     REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, dst_size:%lu, src_size:%zu, ret:0x%X",
+                                       outputDataTmp.length, outputDataTmp.length, ret_value);
                      GELOGE(GE_GRAPH_EXECUTE_FAILED, "Call rt api rtMemcpy failed, ret: 0x%X", ret);
                      return GE_GRAPH_EXECUTE_FAILED);
     GeTensor outTensor;
@@ -344,6 +367,8 @@ Status GraphExecutor::ExecuteGraph(GraphId graph_id, const GeRootModelPtr &ge_ro
   last_graph_id_ = graph_id;
 
   if (!init_flag_) {
+    REPORT_INNER_ERROR("E19999", "No SetCondition called before, graph:%u, check invalid",
+                       graph_id);
     GELOGE(GE_GRAPH_EXECUTE_NOT_INIT, "[GraphExecutor] AI Core Engine without calling SetCondition!");
     return GE_GRAPH_EXECUTE_NOT_INIT;
   }
@@ -358,7 +383,8 @@ Status GraphExecutor::ExecuteGraph(GraphId graph_id, const GeRootModelPtr &ge_ro
 }
 
 Status GraphExecutor::ExecuteGraphAsync(GraphId graph_id, const GeRootModelPtr &ge_root_model,
-                                        const std::vector<InputTensorInfo> &input_tensor) {
+                                        const std::vector<InputTensorInfo> &input_tensor,
+                                        const RunAsyncCallback& callback) {
   GELOGI("[GraphExecutor] Start to async execute graph, graph_id=%u", graph_id);
   if (graph_id != last_graph_id_) {
     auto ret = FreeExecuteMemory();
@@ -368,7 +394,7 @@ Status GraphExecutor::ExecuteGraphAsync(GraphId graph_id, const GeRootModelPtr &
   }
   last_graph_id_ = graph_id;
   GE_CHECK_NOTNULL_EXEC(ge_root_model, return FAILED);
-  Status ret = AsyncExecuteModel(ge_root_model->GetModelId(), input_tensor);
+  Status ret = AsyncExecuteModel(ge_root_model, input_tensor, callback);
   if (ret != SUCCESS) {
     GELOGE(GE_GRAPH_SYNC_MODEL_FAILED, "[GraphExecutor] AsyncExecuteModel Error!");
     return GE_GRAPH_SYNC_MODEL_FAILED;
@@ -378,11 +404,81 @@ Status GraphExecutor::ExecuteGraphAsync(GraphId graph_id, const GeRootModelPtr &
   return SUCCESS;
 }
 
-Status GraphExecutor::AsyncExecuteModel(uint32_t model_id, const std::vector<InputTensorInfo> &inputs) {
+bool CompareByLoad(const Uint32Pair &lhs, const Uint32Pair &rhs) {
+  return lhs.second < rhs.second;
+}
+
+uint32_t GraphExecutor::GetExecuteModelId(const GeRootModelPtr &ge_root_model) {
+  std::vector<uint32_t> model_ids = ge_root_model->GetAllModelId();
+  if (model_ids.empty()) {
+    return kInvalidModelId;
+  }
+  if (model_ids.size() == 1) {
+    return ge_root_model->GetModelId();
+  }
+  std::vector<Uint32Pair> model_id_to_loads;
+  auto model_manager = ModelManager::GetInstance();
+  GE_CHECK_NOTNULL(model_manager);
+  for (auto model_id : model_ids) {
+    auto davinci_model = model_manager->GetModel(model_id);
+    auto hybrid_model = model_manager->GetHybridModel(model_id);
+    if (hybrid_model == nullptr) {
+      GE_CHECK_NOTNULL(davinci_model);
+    }
+    uint32_t input_load = hybrid_model != nullptr ? hybrid_model->GetDataInputerSize() :
+                                                    davinci_model->GetDataInputerSize();
+    uint32_t running_load = hybrid_model != nullptr ? static_cast<uint32_t>(hybrid_model->GetRunningFlag()) :
+                                                      static_cast<uint32_t>(davinci_model->GetRunningFlag());
+    uint32_t load = input_load + running_load;
+    if (load == 0) {
+      return model_id;
+    }
+    model_id_to_loads.emplace_back(model_id, load);
+  }
+  sort(model_id_to_loads.begin(), model_id_to_loads.end(), CompareByLoad);
+  if (model_id_to_loads.empty()) {
+    return kInvalidModelId;
+  }
+  return model_id_to_loads.begin()->first;
+}
+
+Status GraphExecutor::SetCallback(uint32_t model_id, const GeRootModelPtr &ge_root_model,
+                                  const RunAsyncCallback &callback) {
+  auto model_manager = ge::ModelManager::GetInstance();
+  GE_CHECK_NOTNULL(model_manager);
+  if (model_manager->IsNeedHybridLoad(*ge_root_model)) {
+    auto model = model_manager->GetHybridModel(model_id);
+    GE_CHECK_NOTNULL(model);
+    if (model->SetRunAsyncListenerCallback(callback) != SUCCESS) {
+      GELOGE(FAILED, "SetRunAsyncListenerCallback failed.");
+      return FAILED;
+    }
+  } else {
+    auto model = model_manager->GetModel(model_id);
+    GE_CHECK_NOTNULL(model);
+    if (model->SetRunAsyncListenerCallback(callback) != SUCCESS) {
+      GELOGE(FAILED, "SetRunAsyncListenerCallback failed.");
+      return FAILED;
+    }
+  }
+  return SUCCESS;
+}
+
+Status GraphExecutor::AsyncExecuteModel(const GeRootModelPtr &ge_root_model, const std::vector<InputTensorInfo> &inputs,
+                                        const RunAsyncCallback &callback) {
+  uint32_t model_id = GetExecuteModelId(ge_root_model);
+  if (model_id == kInvalidModelId) {
+    GELOGE(INTERNAL_ERROR, "No valid model id.");
+    return INTERNAL_ERROR;
+  }
   try {
     auto model_manager = ge::ModelManager::GetInstance();
     GE_CHECK_NOTNULL(model_manager);
     GELOGI("RunAsync begin.model_id %u", model_id);
+    if (SetCallback(model_id, ge_root_model, callback) != SUCCESS) {
+      GELOGE(FAILED, "RunAsync: SetCallBack for model fail");
+      return FAILED;
+    }
 
     Status ret = model_manager->DataInputTensor(model_id, inputs);
     if (ret != SUCCESS) {
@@ -392,10 +488,12 @@ Status GraphExecutor::AsyncExecuteModel(uint32_t model_id, const std::vector<Inp
 
     GELOGI("RunAsync success.");
   } catch (std::bad_alloc &) {
+    REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed");
     GELOGE(MEMALLOC_FAILED, "RunAsync failed, bad memory allocation occur !");
     CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return MEMALLOC_FAILED;
   } catch (...) {
+    REPORT_INNER_ERROR("E19999", "Some exceptions occur failed");
     GELOGE(FAILED, "RunAsync failed, some exceptions occur !");
     CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return FAILED;
@@ -415,10 +513,12 @@ Status GraphExecutor::DataInput(const InputData &input_data, OutputData &output_
       return ret;
     }
   } catch (std::bad_alloc &) {
+    REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed");
     GELOGE(MEMALLOC_FAILED, "DataInput failed, bad memory allocation occur !");
     CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return MEMALLOC_FAILED;
   } catch (...) {
+    REPORT_INNER_ERROR("E19999", "Some exceptions occur failed");
     GELOGE(FAILED, "DataInput failed, some exceptions occur !");
     CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return FAILED;
@@ -439,10 +539,12 @@ Status GraphExecutor::GetInputOutputDescInfo(const uint32_t model_id, vector<Inp
       return ret;
     }
   } catch (std::bad_alloc &) {
+    REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed");
     GELOGE(MEMALLOC_FAILED, "GetInputOutputDescInfo failed, bad memory allocation occur !");
     CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return MEMALLOC_FAILED;
   } catch (...) {
+    REPORT_INNER_ERROR("E19999", "Some exceptions occur failed");
     GELOGE(FAILED, "GetInputOutputDescInfo failed, some exceptions occur !");
     CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return FAILED;
@@ -466,10 +568,12 @@ Status GraphExecutor::GetInputOutputDescInfo(const uint32_t model_id, vector<Inp
       return ret;
     }
   } catch (std::bad_alloc &) {
+    REPORT_INNER_ERROR("E19999", "Bad memory allocation exception occur failed");
     GELOGE(MEMALLOC_FAILED, "GetInputOutputDescInfo failed, bad memory allocation occur !");
     CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return MEMALLOC_FAILED;
   } catch (...) {
+    REPORT_INNER_ERROR("E19999", "Some exceptions occur failed");
     GELOGE(FAILED, "GetInputOutputDescInfo failed, some exceptions occur !");
     CsaInteract::GetInstance().WriteErrorCode(FAILED, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
     return FAILED;

ge/graph/execute/graph_execute.h

@@ -50,7 +50,7 @@ class GraphExecutor {
                       std::vector<GeTensor> &output_tensor);
 
   ge::Status ExecuteGraphAsync(GraphId graph_id, const GeRootModelPtr &ge_root_model,
-                               const std::vector<InputTensorInfo> &input_tensor);
+                               const std::vector<InputTensorInfo> &input_tensor, const RunAsyncCallback &callback);
 
   Status SetCondition(std::mutex *mutex, std::condition_variable *cond, std::shared_ptr<GraphModelListener> listener);
 
@@ -116,6 +116,8 @@ class GraphExecutor {
 
   static Status GetOpDescInfo(uint32_t device_id, uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info);
 
+  uint32_t GetExecuteModelId(const GeRootModelPtr &ge_root_model);
+
  private:
   Status PrepareInputData(const std::vector<GeTensor> &input_tensor, InputData &graph_input_data,
                           OutputData &graph_output_data, std::vector<InputOutputDescInfo> &output_desc);
@@ -123,7 +125,8 @@ class GraphExecutor {
   Status SyncExecuteModel(uint32_t model_id, const std::vector<GeTensor> &input_tensor,
                           std::vector<GeTensor> &output_tensor);
 
-  Status AsyncExecuteModel(uint32_t model_id, const std::vector<InputTensorInfo> &input_tensor);
+  Status AsyncExecuteModel(const GeRootModelPtr &ge_root_model, const std::vector<InputTensorInfo> &input_tensor,
+                           const RunAsyncCallback &callback);
 
   void InitModelIdInfo(std::vector<uint32_t> &out_model_id_info, std::vector<SubGraphInfoPtr> &sub_graph_vec,
                        uint32_t output_size);
@@ -132,6 +135,9 @@ class GraphExecutor {
 
   Status MallocInOutBuffer(const std::vector<uint64_t> &buffer_size, std::vector<void *> &data_addr);
 
+  static Status SetCallback(uint32_t model_id, const GeRootModelPtr &ge_root_model,
+                            const RunAsyncCallback &callback);
+
   bool init_flag_;
 
   bool train_graph_flag_;

ge/graph/label/case_label_maker.cc

@@ -42,6 +42,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) {
 
   const auto graph_names = case_desc->GetSubgraphInstanceNames();
   if (graph_names.empty() || graph_names.size() > kMaxCaseBranch) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) subgraph size: %zu, check invalid", case_desc->GetName().c_str(),
+                       case_desc->GetType().c_str(), graph_names.size());
     GELOGE(INTERNAL_ERROR, "Node: %s has invalid subgraph, graph size: %zu.", case_desc->GetName().c_str(),
            graph_names.size());
     return FAILED;
@@ -67,6 +69,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) {
       parent_node_->GetName() + "/StreamActive_" + std::to_string(index);  // rtStreamActive
     NodePtr stream_active = AddStreamActive(graph, stream_active_name);
     if (stream_active == nullptr) {
+      REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail",
+                        graph->GetName().c_str());
       GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", graph->GetName().c_str());
       return FAILED;
     }
@@ -75,6 +79,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) {
     std::string label_set_name = parent_node_->GetName() + "/LabelSet_" + std::to_string(index);  // rtLabelSet
     NodePtr label = AddLabelSetEnter(graph, label_set_name, curr_label_index, stream_active);
     if (label == nullptr) {
+      REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail",
+                        graph->GetName().c_str());
       GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", graph->GetName().c_str());
       return FAILED;
     }
@@ -88,6 +94,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) {
       // middle node, add goto node to tail.
       std::string label_goto_name = parent_node_->GetName() + "/LabelGoto_" + std::to_string(index);  // rtLabelGoto
       if (AddLabelGotoLeave(graph, label_goto_name, last_label_index) == nullptr) {
+        REPORT_CALL_ERROR("E19999", "Add LabelGotoLeave node in graph:%s fail",
+                          graph->GetName().c_str());
         GELOGE(INTERNAL_ERROR, "Subgraph: %s add label goto failed.", graph->GetName().c_str());
         return FAILED;
       }
@@ -95,6 +103,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) {
       // last node, add label node to tail.
       std::string last_label_name = parent_node_->GetName() + "/LabelSet_Last";  // rtLabelSet
       if (AddLabelSetLeave(graph, last_label_name, last_label_index) == nullptr) {
+        REPORT_CALL_ERROR("E19999", "Add LabelSetLeave node in graph:%s fail",
+                          graph->GetName().c_str());
         GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", graph->GetName().c_str());
         return FAILED;
       }
@@ -110,12 +120,16 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) {
   const GeTensorDesc &pred_desc = case_desc->GetInputDesc(kCasePredIndex);
   NodePtr switch_node = AddLabelSwitchEnter(first_graph, label_switch_name, pred_desc, switch_labels);
   if (switch_node == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSwitchEnter node in graph:%s fail",
+                      first_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label switch failed.", first_graph->GetName().c_str());
     return FAILED;
   }
 
   // Link control edge to then branch head.
   if (GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), first_label->GetInControlAnchor()) != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", switch_node->GetName().c_str(),
+                      first_label->GetName().c_str(), first_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add ctrl edge to %s failed.", first_label->GetName().c_str());
     return FAILED;
   }
@@ -123,6 +137,8 @@ Status CaseOpLabelMaker::Run(uint32_t &label_index) {
   uint32_t parent_index = 0;  // Case cond input is first.
   const std::string data_name = parent_node_->GetName() + "/SwitchIndexData";
   if (AddLabelSwitchIndex(first_graph, data_name, pred_desc, switch_node, parent_index) == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSwitchIndex node in graph:%s fail",
+                      first_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add switch input failed.", first_graph->GetName().c_str());
     return FAILED;
   }

ge/graph/label/if_label_maker.cc

@@ -43,6 +43,10 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) {
   const std::string then_branch_name = if_desc->GetSubgraphInstanceName(kThenBranchIndex);
   const std::string else_branch_name = if_desc->GetSubgraphInstanceName(kElseBranchIndex);
   if (then_branch_name.empty() || else_branch_name.empty()) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s), check subgraph invalid, "
+                       "then branch graph: %s, else branch graph: %s",
+                       if_desc->GetName().c_str(), if_desc->GetType().c_str(),
+                       then_branch_name.c_str(), else_branch_name.c_str());
     GELOGE(INTERNAL_ERROR, "Node: %s has invalid subgraph, then branch: %s, else branch: %s.",
            if_desc->GetName().c_str(), then_branch_name.c_str(), else_branch_name.c_str());
     return FAILED;
@@ -66,32 +70,44 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) {
 
   NodePtr then_stream_active = AddStreamActive(then_sub_graph, then_active_name);
   if (then_stream_active == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail",
+                      then_sub_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", then_sub_graph->GetName().c_str());
     return FAILED;
   }
 
   NodePtr then_enter_label = AddLabelSetEnter(then_sub_graph, then_label_name, then_enter_index, then_stream_active);
   if (then_enter_label == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail",
+                      then_sub_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", then_sub_graph->GetName().c_str());
     return FAILED;
   }
 
   if (AddLabelGotoLeave(then_sub_graph, then_leave_name, else_leave_index) == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelGotoLeave node in graph:%s fail",
+                      then_sub_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label goto failed.", then_sub_graph->GetName().c_str());
     return FAILED;
   }
 
   NodePtr else_stream_active = AddStreamActive(else_sub_graph, else_active_name);
   if (else_stream_active == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail",
+                      else_stream_active->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", else_sub_graph->GetName().c_str());
     return FAILED;
   }
 
   if (AddLabelSetEnter(else_sub_graph, else_enter_name, else_enter_index, else_stream_active) == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail",
+                      else_sub_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", else_sub_graph->GetName().c_str());
     return FAILED;
   }
   if (AddLabelSetLeave(else_sub_graph, else_leave_name, else_leave_index) == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSetLeave node in graph:%s fail",
+                      else_sub_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", else_sub_graph->GetName().c_str());
     return FAILED;
   }
@@ -103,12 +119,16 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) {
   const GeTensorDesc &pred_desc = if_desc->GetInputDesc(kIfPredIndex);
   NodePtr switch_node = AddLabelSwitchEnter(then_sub_graph, then_enter_name, pred_desc, switch_labels);
   if (switch_node == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSwitchEnter node in graph:%s fail",
+                      then_sub_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label switch failed.", then_sub_graph->GetName().c_str());
     return FAILED;
   }
 
   // Link control edge to then branch head.
   if (GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), then_enter_label->GetInControlAnchor()) != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", switch_node->GetName().c_str(),
+                      then_enter_label->GetName().c_str(), then_sub_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add ctrl edge to %s failed.", then_enter_label->GetName().c_str());
     return FAILED;
   }
@@ -116,6 +136,8 @@ Status IfOpLabelMaker::Run(uint32_t &label_index) {
   uint32_t parent_index = 0;  // If cond input is first.
   const std::string data_name = parent_node_->GetName() + "/SwitchIndexData";
   if (AddLabelSwitchIndex(then_sub_graph, data_name, pred_desc, switch_node, parent_index) == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSwitchIndex node in graph:%s fail",
+                      then_sub_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add switch input failed.", then_sub_graph->GetName().c_str());
     return FAILED;
   }

ge/graph/label/label_maker.cc

@@ -56,6 +56,8 @@ void LabelMaker::LinkToGraphHead(const ComputeGraphPtr &graph, const NodePtr &no
     }
 
     if (GraphUtils::AddEdge(node->GetOutControlAnchor(), n->GetInControlAnchor()) != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", node->GetName().c_str(),
+                        n->GetName().c_str(), graph->GetName().c_str());
       GELOGE(INTERNAL_ERROR, "Add ctrl edge from %s to %s failed.", node->GetName().c_str(), n->GetName().c_str());
     }
   }
@@ -78,6 +80,8 @@ void LabelMaker::LinkToGraphTail(const ComputeGraphPtr &graph, const NodePtr &no
     }
 
     if (GraphUtils::AddEdge(tail->GetOutControlAnchor(), node->GetInControlAnchor()) != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", tail->GetName().c_str(),
+                        node->GetName().c_str(), graph->GetName().c_str());
       GELOGE(INTERNAL_ERROR, "Add ctrl edge from %s to %s failed.", tail->GetName().c_str(), node->GetName().c_str());
     }
     return;
@@ -96,6 +100,7 @@ NodePtr LabelMaker::AddStreamActive(const ComputeGraphPtr &graph, const std::str
 
   const auto &node_list = graph->GetDirectNode();
   if (node_list.empty()) {
+    REPORT_INNER_ERROR("E19999", "Check param graph:%s has no node", graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSet: Graph %s node is empty.", graph->GetName().c_str());
     return nullptr;
   }
@@ -131,6 +136,7 @@ NodePtr LabelMaker::AddLabelSetEnter(const ComputeGraphPtr &graph, const std::st
 
   const auto &node_list = graph->GetDirectNode();
   if (node_list.empty()) {
+    REPORT_INNER_ERROR("E19999", "Check param graph:%s has no node", graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSet: Graph %s node is empty.", graph->GetName().c_str());
     return nullptr;
   }
@@ -145,6 +151,8 @@ NodePtr LabelMaker::AddLabelSetEnter(const ComputeGraphPtr &graph, const std::st
   GE_CHECK_NOTNULL_EXEC(label_set, return nullptr);
 
   if (GraphUtils::AddEdge(label_set->GetOutControlAnchor(), stream_active->GetInControlAnchor()) != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", label_set->GetName().c_str(),
+                      stream_active->GetName().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Add ctrl edge from %s to %s failed.", label_set->GetName().c_str(),
            stream_active->GetName().c_str());
     return nullptr;
@@ -193,6 +201,7 @@ NodePtr LabelMaker::AddLabelGotoEnter(const ComputeGraphPtr &graph, const std::s
   const auto &node_list = graph->GetDirectNode();
   auto it = node_list.begin();
   if (it == node_list.end()) {
+    REPORT_INNER_ERROR("E19999", "Check param graph:%s has no node", graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelGoto: Graph %s node is empty.", graph->GetName().c_str());
     return nullptr;
   }
@@ -205,6 +214,8 @@ NodePtr LabelMaker::AddLabelGotoEnter(const ComputeGraphPtr &graph, const std::s
   (void)AttrUtils::SetInt(op_desc, ATTR_NAME_LABEL_SWITCH_INDEX, index);
   NodePtr label_goto = graph->AddNodeFront(op_desc);
   if (label_goto == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s fail",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelGoto: Add to graph %s failed.", graph->GetName().c_str());
     return nullptr;
   }
@@ -253,6 +264,7 @@ NodePtr LabelMaker::AddLabelSwitchEnter(const ComputeGraphPtr &graph, const std:
   const auto &node_list = graph->GetDirectNode();
   auto it = node_list.begin();
   if (it == node_list.end()) {
+    REPORT_INNER_ERROR("E19999", "Check param graph:%s has no node", graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Graph %s node is empty.", graph->GetName().c_str());
     return nullptr;
   }
@@ -263,17 +275,23 @@ NodePtr LabelMaker::AddLabelSwitchEnter(const ComputeGraphPtr &graph, const std:
 
   GELOGI("LabelSwitchByIndex: Create node %s.", op_desc->GetName().c_str());
   if (op_desc->AddInputDesc(desc) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add input desc into node:%s(%s) in graph:%s fail",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add input desc failed.");
     return nullptr;
   }
 
   if (!AttrUtils::SetListInt(op_desc, ATTR_NAME_LABEL_SWITCH_LIST, labels)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_LABEL_SWITCH_LIST.c_str(),
+                       op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add %s failed.", ATTR_NAME_LABEL_SWITCH_INDEX.c_str());
     return nullptr;
   }
 
   NodePtr label_switch = graph->AddNodeFront(op_desc);
   if (label_switch == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s ahead fail",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add to graph %s failed.", graph->GetName().c_str());
     return nullptr;
   }
@@ -300,11 +318,15 @@ NodePtr LabelMaker::AddLabelSwitchLeave(const ComputeGraphPtr &graph, const std:
 
   GELOGI("LabelSwitchByIndex: Create node %s.", op_desc->GetName().c_str());
   if (op_desc->AddInputDesc(desc) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add input desc into node:%s(%s) in graph:%s fail",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add input desc failed.");
     return nullptr;
   }
 
   if (!AttrUtils::SetListInt(op_desc, ATTR_NAME_LABEL_SWITCH_LIST, labels)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_LABEL_SWITCH_LIST.c_str(),
+                       op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add %s failed.", ATTR_NAME_LABEL_SWITCH_INDEX.c_str());
     return nullptr;
   }
@@ -336,15 +358,21 @@ NodePtr LabelMaker::AddLabelSwitchIndex(const ComputeGraphPtr &graph, const std:
 
   GELOGI("Data: Create node %s.", op_desc->GetName().c_str());
   if (op_desc->AddInputDesc(desc) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add input desc into node:%s(%s) in graph:%s fail",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add data input desc failed.");
     return nullptr;
   }
   if (op_desc->AddOutputDesc(desc) != GRAPH_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add output desc into node:%s(%s) in graph:%s fail",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add data output desc failed.");
     return nullptr;
   }
 
   if (!AttrUtils::SetInt(op_desc, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) {
+    REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_PARENT_NODE_INDEX.c_str(),
+                       op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add %s failed.", ATTR_NAME_PARENT_NODE_INDEX.c_str());
     return nullptr;
   }
@@ -354,6 +382,8 @@ NodePtr LabelMaker::AddLabelSwitchIndex(const ComputeGraphPtr &graph, const std:
 
   // Link control edge to graph head.
   if (GraphUtils::AddEdge(op_data->GetOutDataAnchor(0), sw_node->GetInDataAnchor(0)) != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail", op_data->GetName().c_str(),
+                      sw_node->GetName().c_str(), graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "LabelSwitchByIndex: Add input edge to %s failed.", op_data->GetName().c_str());
     return nullptr;
   }

ge/graph/label/partitioned_call_label_maker.cc

@@ -39,12 +39,17 @@ Status PartitionedCallLabelMaker::Run(uint32_t &label_index) {
 
   std::string sub_graph_name = call_desc->GetSubgraphInstanceName(kSubGraphIndex);
   if (sub_graph_name.empty()) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) subgraph_index:%d name is empty, check invalid",
+                       call_desc->GetName().c_str(), call_desc->GetType().c_str(), kSubGraphIndex);
     GELOGE(INTERNAL_ERROR, "Node: %s has no subgraph name.", sub_graph_name.c_str());
     return FAILED;
   }
 
   ComputeGraphPtr sub_graph = parent_graph_->GetSubgraph(sub_graph_name);
   if (sub_graph == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) subgraph_name:%s is not exist in parent_graph, check invalid",
+                       call_desc->GetName().c_str(), call_desc->GetType().c_str(),
+                       sub_graph_name.c_str());
     GELOGE(INTERNAL_ERROR, "Node: %s has no subgraph.", sub_graph_name.c_str());
     return FAILED;
   }
@@ -52,6 +57,8 @@ Status PartitionedCallLabelMaker::Run(uint32_t &label_index) {
   const std::string stream_active_name = parent_node_->GetName() + "/StreamActive"; // rtStreamActive
   NodePtr stream_active = AddStreamActive(sub_graph, stream_active_name);
   if (stream_active == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail",
+                      sub_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active node failed.", sub_graph->GetName().c_str());
     return FAILED;
   }

ge/graph/label/while_label_maker.cc

@@ -44,6 +44,9 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) {
   std::string cond_name = while_desc->GetSubgraphInstanceName(kCondBranchIndex);
   std::string body_name = while_desc->GetSubgraphInstanceName(kBodyBranchIndex);
   if (cond_name.empty() || body_name.empty()) {
+    REPORT_INNER_ERROR("E19999", "Node:%s(%s) cond subgraph index:%d or body subgraph index:%d name is empty, "
+                       "check invalid", while_desc->GetName().c_str(), while_desc->GetType().c_str(),
+                       kCondBranchIndex, kBodyBranchIndex);
     GELOGE(INTERNAL_ERROR, "Node: %s has invalid subgraph, cond branch: %s, body branch: %s.",
            while_desc->GetName().c_str(), cond_name.c_str(), body_name.c_str());
     return FAILED;
@@ -67,32 +70,44 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) {
 
   NodePtr cond_stream_active = AddStreamActive(cond_graph, cond_active_name);
   if (cond_stream_active == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail",
+                      cond_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", cond_graph->GetName().c_str());
     return FAILED;
   }
 
   if (AddLabelSetEnter(cond_graph, cond_enter_name, cond_enter_index, cond_stream_active) == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail",
+                      cond_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", cond_graph->GetName().c_str());
     return FAILED;
   }
 
   NodePtr body_stream_active = AddStreamActive(body_graph, body_active_name);
   if (body_stream_active == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add StreamActive node in graph:%s fail",
+                      body_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add stream active failed.", body_graph->GetName().c_str());
     return FAILED;
   }
 
   if (AddLabelSetEnter(body_graph, body_enter_name, body_enter_index, body_stream_active) == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSetEnter node in graph:%s fail",
+                      body_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", body_graph->GetName().c_str());
     return FAILED;
   }
 
   if (AddLabelGotoLeave(body_graph, goto_leave_name, cond_enter_index) == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelGotoLeave node in graph:%s fail",
+                      body_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label goto failed.", body_graph->GetName().c_str());
     return FAILED;
   }
 
   if (AddLabelSetLeave(body_graph, body_leave_name, body_leave_index) == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSetLeave node in graph:%s fail",
+                      body_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label set failed.", body_graph->GetName().c_str());
     return FAILED;
   }
@@ -109,6 +124,8 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) {
   const std::vector<uint32_t> switch_labels = {body_leave_index, body_enter_index};
   NodePtr switch_node = AddLabelSwitchLeave(cond_graph, cond_leave_name, pred_desc, switch_labels);
   if (switch_node == nullptr) {
+    REPORT_CALL_ERROR("E19999", "Add LabelSwitchLeave node in graph:%s fail",
+                      cond_graph->GetName().c_str());
     GELOGE(INTERNAL_ERROR, "Subgraph: %s add label switch failed.", cond_graph->GetName().c_str());
     return FAILED;
   }
@@ -124,6 +141,9 @@ Status WhileOpLabelMaker::Run(uint32_t &label_index) {
   InDataAnchorPtr in_anchor = all_in_data.at(kCondOutputIndex);
   GE_CHECK_NOTNULL(in_anchor);
   if (GraphUtils::AddEdge(in_anchor->GetPeerOutAnchor(), switch_node->GetInDataAnchor(kCondOutputIndex)) != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Add ctrl edge from %s to %s in graph:%s fail",
+                      in_anchor->GetPeerOutAnchor()->GetOwnerNode()->GetName().c_str(),
+                      switch_node->GetName().c_str(), cond_graph->GetName().c_str());
     GELOGE(FAILED, "Node: %s Add pred data input failed.", switch_node->GetName().c_str());
     return FAILED;
   }

ge/graph/load/graph_loader.cc

@@ -52,15 +52,17 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge
   GELOGI("Load model online begin.");
   rtError_t rt_ret = rtSetDevice(GetContext().DeviceId());
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, ret:0x%X",
+                      GetContext().DeviceId(), rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_LOAD);
     return RT_FAILED;
   }
   if (ge_root_model_ptr == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param ge_root_model_ptr nullptr, check invalid");
     GELOGE(GE_GRAPH_PARAM_NULLPTR, "[LoadGraph] GE load graph model_ptr is nullptr.");
     return GE_GRAPH_PARAM_NULLPTR;
   }
-  model_id = ge_root_model_ptr->GetModelId();
 
   auto model_manager = ModelManager::GetInstance();
   GE_CHECK_NOTNULL(model_manager);
@@ -71,6 +73,8 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge
 
     rt_ret = rtDeviceReset(GetContext().DeviceId());
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X",
+                        GetContext().DeviceId(), rt_ret);
       GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     }
     return ret;
@@ -84,6 +88,8 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge
 
     rt_ret = rtDeviceReset(GetContext().DeviceId());
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X",
+                        GetContext().DeviceId(), rt_ret);
       GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     }
 
@@ -93,6 +99,8 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge
   }
   rt_ret = rtDeviceReset(GetContext().DeviceId());
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X",
+                      GetContext().DeviceId(), rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     return RT_FAILED;
   }
@@ -121,6 +129,8 @@ Status GraphLoader::LoadDataFromFile(const std::string &path, const std::string
 
   GELOGI("Load model begin, model path is: %s", path.c_str());
   if (!key_path.empty() && !CheckInputPathValid(key_path)) {
+    REPORT_INNER_ERROR("E19999", "Param key_path:%s empty or invalid",
+                       key_path.c_str());
     GELOGE(ACL_ERROR_GE_PARAM_INVALID, "decrypt_key path is invalid: %s", key_path.c_str());
     return ACL_ERROR_GE_PARAM_INVALID;
   }
@@ -147,10 +157,12 @@ Status GraphLoader::CommandHandle(const Command &command) {
       return ret;
     }
   } catch (std::bad_alloc &) {
+    REPORT_INNER_ERROR("E19999", "Bad memory allocation occur");
     GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Command handle failed, bad memory allocation occur !");
 
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   } catch (...) {
+    REPORT_INNER_ERROR("E19999", "Some exceptions occur");
     GELOGE(FAILED, "Command handle failed, some exceptions occur !");
 
     return FAILED;
@@ -232,6 +244,8 @@ Status GraphLoader::ExecuteModel(uint32_t model_id, rtStream_t stream, bool asyn
 Status GraphLoader::GetMemoryInfo(int64_t &free) {
   rtError_t rt_ret = rtSetDevice(GetContext().DeviceId());
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, ret:0x%X",
+                      GetContext().DeviceId(), rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_LOAD);
     return RT_FAILED;
@@ -240,11 +254,14 @@ Status GraphLoader::GetMemoryInfo(int64_t &free) {
   size_t free_mem = 0;
   rt_ret = rtMemGetInfo(&free_mem, &total_mem);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMemGetInfo failed, ret:0x%X", rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     return RT_FAILED;
   }
   rt_ret = rtDeviceReset(GetContext().DeviceId());
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X",
+                      GetContext().DeviceId(), rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     return RT_FAILED;
   }

ge/graph/load/model_manager/cpu_queue_schedule.cc

@@ -51,6 +51,8 @@ CpuTaskInfo::~CpuTaskInfo() {
 ///
 Status CpuTaskModelDequeue::Init(uint32_t queue_id, uintptr_t &in_mbuf) {
   if ((args_ != nullptr) || (args_size_ > 0)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task already initialized, size: %u", args_size_);
     return FAILED;
   }
@@ -58,6 +60,8 @@ Status CpuTaskModelDequeue::Init(uint32_t queue_id, uintptr_t &in_mbuf) {
   args_size_ = sizeof(MbufQueueInfo) + sizeof(uintptr_t);  // sizeof(uintptr_t) for save in_mbuf.
   rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -69,6 +73,8 @@ Status CpuTaskModelDequeue::Init(uint32_t queue_id, uintptr_t &in_mbuf) {
   queue_info.in_mbuf = in_mbuf;  // Placeholder, input mbuf addr will save to this place.
   status = rtMemcpy(args_, args_size_, &queue_info, sizeof(MbufQueueInfo), RT_MEMCPY_HOST_TO_DEVICE);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -78,12 +84,16 @@ Status CpuTaskModelDequeue::Init(uint32_t queue_id, uintptr_t &in_mbuf) {
 
 Status CpuTaskModelDequeue::Distribute() {
   if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_:%u is 0 or stream_ is nullptr,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_);
     return FAILED;
   }
 
   rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskModelDequeue, kCoreDim, args_, args_size_, nullptr, stream_);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X",
+                      status);
     GELOGE(RT_FAILED, "Call rt CpuKernelLaunch ModelDequeue failed, status: 0x%X", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -101,6 +111,8 @@ Status CpuTaskModelDequeue::Distribute() {
 ///
 Status CpuTaskZeroCopy::Init(std::vector<uintptr_t> &mbuf_list, const map<uint32_t, ZeroCopyOffset> &outside_addrs) {
   if ((args_ != nullptr) || (args_size_ > 0)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task already initialized, size: %u", args_size_);
     return FAILED;
   }
@@ -155,12 +167,16 @@ Status CpuTaskZeroCopy::Init(std::vector<uintptr_t> &mbuf_list, const map<uint32
 
 Status CpuTaskZeroCopy::Distribute() {
   if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_:%u is 0 or stream_ is nullptr,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_);
     return FAILED;
   }
 
   rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskZeroCopy, kCoreDim, args_, args_size_, nullptr, stream_);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X",
+                      status);
     GELOGE(RT_FAILED, "Call rt CpuKernelLaunch ZeroCopy failed, status: 0x%X", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -199,6 +215,8 @@ CpuTaskZeroCopy::~CpuTaskZeroCopy() {
 ///
 Status CpuTaskPrepareOutput::Init(uintptr_t addr, uint32_t size, uintptr_t in_mbuf, uintptr_t &out_mbuf) {
   if ((args_ != nullptr) || (args_size_ > 0)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task already initialized, size: %u", args_size_);
     return FAILED;
   }
@@ -206,6 +224,8 @@ Status CpuTaskPrepareOutput::Init(uintptr_t addr, uint32_t size, uintptr_t in_mb
   args_size_ = sizeof(PrepareOutputInfo) + sizeof(uintptr_t);  // sizeof(uintptr_t) for save out_mbuf.
   rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -220,6 +240,8 @@ Status CpuTaskPrepareOutput::Init(uintptr_t addr, uint32_t size, uintptr_t in_mb
   prepare.out_mbuf = out_mbuf;  // Placeholder, output mbuf addr will save to this place.
   status = rtMemcpy(args_, args_size_, &prepare, sizeof(PrepareOutputInfo), RT_MEMCPY_HOST_TO_DEVICE);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -229,12 +251,16 @@ Status CpuTaskPrepareOutput::Init(uintptr_t addr, uint32_t size, uintptr_t in_mb
 
 Status CpuTaskPrepareOutput::Distribute() {
   if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_:%u is 0 or stream_ is nullptr,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_);
     return FAILED;
   }
 
   rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskPrepareOutput, kCoreDim, args_, args_size_, nullptr, stream_);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X",
+                      status);
     GELOGE(RT_FAILED, "Call rt CpuKernelLaunch PrepareOutput failed, status: 0x%X", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -252,6 +278,8 @@ Status CpuTaskPrepareOutput::Distribute() {
 ///
 Status CpuTaskModelEnqueue::Init(uint32_t queue_id, uintptr_t out_mbuf) {
   if ((args_ != nullptr) || (args_size_ > 0)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task already initialized, size: %u", args_size_);
     return FAILED;
   }
@@ -260,6 +288,8 @@ Status CpuTaskModelEnqueue::Init(uint32_t queue_id, uintptr_t out_mbuf) {
   args_size_ = sizeof(MbufQueueInfo);
   rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -270,6 +300,8 @@ Status CpuTaskModelEnqueue::Init(uint32_t queue_id, uintptr_t out_mbuf) {
   queue_info.in_mbuf = out_mbuf;
   status = rtMemcpy(args_, args_size_, &queue_info, args_size_, RT_MEMCPY_HOST_TO_DEVICE);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -279,12 +311,16 @@ Status CpuTaskModelEnqueue::Init(uint32_t queue_id, uintptr_t out_mbuf) {
 
 Status CpuTaskModelEnqueue::Distribute() {
   if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_ is 0 or stream_ is nullptr, arg_size:%u,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_);
     return FAILED;
   }
 
   rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskModelEnqueue, kCoreDim, args_, args_size_, nullptr, stream_);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X",
+                      status);
     GELOGE(RT_FAILED, "Call rt CpuKernelLaunch ModelEnqueue failed, status: 0x%X", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -301,6 +337,7 @@ Status CpuTaskModelEnqueue::Distribute() {
 ///
 Status CpuTaskActiveEntry::Init(rtStream_t stream) {
   if (stream == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Param stream is nullptr, check invalid");
     GELOGE(FAILED, "Task active stream not valid");
     return FAILED;
   }
@@ -311,12 +348,16 @@ Status CpuTaskActiveEntry::Init(rtStream_t stream) {
 
 Status CpuTaskActiveEntry::Distribute() {
   if ((active_stream_ == nullptr) || (stream_ == nullptr)) {
+    REPORT_INNER_ERROR("E19999", "Param stream is nullptr or active_stream_ is nullptr, "
+                       "check invalid");
     GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_);
     return FAILED;
   }
 
   rtError_t ret = rtStreamActive(active_stream_, stream_);
   if (ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtStreamActive failed, ret:0x%X",
+                      ret);
     GELOGE(RT_FAILED, "Call rt StreamActive failed, ret: 0x%X", ret);
     return RT_ERROR_TO_GE_STATUS(ret);
   }
@@ -333,6 +374,8 @@ Status CpuTaskActiveEntry::Distribute() {
 ///
 Status CpuTaskWaitEndGraph::Init(uint32_t model_id) {
   if ((args_ != nullptr) || (args_size_ > 0)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task already initialized, size: %u", args_size_);
     return FAILED;
   }
@@ -340,6 +383,8 @@ Status CpuTaskWaitEndGraph::Init(uint32_t model_id) {
   args_size_ = sizeof(model_id);
   rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -347,6 +392,8 @@ Status CpuTaskWaitEndGraph::Init(uint32_t model_id) {
 
   status = rtMemcpy(args_, args_size_, &model_id, args_size_, RT_MEMCPY_HOST_TO_DEVICE);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -356,12 +403,16 @@ Status CpuTaskWaitEndGraph::Init(uint32_t model_id) {
 
 Status CpuTaskWaitEndGraph::Distribute() {
   if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_:%u is 0 or stream_ is nullptr,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_);
     return FAILED;
   }
 
   rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskWaitEndGraph, kCoreDim, args_, args_size_, nullptr, stream_);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X",
+                      status);
     GELOGE(RT_FAILED, "Call rt CpuKernelLaunch WaitEndGraph failed, status: 0x%X", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -378,6 +429,8 @@ Status CpuTaskWaitEndGraph::Distribute() {
 ///
 Status CpuTaskModelRepeat::Init(uint32_t model_id) {
   if ((args_ != nullptr) || (args_size_ > 0)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is not nullptr or args_size_:%u > 0,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task already initialized, size: %u", args_size_);
     return FAILED;
   }
@@ -385,6 +438,8 @@ Status CpuTaskModelRepeat::Init(uint32_t model_id) {
   args_size_ = sizeof(model_id);
   rtError_t status = rtMalloc(&args_, args_size_, RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt malloc failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -392,6 +447,8 @@ Status CpuTaskModelRepeat::Init(uint32_t model_id) {
 
   status = rtMemcpy(args_, args_size_, &model_id, args_size_, RT_MEMCPY_HOST_TO_DEVICE);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%u, ret:0x%X",
+                      args_size_, status);
     GELOGE(RT_FAILED, "Call rt memcpy failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -401,12 +458,16 @@ Status CpuTaskModelRepeat::Init(uint32_t model_id) {
 
 Status CpuTaskModelRepeat::Distribute() {
   if ((args_ == nullptr) || (args_size_ == 0) || (stream_ == nullptr)) {
+    REPORT_INNER_ERROR("E19999", "Param args_ is nullptr or args_size_:%u is 0 or stream_ is nullptr,"
+                       "check invalid", args_size_);
     GELOGE(FAILED, "Task not initialized, distribute failed, size: %u", args_size_);
     return FAILED;
   }
 
   rtError_t status = rtCpuKernelLaunch(nullptr, kCpuTaskModelRepeat, kCoreDim, args_, args_size_, nullptr, stream_);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtCpuKernelLaunch failed, ret:0x%X",
+                      status);
     GELOGE(RT_FAILED, "Call rt CpuKernelLaunch ModelRepeat failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }

ge/graph/load/model_manager/data_dumper.cc

@@ -72,24 +72,6 @@ static bool ParseNameIndex(const std::string &node_name_index, std::string &node
 static bool IsTensorDescWithSkipDumpAddrType(bool has_mem_type_attr, vector<int64_t> v_memory_type, size_t i) {
   return has_mem_type_attr && (v_memory_type[i] == RT_MEMORY_L1);
 }
-
-static uint64_t GetNowTime() {
-  uint64_t ret = 0;
-  mmTimeval tv;
-  if (mmGetTimeOfDay(&tv, nullptr) == 0) {
-    ret = tv.tv_sec * 1000000ULL + tv.tv_usec;
-  }
-
-  return ret;
-}
-
-static void ReplaceStringElem(std::string &str) {
-  for_each(str.begin(), str.end(), [](char &ch) {
-    if ((ch == ' ') || (ch == '.') || (ch == '/') || (ch == '\\')) {
-      ch = '_';
-    }
-  });
-}
 }  // namespace
 
 static int32_t GetIrDataType(ge::DataType data_type) {
@@ -194,66 +176,6 @@ void DataDumper::SaveOpDebugId(uint32_t task_id, uint32_t stream_id, void *op_de
   is_op_debug_ = is_op_debug;
 }
 
-void DataDumper::SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id,
-                                uint32_t stream_id) {
-  GELOGD("Start SaveDumpOpInfo of task_id: %u, stream_id: %u", task_id, stream_id);
-  OpDescInfo op_desc_info;
-  op_desc_info.op_name = op->GetName();
-  op_desc_info.op_type = op->GetType();
-  op_desc_info.task_id = task_id;
-  op_desc_info.stream_id = stream_id;
-  for (size_t i = 0; i < op->GetAllInputsSize(); ++i) {
-    GeTensorDescPtr input_tensor_desc = op->MutableInputDesc(i);
-    if (input_tensor_desc == nullptr) {
-      continue;
-    }
-    op_desc_info.input_format.emplace_back(input_tensor_desc->GetFormat());
-    op_desc_info.input_shape.emplace_back(input_tensor_desc->GetShape().GetDims());
-    op_desc_info.input_data_type.emplace_back(input_tensor_desc->GetDataType());
-    int64_t input_size = 0;
-
-    if (TensorUtils::GetTensorSizeInBytes(*input_tensor_desc, input_size) != SUCCESS) {
-      GELOGW("Get input size failed");
-      return;
-    }
-    GELOGD("Save dump op info, the input size is %ld", input_size);
-    op_desc_info.input_size.emplace_back(input_size);
-  }
-  for (size_t j = 0; j < op->GetOutputsSize(); ++j) {
-    GeTensorDescPtr output_tensor_desc = op->MutableOutputDesc(j);
-    if (output_tensor_desc == nullptr) {
-      continue;
-    }
-    op_desc_info.output_format.emplace_back(output_tensor_desc->GetFormat());
-    op_desc_info.output_shape.emplace_back(output_tensor_desc->GetShape().GetDims());
-    op_desc_info.output_data_type.emplace_back(output_tensor_desc->GetDataType());
-    int64_t output_size = 0;
-    if (TensorUtils::GetTensorSizeInBytes(*output_tensor_desc, output_size) != SUCCESS) {
-      GELOGW("Get input size failed");
-      return;
-    }
-    GELOGD("Save dump op info, the output size is %ld", output_size);
-    op_desc_info.output_size.emplace_back(output_size);
-  }
-  op_desc_info.input_addrs = ModelUtils::GetInputDataAddrs(model_param, op);
-  op_desc_info.output_addrs = ModelUtils::GetOutputDataAddrs(model_param, op);
-
-  op_desc_info_.emplace_back(op_desc_info);
-}
-
-bool DataDumper::GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const {
-  GELOGI("There are %zu op need to dump.", op_desc_info_.size());
-  for (size_t index = 0; index < op_desc_info_.size(); ++index) {
-    OpDescInfo dump_op_info = op_desc_info_.at(index);
-    if (dump_op_info.task_id == task_id && dump_op_info.stream_id == stream_id) {
-      GELOGI("find exception op of task_id: %u, stream_id: %u.", task_id, stream_id);
-      op_desc_info = dump_op_info;
-      return true;
-    }
-  }
-  return false;
-}
-
 void DataDumper::SaveDumpTask(uint32_t task_id, uint32_t stream_id, const std::shared_ptr<OpDesc> &op_desc,
                               uintptr_t args) {
   if (op_desc == nullptr) {
@@ -325,6 +247,7 @@ Status DataDumper::GenerateOutput(aicpu::dump::Output &output, const OpDesc::Vis
   }
   int64_t output_size = 0;
   if (TensorUtils::GetTensorSizeInBytes(tensor_descs.at(index), output_size) != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Get tensor size fail");
     GELOGE(PARAM_INVALID, "Get output size filed");
     return PARAM_INVALID;
   }
@@ -385,8 +308,11 @@ Status DataDumper::DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_inf
 
 Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task) {
   const auto &output_descs = inner_dump_info.op->GetAllOutputsDesc();
-  const std::vector<void *> output_addrs = ModelUtils::GetOutputDataAddrs(runtime_param_, inner_dump_info.op);
+  const std::vector<void *> output_addrs = ModelUtils::GetOutputDataAddrs(*runtime_param_, inner_dump_info.op);
   if (output_descs.size() != output_addrs.size()) {
+    REPORT_INNER_ERROR("E19999", "output_desc size:%zu != output addr size:%zu in op:%s(%s)",
+                       output_descs.size(), output_addrs.size(),
+                       inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str());
     GELOGE(PARAM_INVALID, "Invalid output desc addrs size %zu, op %s has %zu output desc.", output_addrs.size(),
            inner_dump_info.op->GetName().c_str(), output_descs.size());
     return PARAM_INVALID;
@@ -411,6 +337,8 @@ Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicp
         GELOGI("[L1Fusion] DumpOutputWithTask[%s] output[%zu] is l1 addr.", inner_dump_info.op->GetName().c_str(), i);
         int64_t output_size = 0;
         if (TensorUtils::GetTensorSizeInBytes(output_descs.at(i), output_size) != SUCCESS) {
+          REPORT_CALL_ERROR("E19999", "Get output tensor size fail in op:%s(%s), index:%zu",
+                            inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(), i);
           GELOGE(PARAM_INVALID, "Get output size failed.");
           return PARAM_INVALID;
         }
@@ -436,8 +364,12 @@ Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, aicpu::dump:
   // else data, const or variable op
   aicpu::dump::Output output;
   auto output_tensor = inner_dump_info.op->GetOutputDescPtr(inner_dump_info.output_anchor_index);
-  const std::vector<void *> output_addrs = ModelUtils::GetOutputDataAddrs(runtime_param_, inner_dump_info.op);
+  const std::vector<void *> output_addrs = ModelUtils::GetOutputDataAddrs(*runtime_param_, inner_dump_info.op);
   if (output_tensor == nullptr) {
+    REPORT_INNER_ERROR("E19999", "output_desc tensor is nullptr in op:%s(%s), index:%u, "
+                       "check invalid",
+                       inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(),
+                       inner_dump_info.output_anchor_index);
     GELOGE(PARAM_INVALID, "output_tensor is null, index: %d, size: %zu.", inner_dump_info.output_anchor_index,
            inner_dump_info.op->GetOutputsSize());
     return PARAM_INVALID;
@@ -461,6 +393,9 @@ Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, aicpu::dump:
   output.set_original_output_data_type(static_cast<int32_t>(output_tensor->GetOriginDataType()));
   // due to lhisi virtual addr bug, cannot use args now
   if (inner_dump_info.output_anchor_index >= static_cast<int>(output_addrs.size())) {
+    REPORT_INNER_ERROR("E19999", "output_anchor_index:%u >= output addr size:%zu in op:%s(%s), "
+                       "check invalid", inner_dump_info.output_anchor_index, output_addrs.size(),
+                       inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str());
     GELOGE(FAILED, "Index is out of range.");
     return FAILED;
   }
@@ -487,6 +422,7 @@ Status DataDumper::GenerateInput(aicpu::dump::Input &input, const OpDesc::Vistor
   if (AttrUtils::GetInt(tensor_descs.at(index), ATTR_NAME_INPUT_ORIGIN_SIZE, input_size)) {
     GELOGI("Get aipp input size according to attr is %ld", input_size);
   } else if (TensorUtils::GetTensorSizeInBytes(tensor_descs.at(index), input_size) != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Get tensor size fail");
     GELOGE(PARAM_INVALID, "Get input size filed");
     return PARAM_INVALID;
   }
@@ -540,8 +476,11 @@ Status DataDumper::DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info
 Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task) {
   GELOGI("Start dump input");
   const auto &input_descs = inner_dump_info.op->GetAllInputsDesc();
-  const std::vector<void *> input_addrs = ModelUtils::GetInputDataAddrs(runtime_param_, inner_dump_info.op);
+  const std::vector<void *> input_addrs = ModelUtils::GetInputDataAddrs(*runtime_param_, inner_dump_info.op);
   if (input_descs.size() != input_addrs.size()) {
+    REPORT_INNER_ERROR("E19999", "input_desc size:%zu != input addr size:%zu in op:%s(%s)",
+                       input_descs.size(), input_addrs.size(),
+                       inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str());
     GELOGE(PARAM_INVALID, "Invalid input desc addrs size %zu, op %s has %zu input desc.", input_addrs.size(),
            inner_dump_info.op->GetName().c_str(), input_descs.size());
     return PARAM_INVALID;
@@ -567,6 +506,8 @@ Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump::
         if (AttrUtils::GetInt(input_descs.at(i), ATTR_NAME_INPUT_ORIGIN_SIZE, input_size)) {
           GELOGI("Get aipp input size according to attr is %ld", input_size);
         } else if (TensorUtils::GetTensorSizeInBytes(input_descs.at(i), input_size) != SUCCESS) {
+          REPORT_CALL_ERROR("E19999", "Get input tensor size fail in op:%s(%s), index:%zu",
+                            inner_dump_info.op->GetName().c_str(), inner_dump_info.op->GetType().c_str(), i);
           GELOGE(PARAM_INVALID, "Get input size failed.");
           return PARAM_INVALID;
         }
@@ -595,6 +536,7 @@ Status DataDumper::ExecuteLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_in
   size_t proto_size = op_mapping_info.ByteSizeLong();
   bool ret = op_mapping_info.SerializeToString(&proto_str);
   if (!ret || proto_size == 0) {
+    REPORT_INNER_ERROR("E19999", "Serialize proto to string fail");
     GELOGE(PARAM_INVALID, "Protobuf SerializeToString failed, proto size %zu.", proto_size);
     return PARAM_INVALID;
   }
@@ -606,6 +548,8 @@ Status DataDumper::ExecuteLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_in
 
   rtError_t rt_ret = rtMalloc(&dev_mem_load_, proto_size, RT_MEMORY_HBM);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X",
+                      proto_size, rt_ret);
     GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
@@ -613,12 +557,15 @@ Status DataDumper::ExecuteLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_in
 
   rt_ret = rtMemcpy(dev_mem_load_, proto_size, proto_str.c_str(), proto_size, RT_MEMCPY_HOST_TO_DEVICE);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X",
+                      proto_size, rt_ret);
     GELOGE(RT_FAILED, "Call rtMemcpy failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
 
   rt_ret = rtDatadumpInfoLoad(dev_mem_load_, proto_size);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtDatadumpInfoLoad failed, ret:0x%X", rt_ret);
     GELOGE(RT_FAILED, "Call rtDatadumpInfoLoad failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
@@ -633,6 +580,7 @@ Status DataDumper::ExecuteUnLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_
   size_t proto_size = op_mapping_info.ByteSizeLong();
   bool ret = op_mapping_info.SerializeToString(&proto_str);
   if (!ret || proto_size == 0) {
+    REPORT_INNER_ERROR("E19999", "Serialize proto to string fail");
     GELOGE(PARAM_INVALID, "Protobuf SerializeToString failed, proto size %zu.", proto_size);
     return PARAM_INVALID;
   }
@@ -644,6 +592,8 @@ Status DataDumper::ExecuteUnLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_
 
   rtError_t rt_ret = rtMalloc(&dev_mem_unload_, proto_size, RT_MEMORY_HBM);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X",
+                      proto_size, rt_ret);
     GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
@@ -651,12 +601,15 @@ Status DataDumper::ExecuteUnLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_
 
   rt_ret = rtMemcpy(dev_mem_unload_, proto_size, proto_str.c_str(), proto_size, RT_MEMCPY_HOST_TO_DEVICE);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X",
+                      proto_size, rt_ret);
     GELOGE(RT_FAILED, "Call rtMemcpy failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
 
   rt_ret = rtDatadumpInfoLoad(dev_mem_unload_, proto_size);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtDatadumpInfoLoad failed, ret:0x%X", rt_ret);
     GELOGE(RT_FAILED, "Call rtDatadumpInfoLoad failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
@@ -873,97 +826,4 @@ void DataDumper::PrintCheckLog(string &dump_list_key) {
     }
   }
 }
-
-Status DataDumper::DumpExceptionInput(const OpDescInfo &op_desc_info, const string &dump_file) {
-  GELOGI("Start to dump exception input");
-  for (size_t i = 0; i < op_desc_info.input_addrs.size(); i++) {
-    if (Debug::DumpDevMem(dump_file.data(), op_desc_info.input_addrs.at(i), op_desc_info.input_size.at(i)) != SUCCESS) {
-      GELOGE(PARAM_INVALID, "Dump the %zu input data failed", i);
-      return PARAM_INVALID;
-    }
-  }
-  return SUCCESS;
-}
-
-Status DataDumper::DumpExceptionOutput(const OpDescInfo &op_desc_info, const string &dump_file) {
-  GELOGI("Start to dump exception output");
-  for (size_t i = 0; i < op_desc_info.output_addrs.size(); i++) {
-    if (Debug::DumpDevMem(dump_file.data(), op_desc_info.output_addrs.at(i), op_desc_info.output_size.at(i)) !=
-        SUCCESS) {
-      GELOGE(PARAM_INVALID, "Dump the %zu input data failed", i);
-      return PARAM_INVALID;
-    }
-  }
-  return SUCCESS;
-}
-
-Status DataDumper::DumpExceptionInfo(const std::vector<rtExceptionInfo> exception_infos) {
-  GELOGI("Start to dump exception info");
-  for (const rtExceptionInfo &iter : exception_infos) {
-    OpDescInfo op_desc_info;
-    if (GetOpDescInfo(iter.streamid, iter.taskid, op_desc_info)) {
-      toolkit::dumpdata::DumpData dump_data;
-      dump_data.set_version("2.0");
-      dump_data.set_dump_time(GetNowTime());
-      dump_data.set_op_name(op_desc_info.op_name);
-      for (size_t i = 0; i < op_desc_info.input_format.size(); ++i) {
-        toolkit::dumpdata::OpInput input;
-        input.set_data_type(toolkit::dumpdata::OutputDataType(GetIrDataType(op_desc_info.input_data_type[i])));
-        input.set_format(toolkit::dumpdata::OutputFormat(op_desc_info.input_format[i]));
-        for (auto dim : op_desc_info.input_shape[i]) {
-          input.mutable_shape()->add_dim(dim);
-        }
-        input.set_size(op_desc_info.input_size[i]);
-        GELOGI("The input size int exception is %ld", op_desc_info.input_size[i]);
-        dump_data.mutable_input()->Add(std::move(input));
-      }
-      for (size_t j = 0; j < op_desc_info.output_format.size(); ++j) {
-        toolkit::dumpdata::OpOutput output;
-        output.set_data_type(toolkit::dumpdata::OutputDataType(GetIrDataType(op_desc_info.output_data_type[j])));
-        output.set_format(toolkit::dumpdata::OutputFormat(op_desc_info.output_format[j]));
-        for (auto dim : op_desc_info.output_shape[j]) {
-          output.mutable_shape()->add_dim(dim);
-        }
-        output.set_size(op_desc_info.output_size[j]);
-        GELOGI("The output size int exception is %ld", op_desc_info.output_size[j]);
-        dump_data.mutable_output()->Add(std::move(output));
-      }
-      uint64_t now_time = GetNowTime();
-      std::string op_name = op_desc_info.op_name;
-      std::string op_type = op_desc_info.op_type;
-      ReplaceStringElem(op_name);
-      ReplaceStringElem(op_type);
-      string dump_file_path =
-          "./" + op_type + "." + op_name + "." + std::to_string(op_desc_info.task_id) + "." + std::to_string(now_time);
-      GELOGI("The exception dump file path is %s", dump_file_path.c_str());
-
-      uint64_t proto_size = dump_data.ByteSizeLong();
-      std::unique_ptr<char[]> proto_msg(new (std::nothrow) char[proto_size]);
-      bool ret = dump_data.SerializeToArray(proto_msg.get(), proto_size);
-      if (!ret || proto_size == 0) {
-        GELOGE(PARAM_INVALID, "Dump data proto serialize failed");
-        return PARAM_INVALID;
-      }
-
-      GE_CHK_STATUS_RET(MemoryDumper::DumpToFile(dump_file_path.c_str(), &proto_size, sizeof(uint64_t)),
-                        "Failed to dump proto size");
-      GE_CHK_STATUS_RET(MemoryDumper::DumpToFile(dump_file_path.c_str(), proto_msg.get(), proto_size),
-                        "Failed to dump proto msg");
-      if (DumpExceptionInput(op_desc_info, dump_file_path) != SUCCESS) {
-        GELOGE(PARAM_INVALID, "Dump exception input failed");
-        return PARAM_INVALID;
-      }
-
-      if (DumpExceptionOutput(op_desc_info, dump_file_path) != SUCCESS) {
-        GELOGE(PARAM_INVALID, "Dump exception output failed");
-        return PARAM_INVALID;
-      }
-      GELOGI("Dump exception info SUCCESS");
-    } else {
-      GELOGE(PARAM_INVALID, "Get op desc info failed,task id:%u,stream id:%u", iter.taskid, iter.streamid);
-      return PARAM_INVALID;
-    }
-  }
-  return SUCCESS;
-}
 }  // namespace ge

ge/graph/load/model_manager/data_dumper.h

@@ -36,9 +36,21 @@
 namespace ge {
 class DataDumper {
  public:
-  DataDumper() : runtime_param_{} {}
-
-  explicit DataDumper(const RuntimeParam &rsh) : runtime_param_(rsh) {}
+  explicit DataDumper(RuntimeParam *rsh)
+      : model_name_(),
+        model_id_(0),
+        runtime_param_(rsh),
+        dev_mem_load_(nullptr),
+        dev_mem_unload_(nullptr),
+        op_list_(),
+        input_map_(),
+        load_flag_(false),
+        device_id_(0),
+        global_step_(0),
+        loop_per_iter_(0),
+        loop_cond_(0),
+        compute_graph_(nullptr),
+        ref_info_() {}
 
   ~DataDumper();
 
@@ -58,8 +70,6 @@ class DataDumper {
 
   void SaveDumpInput(const std::shared_ptr<Node> &node);
 
-  void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id);
-
   // args is device memory stored first output addr
   void SaveDumpTask(uint32_t task_id, uint32_t stream_id, const std::shared_ptr<OpDesc> &op_desc, uintptr_t args);
   void SaveEndGraphId(uint32_t task_id, uint32_t stream_id);
@@ -75,14 +85,8 @@ class DataDumper {
 
   void SetDumpProperties(const DumpProperties &dump_properties) { dump_properties_ = dump_properties; }
   const DumpProperties &GetDumpProperties() const { return dump_properties_; }
-  bool GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const;
   const std::vector<OpDescInfo> &GetAllOpDescInfo() const { return op_desc_info_; }
 
-  // Dump exception info
-  Status DumpExceptionInput(const OpDescInfo &op_desc_info, const string &dump_file);
-  Status DumpExceptionOutput(const OpDescInfo &op_desc_info, const string &dump_file);
-  Status DumpExceptionInfo(const std::vector<rtExceptionInfo> exception_infos);
-
  private:
   void ReleaseDevMem(void **ptr) noexcept;
 
@@ -93,10 +97,10 @@ class DataDumper {
   // for inference data dump
   std::string om_name_;
 
-  uint32_t model_id_ = 0;
-  const RuntimeParam &runtime_param_;
-  void *dev_mem_load_ = nullptr;
-  void *dev_mem_unload_ = nullptr;
+  uint32_t model_id_;
+  RuntimeParam *runtime_param_;
+  void *dev_mem_load_;
+  void *dev_mem_unload_;
 
   struct InnerDumpInfo;
   struct InnerInputMapping;
@@ -107,12 +111,12 @@ class DataDumper {
   uint32_t end_graph_stream_id_ = 0;
   bool is_end_graph_ = false;
   std::multimap<std::string, InnerInputMapping> input_map_;  // release after DavinciModel::Init
-  bool load_flag_ = false;
-  uint32_t device_id_ = 0;
-  uintptr_t global_step_ = 0;
-  uintptr_t loop_per_iter_ = 0;
-  uintptr_t loop_cond_ = 0;
-  ComputeGraphPtr compute_graph_ = nullptr;  // release after DavinciModel::Init
+  bool load_flag_;
+  uint32_t device_id_;
+  uintptr_t global_step_;
+  uintptr_t loop_per_iter_;
+  uintptr_t loop_cond_;
+  ComputeGraphPtr compute_graph_;  // release after DavinciModel::Init
   std::map<OpDescPtr, void *> ref_info_;     // release after DavinciModel::Init
   void *l1_fusion_addr_ = nullptr;
 

ge/graph/load/model_manager/data_inputer.h

@@ -134,6 +134,8 @@ class DataInputer {
   ///
   void Stop() { queue_.Stop(); }
 
+  uint32_t Size() { return queue_.Size(); }
+
  private:
   ///
   /// @ingroup domi_ome

ge/graph/load/model_manager/davinci_model.h

@@ -29,6 +29,7 @@
 #include "common/helper/om_file_helper.h"
 #include "common/opskernel/ge_task_info.h"
 #include "common/properties_manager.h"
+#include "common/dump/exception_dumper.h"
 #include "common/dump/opdebug_register.h"
 #include "common/types.h"
 #include "framework/common/util.h"
@@ -221,6 +222,11 @@ class DavinciModel {
   ///
   DataInputer *const GetDataInputer() const { return data_inputer_; }
 
+  uint32_t GetDataInputerSize() {
+    GE_CHECK_NOTNULL(data_inputer_);
+    return data_inputer_->Size();
+  }
+
   // get Stream number
   uint32_t StreamNum() const { return runtime_param_.stream_num; }
 
@@ -248,7 +254,10 @@ class DavinciModel {
   string Name() const { return name_; }
 
   // om_name
-  string OmName() const { return om_name_; }
+  const string &OmName() const { return om_name_; }
+
+  // dump_model_name
+  const string &DumpModelName() const { return dump_model_name_; }
 
   // version
   uint32_t Version() const { return version_; }
@@ -273,6 +282,8 @@ class DavinciModel {
 
   const vector<rtLabel_t> &GetLabelList() const { return label_list_; }
 
+  Status GetLabelGotoAddr(uint32_t label_index, rtMemType_t memory_type, void *&addr, uint32_t &size);
+
   Status DestroyThread();
 
   // get Op
@@ -466,13 +477,17 @@ class DavinciModel {
   Status ReportProfilingData();
 
   void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id) {
-    data_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id);
+    exception_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id);
   }
 
   void SaveDumpTask(uint32_t task_id, uint32_t stream_id, const shared_ptr<OpDesc> &op_desc, uintptr_t args) {
     data_dumper_.SaveDumpTask(task_id, stream_id, op_desc, args);
   }
 
+  Status DumpExceptionInfo(const std::vector<rtExceptionInfo> &exception_infos) const {
+    return exception_dumper_.DumpExceptionInfo(exception_infos);
+  }
+
   void SetKnownShapeGlobalStep(void *global_step) {
     known_shape_global_step_ = global_step;
   }
@@ -481,6 +496,12 @@ class DavinciModel {
     data_dumper_.DumpShrink();
   }
 
+  bool OpNeedDump(const string &op_name) {
+    return GetDumpProperties().IsLayerNeedDump(dump_model_name_, om_name_, op_name);
+  }
+
+  bool ModelNeedDump();
+
   void SetEndGraphId(uint32_t task_id, uint32_t stream_id);
   DavinciModel &operator=(const DavinciModel &model) = delete;
 
@@ -529,10 +550,10 @@ class DavinciModel {
   void SetKnownNode(bool known_node) { known_node_ = known_node; }
   bool IsKnownNode() { return known_node_; }
   Status MallocKnownArgs();
+  Status CheckCapability(rtFeatureType_t featureType, int32_t featureInfo, bool &is_support) const;
   Status UpdateKnownNodeArgs(const vector<void *> &inputs, const vector<void *> &outputs);
   Status CreateKnownZeroCopyMap(const vector<void *> &inputs, const vector<void *> &outputs);
   Status UpdateKnownZeroCopyAddr(vector<void *> &total_io_addrs, bool update_args = true);
-  void SetKnownNodeAddrNotChanged(bool base_addr_not_changed) { base_addr_not_changed_ = base_addr_not_changed; }
 
   Status GetOrigInputInfo(uint32_t index, OriginInputInfo &orig_input_info) const;
   Status GetAllAippInputOutputDims(uint32_t index, vector<InputOutputDims> &input_dims,
@@ -540,13 +561,19 @@ class DavinciModel {
 
   // om file name
   void SetOmName(const string &om_name) { om_name_ = om_name; }
+  void SetDumpModelName(const string &dump_model_name) { dump_model_name_ = dump_model_name; }
 
   void SetDumpProperties(const DumpProperties &dump_properties) { data_dumper_.SetDumpProperties(dump_properties); }
   const DumpProperties &GetDumpProperties() const { return data_dumper_.GetDumpProperties(); }
 
   bool GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const {
-    return data_dumper_.GetOpDescInfo(stream_id, task_id, op_desc_info);
+    return exception_dumper_.GetOpDescInfo(stream_id, task_id, op_desc_info);
   }
+  void UpdateOpIOAddrs(uint32_t task_id, uint32_t stream_id, const std::vector<void *> &io_addrs);
+
+  bool GetRunningFlag() const { return running_flg_; }
+  void SetRunningFlag(bool flag) { running_flg_ = flag; }
+  Status SetRunAsyncListenerCallback(const RunAsyncCallback &callback);
 
  private:
   // memory address of weights
@@ -886,6 +913,7 @@ class DavinciModel {
 
   // used for inference data dump
   string om_name_;
+  string dump_model_name_;
 
   uint32_t version_;
   GeModelPtr ge_model_;  // release after DavinciModel::Init
@@ -911,6 +939,8 @@ class DavinciModel {
   shared_ptr<ModelListener> listener_;
 
   bool run_flg_;
+  // check whether model is running with data
+  bool running_flg_ = false;
 
   mutex mux_run_flg_;
 
@@ -930,6 +960,9 @@ class DavinciModel {
   vector<rtLabel_t> label_list_;
   set<uint32_t> label_id_indication_;
 
+  mutex label_args_mutex_;
+  map<uint32_t, pair<void *, uint32_t>> label_goto_args_;
+
   mutex outside_addrs_mutex_;
   vector<ZeroCopyTask> zero_copy_tasks_;  // Task used Data or NetOutput addr.
   set<const void *> copy_only_addrs_;     // Address need copy to original place.
@@ -985,6 +1018,7 @@ class DavinciModel {
   int64_t maxDumpOpNum_;
   // for data dump
   DataDumper data_dumper_;
+  ExceptionDumper exception_dumper_;
   OpdebugRegister opdebug_register_;
   uint64_t iterator_count_;
   bool is_l1_fusion_enable_;
@@ -1002,8 +1036,6 @@ class DavinciModel {
   map<const void *, void *> known_input_data_info_;
   map<const void *, void *> known_output_data_info_;
   vector<void *> total_io_addrs_;
-  vector<void *> orig_total_io_addrs_;
-  bool base_addr_not_changed_ = false;
 
   vector<vector<int64_t>> batch_info_;
   vector<vector<int64_t>> combined_batch_info_;

ge/graph/load/model_manager/model_manager.cc

@@ -99,11 +99,17 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u
 
       auto kernel_size = sizeof(uint64_t) * (v_aicpu_kernel.size());
       rtError_t rt_ret = rtMalloc(&aicpu_kernel_addr, kernel_size, RT_MEMORY_HBM);
-      GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret);
+      GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE,
+                      REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret: 0x%X",
+                                        kernel_size, rt_ret);
+                      GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret);
                       return RT_ERROR_TO_GE_STATUS(rt_ret);)
 
       rt_ret = rtMemcpy(aicpu_kernel_addr, kernel_size, v_aicpu_kernel.data(), kernel_size, RT_MEMCPY_HOST_TO_DEVICE);
-      GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, GELOGE(RT_FAILED, "rtMemcpy to input_output_addr_ error: 0x%X", rt_ret);
+      GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE,
+                      REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret: 0x%X",
+                                        kernel_size, rt_ret);
+                      GELOGE(RT_FAILED, "rtMemcpy to input_output_addr_ error: 0x%X", rt_ret);
                       GE_CHK_RT(rtFree(aicpu_kernel_addr)); return RT_ERROR_TO_GE_STATUS(rt_ret);)
       uint64_t kernel_id_addr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(aicpu_kernel_addr));
       param_base.fwkKernelBase.fwk_kernel.kernelID = kernel_id_addr;
@@ -114,6 +120,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u
 
   rtError_t rt_ret = rtMalloc(&(devicebase), sizeof(STR_FWK_OP_KERNEL), RT_MEMORY_HBM);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret: 0x%X",
+                      sizeof(STR_FWK_OP_KERNEL), rt_ret);
     GELOGE(RT_FAILED, "malloc device memory failed. ret: 0x%X", rt_ret);
     GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr)));
     return RT_ERROR_TO_GE_STATUS(rt_ret);
@@ -122,6 +130,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u
   rt_ret =
       rtMemcpy(devicebase, sizeof(STR_FWK_OP_KERNEL), &param_base, sizeof(STR_FWK_OP_KERNEL), RT_MEMCPY_HOST_TO_DEVICE);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret: 0x%X",
+                      sizeof(STR_FWK_OP_KERNEL), rt_ret);
     GELOGE(RT_FAILED, "memory copy to device failed. ret: 0x%X", rt_ret);
     GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr)));
     GE_CHK_RT(rtFree(devicebase));
@@ -131,6 +141,7 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u
   rtStream_t stream = nullptr;
   rt_ret = rtStreamCreate(&stream, 0);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtStreamCreate failed, ret: 0x%X", rt_ret);
     GELOGE(RT_FAILED, "create stream failed. ret: 0x%X", rt_ret);
     GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr)));
     GE_CHK_RT(rtFree(devicebase));
@@ -139,6 +150,7 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u
 
   rt_ret = rtKernelLaunchEx(devicebase, sizeof(STR_FWK_OP_KERNEL), 0, stream);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchEx failed, ret: 0x%X", rt_ret);
     GELOGE(RT_FAILED, "rtKernelLaunchEx failed. ret: 0x%X", rt_ret);
     GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr)));
     GE_CHK_RT(rtFree(devicebase));
@@ -147,6 +159,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u
   }
   rt_ret = rtStreamSynchronize(stream);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize failed, ret: 0x%X",
+                      rt_ret);
     GELOGE(RT_FAILED, "rtStreamSynchronize failed. ret: 0x%X", rt_ret);
     GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr)));
     GE_CHK_RT(rtFree(devicebase));
@@ -156,6 +170,7 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u
   if (aicpu_kernel_addr != nullptr) {
     rt_ret = rtFree(aicpu_kernel_addr);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret: 0x%X", rt_ret);
       GELOGE(RT_FAILED, "free memory failed. ret: 0x%X", rt_ret);
       GE_CHK_RT(rtFree(devicebase));
       GE_CHK_RT(rtStreamDestroy(stream));
@@ -164,12 +179,14 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u
   }
   rt_ret = rtFree(devicebase);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret: 0x%X", rt_ret);
     GELOGE(RT_FAILED, "free memory failed. ret: 0x%X", rt_ret);
     GE_CHK_RT(rtStreamDestroy(stream));
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
   rt_ret = rtStreamDestroy(stream);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtStreamDestroy failed, ret: 0x%X", rt_ret);
     GELOGE(RT_FAILED, "rtStreamDestroy failed. ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
@@ -216,6 +233,8 @@ ge::Status ModelManager::DestroyAicpuSessionForInfer(uint32_t model_id) {
 
   auto it = model_map_.find(model_id);
   if (it == model_map_.end()) {
+    REPORT_INNER_ERROR("E19999", "Param model_id:%u can't find in model_map, check invalid",
+                       model_id);
     GELOGE(ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, "model id %u does not exists.", model_id);
     return ACL_ERROR_GE_EXEC_MODEL_ID_INVALID;
   }
@@ -233,6 +252,8 @@ ge::Status ModelManager::DestroyAicpuKernel(uint64_t session_id, uint32_t model_
     Status ret = KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType::FWK_ADPT_KERNEL_DESTROY, session_id, model_id,
                                 sub_model_id);
     if (ret != SUCCESS) {
+      REPORT_CALL_ERROR("E19999", "Call KernelLaunchEx fail, model_id:%u, sub_model_id:%u, session_id:%lu",
+                        model_id, sub_model_id, session_id);
       GELOGE(FAILED, "Destroy aicpu kernel failed.");
       return FAILED;
     }
@@ -259,6 +280,7 @@ ModelManager::~ModelManager() {
   model_map_.clear();
   model_aicpu_kernel_.clear();
   cust_aicpu_so_.clear();
+  dump_exception_flag_ = false;
 
   GE_IF_BOOL_EXEC(device_count > 0, GE_CHK_RT(rtDeviceReset(0)));
 }
@@ -271,7 +293,7 @@ ge::Status ModelManager::SetDynamicSize(uint32_t model_id, const std::vector<uin
   return SUCCESS;
 }
 
-ge::Status ModelManager::DoLoadHybridModelOnline(uint32_t model_id, const string &model_name,
+ge::Status ModelManager::DoLoadHybridModelOnline(uint32_t model_id, const string &om_name,
                                                  const shared_ptr<ge::GeRootModel> &ge_root_model,
                                                  const shared_ptr<ModelListener> &listener) {
   auto hybrid_model = hybrid::HybridDavinciModel::Create(ge_root_model);
@@ -279,13 +301,26 @@ ge::Status ModelManager::DoLoadHybridModelOnline(uint32_t model_id, const string
   hybrid_model->SetListener(listener);
   hybrid_model->SetModelId(model_id);
   hybrid_model->SetDeviceId(GetContext().DeviceId());
-  hybrid_model->SetModelName(model_name);
+  hybrid_model->SetOmName(om_name);
   GE_CHK_STATUS_RET(hybrid_model->Init(), "Failed to init hybrid model. model_id = %u", model_id);
   auto shared_model = std::shared_ptr<hybrid::HybridDavinciModel>(hybrid_model.release());
   InsertModel(model_id, shared_model);
   return SUCCESS;
 }
 
+bool ModelManager::IsNeedHybridLoad(ge::GeRootModel &ge_root_model) {
+  auto root_graph = ge_root_model.GetRootGraph();
+  if (root_graph == nullptr) {
+    REPORT_INNER_ERROR("E19999", "root graph in param ge_root_model is nullptr, model_id:%u, "
+                       "check invalid", ge_root_model.GetModelId());
+    GELOGE(FAILED, "no model on root model");
+    return false;
+  }
+  bool is_shape_unknown = root_graph->GetGraphUnknownFlag();
+  bool is_dsp_partitioned_graph = false;
+  (void)AttrUtils::GetBool(root_graph, ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED, is_dsp_partitioned_graph);
+  return is_shape_unknown || is_dsp_partitioned_graph || GetContext().GetHostExecFlag();
+}
 ///
 /// @ingroup domi_ome
 /// @brief load model online
@@ -296,19 +331,18 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
   GE_CHK_BOOL_RET_STATUS(listener.get() != nullptr, PARAM_INVALID, "Param incorrect, listener is null");
   if (model_id == INVALID_MODEL_ID) {
     GenModelId(&model_id);
+    GELOGD("Generate new model_id:%u", model_id);
   }
-
-  bool is_shape_unknown = false;
-  string model_name = "";
-  GE_CHK_STATUS_RET(ge_root_model->CheckIsUnknownShape(is_shape_unknown), "CheckIsUnknownShape failed, model id:%u",
-                    model_id);
-  if (is_shape_unknown || GetContext().GetHostExecFlag()) {
-    return DoLoadHybridModelOnline(model_id, model_name, ge_root_model, listener);
+  auto name_to_model = ge_root_model->GetSubgraphInstanceNameToModel();
+  string om_name;
+  if (IsNeedHybridLoad(*ge_root_model)) {
+    return DoLoadHybridModelOnline(model_id, om_name, ge_root_model, listener);
   }
 
   mmTimespec timespec = mmGetTickCount();
   std::shared_ptr<DavinciModel> davinci_model = MakeShared<DavinciModel>(0, listener);
   if (davinci_model == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New DavinciModel fail, model_id:%u", model_id);
     GELOGE(FAILED, "davinci_model is nullptr");
     return FAILED;
   }
@@ -324,7 +358,6 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
   auto root_graph = ge_root_model->GetRootGraph();
   GE_CHECK_NOTNULL(root_graph);
   string root_model_name = root_graph->GetName();
-  auto name_to_model = ge_root_model->GetSubgraphInstanceNameToModel();
   GeModelPtr ge_model = name_to_model[root_model_name];
   Status ret = SUCCESS;
   do {
@@ -332,7 +365,18 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
     GE_IF_BOOL_EXEC(SUCCESS != (ret = davinci_model->Assign(ge_model)), GELOGW("assign model to modeldef failed.");
                     break;);
     GE_TIMESTAMP_END(Assign, "GraphLoader::ModelAssign");
-
+    /// In multi-threaded inference,  using the same session_id among multiple threads may cause some threads to fail.
+    /// These session_ids come from the same model, so the values of session_id are the same.
+    /// Update session_id for infer in load model to avoid the same session_id.
+    if (!ge_root_model->GetTrainFlag()) {
+      uint64_t new_session_id;
+      ret = GenSessionId(new_session_id);
+      GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Generate session_id for infer failed.");
+      ret = davinci_model->UpdateSessionId(new_session_id);
+      GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Update session_id for infer failed.");
+      ge_model->InsertSessionMap(model_id, new_session_id);
+      GELOGD("Update new session id: %lu.", new_session_id);
+    }
     GE_TIMESTAMP_START(Init);
     GE_IF_BOOL_EXEC(SUCCESS != (ret = davinci_model->Init()), GELOGW("DavinciInit failed."); break;);
     GE_TIMESTAMP_END(Init, "GraphLoader::ModelInit");
@@ -345,16 +389,16 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
   return ret;
 }
 
-void ModelManager::InsertModel(uint32_t id, std::shared_ptr<DavinciModel> &davinci_model) {
-  GE_CHK_BOOL_EXEC(davinci_model != nullptr, return, "davinci_model ptr is null, id: %u", id);
+void ModelManager::InsertModel(uint32_t model_id, std::shared_ptr<DavinciModel> &davinci_model) {
+  GE_CHK_BOOL_EXEC(davinci_model != nullptr, return, "davinci_model ptr is null, id: %u", model_id);
   std::lock_guard<std::recursive_mutex> lock(map_mutex_);
-  model_map_[id] = davinci_model;
+  model_map_[model_id] = davinci_model;
 }
 
-void ModelManager::InsertModel(uint32_t id, shared_ptr<hybrid::HybridDavinciModel> &hybrid_model) {
-  GE_CHK_BOOL_EXEC(hybrid_model != nullptr, return, "hybrid_model ptr is null, id: %u", id);
+void ModelManager::InsertModel(uint32_t model_id, shared_ptr<hybrid::HybridDavinciModel> &hybrid_model) {
+  GE_CHK_BOOL_EXEC(hybrid_model != nullptr, return, "hybrid_model ptr is null, id: %u", model_id);
   std::lock_guard<std::recursive_mutex> lock(map_mutex_);
-  hybrid_model_map_[id] = hybrid_model;
+  hybrid_model_map_[model_id] = hybrid_model;
 }
 
 Status ModelManager::DeleteModel(uint32_t id) {
@@ -374,6 +418,8 @@ Status ModelManager::DeleteModel(uint32_t id) {
   } else if (hybrid_model_it != hybrid_model_map_.end()) {
     (void)hybrid_model_map_.erase(hybrid_model_it);
   } else {
+    REPORT_INNER_ERROR("E19999", "model_id:%u not exist in model_map, check invalid",
+                       id);
     GELOGE(ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, "model id %u does not exists.", id);
     return ACL_ERROR_GE_EXEC_MODEL_ID_INVALID;
   }
@@ -420,6 +466,7 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d
 
   Status status = data_wrap->Init(input_data, output_data);
   if (status != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Init InputDataWrapper failed, input data index: %u", input_data.index);
     GELOGE(domi::PUSH_DATA_FAILED, "Init InputDataWrapper failed, input data index: %u.", input_data.index);
     return domi::PUSH_DATA_FAILED;
   }
@@ -436,6 +483,7 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d
   DataInputer *inputer = model->GetDataInputer();
   GE_CHECK_NOTNULL(inputer);
   if (inputer->Push(data_wrap) != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "DataInputer queue is full, please call again later, model_id %u", model_id);
     GELOGE(domi::DATA_QUEUE_ISFULL, "Data queue is full, please call again later, model_id %u ", model_id);
     return domi::DATA_QUEUE_ISFULL;
   }
@@ -449,6 +497,9 @@ Status ModelManager::GetCurDynamicDims(const vector<vector<int64_t>> &user_real_
                                        vector<int32_t> &cur_dynamic_dims) {
   GELOGD("Start get cur dynamic dims.");
   if (user_real_input_dims.size() != user_input_dims.size()) {
+    REPORT_INNER_ERROR("E19999", "Param user_real_input_dims.size:%zu != user_input_dims.size:%zu, "
+                       "check invalid",
+                       user_real_input_dims.size(), user_input_dims.size());
     GELOGE(INTERNAL_ERROR,
            "The input count of user: %zu should be equal to the data count of graph: %zu",
            user_real_input_dims.size(), user_input_dims.size());
@@ -457,6 +508,9 @@ Status ModelManager::GetCurDynamicDims(const vector<vector<int64_t>> &user_real_
 
   for (size_t i = 0; i < user_input_dims.size(); ++i) {
     if (user_real_input_dims[i].size() != user_input_dims[i].second.size()) {
+      REPORT_INNER_ERROR("E19999", "Param user_real_input_dims[%zu].size:%zu != user_input_dims[%zu].size:%zu, "
+                         "check invalid", i, user_real_input_dims[i].size(),
+                         i, user_input_dims[i].second.size());
       GELOGE(INTERNAL_ERROR,
              "The shape size: %zu of dynamic input: %s should be equal to the shape size of input shape: %zu.",
              user_real_input_dims[i].size(), user_input_dims[i].first.c_str(), user_input_dims[i].second.size());
@@ -478,6 +532,8 @@ Status ModelManager::GetCurDynamicDims(const vector<vector<int64_t>> &user_real_
     }
   }
   if (!cur_dynamic_dims_valid) {
+    REPORT_INNER_ERROR("E19999", "cur dynamic dims is %s, not exist in options, check invalid",
+                       formats::JoinToString(cur_dynamic_dims).c_str());
     GELOGE(INTERNAL_ERROR, "Cur dynamic dims is %s, not exist in options.",
            formats::JoinToString(cur_dynamic_dims).c_str());
     return INTERNAL_ERROR;
@@ -629,6 +685,8 @@ Status ModelManager::HandleCommand(const Command &command) {
 
   auto iter = cmds.find(command.cmd_type);
   if (iter == cmds.end()) {
+    REPORT_INNER_ERROR("E19999", "Unsupported command:%s check",
+                       command.cmd_type.c_str());
     GELOGE(PARAM_INVALID, "Unsupported command: %s", command.cmd_type.c_str());
     return PARAM_INVALID;
   } else {
@@ -639,6 +697,9 @@ Status ModelManager::HandleCommand(const Command &command) {
 Status ModelManager::GetModelByCmd(const Command &command,
                                    std::shared_ptr<DavinciModel> &davinci_model) {
   if (command.cmd_params.size() < kCmdParSize) {
+    REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu < kCmdParSize:%u, command_type:%s, "
+                       "check invalid", command.cmd_params.size(), kCmdParSize,
+                       command.cmd_type.c_str());
     GELOGE(PARAM_INVALID, "When the cmd_type is '%s', the size of cmd_params must larger than 2.",
         command.cmd_type.c_str());
     return PARAM_INVALID;
@@ -651,12 +712,18 @@ Status ModelManager::GetModelByCmd(const Command &command,
     try {
       model_id = std::stoi(value);
     } catch (std::invalid_argument &) {
+      REPORT_INNER_ERROR("E19999", "%s param:%s, check invalid", PROFILE_MODEL_ID.c_str(),
+                         value.c_str());
       GELOGE(PARAM_INVALID, "Model id: %s is invalid.", value.c_str());
       return PARAM_INVALID;
     } catch (std::out_of_range &) {
+      REPORT_INNER_ERROR("E19999", "%s param:%s, check out of range", PROFILE_MODEL_ID.c_str(),
+                         value.c_str());
       GELOGE(PARAM_INVALID, "Model id: %s is out of range.", value.c_str());
       return PARAM_INVALID;
     } catch (...) {
+      REPORT_INNER_ERROR("E19999", "%s param:%s, check cannot change to int",
+                         PROFILE_MODEL_ID.c_str(), value.c_str());
       GELOGE(FAILED, "Model id: %s cannot change to int.", value.c_str());
       return FAILED;
     }
@@ -665,10 +732,14 @@ Status ModelManager::GetModelByCmd(const Command &command,
     GE_CHECK_NOTNULL(model_manager);
     davinci_model = model_manager->GetModel(static_cast<uint32_t>(model_id));
     if (davinci_model == nullptr) {
+      REPORT_INNER_ERROR("E19999", "GetModel from model_manager fail, model_id:%u",
+                         model_id);
       GELOGE(FAILED, "Model id: %d is invaild or model is not loaded.", model_id);
       return FAILED;
     }
   } else {
+    REPORT_INNER_ERROR("E19999", "Fisrt cmd_param not %s, check invalid",
+                       PROFILE_MODEL_ID.c_str());
     GELOGE(FAILED, "The model_id parameter is not found in the command.");
     return FAILED;
   }
@@ -732,10 +803,14 @@ Status ModelManager::HandleProfFinalizeCommand(const Command &command) {
  */
 Status ModelManager::HandleProfStartCommand(const Command &command) {
   if (command.cmd_params.size() < kProfStartCmdParaSize) {
+    REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu < %zu, check invalid",
+                       command.cmd_params.size(), kProfStartCmdParaSize);
     GELOGE(PARAM_INVALID, "When the cmd_type is 'profile start', the size of cmd_params must larger than 2.");
     return PARAM_INVALID;
   }
   if (command.cmd_params.size() > kProfCmdParaMaxSize) {
+    REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu > %zu, check invalid",
+                       command.cmd_params.size(), kProfCmdParaMaxSize);
     GELOGE(PARAM_INVALID, "Command para size[%zu] larger than max[1000].", command.cmd_params.size());
     return PARAM_INVALID;
   }
@@ -758,10 +833,14 @@ Status ModelManager::HandleProfStartCommand(const Command &command) {
 
 Status ModelManager::HandleProfStopCommand(const Command &command) {
   if (command.cmd_params.size() < kProfStartCmdParaSize) {
+    REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu < %zu, check invalid",
+                       command.cmd_params.size(), kProfStartCmdParaSize);
     GELOGE(PARAM_INVALID, "When the cmd_type is 'profile stop', the size of cmd_params must larger than 2.");
     return PARAM_INVALID;
   }
   if (command.cmd_params.size() > kProfCmdParaMaxSize) {
+    REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu > %zu, check invalid",
+                       command.cmd_params.size(), kProfCmdParaMaxSize);
     GELOGE(PARAM_INVALID, "Command para size[%zu] larger than max[1000].", command.cmd_params.size());
     return PARAM_INVALID;
   }
@@ -787,6 +866,8 @@ static Status ParserPara(const Command &command, const string &dump_key, string
   if (iter != command.cmd_params.end()) {
     ++iter;
     if (iter == command.cmd_params.end()) {
+      REPORT_INNER_ERROR("E19999", "dump_key:%s can't find in command.param, check invalid",
+                         dump_key.c_str());
       GELOGE(PARAM_INVALID, "Invalid access.");
       return PARAM_INVALID;
     }
@@ -797,6 +878,8 @@ static Status ParserPara(const Command &command, const string &dump_key, string
 
 Status ModelManager::HandleDumpCommand(const Command &command) {
   if (command.cmd_params.size() % kDumpCmdPairSize != 0) {
+    REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu MOD 2 != 0, check invalid",
+                       command.cmd_params.size());
     GELOGE(PARAM_INVALID, "When the cmd_type is 'dump', the size of cmd_params must be a even number.");
     return PARAM_INVALID;
   }
@@ -1013,6 +1096,7 @@ Status ModelManager::GenSessionId(uint64_t &session_id) {
 
   mmTimeval tv;
   if (mmGetTimeOfDay(&tv, nullptr) != 0) {
+    REPORT_CALL_ERROR("E19999", "Call mmGetTimeOfDay fail");
     GELOGE(INTERNAL_ERROR, "Failed to get current time.");
     return INTERNAL_ERROR;
   }
@@ -1057,6 +1141,7 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model
     GeModelPtr ge_model = model_helper.GetGeModel();
     shared_ptr<DavinciModel> davinci_model = MakeShared<DavinciModel>(model.priority, listener);
     if (davinci_model == nullptr) {
+      REPORT_CALL_ERROR("E19999", "New DavinciModel fail");
       GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Make shared failed");
       return ACL_ERROR_GE_MEMORY_ALLOCATION;
     }
@@ -1072,6 +1157,7 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model
     int32_t device_id = 0;
     rtError_t rt_ret = rtGetDevice(&device_id);
     if (rt_ret != RT_ERROR_NONE || device_id < 0) {
+      REPORT_CALL_ERROR("E19999", "Call rtGetDevice failed, ret = 0x%X", rt_ret);
       GELOGE(rt_ret, "Call rtGetDevice failed, ret = 0x%X, device_id = %d.", rt_ret, device_id);
       return RT_ERROR_TO_GE_STATUS(rt_ret);
     }
@@ -1130,6 +1216,7 @@ Status ModelManager::LoadModelWithQ(uint32_t &model_id, const ModelData &model_d
 
   shared_ptr<DavinciModel> davinci_model = MakeShared<DavinciModel>(model_data.priority, nullptr);
   if (davinci_model == nullptr) {
+    REPORT_CALL_ERROR("E19999", "New DavinciModel fail");
     GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "create model failed.");
     return ACL_ERROR_GE_MEMORY_ALLOCATION;
   }
@@ -1250,6 +1337,8 @@ Status ModelManager::LoadCustAicpuSo(const OpDescPtr &op_desc, const string &so_
   rtContext_t rt_cur_ctx = nullptr;
   auto rt_error = rtCtxGetCurrent(&rt_cur_ctx);
   if (rt_error != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X",
+                      rt_error);
     GELOGE(RT_FAILED, "get current context failed, runtime result is %d", static_cast<int>(rt_error));
     return RT_FAILED;
   }
@@ -1285,6 +1374,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) {
   rtContext_t rt_cur_ctx = nullptr;
   auto rt_error = rtCtxGetCurrent(&rt_cur_ctx);
   if (rt_error != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X",
+                      rt_error);
     GELOGE(RT_FAILED, "get current context failed, runtime result is %d", static_cast<int>(rt_error));
     return RT_FAILED;
   }
@@ -1310,12 +1401,16 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) {
 
     status = rtMalloc(&d_aicpu_data, aicpu_data_length, RT_MEMORY_HBM);
     if (status != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret = 0x%X",
+                        aicpu_data_length, status);
       GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
       return RT_ERROR_TO_GE_STATUS(status);
     }
     allocated_mem.push_back(d_aicpu_data);
     status = rtMalloc(&d_so_name, so_name.size(), RT_MEMORY_HBM);
     if (status != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X",
+                        so_name.size(), status);
       GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
       return RT_ERROR_TO_GE_STATUS(status);
     }
@@ -1338,6 +1433,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) {
   uint32_t args_size = sizeof(CustAicpuSoBuf) * v_cust_so.size();
   status = rtMalloc(&args, args_size, RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X",
+                      args_size, status);
     GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -1352,6 +1449,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) {
   uint32_t batch_args_size = sizeof(BatchLoadOpFromBufArgs);
   status = rtMalloc(&batch_args, batch_args_size, RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X",
+                      batch_args_size, status);
     GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -1364,6 +1463,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) {
 
   status = rtStreamSynchronize(stream);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X",
+                      status);
     GELOGE(RT_FAILED, "Call rt stream sync failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -1408,6 +1509,7 @@ Status ModelManager::GetModelMemAndWeightSize(const ModelData &model, size_t &me
 
   auto partition_table = reinterpret_cast<ModelPartitionTable *>(model_data);
   if (partition_table->num == 1) {
+    REPORT_INNER_ERROR("E19999", "partition_table num in model_data is 1, check invalid");
     GELOGE(ACL_ERROR_GE_PARAM_INVALID, "om model is error,please use executable om model");
     return ACL_ERROR_GE_PARAM_INVALID;
   }
@@ -1474,6 +1576,7 @@ ge::Status ModelManager::SyncExecuteModel(uint32_t model_id, const vector<GeTens
                                           vector<GeTensor> &outputs) {
   auto model = GetHybridModel(model_id);
   if (model == nullptr) {
+    REPORT_INNER_ERROR("E19999", "partition_table num in model_data is 1, check invalid");
     GELOGE(FAILED, "Hybrid model not found. model id = %u.", model_id);
     return FAILED;
   }
@@ -1485,9 +1588,21 @@ Status ModelManager::GetOpDescInfo(uint32_t device_id, uint32_t stream_id, uint3
   for (const auto &model : model_map_) {
     auto davinci_model = model.second;
     if (davinci_model->GetDeviceId() == device_id) {
-      GELOGI("Start to GetOpDescInfo of device_id: %u.", device_id);
+      GELOGI("[Get][OpDescInfo] Start to GetOpDescInfo of device_id: %u in davinci model.", device_id);
       if (davinci_model->GetOpDescInfo(stream_id, task_id, op_desc_info)) {
-        GELOGI("Find specific node of stream_id: %u, task_id: %u.", stream_id, task_id);
+        GELOGI("[Get][OpDescInfo] Find specific node of stream_id: %u, task_id: %u in davinci model.",
+               stream_id, task_id);
+        return SUCCESS;
+      }
+    }
+  }
+  for (const auto &model : hybrid_model_map_) {
+    auto hybrid_model = model.second;
+    if (hybrid_model->GetDeviceId() == device_id) {
+      GELOGI("[Get][OpDescInfo] Start to GetOpDescInfo of device_id: %u in hybrid model.", device_id);
+      if (hybrid_model->GetOpDescInfo(stream_id, task_id, op_desc_info)) {
+        GELOGI("[Get][OpDescInfo] Find specific node of stream_id: %u, task_id: %u in hybrid model.",
+               stream_id, task_id);
         return SUCCESS;
       }
     }
@@ -1500,8 +1615,11 @@ Status ModelManager::EnableExceptionDump(const std::map<string, string> &options
   if (iter != options.end()) {
     GELOGI("Find option enable_exeception_dump is %s", iter->second.c_str());
     if (iter->second == "1") {
+      dump_exception_flag_ = true;
       rtError_t rt_ret = rtSetTaskFailCallback(reinterpret_cast<rtTaskFailCallback>(ExceptionCallback));
       if (rt_ret != RT_ERROR_NONE) {
+        REPORT_CALL_ERROR("E19999", "Call rtSetTaskFailCallback fail, ret = 0x%X",
+                          rt_ret);
         GELOGE(RT_FAILED, "rtSetTaskFailCallback failed");
         return RT_ERROR_TO_GE_STATUS(rt_ret);
       }
@@ -1549,6 +1667,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_op
   // malloc sysOpInfoList in SysOpCheckInfo
   status = rtMalloc(&d_req_op_list, op_nums * sizeof(SysOpInfo), RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X",
+                      op_nums * sizeof(SysOpInfo), status);
     GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -1557,6 +1677,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_op
   // malloc sysOpInfoList in SysOpCheckResp
   status = rtMalloc(&d_res_op_list, op_nums * sizeof(SysOpInfo), RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X",
+                      op_nums * sizeof(SysOpInfo), status);
     GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -1565,6 +1687,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_op
   // malloc returnCodeList in SysOpCheckResp
   status = rtMalloc(&d_ret_code_list, op_nums * sizeof(ReturnCode), RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X",
+                      op_nums * sizeof(ReturnCode), status);
     GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -1576,6 +1700,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_op
     void *d_op_type_name = nullptr;
     status = rtMalloc(&d_op_type_name, op_type.length(), RT_MEMORY_HBM);
     if (status != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X",
+                        op_type.length(), status);
       GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
       return RT_ERROR_TO_GE_STATUS(status);
     }
@@ -1593,6 +1719,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_op
     void *d_op_type_name = nullptr;
     status = rtMalloc(&d_op_type_name, op_type.size(), RT_MEMORY_HBM);
     if (status != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X",
+                        op_type.length(), status);
       GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
       return RT_ERROR_TO_GE_STATUS(status);
     }
@@ -1621,6 +1749,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_op
   uint32_t args_size = sizeof(SysOpCheckInfo) + sizeof(SysOpCheckResp);
   status = rtMalloc(&args, args_size, RT_MEMORY_HBM);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X",
+                      args_size, status);
     GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status);
     return RT_ERROR_TO_GE_STATUS(status);
   }
@@ -1636,6 +1766,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_op
 
   status = rtStreamSynchronize(stream);
   if (status != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X",
+                      status);
     GELOGE(RT_FAILED, "Call rt stream sync failed, status: 0x%x", status);
     GE_CHK_RT(rtStreamDestroy(stream));
     return RT_ERROR_TO_GE_STATUS(status);
@@ -1668,6 +1800,9 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_op
                        reinterpret_cast<void *>(static_cast<uintptr_t>(op_check_info_res.sysOpInfoList)),
                        sizeof(SysOpInfo) * res_op_nums, RT_MEMCPY_DEVICE_TO_HOST));
     if (res_ret_code_list.size() != res_aicpu_op_info_list.size() || res_ret_code_list.size() != res_op_nums) {
+      REPORT_INNER_ERROR("E19999", "res_ret_code_list.size:%zu res_aicpu_op_info_list.size:%zu res_op_nums:%lu "
+                         "not equal, check invalid",
+                         res_ret_code_list.size(), res_aicpu_op_info_list.size(), res_op_nums);
       GELOGE(FAILED, "Number of retcode is not equal to number of op type.");
       GE_CHK_RT(rtStreamDestroy(stream));
       return FAILED;
@@ -1691,6 +1826,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector<std::string> &aicpu_op
                      "<0: op_type, 1: format, 2: datatype> \n";
     }
     fail_reason += "not support.";
+    REPORT_INNER_ERROR("E19999", "Check aicpu op_type failed, details:%s",
+                       fail_reason.c_str());
     GELOGE(FAILED, "Check aicpu op_type failed. details: %s", fail_reason.c_str());
     GE_CHK_RT(rtStreamDestroy(stream));
     return FAILED;

ge/graph/load/model_manager/model_manager.h

@@ -294,6 +294,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
                                        std::vector<InputOutputDims> &output_dims);
 
   bool IsDynamicShape(uint32_t model_id);
+  bool IsNeedHybridLoad(ge::GeRootModel &ge_root_model);
   ge::Status GetOpDescInfo(uint32_t device_id, uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info);
 
   ge::Status EnableExceptionDump(const std::map<string, string> &options);
@@ -312,6 +313,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
     instance->AddExceptionInfo(*rt_exception_info);
   }
 
+  bool IsDumpExceptionOpen() { return dump_exception_flag_; }
  private:
   ///
   /// @ingroup domi_ome
@@ -329,8 +331,8 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
   /// @ingroup domi_ome
   /// @brief insert new model into model manager set
   ///
-  void InsertModel(uint32_t id, std::shared_ptr<DavinciModel> &davinci_model);
-  void InsertModel(uint32_t id, std::shared_ptr<hybrid::HybridDavinciModel> &hybrid_model);
+  void InsertModel(uint32_t model_id, std::shared_ptr<DavinciModel> &davinci_model);
+  void InsertModel(uint32_t model_id, std::shared_ptr<hybrid::HybridDavinciModel> &hybrid_model);
 
   ///
   /// @ingroup domi_ome
@@ -340,6 +342,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
 
   void GenModelId(uint32_t *id);
 
+
   std::map<uint32_t, std::shared_ptr<DavinciModel>> model_map_;
   std::map<uint32_t, std::shared_ptr<hybrid::HybridDavinciModel>> hybrid_model_map_;
   std::map<std::string, std::vector<uint64_t>> model_aicpu_kernel_;
@@ -354,6 +357,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
   std::map<uintptr_t, std::map<std::string, CustAICPUKernelPtr>> cust_aicpu_so_;
 
   static DumpProperties dump_properties_;
+  bool dump_exception_flag_ = false;
 };
 }  // namespace ge
 

ge/graph/load/model_manager/model_utils.cc

@@ -25,6 +25,9 @@
 #define VALIDATE_MEM_RANGE(OP, SIZE, OFFSET)                                                                 \
   do {                                                                                                       \
     if (SIZE <= static_cast<uint64_t>(OFFSET)) {                                                             \
+      REPORT_INNER_ERROR("E19999",                                                                           \
+                         "Node:%s(%s) offset:%ld out of range size:%lu, check invalid",                      \
+                         OP->GetName().c_str(), OP->GetType().c_str(), OFFSET, SIZE);                        \
       GELOGE(OUT_OF_MEMORY, "Node: %s, memory out of range[%lu: %ld]", OP->GetName().c_str(), SIZE, OFFSET); \
       return {};                                                                                             \
     }                                                                                                        \
@@ -305,6 +308,9 @@ vector<void *> ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co
   vector<int64_t> v_memory_type;
   bool has_mem_type_attr = ge::AttrUtils::GetListInt(op_desc, ATTR_NAME_INPUT_MEM_TYPE_LIST, v_memory_type);
   if (has_mem_type_attr && (v_memory_type.size() != inputs_size)) {
+    REPORT_INNER_ERROR("E19999", "Attr:%s, memory_type.size:%zu != input_desc.size:%zu, op:%s(%s), check invalid",
+                       ATTR_NAME_INPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), inputs_size,
+                       op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(PARAM_INVALID, "Fusion: check input size failed, op: %s, input v_memory_type size: %zu input numbers: %zu",
            op_desc->GetName().c_str(), v_memory_type.size(), inputs_size);
     return v_input_data_addr;
@@ -384,7 +390,9 @@ Status ModelUtils::GetVarAddr(const RuntimeParam &model_param, const ConstOpDesc
   switch (mem_type) {
     case RT_MEMORY_RDMA_HBM:
       if (offset < 0) {
-        GELOGE(PARAM_INVALID, "rdma var addr is invalid, addr=%p", reinterpret_cast<uint8_t *>(offset));
+        REPORT_INNER_ERROR("E19999", "Param offset:%ld < 0, check invalid", offset);
+        GELOGE(PARAM_INVALID, "rdma var addr is invalid, addr=%p",
+               reinterpret_cast<uint8_t *>(static_cast<uintptr_t>(offset)));
         return PARAM_INVALID;
       }
       var_addr = reinterpret_cast<uint8_t *>(static_cast<uintptr_t>(offset));
@@ -394,6 +402,8 @@ Status ModelUtils::GetVarAddr(const RuntimeParam &model_param, const ConstOpDesc
       var_addr = model_param.var_base + offset - model_param.logic_var_base;
       break;
     default:
+      REPORT_INNER_ERROR("E19999", "Get mem_type:%d for offset:%ld is unsupported, check invalid",
+                         mem_type, offset);
       GELOGE(PARAM_INVALID, "unsupported memory type %u", mem_type);
       return PARAM_INVALID;
   }
@@ -419,6 +429,9 @@ vector<void *> ModelUtils::GetOutputDataAddrs(const RuntimeParam &model_param, C
   vector<int64_t> v_memory_type;
   bool has_mem_type_attr = ge::AttrUtils::GetListInt(op_desc, ATTR_NAME_OUTPUT_MEM_TYPE_LIST, v_memory_type);
   if (has_mem_type_attr && (v_memory_type.size() != outputs_size)) {
+    REPORT_INNER_ERROR("E19999", "Attr:%s, memory_type.size:%zu != output_desc.size:%zu, op:%s(%s), check invalid",
+                       ATTR_NAME_OUTPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), outputs_size,
+                       op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(PARAM_INVALID,
            "Fusion: check output size failed, op: %s, output v_memory_type size: %lu output numbers: %zu",
            op_desc->GetName().c_str(), v_memory_type.size(), outputs_size);
@@ -567,6 +580,7 @@ Status ModelUtils::GetRtAddress(const RuntimeParam &param, uintptr_t logic_addr,
            param.var_size);
   } else if (logic_addr != 0) {
     mem_addr = nullptr;
+    REPORT_INNER_ERROR("E19999", "Check param logic addr:0x%lx abnormal", logic_addr);
     GELOGE(PARAM_INVALID, "The logic addr:0x%lx is abnormal", logic_addr);
     return PARAM_INVALID;
   }

ge/graph/load/model_manager/task_info/end_graph_task_info.cc

@@ -27,6 +27,7 @@ namespace ge {
 Status EndGraphTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) {
   GELOGI("InitEndGraphTaskInfo Init Start.");
   if (davinci_model == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr");
     GELOGE(PARAM_INVALID, "davinci_model is null!");
     return PARAM_INVALID;
   }
@@ -45,13 +46,11 @@ Status EndGraphTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin
 Status EndGraphTaskInfo::Distribute() {
   GELOGI("EndGraphTaskInfo Distribute Start.");
   GE_CHECK_NOTNULL(davinci_model_);
-  auto all_dump_model = davinci_model_->GetDumpProperties().GetAllDumpModel();
-  if (all_dump_model.find(ge::DUMP_ALL_MODEL) != all_dump_model.end() ||
-      all_dump_model.find(davinci_model_->Name()) != all_dump_model.end() ||
-      all_dump_model.find(davinci_model_->OmName()) != all_dump_model.end()) {
+  if (davinci_model_->ModelNeedDump()) {
     GELOGI("Start to call rtEndGraphEx");
     rtError_t rt_ret = rtEndGraphEx(model_, stream_, kDumpFlag);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtEndGraphEx failed, ret:0x%X", rt_ret);
       GELOGE(RT_FAILED, "Call rtEndGraphEx failed, ret: 0x%x", rt_ret);
       return RT_ERROR_TO_GE_STATUS(rt_ret);
     }
@@ -59,6 +58,7 @@ Status EndGraphTaskInfo::Distribute() {
     GELOGI("Start to call rtEndGraph");
     rtError_t rt_ret = rtEndGraph(model_, stream_);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtEndGraph failed, ret:0x%X", rt_ret);
       GELOGE(RT_FAILED, "Call rtEndGraph failed, ret: 0x%x", rt_ret);
       return RT_ERROR_TO_GE_STATUS(rt_ret);
     }
@@ -68,6 +68,8 @@ Status EndGraphTaskInfo::Distribute() {
   uint32_t stream_id = 0;
   rtError_t rt_ret = rtModelGetTaskId(davinci_model_->GetRtModelHandle(), &task_id, &stream_id);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtModelGetTaskId failed, ret:0x%X",
+                      rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }

ge/graph/load/model_manager/task_info/event_record_task_info.cc

@@ -23,6 +23,7 @@ namespace ge {
 Status EventRecordTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) {
   GELOGI("EventRecordTaskInfo Init Start.");
   if (davinci_model == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr");
     GELOGE(PARAM_INVALID, "davinci_model is null!");
     return PARAM_INVALID;
   }
@@ -34,6 +35,8 @@ Status EventRecordTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *da
 
   const auto &eventList = davinci_model->GetEventList();
   if (task_def.event_id() >= eventList.size()) {
+    REPORT_INNER_ERROR("E19999", "Task event_id:%u > model event size:%zu, check invalid",
+                       task_def.event_id(), eventList.size());
     GELOGE(INTERNAL_ERROR, "event list size:%zu, cur:%u!", eventList.size(), task_def.event_id());
     return INTERNAL_ERROR;
   }
@@ -47,6 +50,8 @@ Status EventRecordTaskInfo::Distribute() {
   GELOGI("EventRecordTaskInfo Distribute Start.");
   rtError_t rt_ret = rtEventRecord(event_, stream_);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtEventRecord failed, ret:0x%X",
+                      rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }

ge/graph/load/model_manager/task_info/event_wait_task_info.cc

@@ -23,6 +23,7 @@ namespace ge {
 Status EventWaitTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) {
   GELOGI("EventWaitTaskInfo Init Start.");
   if (davinci_model == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr");
     GELOGE(PARAM_INVALID, "davinci_model is null!");
     return PARAM_INVALID;
   }
@@ -34,6 +35,8 @@ Status EventWaitTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davi
 
   const auto &eventList = davinci_model->GetEventList();
   if (task_def.event_id() >= eventList.size()) {
+    REPORT_INNER_ERROR("E19999", "Task event_id:%u > model event size:%zu, check invalid",
+                       task_def.event_id(), eventList.size());
     GELOGE(INTERNAL_ERROR, "event list size:%zu, cur:%u!", eventList.size(), task_def.event_id());
     return INTERNAL_ERROR;
   }
@@ -48,12 +51,16 @@ Status EventWaitTaskInfo::Distribute() {
   GELOGI("EventWaitTaskInfo Distribute Start.");
   rtError_t rt_ret = rtStreamWaitEvent(stream_, event_);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtStreamWaitEvent failed, ret:0x%X",
+                      rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
 
   rt_ret = rtEventReset(event_, stream_);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtEventReset failed, ret:0x%X",
+                      rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }

ge/graph/load/model_manager/task_info/fusion_start_task_info.cc

@@ -23,6 +23,7 @@ namespace ge {
 Status FusionStartTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) {
   GELOGI("FusionStartTaskInfo Init Start.");
   if (davinci_model == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr");
     GELOGE(PARAM_INVALID, "davinci_model is null!");
     return PARAM_INVALID;
   }
@@ -39,6 +40,8 @@ Status FusionStartTaskInfo::Distribute() {
   GELOGI("FusionStartTaskInfo Distribute Start.");
   rtError_t rt_ret = rtKernelFusionStart(stream_);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtKernelFusionStart failed, ret:0x%X",
+                      rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }

ge/graph/load/model_manager/task_info/fusion_stop_task_info.cc

@@ -23,6 +23,7 @@ namespace ge {
 Status FusionStopTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) {
   GELOGI("FusionStopTaskInfo Init Start.");
   if (davinci_model == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr");
     GELOGE(PARAM_INVALID, "davinci_model is null!");
     return PARAM_INVALID;
   }
@@ -39,6 +40,7 @@ Status FusionStopTaskInfo::Distribute() {
   GELOGI("FusionStopTaskInfo Distribute Start.");
   rtError_t rt_ret = rtKernelFusionEnd(stream_);
   if (rt_ret != RT_ERROR_NONE) {
+    REPORT_CALL_ERROR("E19999", "Call rtKernelFusionEnd failed, ret:0x%X", rt_ret);
     GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }

ge/graph/load/model_manager/task_info/hccl_task_info.cc

@@ -30,6 +30,7 @@ HcclTaskInfo::~HcclTaskInfo() {
   if (private_def_ != nullptr) {
     rtError_t ret = rtFreeHost(private_def_);
     if (ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtFreeHost failed, ret:0x%X", ret);
       GELOGE(RT_FAILED, "Call rtFree Fail, ret = 0x%X.", ret);
     }
     private_def_ = nullptr;
@@ -41,6 +42,7 @@ HcclTaskInfo::~HcclTaskInfo() {
 Status HcclTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_model) {
   GELOGI("HcclTaskInfo Init Start.");
   if (davinci_model == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param davinci_model nullptr");
     GELOGE(PARAM_INVALID, "davinci_model is null!");
     return PARAM_INVALID;
   }
@@ -67,22 +69,30 @@ Status HcclTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci_m
   // Only in Horovod scenario should get the inputName and GeShape
   ret = HcomOmeUtil::GetHorovodInputs(op_desc, kernel_hccl_infos_);
   if (ret != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Call GetHorovodInputs fail for op:%s(%s)",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(ret, "davinci_model: GetHorovodInputs fail! domi error: %u", ret);
     return ret;
   }
   Status dmrt = HcomOmeUtil::GetHcclDataType(op_desc, kernel_hccl_infos_);
   if (dmrt != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Call GetHcclDataType fail for op:%s(%s)",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(dmrt, "davinci_model: GetHcomDataType fail! domi error: %u", dmrt);
     return dmrt;
   }
   dmrt = HcomOmeUtil::GetHcclCount(op_desc, kernel_hccl_infos_);
   if (dmrt != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Call GetHcclCount fail for op:%s(%s)",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(dmrt, "davinci_model: GetHcomCount fail! domi error: %u", dmrt);
     return dmrt;
   }
   // Only HCOMBROADCAST and HVDCALLBACKBROADCAST need to get the rootId
   dmrt = HcomOmeUtil::GetAllRootId(op_desc, kernel_hccl_infos_);
   if (dmrt != SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Call GetAllRootId fail for op:%s(%s)",
+                      op_desc->GetName().c_str(), op_desc->GetType().c_str());
     GELOGE(dmrt, "davinci_model: Get rootId fail! domi error: %u", dmrt);
     return dmrt;
   }
@@ -169,12 +179,16 @@ Status HcclTaskInfo::CreateStream(int64_t stream_num, DavinciModel *davinci_mode
     rtError_t rt_ret =
         rtStreamCreateWithFlags(&stream, davinci_model->Priority(), RT_STREAM_PERSISTENT | RT_STREAM_FORCE_COPY);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtStreamCreateWithFlags failed, ret:0x%X, stream_idx:%ld, stream_num:%ld",
+                        rt_ret, i, stream_num);
       GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
       return RT_ERROR_TO_GE_STATUS(rt_ret);
     }
     // Create slave stream, inactive by default, activated by hccl
     rt_ret = rtModelBindStream(davinci_model->GetRtModelHandle(), stream, RT_MODEL_WAIT_ACTIVE_STREAM);
     if (rt_ret != RT_ERROR_NONE) {
+      REPORT_CALL_ERROR("E19999", "Call rtModelBindStream failed, ret:0x%X, stream_idx:%ld, stream_num:%ld",
+                        rt_ret, i, stream_num);
       GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret);
       (void)rtStreamDestroy(stream);
       return RT_ERROR_TO_GE_STATUS(rt_ret);
@@ -192,6 +206,7 @@ Status HcclTaskInfo::CreateStream(int64_t stream_num, DavinciModel *davinci_mode
 Status HcclTaskInfo::Distribute() {
   GELOGI("HcclTaskInfo Distribute Start. begin to call function LoadTask in hccl.");
   if (ops_kernel_store_ == nullptr) {
+    REPORT_INNER_ERROR("E19999", "Check param ops_kernel_store_ nullptr");
     GELOGE(INTERNAL_ERROR, "ops kernel store is null.");
     return INTERNAL_ERROR;
   }
@@ -201,6 +216,7 @@ Status HcclTaskInfo::Distribute() {
   TransToGETaskInfo(ge_task);
   auto result = ops_kernel_info_store->LoadTask(ge_task);
   if (result != HCCL_SUCCESS) {
+    REPORT_CALL_ERROR("E19999", "Call ops_kernel_info_store LoadTask fail");
     GELOGE(INTERNAL_ERROR, "davinci_model : load task fail, return ret: %u", result);
     return INTERNAL_ERROR;
   }
@@ -316,6 +332,8 @@ void HcclTaskInfo::GetPrivateDefByTaskDef(const domi::TaskDef &task) {
       private_def_len_ = private_def_temp.size();
       rtError_t ret = rtMallocHost(&private_def_, private_def_len_);
       if (ret != RT_ERROR_NONE) {
+        REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, ret:0x%X, size:%u",
+                          ret, private_def_len_);
         GELOGE(RT_FAILED, "Call rtMallocHost Fail, ret = 0x%X.", ret);
         return;
       }
@@ -323,6 +341,8 @@ void HcclTaskInfo::GetPrivateDefByTaskDef(const domi::TaskDef &task) {
       ret = rtMemcpy(private_def_, private_def_len_, task.private_def().c_str(), private_def_len_,
                      RT_MEMCPY_HOST_TO_HOST);
       if (ret != RT_ERROR_NONE) {
+        REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, ret:0x%X, size:%u",
+                          ret, private_def_len_);
         GELOGE(RT_FAILED, "Call rtMemcpy Fail, ret = 0x%X.", ret);
         return;
       }