add errlog and modify geloge --1

ge/hybrid/common/npu_memory_allocator.cc

@@ -39,7 +39,8 @@ AllocationAttr::AllocationAttr(void *try_reuse_addr) : AllocationAttr(0, try_reu
 NpuMemoryAllocator *NpuMemoryAllocator::GetAllocator() {
   int32_t device_id = 0;
   if (rtGetDevice(&device_id) != RT_ERROR_NONE) {
-    GELOGE(RT_FAILED, "Failed to get device id");
+    GELOGE(RT_FAILED, "[Get][Device] Failed when %s.", __FUNCTION__);
+    REPORT_INNER_ERROR("E19999", "rtGetDevice failed when %s.", __FUNCTION__);
     return nullptr;
   }
 
@@ -57,7 +58,10 @@ void *NpuMemoryAllocator::Allocate(std::size_t size, AllocationAttr *attr) {
   }
 
   if (allocate_size == 0) {
-    GELOGE(MEMALLOC_FAILED, "Memory size is 0, device_id = %u, size = %zu", device_id_, allocate_size);
+    GELOGE(MEMALLOC_FAILED, "[Check][Param:size_t]Memory size is 0, device_id = %u, size = %zu when %s.", 
+        device_id_, allocate_size, __FUNCTION__);
+    REPORT_INNER_ERROR("E19999", "Memory size is 0, device_id = %u, size = %zu when %s.", 
+        device_id_, allocate_size, __FUNCTION__);
     return nullptr;
   }
 
@@ -68,7 +72,8 @@ void *NpuMemoryAllocator::Allocate(std::size_t size, AllocationAttr *attr) {
     buffer = MemManager::Instance().HostMemInstance(RT_MEMORY_HBM).Malloc(allocate_size);
   } else {
     if (allocate_size > kMaxHbmMemorySize) {
-      GELOGE(PARAM_INVALID, "Invalid HBM memory size: %zu", allocate_size);
+      GELOGE(PARAM_INVALID, "[Check][Param:size_t]Invalid HBM memory size: %zu when %s.", allocate_size, __FUNCTION__);
+      REPORT_CALL_ERROR("E19999", "Invalid HBM memory size: %zu when %s.", allocate_size, __FUNCTION__);
       return nullptr;
     }
     void *try_reuse_addr = nullptr;
@@ -87,7 +92,10 @@ void *NpuMemoryAllocator::Allocate(std::size_t size, AllocationAttr *attr) {
                  .Malloc(allocate_size, reinterpret_cast<uint8_t *>(try_reuse_addr), device_id_);
   }
   if (buffer == nullptr) {
-    GELOGE(MEMALLOC_FAILED, "Failed to malloc memory, device_id = %u, size = %zu", device_id_, allocate_size);
+    GELOGE(MEMALLOC_FAILED, "[Malloc][Memory] Failed, device_id = %u, size = %zu when %s.", 
+        device_id_, allocate_size, __FUNCTION__);
+    REPORT_CALL_ERROR("E19999", "malloc memory failed, device_id = %u, size = %zu when %s.", 
+        device_id_, allocate_size, __FUNCTION__);
     return nullptr;
   }
 

ge/hybrid/common/tensor_value.cc

@@ -32,7 +32,8 @@ std::unique_ptr<TensorBuffer> TensorBuffer::Create(NpuMemoryAllocator *allocator
   }
 
   if (allocator == nullptr) {
-    GELOGE(INTERNAL_ERROR, "allocator is NULL");
+    GELOGE(INTERNAL_ERROR, "[Check][Param:NpuMemoryAllocator] allocator is NULL, when %s.", __FUNCTION__);
+    REPORT_INNER_ERROR("E19999", "input allocator is NULL, when %s.", __FUNCTION__);
     return nullptr;
   }
 
@@ -42,7 +43,8 @@ std::unique_ptr<TensorBuffer> TensorBuffer::Create(NpuMemoryAllocator *allocator
   }
   buffer = allocator->Allocate(size, attr);
   if (buffer == nullptr) {
-    GELOGE(MEMALLOC_FAILED, "Failed to allocate memory. size = %zu", size);
+    GELOGE(MEMALLOC_FAILED, "[Allocate][Memory] Failed. size = %zu, when %s.", size, __FUNCTION__);
+    REPORT_CALL_ERROR("E19999", "allocate failed, size = %zu, when %s.", size, __FUNCTION__);
     return nullptr;
   }
 

ge/hybrid/executor/hybrid_execution_context.cc

@@ -59,7 +59,10 @@ Status GraphExecutionContext::Synchronize(rtStream_t rt_stream) {
     return SUCCESS;
   }
 
-  GELOGE(RT_FAILED, "Failed to invoke rtStreamSynchronize, ret = %d", rt_ret);
+  GELOGE(RT_FAILED, 
+      "[Invoke][rtStreamSynchronize] failed when GraphExecutionContext %s, ret = %d", __FUNCTION__, rt_ret);
+  REPORT_CALL_ERROR("E19999", 
+      "invoke rtStreamSynchronize failed when GraphExecutionContext %s, ret = %d", __FUNCTION__, rt_ret);
   return RT_FAILED;
 }
 }  // namespace hybrid

ge/hybrid/executor/hybrid_model_async_executor.cc

@@ -52,7 +52,7 @@ void HybridModelAsyncExecutor::SetModelName(const string &model_name) {
 
 Status HybridModelAsyncExecutor::EnqueueData(const shared_ptr<InputDataWrapper> &data) {
   GE_CHK_STATUS_EXEC(data_inputer_->Push(data), return domi::DATA_QUEUE_ISFULL,
-                     "Data queue is full, please call again later, model_id %u ", model_id_);
+                     "[Push][Data] Data queue is full, please call again later, model_id %u ", model_id_);
   GELOGD("EnqueueData successfully. model_id = %u, data_index = %u", data->GetInput().model_id, data->GetInput().index);
   return SUCCESS;
 }
@@ -60,7 +60,8 @@ Status HybridModelAsyncExecutor::EnqueueData(const shared_ptr<InputDataWrapper>
 Status HybridModelAsyncExecutor::Start(const std::shared_ptr<ModelListener> &listener) {
   GELOGD("HybridModelExecutor::Start IN, has listener = %d", listener != nullptr);
   std::lock_guard<std::mutex> lk(mu_);
-  GE_CHK_BOOL_RET_STATUS(!run_flag_, INTERNAL_ERROR, "Model already started.");
+  GE_CHK_BOOL_RET_STATUS(!run_flag_, INTERNAL_ERROR, 
+      "[Check][RunState] Model already started when HybridModelAsyncExecutor %s.", __FUNCTION__);
 
   run_flag_ = true;
   listener_ = listener;
@@ -71,7 +72,8 @@ Status HybridModelAsyncExecutor::Start(const std::shared_ptr<ModelListener> &lis
     return RunInternal();
   });
 
-  GE_CHK_BOOL_RET_STATUS(future_.valid(), INTERNAL_ERROR, "Failed to start.");
+  GE_CHK_BOOL_RET_STATUS(future_.valid(), INTERNAL_ERROR, 
+      "[Check][RunState] Failed to start when HybridModelAsyncExecutor %s.", __FUNCTION__);
   GELOGD("HybridModelExecutor::Start successfully");
   return SUCCESS;
 }
@@ -105,26 +107,29 @@ Status HybridModelAsyncExecutor::Init() {
 
   executor_ = std::unique_ptr<HybridModelExecutor>(new(std::nothrow) HybridModelExecutor(model_, device_id_, stream_));
   GE_CHECK_NOTNULL(executor_);
-  GE_CHK_STATUS_RET(executor_->Init(), "Failed to init hybrid engine");
-  GE_CHK_STATUS_RET(DumpOpDebug(), "Dump op debug failed in hybrid engine");
+  GE_CHK_STATUS_RET(executor_->Init(), 
+      "[Init][HybridModelExecutor] failed when HybridModelAsyncExecutor %s.", __FUNCTION__);
+  GE_CHK_STATUS_RET(DumpOpDebug(), "[Dump][OpDebug] failed when HybridModelAsyncExecutor %s.", __FUNCTION__);
 
   GELOGI("HybridModel stage nums:%zu", model_->GetRootGraphItem()->NumGroups());
   if (model_->GetRootGraphItem()->NumGroups() >= kMinimumPiplineStages) {
     pipe_executor_ =
         std::unique_ptr<HybridModelPipelineExecutor>(new(std::nothrow) HybridModelPipelineExecutor(model_, device_id_));
     GE_CHECK_NOTNULL(pipe_executor_);
-    GE_CHK_STATUS_RET(pipe_executor_->Init(), "Failed to init hybrid engine");
+    GE_CHK_STATUS_RET(pipe_executor_->Init(), 
+        "[Init][HybridModelPipelineExecutor] failed when HybridModelAsyncExecutor %s.", __FUNCTION__);
   }
 
-  GE_CHK_STATUS_RET(InitInputDesc(), "Failed to init input tensors");
+  GE_CHK_STATUS_RET(InitInputDesc(), "[Init][InputDesc] failed when HybridModelAsyncExecutor %s.", __FUNCTION__);
 
   return SUCCESS;
 }
 
 Status HybridModelAsyncExecutor::PreRun(InputData &current_data, HybridModelExecutor::ExecuteArgs &args) {
-  GE_CHK_STATUS_RET(SyncVarData(), "Failed to sync var data");
+  GE_CHK_STATUS_RET(SyncVarData(), "[Invoke][SyncVarData] failed when HybridModelAsyncExecutor %s.", __FUNCTION__);
   RECORD_MODEL_EXECUTION_EVENT(executor_->GetContext(), "[SyncVarData] End");
-  GE_CHK_STATUS_RET(PrepareInputs(current_data, args), "Failed to copy input data to model");
+  GE_CHK_STATUS_RET(PrepareInputs(current_data, args), 
+      "[Invoke][PrepareInputs] failed to copy input data to model when HybridModelAsyncExecutor %s.", __FUNCTION__);
   RECORD_MODEL_EXECUTION_EVENT(executor_->GetContext(), "[CopyInputData] End");
   return SUCCESS;
 }
@@ -155,7 +160,7 @@ Status HybridModelAsyncExecutor::RunInternal() {
     GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
         ret != SUCCESS, (void) HandleResult(ret, current_data.index, args, data_wrapper->GetOutput());
         CsaInteract::GetInstance().StoreInternalErrorCode(ret, ERROR_MODULE_FMK, JOBSUBSTATE_GRAPH_EXEC);
-        continue, "PreRun failed.");  // [No need to check value]
+        continue, "[Invoke][PreRun] failed when HybridModelAsyncExecutor %s.", __FUNCTION__);  // [No need to check value]
 
     if (pipe_executor_ != nullptr) {
       GELOGI("HybridModel will execute in pipeline mode");
@@ -199,7 +204,11 @@ Status HybridModelAsyncExecutor::HandleResult(Status exec_ret,
   }
 
   if (exec_ret != SUCCESS) {
-    GELOGE(exec_ret, "Failed to execute graph. model_id = %u", model_id_);
+    GELOGE(exec_ret, 
+        "[Check][Param:Status] failed to execute graph when HybridModelAsyncExecutor %s. model_id = %u", 
+        __FUNCTION__, model_id_);
+    REPORT_INNER_ERROR("E19999", 
+        "failed to execute graph when HybridModelAsyncExecutor %s. model_id = %u", __FUNCTION__, model_id_);
     return OnComputeDone(data_id, INTERNAL_ERROR, output_tensor_info_list);
   }
 
@@ -235,8 +244,12 @@ Status HybridModelAsyncExecutor::SyncVarData() {
 
 Status HybridModelAsyncExecutor::PrepareInputs(const InputData &current_data, HybridModelExecutor::ExecuteArgs &args) {
   if (current_data.blobs.size() < input_tensor_desc_.size()) {
-    GELOGE(PARAM_INVALID, "Blob size mismatches, expect at least %zu, but got %zu",
-           input_tensor_desc_.size(), current_data.blobs.size());
+    GELOGE(PARAM_INVALID, 
+        "[Check][Size]Blob size mismatches, expect at least %zu, but got %zu when HybridModelAsyncExecutor %s.",
+        input_tensor_desc_.size(), current_data.blobs.size(), __FUNCTION__);
+    REPORT_INNER_ERROR("E19999", 
+        "Blob size mismatches, expect at least %zu, but got %zu when HybridModelAsyncExecutor %s.",
+        input_tensor_desc_.size(), current_data.blobs.size(), __FUNCTION__);
     return PARAM_INVALID;
   }
 
@@ -248,8 +261,12 @@ Status HybridModelAsyncExecutor::PrepareInputs(const InputData &current_data, Hy
     auto tensor_size = input_sizes_[input_index];
     if (is_input_dynamic_[input_index]) {
       if (input_index >= current_data.shapes.size()) {
-        GELOGE(PARAM_INVALID, "Shape index out of range, index = %zu, shape size = %zu",
-               input_index, current_data.shapes.size());
+        GELOGE(PARAM_INVALID, 
+            "[Check][Range]Shape index out of range, index = %zu, shape size = %zu when HybridModelAsyncExecutor %s.",
+            input_index, current_data.shapes.size(), __FUNCTION__);
+        REPORT_INNER_ERROR("E19999", 
+            "Shape index out of range, index = %zu, shape size = %zu when HybridModelAsyncExecutor %s.",
+            input_index, current_data.shapes.size(), __FUNCTION__);    
         return PARAM_INVALID;
       }
       auto &tensor_desc = input_tensor_desc_[input_index];
@@ -257,15 +274,19 @@ Status HybridModelAsyncExecutor::PrepareInputs(const InputData &current_data, Hy
       std::vector<std::pair<int64_t, int64_t>> range;
       auto range_ret = tensor_desc->GetShapeRange(range);
       GE_CHK_BOOL_RET_STATUS(range_ret == GRAPH_SUCCESS, INTERNAL_ERROR,
-                             "Get shape range failed, ret=%u.", range_ret);
+                             "[Invoke][GetShapeRange] failed, ret=%u.", range_ret);
       for (size_t k = 0; k < range.size(); ++k) {
         if (k >= shape.GetDimNum()) {
           break;
         }
         // range[k].second can be -1
         if (shape.GetDim(k) < range[k].first || (range[k].second >= 0 && shape.GetDim(k) > range[k].second)) {
-          GELOGE(PARAM_INVALID, "Dim out of range, shape idx = %zu, dim idx = %zu, dim = %ld, range = [%ld, %ld]",
-                 input_index, k, shape.GetDim(k), range[k].first, range[k].second);
+          GELOGE(PARAM_INVALID, 
+              "[Check][Range]Dim out of range, shape idx = %zu, dim idx = %zu, dim = %ld, range = [%ld, %ld]",
+              input_index, k, shape.GetDim(k), range[k].first, range[k].second);
+          REPORT_INNER_ERROR("E19999", 
+              "Dim out of range, shape idx = %zu, dim idx = %zu, dim = %ld, range = [%ld, %ld]",
+              input_index, k, shape.GetDim(k), range[k].first, range[k].second);    
           return PARAM_INVALID;
         }
       }
@@ -273,9 +294,8 @@ Status HybridModelAsyncExecutor::PrepareInputs(const InputData &current_data, Hy
       args.input_desc[input_index] = tensor_desc;
       GELOGD("Update shape of input[%zu] to [%s]", input_index, tensor_desc->MutableShape().ToString().c_str());
       GE_CHK_GRAPH_STATUS_RET(TensorUtils::GetTensorMemorySizeInBytes(*tensor_desc, tensor_size),
-                              "Failed to calc tensor size, index = %zu, shape = [%s]",
-                              input_index,
-                              tensor_desc->GetShape().ToString().c_str());
+          "[Invoke][GetTensorMemorySizeInBytes]Failed to calc tensor size, index = %zu, shape = [%s]",
+          input_index, tensor_desc->GetShape().ToString().c_str());
       GELOGD("Input tensor[%zu] size = %zu", input_index, tensor_size);
     }
 
@@ -293,7 +313,7 @@ Status HybridModelAsyncExecutor::PrepareInputs(const InputData &current_data, Hy
     auto mem_size = static_cast<uint64_t>(tensor_size);
     GE_CHK_BOOL_RET_STATUS(mem_size >= data_buf.length,
                            PARAM_INVALID,
-                           "input data size(%lu) does not match model required size(%lu), ret failed.",
+                           "[Check][Size]input data size(%lu) does not match model required size(%lu), ret failed.",
                            data_buf.length,
                            mem_size);
 
@@ -351,7 +371,7 @@ Status HybridModelAsyncExecutor::OnComputeDone(uint32_t data_index, uint32_t res
   GELOGD("OnComputeDone. model id = %u, data index = %u, execution ret = %u", model_id_, data_index, result_code);
   if (listener_ != nullptr) {
     GE_CHK_STATUS(listener_->OnComputeDone(model_id_, data_index, result_code, outputs),
-                  "OnComputeDone failed");
+                  "[Invoke][OnComputeDone] failed.");
   }
 
   return result_code;
@@ -365,9 +385,12 @@ Status HybridModelAsyncExecutor::CopyOutputs(HybridModelExecutor::ExecuteArgs &a
   std::vector<TensorValue> &output_tensors = args.outputs;
   if (output_tensor_desc_list.size() != output_tensors.size()) {
     GELOGE(INTERNAL_ERROR,
-           "Output sizes mismatch. From op_desc = %zu, and from output tensors = %zu",
-           output_tensor_desc_list.size(),
-           output_tensors.size());
+        "[Check][Size]Output sizes mismatch. From op_desc = %zu, and from output tensors = %zu "
+        "when HybridModelAsyncExecutor %s.",
+        output_tensor_desc_list.size(), output_tensors.size(), __FUNCTION__);
+    REPORT_INNER_ERROR("E19999", "Output sizes mismatch. From op_desc = %zu, and from output tensors = %zu "
+        "when HybridModelAsyncExecutor %s.",
+        output_tensor_desc_list.size(), output_tensors.size(), __FUNCTION__);
     return INTERNAL_ERROR;
   }
 
@@ -399,8 +422,10 @@ Status HybridModelAsyncExecutor::CopyOutputs(HybridModelExecutor::ExecuteArgs &a
     GE_CHECK_LE(output_size, UINT32_MAX);
     if (output_tensor.GetSize() < static_cast<size_t>(output_size)) {
       GELOGE(INTERNAL_ERROR,
-             "output[%zu] tensor size(%zu) is not enough for output shape [%s]",
-             i, output_tensor.GetSize(), tensor_desc->GetShape().ToString().c_str());
+          "[Check][Size]output[%zu] tensor size(%zu) is not enough for output shape [%s]",
+          i, output_tensor.GetSize(), tensor_desc->GetShape().ToString().c_str());
+      REPORT_INNER_ERROR("E19999", "output[%zu] tensor size(%zu) is not enough for output shape [%s]",
+          i, output_tensor.GetSize(), tensor_desc->GetShape().ToString().c_str());
       return INTERNAL_ERROR;
     }
 
@@ -456,7 +481,7 @@ Status HybridModelAsyncExecutor::Execute(const std::vector<DataBuffer> &inputs,
     args.input_desc.emplace_back(tensor_desc_ptr);
   }
 
-  GE_CHK_STATUS_RET(executor_->Execute(args), "Failed to execute model.");
+  GE_CHK_STATUS_RET(executor_->Execute(args), "[Invoke][Execute] Failed when HybridModelAsyncExecutor %s.", __FUNCTION__);
   for (const auto &output_tensor_desc : args.output_desc) {
     output_desc.emplace_back(*output_tensor_desc);
   }
@@ -477,13 +502,14 @@ Status HybridModelAsyncExecutor::Execute(const vector<GeTensor> &inputs, vector<
   }
 
   HybridModelExecutor::ExecuteArgs args;
-  GE_CHK_STATUS_RET(PrepareInputs(input_data, args), "Failed to copy input data to model");
+  GE_CHK_STATUS_RET(PrepareInputs(input_data, args), "[Invoke][PrepareInputs]Failed to copy input data to model");
   GELOGD("Done copying input data successfully.");
-  GE_CHK_STATUS_RET(executor_->Execute(args), "Failed to execute model.");
+  GE_CHK_STATUS_RET(executor_->Execute(args), "[Invoke][Execute] Failed.");
 
   std::vector<ge::OutputTensorInfo> output_tensor_info_list;
   OutputData output_data;
-  GE_CHK_STATUS_RET(CopyOutputs(args, &output_data, output_tensor_info_list), "Failed to copy outputs.");
+  GE_CHK_STATUS_RET(CopyOutputs(args, &output_data, output_tensor_info_list), 
+      "[Invoke][CopyOutputs]Failed to copy outputs.");
   GELOGD("Done copying output data successfully. output count = %zu", output_tensor_info_list.size());
 
   int out_index = 0;
@@ -534,7 +560,7 @@ Status HybridModelAsyncExecutor::DumpOpDebug() {
       loop_cond = const_cast<void *>(varible_loop_cond->GetData());
     }
     data_dumper_.SetLoopAddr(global_step, loop_per_iter, loop_cond);
-    GE_CHK_STATUS_RET(data_dumper_.LoadDumpInfo(), "LoadDumpInfo failed in hybrid engine");
+    GE_CHK_STATUS_RET(data_dumper_.LoadDumpInfo(), "[Invoke][LoadDumpInfo] failed in hybrid engine");
     GELOGD("Dump op debug SUCCESS in hybrid engine");
   }
   return SUCCESS;

ge/hybrid/executor/worker/execution_engine.cc

@@ -102,11 +102,13 @@ Status NodeDoneCallback::PrepareConstInputs(const NodeItem &node_item) {
 
     if (output_tensor->GetSize() < static_cast<size_t>(tensor_size)) {
       GELOGE(INTERNAL_ERROR,
-             "[%s] Tensor size is not enough. output index = %d, required size = %ld, tensor = %s",
-             node_item.NodeName().c_str(),
-             output_idx,
-             tensor_size,
-             output_tensor->DebugString().c_str());
+          "[Check][Size][%s] Tensor size is not enough. output index = %d, required size = %ld, tensor = %s when %s.",
+          node_item.NodeName().c_str(), output_idx, tensor_size,
+          output_tensor->DebugString().c_str(), __FUNCTION__);
+      REPORT_INNER_ERROR("E19999", 
+          "[%s] Tensor size is not enough. output index = %d, required size = %ld, tensor = %s when %s.",
+          node_item.NodeName().c_str(), output_idx, tensor_size,
+          output_tensor->DebugString().c_str(), __FUNCTION__);
       return INTERNAL_ERROR;
     }
 
@@ -128,7 +130,7 @@ Status NodeDoneCallback::PrepareConstInputs(const NodeItem &node_item) {
     GE_CHK_GRAPH_STATUS_RET(RuntimeInferenceContext::GetContext(context_id, &runtime_infer_ctx),
                             "Failed to get RuntimeInferenceContext, context_id = %s", context_id.c_str());
     GE_CHK_STATUS_RET(runtime_infer_ctx->SetTensor(node_item.node_id, output_idx, std::move(tensor)),
-                      "Failed to SetTensor, node = %s, output_index = %d", node_item.NodeName().c_str(), output_idx);
+                      "[Set][Tensor] Failed, node = %s, output_index = %d", node_item.NodeName().c_str(), output_idx);
     GELOGD("[%s] Output[%d] cached successfully in context: %s. node_id = %d, shape = [%s]",
            node_item.NodeName().c_str(),
            output_idx,
@@ -173,7 +175,8 @@ Status NodeDoneCallback::GetTaskDescInfo(const NodePtr node, const HybridModel *
 Status NodeDoneCallback::ProfilingReport() {
   auto node = context_->GetNodeItem().node;
   if (node == nullptr) {
-    GELOGE(PARAM_INVALID, "Get node is nullptr");
+    GELOGE(PARAM_INVALID, "[Get][Node] value is nullptr when %s.", __FUNCTION__);
+    REPORT_INNER_ERROR("E19999", "Get node failed, when %s.", __FUNCTION__);
     return PARAM_INVALID;
   }
 
@@ -190,7 +193,8 @@ Status NodeDoneCallback::ProfilingReport() {
   std::vector<TaskDescInfo> task_desc_info;
   auto profiling_ret = GetTaskDescInfo(node, model, task_desc_info);
   if (profiling_ret != RT_ERROR_NONE) {
-    GELOGE(profiling_ret, "Get task info of node[%s] failed.", node->GetName().c_str());
+    GELOGE(profiling_ret, "[Get][TaskDescInfo] of node:%s failed, when %s.", node->GetName().c_str(), __FUNCTION__);
+    REPORT_CALL_ERROR("E19999", "GetTaskDescInfo of node:%s failed, when %s.", node->GetName().c_str(), __FUNCTION__);
     return profiling_ret;
   }
 
@@ -202,7 +206,8 @@ Status NodeDoneCallback::ProfilingReport() {
 Status NodeDoneCallback::DumpDynamicNode() {
   auto node = context_->GetNodeItem().node;
   if (node == nullptr) {
-    GELOGE(PARAM_INVALID, "Get node is nullptr");
+    GELOGE(PARAM_INVALID, "[Get][Node] value is nullptr when %s.", __FUNCTION__);
+    REPORT_INNER_ERROR("E19999", "get node is nullptr when %s.", __FUNCTION__);
     return PARAM_INVALID;
   }
   auto op_desc = node->GetOpDesc();
@@ -211,13 +216,13 @@ Status NodeDoneCallback::DumpDynamicNode() {
   vector<uintptr_t> output_addrs;
   for (int i = 0; i < context_->NumInputs(); i++) {
     auto tensor_value = context_->GetInput(i);
-    GE_CHK_BOOL_RET_STATUS(tensor_value != nullptr, PARAM_INVALID, "Tensor value is nullptr");
+    GE_CHK_BOOL_RET_STATUS(tensor_value != nullptr, PARAM_INVALID, "[Get][Tensor] value is nullptr.");
     uint64_t input_addr = reinterpret_cast<uintptr_t>(tensor_value->GetData());
     input_addrs.emplace_back(input_addr);
   }
   for (int j = 0; j < context_->NumOutputs(); j++) {
     auto tensor_value = context_->GetOutput(j);
-    GE_CHK_BOOL_RET_STATUS(tensor_value != nullptr, PARAM_INVALID, "Tensor value is nullptr");
+    GE_CHK_BOOL_RET_STATUS(tensor_value != nullptr, PARAM_INVALID, "[Get][Tensor] value is nullptr.");
     uint64_t output_addr = reinterpret_cast<uintptr_t>(tensor_value->GetData());
     output_addrs.emplace_back(output_addr);
   }
@@ -245,11 +250,12 @@ Status NodeDoneCallback::DumpDynamicNode() {
   void *global_step = context_->GetExecutionContext()->global_step;
   dump_op_.SetLoopAddr(global_step, loop_per_iter, loop_cond);
 
-  GE_CHK_STATUS_RET(dump_op_.LaunchDumpOp(), "Failed to launch dump op in hybird model");
+  GE_CHK_STATUS_RET(dump_op_.LaunchDumpOp(), "[Launch][DumpOp] failed in hybird model when %s.", __FUNCTION__);
 
   auto rt_ret = rtStreamSynchronize(stream);
   if (rt_ret != RT_ERROR_NONE) {
-    GELOGE(rt_ret, "rtStreamSynchronize failed");
+    GELOGE(rt_ret, "[Call][rtStreamSynchronize] failed when %s.", __FUNCTION__);
+    REPORT_CALL_ERROR("E19999", "call rtStreamSynchronize failed when %s.", __FUNCTION__);
     return rt_ret;
   }
   return SUCCESS;
@@ -264,12 +270,12 @@ Status NodeDoneCallback::OnNodeDone() {
   const DumpProperties &dump_properties = context_->GetDumpProperties();
   if (dump_properties.IsDumpOpen() || context_->IsOverFlow()) {
     GELOGI("Start to dump dynamic shape op");
-    GE_CHK_STATUS_RET(DumpDynamicNode(), "Failed to dump dynamic node");
+    GE_CHK_STATUS_RET(DumpDynamicNode(), "[Call][DumpDynamicNode] Failed when %s.", __FUNCTION__);
   }
 
   if (ProfilingManager::Instance().ProfilingModelExecuteOn()) {
-    GE_CHK_STATUS_RET(ProfilingReport(), "Report node[%s] to profiling failed.",
-                      node_item.NodeName().c_str());
+    GE_CHK_STATUS_RET(ProfilingReport(), "[Report][Profiling] of node[%s] failed when %s.",
+        node_item.NodeName().c_str(), __FUNCTION__);
   }
 
   // release workspace
@@ -292,8 +298,7 @@ Status NodeDoneCallback::OnNodeDone() {
     }
 
     GE_CHK_STATUS_RET(context_->PropagateOutputs(),
-                      "[%s] Failed to propagate outputs failed",
-                      node_item.NodeName().c_str());
+        "[Propagate][Outputs] of [%s] failed when %s.", node_item.NodeName().c_str(), __FUNCTION__);
 
     RECORD_CALLBACK_EVENT(graph_context_, context_->GetNodeName(), "[PropagateOutputs] End");
   }
@@ -333,7 +338,8 @@ Status ExecutionEngine::DoExecuteAsync(NodeState &node_state,
                                        const std::function<void()> &callback) {
   const auto &task = node_state.GetKernelTask();
   if (task == nullptr) {
-    GELOGE(INTERNAL_ERROR, "[%s] NodeTask is null.", node_state.GetName().c_str());
+    GELOGE(INTERNAL_ERROR, "[Get][KernelTask] of [%s] is null when %s.", node_state.GetName().c_str(), __FUNCTION__);
+    REPORT_CALL_ERROR("E19999", "GetKernelTask of %s is null when %s.", node_state.GetName().c_str(), __FUNCTION__);
     return INTERNAL_ERROR;
   }
 
@@ -348,8 +354,7 @@ Status ExecutionEngine::DoExecuteAsync(NodeState &node_state,
   GE_CHECK_NOTNULL(executor);
   RECORD_EXECUTION_EVENT(&context, task_context.GetNodeName(), "[PrepareTask] Start");
   GE_CHK_STATUS_RET(executor->PrepareTask(*task, task_context),
-                    "[%s] Failed to prepare task",
-                    node_state.GetName().c_str());
+      "[Prepare][Task] for [%s] failed when %s", node_state.GetName().c_str(), __FUNCTION__);
   RECORD_EXECUTION_EVENT(&context, task_context.GetNodeName(), "[PrepareTask] End");
   GELOGD("[%s] Done task preparation successfully.", node_state.GetName().c_str());
 
@@ -360,7 +365,8 @@ Status ExecutionEngine::DoExecuteAsync(NodeState &node_state,
     }
   }
 
-  GE_CHK_STATUS_RET(ValidateInputTensors(node_state, task_context), "Failed to validate input tensors.");
+  GE_CHK_STATUS_RET(ValidateInputTensors(node_state, task_context), "[Validate][InputTensors] for %s failed when %s.",
+      node_state.GetName().c_str(), __FUNCTION__);
   RECORD_EXECUTION_EVENT(&context, task_context.GetNodeName(), "[ValidateInputTensors] End");
 
   if (context.profiling_level > 0) {
@@ -414,11 +420,10 @@ Status ExecutionEngine::ValidateInputTensors(const NodeState &node_state, const
                input_tensor->GetSize());
       } else {
         GELOGE(INTERNAL_ERROR,
-               "[%s] Input[%d]: tensor size mismatches. expected: %ld, but given %zu",
-               task_context.GetNodeName(),
-               i,
-               expected_size,
-               input_tensor->GetSize());
+            "[Check][Size] for [%s] Input[%d]: tensor size mismatches. expected: %ld, but given %zu when %s.",
+            task_context.GetNodeName(), i, expected_size, input_tensor->GetSize(), __FUNCTION__);
+        REPORT_INNER_ERROR("E19999", "[%s] Input[%d]: tensor size mismatches. expected: %ld, but given %zu when %s.",
+            task_context.GetNodeName(), i, expected_size, input_tensor->GetSize(), __FUNCTION__);    
         return INTERNAL_ERROR;
       }
     }
@@ -432,8 +437,7 @@ Status ExecutionEngine::PropagateOutputs(const NodeItem &node_item,
                                          GraphExecutionContext &context) {
   if (node_item.shape_inference_type != DEPEND_COMPUTE) {
     GE_CHK_STATUS_RET(task_context.PropagateOutputs(),
-                      "[%s] Failed to propagate outputs.",
-                      node_item.NodeName().c_str());
+        "[Propagate][Outputs] for [%s] failed when ExecutionEngine %s.", node_item.NodeName().c_str(), __FUNCTION__);
     RECORD_EXECUTION_EVENT(&context, task_context.GetNodeName(), "[PropagateOutputs] End");
     GELOGD("[%s] Done propagating outputs successfully.", node_item.NodeName().c_str());
   }

ge/hybrid/executor/worker/shape_inference_engine.cc

@@ -70,7 +70,7 @@ Status ShapeInferenceEngine::InferShape(NodeState &node_state) {
   {
     RECORD_SHAPE_INFERENCE_EVENT(execution_context_, node_item.NodeName().c_str(), "[InferShapeAndType] Start");
     GE_CHK_STATUS_RET(ShapeRefiner::InferShapeAndTypeForRunning(node_item.node, true),
-                      "Invoke InferShapeAndType failed.");
+        "[Invoke][InferShapeAndType] for %s failed when %s.", node_item.NodeName().c_str(), __FUNCTION__);
     RECORD_SHAPE_INFERENCE_EVENT(execution_context_, node_item.NodeName().c_str(), "[InferShapeAndType] End");
   }
 
@@ -172,8 +172,7 @@ Status ShapeInferenceEngine::InferShapeForSubgraph(const NodeItem &node_item, co
     GE_CHK_STATUS_RET(ShapeRefiner::InferShapeAndType(node));
     GELOGD("[%s] Done invoking InferShapeAndType", node->GetName().c_str());
     GE_CHK_STATUS_RET(UpdatePeerNodeShape(*node),
-                      "[%s] Failed to update shapes of peer node.",
-                      node->GetName().c_str());
+        "[Update][PeerNodeShape] failed for [%s] when %s.", node->GetName().c_str(), __FUNCTION__);
   }
 
   for (auto &it : fused_subgraph.output_mapping) {
@@ -205,7 +204,10 @@ Status ShapeInferenceEngine::UpdatePeerNodeShape(const Node &node) {
       GE_CHECK_NOTNULL(peer_op_desc);
       auto peer_input_desc = peer_op_desc->MutableInputDesc(peer_anchor->GetIdx());
       if (peer_input_desc == nullptr) {
-        GELOGE(GRAPH_FAILED, "peer_input_desc is nullptr");
+        GELOGE(GRAPH_FAILED, "[Call][MutableInputDesc] for %s return nullptr when ShapeInferenceEngine %s.", 
+            peer_op_desc->GetName().c_str(), __FUNCTION__);
+        REPORT_CALL_ERROR("E19999", "%s call MutableInputDesc return nullptr when ShapeInferenceEngine %s.", 
+            peer_op_desc->GetName().c_str(), __FUNCTION__);    
         continue;
       }
 
@@ -230,8 +232,11 @@ Status ShapeInferenceEngine::CanonicalizeShape(GeTensorDesc &tensor_desc,
   const auto &tensor_shape = tensor_desc.MutableShape();
   if (tensor_shape.IsUnknownShape()) {
     if (!fallback_with_range) {
-      GELOGE(INTERNAL_ERROR, "Output shape is still unknown after shape inference. shape = [%s]",
-             tensor_shape.ToString().c_str());
+      GELOGE(INTERNAL_ERROR, 
+          "[Is][UnknownShape] Output shape is still unknown after shape inference. "
+          "shape = [%s] when ShapeInferenceEngine %s.", tensor_shape.ToString().c_str(), __FUNCTION__);
+      REPORT_INNER_ERROR("E19999", "Output shape is still unknown after shape inference. "
+          "shape = [%s] when ShapeInferenceEngine %s.", tensor_shape.ToString().c_str(), __FUNCTION__);    
       return INTERNAL_ERROR;
     }
 
@@ -239,9 +244,10 @@ Status ShapeInferenceEngine::CanonicalizeShape(GeTensorDesc &tensor_desc,
     std::vector<std::pair<int64_t, int64_t>> shape_range;
     GE_CHK_GRAPH_STATUS_RET(tensor_desc.GetShapeRange(shape_range), "Failed to get shape range");
     if (shape_range.size() != shape.size()) {
-      GELOGE(INTERNAL_ERROR, "Number of shape ranges (%zu) mismatches that of dims (%zu)",
-             shape_range.size(),
-             shape.size());
+      GELOGE(INTERNAL_ERROR, "[Check][Size] Number of shape ranges (%zu) mismatches that of dims (%zu)" 
+          " when ShapeInferenceEngine %s.", shape_range.size(), shape.size(), __FUNCTION__);
+      REPORT_INNER_ERROR("E19999", "Number of shape ranges (%zu) mismatches that of dims (%zu)" 
+          " when ShapeInferenceEngine %s.", shape_range.size(), shape.size(), __FUNCTION__);    
       return INTERNAL_ERROR;
     }
 
@@ -265,7 +271,10 @@ Status ShapeInferenceEngine::CalcTensorSize(DataType data_type,
   GELOGD("To calc tensor size by shape = [%s]", GeShape(shape).ToString().c_str());
   uint32_t type_size;
   if (!TypeUtils::GetDataTypeLength(data_type, type_size)) {
-    GELOGE(INTERNAL_ERROR, "Failed to get data type size");
+    GELOGE(INTERNAL_ERROR, "[Get][DataTypeLength] failed for type:%s when ShapeInferenceEngine %s.", 
+        TypeUtils::DataTypeToSerialString(data_type).c_str(), __FUNCTION__);
+    REPORT_CALL_ERROR("E19999", "GetDataTypeLength failed for type:%s when ShapeInferenceEngine %s.", 
+        TypeUtils::DataTypeToSerialString(data_type).c_str(), __FUNCTION__);    
     return INTERNAL_ERROR;
   }
 
@@ -273,15 +282,13 @@ Status ShapeInferenceEngine::CalcTensorSize(DataType data_type,
   for (const auto &dim : shape) {
     GE_CHECK_GE(dim, 0);
     GE_CHK_STATUS_RET(Int64MulCheckOverflow(tensor_size, dim),
-                      "Shape size overflow, shape = [%s]",
-                      GeShape(shape).ToString().c_str());
+        "[Check][Overflow] Shape size overflow, shape = [%s]", GeShape(shape).ToString().c_str());
     tensor_size *= dim;
   }
 
   GE_CHK_STATUS_RET(CheckInt64AddOverflow(tensor_size, kAlignment - 1),
-                    "Tensor size is too large: %ld, shape = [%s]",
-                    tensor_size,
-                    GeShape(shape).ToString().c_str());
+      "[Check][Overflow]Tensor size is too large: %ld, shape = [%s]",
+      tensor_size, GeShape(shape).ToString().c_str());
   tensor_size = (tensor_size + kAlignment - 1) / kAlignment * kAlignment;
   return SUCCESS;
 }
@@ -295,15 +302,13 @@ Status ShapeInferenceEngine::CalcOutputTensorSizes(const NodeItem &node_item, bo
     // modify on copy
     auto dims = shape.GetDims();
     GE_CHK_STATUS_RET(CanonicalizeShape(*tensor_desc, dims, fallback_with_range),
-                      "[%s] Failed to canonicalize shape for output %zu",
-                      node_item.NodeName().c_str(),
-                      output_index);
+        "[Canonicalize][Shape] failed for [%s], output %zu, when ShapeInferenceEngine %s.",
+        node_item.NodeName().c_str(), output_index, __FUNCTION__);
 
     int64_t tensor_size;
     GE_CHK_STATUS_RET(CalcTensorSize(tensor_desc->GetDataType(), dims, tensor_size),
-                      "[%s] Failed to calc tensor size for output %zu",
-                      node_item.NodeName().c_str(),
-                      output_index);
+        "[Calc][TensorSize] failed for [%s], output %zu when ShapeInferenceEngine %s.",
+        node_item.NodeName().c_str(), output_index, __FUNCTION__);
     GELOGD("[%s] Tensor size of output %zu = %ld", node_item.NodeName().c_str(), output_index, tensor_size);
     (void) TensorUtils::SetSize(*tensor_desc, tensor_size);
   }

ge/hybrid/executor/worker/task_compile_engine.cc

@@ -32,7 +32,8 @@ Status TaskCompileEngine::Compile(NodeState &node_state, GraphExecutionContext *
   shared_ptr<NodeTask> kernel_task;
   auto ret = node_item.node_executor->CompileTask(*context->model, node_item.node, kernel_task);
   RECORD_COMPILE_EVENT(context, node_state.GetName().c_str(), "[Compile] End");
-  GE_CHK_STATUS_RET(ret, "Failed to create task for node: %s", node_item.NodeName().c_str());
+  GE_CHK_STATUS_RET(ret, "[Compile][Task] failed for node: %s, when TaskCompileEngine %s.", 
+      node_item.NodeName().c_str(), __FUNCTION__);
   node_state.SetKernelTask(kernel_task);
   GELOGI("Compiling node %s successfully", node_state.GetName().c_str());
   return SUCCESS;