@@ -72,7 +72,7 @@ Status HybridModelExecutor::Execute(HybridModelExecutor::ExecuteArgs &args) { | |||||
if (ret == END_OF_SEQUENCE) { | if (ret == END_OF_SEQUENCE) { | ||||
args.is_eos = true; | args.is_eos = true; | ||||
} else { | } else { | ||||
GE_CHK_STATUS_RET(ret, "Failed to execute model"); | |||||
GE_CHK_STATUS_RET(ret, "[Invoke][ExecuteGraphInternal]Failed when HybridModelExecutor %s.", __FUNCTION__); | |||||
} | } | ||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
@@ -59,7 +59,10 @@ Status StageExecutor::Start(const std::vector<TensorValue> &inputs, const std::v | |||||
task_queue_.Pop(task_info); | task_queue_.Pop(task_info); | ||||
GELOGD("[Executor: %d] Got task, stage = %d, iteration = %ld", id_, task_info.stage, task_info.iteration); | GELOGD("[Executor: %d] Got task, stage = %d, iteration = %ld", id_, task_info.stage, task_info.iteration); | ||||
if (task_info.iteration >= pipe_config_->iteration_end) { | if (task_info.iteration >= pipe_config_->iteration_end) { | ||||
GELOGE(INTERNAL_ERROR, "[Executor: %d] Unexpected iteration: %d", id_, task_info.iteration); | |||||
GELOGE(INTERNAL_ERROR, "[Check][Range][Executor: %d] Unexpected iteration: %d when StageExecutor %s.", | |||||
id_, task_info.iteration, __FUNCTION__); | |||||
REPORT_INNER_ERROR("E19999", "[Executor: %d] Unexpected iteration: %d when StageExecutor %s.", | |||||
id_, task_info.iteration, __FUNCTION__); | |||||
return INTERNAL_ERROR; | return INTERNAL_ERROR; | ||||
} | } | ||||
@@ -75,7 +78,8 @@ Status StageExecutor::Start(const std::vector<TensorValue> &inputs, const std::v | |||||
if (task_info.stage == 0) { | if (task_info.stage == 0) { | ||||
GELOGD("[Executor: %d] To ResetExecutionContext", id_); | GELOGD("[Executor: %d] To ResetExecutionContext", id_); | ||||
GE_CHK_STATUS_RET(ResetExecutionContext(context_), "[Executor: %d] Failed to reset context", id_); | |||||
GE_CHK_STATUS_RET(ResetExecutionContext(context_), | |||||
"[Invoke][ResetExecutionContext][Executor: %d] Failed to reset context", id_); | |||||
context_.iteration = task_info.iteration; | context_.iteration = task_info.iteration; | ||||
GE_CHK_STATUS_RET_NOLOG(SetInputs(inputs, input_desc)); | GE_CHK_STATUS_RET_NOLOG(SetInputs(inputs, input_desc)); | ||||
} | } | ||||
@@ -92,8 +96,11 @@ Status StageExecutor::Start(const std::vector<TensorValue> &inputs, const std::v | |||||
auto sync_result = Synchronize(); | auto sync_result = Synchronize(); | ||||
if (sync_result != SUCCESS) { | if (sync_result != SUCCESS) { | ||||
GELOGE(sync_result, "[Executor: %d] Failed to sync result. iteration = %d", id_, task_info.iteration); | |||||
GELOGE(sync_result, | |||||
"[Invoke][Synchronize][Executor: %d] Failed to sync result when StageExecutor %s. iteration = %d", | |||||
id_, __FUNCTION__, task_info.iteration); | |||||
REPORT_CALL_ERROR("E19999", "[Executor: %d] Failed to sync result when StageExecutor %s. iteration = %d", | |||||
id_, __FUNCTION__, task_info.iteration); | |||||
context_.profiler->Dump(std::cout); | context_.profiler->Dump(std::cout); | ||||
context_.callback_manager->Destroy(); | context_.callback_manager->Destroy(); | ||||
RuntimeInferenceContext::DestroyContext(std::to_string(context_.context_id)); | RuntimeInferenceContext::DestroyContext(std::to_string(context_.context_id)); | ||||
@@ -242,7 +249,10 @@ Status HybridModelPipelineExecutor::Execute(HybridModelExecutor::ExecuteArgs &ar | |||||
GELOGD("Start to sync result of executor[%zu]", i); | GELOGD("Start to sync result of executor[%zu]", i); | ||||
auto ret = futures[i].get(); | auto ret = futures[i].get(); | ||||
if (ret != SUCCESS) { | if (ret != SUCCESS) { | ||||
GELOGE(ret, "[Executor: %zu] Failed to schedule tasks.", i); | |||||
GELOGE(ret, "[Check][Result][Executor: %zu] Failed to schedule tasks when HybridModelPipelineExecutor %s.", | |||||
i, __FUNCTION__); | |||||
REPORT_INNER_ERROR("E19999", "[Executor: %zu] Failed to schedule tasks when HybridModelPipelineExecutor %s.", | |||||
i, __FUNCTION__); | |||||
has_error = true; | has_error = true; | ||||
continue; | continue; | ||||
} | } | ||||
@@ -250,7 +260,10 @@ Status HybridModelPipelineExecutor::Execute(HybridModelExecutor::ExecuteArgs &ar | |||||
ret = stage_executors_[i]->Synchronize(); | ret = stage_executors_[i]->Synchronize(); | ||||
if (ret != SUCCESS) { | if (ret != SUCCESS) { | ||||
GELOGE(ret, "[Executor: %zu] Failed to synchronize result.", i); | |||||
GELOGE(ret, "[Invoke][Synchronize] failed for [Executor: %zu] when HybridModelPipelineExecutor %s.", | |||||
i, __FUNCTION__); | |||||
REPORT_CALL_ERROR("E19999", "[Executor: %zu] failed to Synchronize result when HybridModelPipelineExecutor %s.", | |||||
i, __FUNCTION__); | |||||
has_error = true; | has_error = true; | ||||
continue; | continue; | ||||
} | } | ||||
@@ -266,13 +279,14 @@ Status HybridModelPipelineExecutor::Execute(HybridModelExecutor::ExecuteArgs &ar | |||||
iteration_ = config_.iteration_end; | iteration_ = config_.iteration_end; | ||||
if (has_error) { | if (has_error) { | ||||
GELOGE(FAILED, "Error occurred while execution"); | |||||
GELOGE(FAILED, "[Check][Error]Error occurred while execution when HybridModelPipelineExecutor %s.", __FUNCTION__); | |||||
REPORT_INNER_ERROR("E19999", "Error occurred while execution when HybridModelPipelineExecutor %s.", __FUNCTION__); | |||||
return FAILED; | return FAILED; | ||||
} | } | ||||
auto last_iter_executor_idx = loop_count % stage_executors_.size(); | auto last_iter_executor_idx = loop_count % stage_executors_.size(); | ||||
GE_CHK_STATUS_RET(stage_executors_[last_iter_executor_idx]->GetOutputs(args.outputs, args.output_desc), | GE_CHK_STATUS_RET(stage_executors_[last_iter_executor_idx]->GetOutputs(args.outputs, args.output_desc), | ||||
"Failed to get output from executor[%zu]", last_iter_executor_idx); | |||||
"[Get][Outputs]Failed from executor[%zu]", last_iter_executor_idx); | |||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
@@ -40,7 +40,8 @@ void HybridProfiler::RecordEvent(EventType event_type, const char *fmt, ...) { | |||||
char buf[kEventDescMax]; | char buf[kEventDescMax]; | ||||
if (vsnprintf_s(buf, kEventDescMax, kEventDescMax - 1, fmt, args) == -1) { | if (vsnprintf_s(buf, kEventDescMax, kEventDescMax - 1, fmt, args) == -1) { | ||||
GELOGE(FAILED, "Format %s failed.", fmt); | |||||
GELOGE(FAILED, "[Parse][Param:fmt]Format %s failed when HybridProfiler %s.", fmt, __FUNCTION__); | |||||
REPORT_INNER_ERROR("E19999", "Parse Format %s failed when HybridProfiler %s.", fmt, __FUNCTION__); | |||||
va_end(args); | va_end(args); | ||||
return; | return; | ||||
} | } | ||||
@@ -48,7 +49,10 @@ void HybridProfiler::RecordEvent(EventType event_type, const char *fmt, ...) { | |||||
va_end(args); | va_end(args); | ||||
auto index = counter_++; | auto index = counter_++; | ||||
if (index >= static_cast<int>(events_.size())) { | if (index >= static_cast<int>(events_.size())) { | ||||
GELOGE(INTERNAL_ERROR, "index out of range. index = %d, max event size = %zu", index, events_.size()); | |||||
GELOGE(INTERNAL_ERROR, | |||||
"[Check][Range]index out of range. index = %d, max event size = %zu", index, events_.size()); | |||||
REPORT_INNER_ERROR("E19999", "index out of range when HybridProfiler %s. index = %d, max event size = %zu", | |||||
__FUNCTION__, index, events_.size()); | |||||
return; | return; | ||||
} | } | ||||
auto &evt = events_[index]; | auto &evt = events_[index]; | ||||
@@ -28,7 +28,8 @@ bool NodeDoneManager::Cond::Await() { | |||||
if (!cv_.wait_for(lk, | if (!cv_.wait_for(lk, | ||||
std::chrono::seconds(kDefaultWaitTimeoutInSec), | std::chrono::seconds(kDefaultWaitTimeoutInSec), | ||||
[&]() { return is_released_ || is_cancelled_; })) { | [&]() { return is_released_ || is_cancelled_; })) { | ||||
GELOGE(INTERNAL_ERROR, "Wait timed out."); | |||||
GELOGE(INTERNAL_ERROR, "[Invoke][wait_for]Wait timed out when %s.", __FUNCTION__); | |||||
REPORT_INNER_ERROR("E19999", "wait timed out when %s.", __FUNCTION__); | |||||
return false; | return false; | ||||
} | } | ||||
@@ -67,7 +67,10 @@ Status ShapeInferenceState::UpdateInputShape(int idx, const GeTensorDesc &target | |||||
Format format = input_desc.GetFormat(); | Format format = input_desc.GetFormat(); | ||||
DataType data_type = input_desc.GetDataType(); | DataType data_type = input_desc.GetDataType(); | ||||
if (TensorUtils::CalcTensorMemSize(shape, format, data_type, tensor_size) != GRAPH_SUCCESS) { | if (TensorUtils::CalcTensorMemSize(shape, format, data_type, tensor_size) != GRAPH_SUCCESS) { | ||||
GELOGE(FAILED, "[%s] Calculate tensor memory size failed.", node_item.NodeName().c_str()); | |||||
GELOGE(FAILED, "[Invoke][CalcTensorMemSize] failed for [%s] when ShapeInferenceState %s.", | |||||
node_item.NodeName().c_str(), __FUNCTION__); | |||||
REPORT_CALL_ERROR("E19999", "CalcTensorMemSize failed for [%s] when ShapeInferenceState %s.", | |||||
node_item.NodeName().c_str(), __FUNCTION__); | |||||
return FAILED; | return FAILED; | ||||
} | } | ||||
} | } | ||||
@@ -121,13 +124,19 @@ Status ShapeInferenceState::AwaitShapesReady(const GraphExecutionContext &contex | |||||
} | } | ||||
if (context.GetStatus() != SUCCESS) { | if (context.GetStatus() != SUCCESS) { | ||||
GELOGE(FAILED, "[%s] Await pending shape cancelled", node_item.NodeName().c_str()); | |||||
GELOGE(FAILED, "[Check][Status][%s] Await pending shape cancelled when %s.", | |||||
node_item.NodeName().c_str(), __FUNCTION__); | |||||
REPORT_CALL_ERROR("E19999", "[%s] Await pending shape cancelled when %s.", | |||||
node_item.NodeName().c_str(), __FUNCTION__); | |||||
break; | break; | ||||
} | } | ||||
} | } | ||||
if (!wait_success) { | if (!wait_success) { | ||||
GELOGE(FAILED, "[%s] Wait for shape timeout.", node_item.NodeName().c_str()); | |||||
GELOGE(FAILED, "[Check][Status][%s] Wait for shape timeout when %s.", | |||||
node_item.NodeName().c_str(), __FUNCTION__); | |||||
REPORT_CALL_ERROR("E19999", "[%s] Wait for shape timeout when %s.", | |||||
node_item.NodeName().c_str(), __FUNCTION__); | |||||
return FAILED; | return FAILED; | ||||
} | } | ||||
} | } | ||||
@@ -232,7 +241,7 @@ Status NodeState::WaitForPrepareDone() { | |||||
if (prepare_future_.valid()) { | if (prepare_future_.valid()) { | ||||
GELOGD("[%s] Start to wait for prepare future.", GetName().c_str()); | GELOGD("[%s] Start to wait for prepare future.", GetName().c_str()); | ||||
GE_CHK_STATUS_RET(prepare_future_.get(), | GE_CHK_STATUS_RET(prepare_future_.get(), | ||||
"[%s] PreRun failed.", GetName().c_str()); | |||||
"[Check][Status][%s] PreRun failed when NodeState %s.", GetName().c_str(), __FUNCTION__); | |||||
} | } | ||||
return SUCCESS; | return SUCCESS; | ||||
@@ -1 +1 @@ | |||||
Subproject commit 620e9b9ac3210db3e4cf47babfb23d248bb9f17e | |||||
Subproject commit 4ff5e3987f2e5d2980019defacaf0891861c84fc |
@@ -1 +1 @@ | |||||
Subproject commit d744541c6ca7f6966c1befacc9f83f53b0829e0a | |||||
Subproject commit 51fb6c4850906e8342598d47eccfca0b87ffea59 |