From 3241cb6f29e01ad72ed38fffef384b90c57eb7b5 Mon Sep 17 00:00:00 2001 From: wangxiaotian22 Date: Tue, 30 Mar 2021 11:20:44 +0800 Subject: [PATCH] add error msg --- ge/graph/manager/graph_manager_utils.cc | 2 + ge/graph/manager/graph_mem_allocator.cc | 9 ++ ge/graph/manager/graph_var_manager.cc | 51 +++++++ ge/graph/manager/host_mem_allocator.cc | 4 + ge/graph/manager/host_mem_manager.cc | 9 ++ ge/graph/manager/memory_api.cc | 6 + ge/graph/manager/rdma_pool_allocator.cc | 11 ++ ge/graph/manager/trans_var_data_utils.cc | 53 ++++++- ge/graph/preprocess/graph_preprocess.cc | 129 ++++++++++++++++++ ge/graph/preprocess/insert_op/ge_aipp_op.cc | 46 +++++++ .../insert_op/util_insert_aipp_op.cc | 27 ++++ 11 files changed, 344 insertions(+), 3 deletions(-) diff --git a/ge/graph/manager/graph_manager_utils.cc b/ge/graph/manager/graph_manager_utils.cc index fe7e5b34..e9270401 100644 --- a/ge/graph/manager/graph_manager_utils.cc +++ b/ge/graph/manager/graph_manager_utils.cc @@ -84,6 +84,7 @@ Status SubGraphInfo::FreeInOutBuffer() { rtError_t rt_ret; rt_ret = rtFreeHost(*iter); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtFreeHost fail when SubGraphInfo %s", __FUNCTION__); GELOGE(rt_ret, "[GraphManager] subgraph free buffer failed, modelId = %u", model_id_info_.model_id); buffer_addr_.erase(buffer_addr_.begin(), iter); return GE_GRAPH_FREE_FAILED; @@ -119,6 +120,7 @@ Status GraphModelListener::OnComputeDone(uint32_t model_id, uint32_t task_id, ui uint32_t GraphModelListener::GetResultCode() const { if (!is_finished_) { + REPORT_CALL_ERROR("E19999", "Model not run finish, fail for %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "[GraphManager] model not run finish."); return INTERNAL_ERROR; } diff --git a/ge/graph/manager/graph_mem_allocator.cc b/ge/graph/manager/graph_mem_allocator.cc index 428b08ae..56b455da 100755 --- a/ge/graph/manager/graph_mem_allocator.cc +++ b/ge/graph/manager/graph_mem_allocator.cc @@ -49,6 +49,8 @@ uint8_t *MemoryAllocator::MallocMemory(const string &purpose, size_t memory_size uint8_t *memory_addr = nullptr; if (rtMalloc(reinterpret_cast(&memory_addr), memory_size, memory_type_) != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, purpose:%s, size:%zu, device_id:%u, when MemoryAllocator %s", + purpose.c_str(), memory_size, device_id, __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "MemoryAllocator::MallocMemory device_id = %u," " size= %lu", @@ -66,6 +68,7 @@ Status MemoryAllocator::FreeMemory(uint8_t *memory_addr, uint32_t device_id) con GELOGI("MemoryAllocator::FreeMemory device_id = %u", device_id); auto rtRet = rtFree(memory_addr); if (rtRet != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtFree fail, device_id:%u, when MemoryAllocator %s", device_id, __FUNCTION__); GELOGE(rtRet, "MemoryAllocator::MallocMemory device_id = %u", device_id); return RT_ERROR_TO_GE_STATUS(rtRet); } @@ -84,6 +87,9 @@ uint8_t *MemoryAllocator::MallocMemory(const string &purpose, const string &memo uint8_t *memory_addr = MallocMemory(purpose, memory_size, device_id); if (memory_addr == nullptr) { + REPORT_CALL_ERROR("E19999", "Malloc Memory fail, purpose:%s, memory_key:%s, memory_size:%zu, device_id:%u, " + "when MemoryAllocator %s", purpose.c_str(), memory_key.c_str(), + memory_size, device_id, __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "MemoryAllocator::MallocMemory failed," " memory_key[%s], size = %lu.", @@ -120,6 +126,8 @@ Status MemoryAllocator::FreeMemory(const string &memory_key, uint32_t device_id) } if (FreeMemory(it->second.memory_addr_, device_id) != ge::SUCCESS) { + REPORT_CALL_ERROR("E19999", "Free Memory fail, memory_key:%s, device_id:%u, when MemoryAllocator %s", + memory_key.c_str(), device_id, __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "MemoryAllocator::FreeMemory rtFree failed," " memory_key[%s]", @@ -169,6 +177,7 @@ Status MemManager::Initialize(const std::vector &memory_type) { memory_allocator_map_[index] = memory_allocator; GELOGI("Create MemoryAllocator memory type[%u] success.", index); } else { + REPORT_CALL_ERROR("E19999", "New MemoryAllocator fail, index:%u, when MemoryAllocator %s", index, __FUNCTION__); GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc MemoryAllocator failed."); } } else { diff --git a/ge/graph/manager/graph_var_manager.cc b/ge/graph/manager/graph_var_manager.cc index de75344d..f83353ff 100755 --- a/ge/graph/manager/graph_var_manager.cc +++ b/ge/graph/manager/graph_var_manager.cc @@ -39,6 +39,8 @@ VarResource::~VarResource() { ge::Status VarResource::GetVarAddr(const std::string &var_name, const ge::GeTensorDesc &tensor_desc, uint8_t **dev_ptr, rtMemType_t &memory_type) { if (dev_ptr == nullptr) { + REPORT_INNER_ERROR("E19999", "Param dev_ptr is nullptr, var_name:%s, session_id:%lu, " + "check invalid when VarResource %s", var_name.c_str(), session_id_, __FUNCTION__); GELOGE(FAILED, "[GetVarAddr] dev_ptr is null!"); return FAILED; } @@ -47,6 +49,9 @@ ge::Status VarResource::GetVarAddr(const std::string &var_name, const ge::GeTens auto iter = var_addr_mgr_map_.find(var_key); if (iter == var_addr_mgr_map_.end()) { + REPORT_INNER_ERROR("E19999", "var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, " + "check invalid when VarResource %s", var_key.c_str(), var_name.c_str(), + session_id_, __FUNCTION__); GELOGE(FAILED, "VarResource::GetVarAddr failed, var_key %s", var_key.c_str()); return FAILED; } @@ -102,6 +107,9 @@ ge::Status VarResource::SaveVarAddr(const std::string &var_name, const ge::GeTen return SUCCESS; } + REPORT_INNER_ERROR("E19999", "var_key:%s conflict in var_addr_mgr_map_, var_name:%s, session_id:%lu, " + "check invalid when VarResource %s", var_key.c_str(), var_name.c_str(), + session_id_, __FUNCTION__); GELOGE(FAILED, "VarResource::SaveVarAddr, var_key %s save addr conflict", var_key.c_str()); return FAILED; } @@ -136,6 +144,8 @@ ge::Status VarResource::RenewCurVarDesc(const std::string &var_name, const ge::O } if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "Param op_desc is nullptr, var_name:%s, session_id:%lu, check invalid " + "when VarResource %s", var_name.c_str(), session_id_, __FUNCTION__); GELOGE(FAILED, "[RenewCurVarDesc] renew var desc fail! input opdesc is null!"); return FAILED; } @@ -152,6 +162,9 @@ ge::Status VarResource::RenewCurVarDesc(const std::string &var_name, const ge::O cur_var_tensor_desc_map_[var_name] = curr_desc; auto iter = var_addr_mgr_map_.find(key); if (iter == var_addr_mgr_map_.end()) { + REPORT_INNER_ERROR("E19999", "var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, op:%s(%s), " + "check invalid when VarResource %s", key.c_str(), var_name.c_str(), + session_id_, op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "[RenewCurVarDesc] can't find ele with key [%s]", key.c_str()); return FAILED; } @@ -271,11 +284,15 @@ Status HbmMemResource::AssignVarMem(const std::string &var_name, uint64_t size, uint64_t real_size = size; total_size_ = VarManager::Instance(session_id)->GetVarMemMaxSize(); if (total_size_ < var_mem_size_) { + REPORT_INNER_ERROR("E19999", "VarMemMaxSize:%lu < var_mem_size_:%lu, var_size:%lu, var_name:%s, check invalid" + " when HbmMemResource %s", total_size_, var_mem_size_, size, var_name.c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "total_size_: %lu is smaller than var_mem_size_: %lu", total_size_, var_mem_size_); return PARAM_INVALID; } uint64_t free_size = total_size_ - var_mem_size_; if (free_size < (size + kSessionMemAlignSize * kSessionMemAlignUnit)) { + REPORT_INNER_ERROR("E19999", "free_size:%lu not enough, var_align_size:%lu, var_name:%s, check invalid " + "when HbmMemResource %s", free_size, size, var_name.c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "Out of memory : current var size[%lu] exceeds total var size[%lu]", size + kSessionMemAlignSize * kSessionMemAlignUnit + var_mem_size_, total_size_); return PARAM_INVALID; @@ -299,6 +316,8 @@ Status HbmMemResource::AssignVarMem(const std::string &var_name, uint64_t size, Status RdmaMemResource::AssignVarMem(const std::string &var_name, uint64_t size, uint64_t session_id, size_t &address) { uint8_t *buffer = MemManager::Instance().RdmaPoolInstance(RT_MEMORY_HBM).Malloc(size); if (buffer == nullptr) { + REPORT_CALL_ERROR("E19999", "malloc rdma memory fail, var_size:%lu, var_name:%s when RdmaMemResource %s", + size, var_name.c_str(), __FUNCTION__); GELOGE(MEMALLOC_FAILED, "Failed to malloc rdma memory for node %s, size = %lu", var_name.c_str(), size); return MEMALLOC_FAILED; } @@ -448,6 +467,8 @@ int64_t VarManager::GetVarMemSize(rtMemType_t memory_type) { } if (mem_resource == nullptr) { + REPORT_INNER_ERROR("E19999", "Find no mem_resource in map, memory_type:%d, session_id:%lu when VarManager %s", + memory_type, session_id_, __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid."); return 0; } @@ -461,6 +482,8 @@ Status VarManager::UpdateVarMemSize(rtMemType_t memory_type, int64_t mem_size) { if (iter == mem_resource_map_.end()) { mem_resource = MemResource::BuildMemResourceFromType(memory_type); if (mem_resource == nullptr) { + REPORT_INNER_ERROR("E19999", "memory_type:%d invalid or New MemResource fail, session_id:%lu when VarManager %s", + memory_type, session_id_, __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "Alloc MemResource failed, memory_type = %u.", memory_type); return ge::INTERNAL_ERROR; } else { @@ -471,6 +494,8 @@ Status VarManager::UpdateVarMemSize(rtMemType_t memory_type, int64_t mem_size) { } if (mem_resource == nullptr) { + REPORT_INNER_ERROR("E19999", "MemResource is invalid, memory_type:%d, session_id:%lu when VarManager %s", + memory_type, session_id_, __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid."); return FAILED; } @@ -489,6 +514,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen size_t mem_offset = 0; ge::Status result = TensorUtils::GetSize(tensor_desc, tensor_desc_size); if (result != ge::SUCCESS) { + REPORT_INNER_ERROR("E19999", "Get size from tensor fail, var_name:%s, memory_type:%d, session_id:%lu, " + "when VarManager %s", var_name.c_str(), memory_type, session_id_, __FUNCTION__); GELOGE(result, "get size from TensorDesc failed"); return result; } @@ -498,6 +525,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen if (it == mem_resource_map_.end()) { mem_resource = MemResource::BuildMemResourceFromType(memory_type); if (mem_resource == nullptr) { + REPORT_INNER_ERROR("E19999", "memory_type:%d invalid or New MemResource fail, session_id:%lu when VarManager %s", + memory_type, session_id_, __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "Alloc MemResource failed, memory_type = %u.", memory_type); return ge::INTERNAL_ERROR; } else { @@ -508,6 +537,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen } if (mem_resource == nullptr) { + REPORT_INNER_ERROR("E19999", "MemResource is invalid, memory_type:%d, session_id:%lu when VarManager %s", + memory_type, session_id_, __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid, memory_type = %u.", memory_type); return ge::INTERNAL_ERROR; } @@ -517,6 +548,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen return ge::INTERNAL_ERROR; } if (var_resource_ == nullptr) { + REPORT_INNER_ERROR("E19999", "VarManager has not been init, memory_type:%d, session_id:%lu, " + "check invalid when VarManager %s", memory_type, session_id_, __FUNCTION__); GELOGW("VarManager has not been init."); return ge::INTERNAL_ERROR; } @@ -635,6 +668,9 @@ ge::Status VarManager::RenewCurVarDesc(const std::string &var_name, ge::OpDescPt GELOGD("VarManager::RenewCurVarDesc var_name = %s.", var_name.c_str()); if (var_resource_ == nullptr) { + REPORT_INNER_ERROR("E19999", "VarManager has not been init, op:%s(%s), session_id:%lu, check invalid " + "when VarManager %s", op_desc->GetName().c_str(), op_desc->GetType().c_str(), + session_id_, __FUNCTION__); GELOGE(ge::INTERNAL_ERROR, "VarManager has not been init."); return ge::INTERNAL_ERROR; } @@ -786,6 +822,8 @@ Status VarManager::SetMemoryMallocSize(const map &options) { var_mem_logic_base_ = graph_mem_max_size_ + kGraphMemoryBuffer; if (var_mem_logic_base_ > kMaxMemorySize) { + REPORT_INNER_ERROR("E19999", "var_login_base:%zu can not exeed limit:%zu, session_id:%lu, check invalid " + "when VarManager %s", var_mem_logic_base_, kMaxMemorySize, session_id_, __FUNCTION__); GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "kMemoryVarLogicBase : %zu can not exceed max memory size : %zu.", var_mem_logic_base_, kMaxMemorySize); return ge::GE_GRAPH_OPTIONS_INVALID; @@ -793,6 +831,8 @@ Status VarManager::SetMemoryMallocSize(const map &options) { use_max_mem_size_ = graph_mem_max_size_ + var_mem_max_size_; if (use_max_mem_size_ > kMaxMemorySize) { + REPORT_INNER_ERROR("E19999", "all mem_use size:%zu can not exeed limit:%zu, session_id:%lu, check invalid " + "when VarManager %s", use_max_mem_size_, kMaxMemorySize, session_id_, __FUNCTION__); GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "kUseMaxMemorySize : %zu can not exceed max memory size : %zu.", use_max_mem_size_, kMaxMemorySize); return ge::GE_GRAPH_OPTIONS_INVALID; @@ -803,6 +843,8 @@ Status VarManager::SetMemoryMallocSize(const map &options) { Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) { if (memory_size.empty()) { + REPORT_INNER_ERROR("E19999", "Param memory_size is empty, session_id:%lu, check invalid when VarManager %s", + session_id_, __FUNCTION__); GELOGE(GE_GRAPH_OPTIONS_INVALID, "Memory malloc size input is empty."); return GE_GRAPH_OPTIONS_INVALID; } @@ -828,15 +870,23 @@ Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) { for (char c : split) { if (!isdigit(c)) { + REPORT_INNER_ERROR("E19999", "Param memory_size:%s contains non digit, session_id:%lu, check invalid " + "when VarManager %s", memory_size.c_str(), session_id_, __FUNCTION__); GELOGE(GE_GRAPH_OPTIONS_INVALID, "Memory malloc size input contains non digit."); return GE_GRAPH_OPTIONS_INVALID; } } uint64_t num = std::strtoul(split.c_str(), nullptr, 0); GE_IF_BOOL_EXEC(TypeUtils::CheckUint64MulOverflow(result, static_cast(num)), + REPORT_INNER_ERROR("E19999", "Param memory_size:%s will overflow after multi all, session_id:%lu, " + "check invalid when VarManager %s", memory_size.c_str(), + session_id_, __FUNCTION__); GELOGE(FAILED, "Input memory size is out of range."); return FAILED); if ((num > kMaxMemorySize) || (result * static_cast(num) > kMaxMemorySize)) { + REPORT_INNER_ERROR("E19999", "Param memory_size:%s after multi will exceed limit:%lu, session_id:%lu, " + "check invalid when VarManager %s", memory_size.c_str(), kMaxMemorySize, + session_id_, __FUNCTION__); GELOGE(FAILED, "Input memory size can not exceed max memory size : %zu.", kMaxMemorySize); return FAILED; } @@ -940,6 +990,7 @@ VarManager *VarManagerPool::GetVarManager(uint64_t session_id) { VarManager *var_manager = new (std::nothrow) VarManager(session_id); if (var_manager == nullptr) { + REPORT_INNER_ERROR("E19999", "New VarManager fali, session_id:%lu, when VarManager %s", session_id, __FUNCTION__); GELOGE(INTERNAL_ERROR, "VarManager::Instance find session by " "session_id[%lu] failed.", diff --git a/ge/graph/manager/host_mem_allocator.cc b/ge/graph/manager/host_mem_allocator.cc index ca2b5124..ab272670 100644 --- a/ge/graph/manager/host_mem_allocator.cc +++ b/ge/graph/manager/host_mem_allocator.cc @@ -34,6 +34,7 @@ uint8_t *HostMemAllocator::Malloc(size_t size) { std::lock_guard lock(mutex_); std::shared_ptr aligned_ptr = MakeShared(size); if (aligned_ptr == nullptr) { + REPORT_INNER_ERROR("E19999", "New AlignedPtr fail, when HostMemAllocator %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "make shared_ptr for AlignedPtr failed"); return nullptr; } @@ -44,6 +45,7 @@ uint8_t *HostMemAllocator::Malloc(size_t size) { Status HostMemAllocator::Free(const void *memory_addr) { if (memory_addr == nullptr) { + REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, check invalid when HostMemAllocator %s", __FUNCTION__); GELOGE(GE_GRAPH_FREE_FAILED, "Invalid memory pointer"); return GE_GRAPH_FREE_FAILED; } @@ -51,6 +53,8 @@ Status HostMemAllocator::Free(const void *memory_addr) { std::lock_guard lock(mutex_); auto it = allocated_blocks_.find(memory_addr); if (it == allocated_blocks_.end()) { + REPORT_INNER_ERROR("E19999", "Memory_addr is not alloc before, check invalid when HostMemAllocator %s", + __FUNCTION__); GELOGE(PARAM_INVALID, "Invalid memory pointer"); return PARAM_INVALID; } diff --git a/ge/graph/manager/host_mem_manager.cc b/ge/graph/manager/host_mem_manager.cc index 60a7586d..6461b77c 100644 --- a/ge/graph/manager/host_mem_manager.cc +++ b/ge/graph/manager/host_mem_manager.cc @@ -39,6 +39,8 @@ Status SharedMemAllocator::Allocate(SharedMemInfo &mem_info) { rtMallocHostSharedMemoryOut output_para; rtError_t rt_ret = rtMallocHostSharedMemory(&input_para, &output_para); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMallocHostSharedMemory fail, ret:0x%X, when SharedMemAllocator %s", + rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api(rtMallocHostSharedMemory) failed, devid:[%u].", device_id); return GE_GRAPH_MEMORY_ALLOC_FAILED; } @@ -59,6 +61,8 @@ Status SharedMemAllocator::DeAllocate(SharedMemInfo &mem_info) { mem_info.host_aligned_ptr->MutableGet(), mem_info.device_address}; rtError_t rt_ret = rtFreeHostSharedMemory(&free_para); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtFreeHostSharedMemory fail, ret:0x%X, when SharedMemAllocator %s", + rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Call rt api(rtFreeHostSharedMemory) failed, ret: 0x%X.", rt_ret); return RT_FAILED; } @@ -74,6 +78,7 @@ Status HostMemManager::Initialize() { std::lock_guard lock(mutex_); allocator_ = std::unique_ptr(new (std::nothrow) SharedMemAllocator()); if (allocator_ == nullptr) { + REPORT_CALL_ERROR("E19999", "New SharedMemAllocator fail when SharedMemAllocator %s", __FUNCTION__); GELOGE(GE_GRAPH_MALLOC_FAILED, "Shared memory allocator init failed!"); return GE_GRAPH_MALLOC_FAILED; } @@ -94,6 +99,8 @@ Status HostMemManager::MallocSharedMemory(SharedMemInfo &mem_info) { std::lock_guard lock(mutex_); auto iter = var_memory_base_map_.find(mem_info.op_name); if (iter != var_memory_base_map_.end()) { + REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_ when HostMemManager %s", + mem_info.op_name.c_str(), __FUNCTION__); GELOGE(FAILED, "Host shared memory for op %s has been malloced", mem_info.op_name.c_str()); return FAILED; } @@ -107,6 +114,8 @@ Status HostMemManager::MallocSharedMemory(SharedMemInfo &mem_info) { Status HostMemManager::QueryVarMemInfo(const string &op_name, uint64_t &base_addr, uint64_t &data_size) { std::lock_guard lock(mutex_); if (var_memory_base_map_.find(op_name) == var_memory_base_map_.end()) { + REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_ when HostMemManager %s", + op_name.c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Find host base base_addr failed,node name:%s!", op_name.c_str()); return INTERNAL_ERROR; } diff --git a/ge/graph/manager/memory_api.cc b/ge/graph/manager/memory_api.cc index 0798eb51..9843634d 100644 --- a/ge/graph/manager/memory_api.cc +++ b/ge/graph/manager/memory_api.cc @@ -50,6 +50,8 @@ Status RdmaRemoteRegister(const std::vector &var_info, rtMemType_t path.append(file_name); string canonical_path = RealPath(path.c_str()); if (canonical_path.empty()) { + REPORT_INNER_ERROR("E19999", "canonical_path:%s is empty, check invalid when %s", + canonical_path.c_str(), __FUNCTION__); GELOGE(FAILED, "Failed to get realpath of %s", path.c_str()); return FAILED; } @@ -65,12 +67,16 @@ Status RdmaRemoteRegister(const std::vector &var_info, rtMemType_t auto hcom_remote_mem_register = (HcclResult(*)(const MemRegisterAddr *, uint32_t))dlsym(handle, "HcomRegRemoteAccessMem"); if (hcom_remote_mem_register == nullptr) { + REPORT_CALL_ERROR("E19999", "Symbol HcomRegRemoteAccessMem can't find in %s, check invalid when %s", + canonical_path.c_str(), __FUNCTION__); GELOGE(FAILED, "Failed to invoke hcom_remote_mem_register function."); return FAILED; } HcclResult hccl_ret = hcom_remote_mem_register(reg_addrs.get(), table_len); if (hccl_ret != HCCL_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Call hcom_remote_mem_register failed, ret:%u, when %s", + hccl_ret.c_str(), __FUNCTION__); GELOGE(HCCL_E_INTERNAL, "Rdma mem register failed, ret: 0x%X", hccl_ret); return HCCL_E_INTERNAL; } diff --git a/ge/graph/manager/rdma_pool_allocator.cc b/ge/graph/manager/rdma_pool_allocator.cc index ed243801..6f9c4d31 100644 --- a/ge/graph/manager/rdma_pool_allocator.cc +++ b/ge/graph/manager/rdma_pool_allocator.cc @@ -81,6 +81,8 @@ Status RdmaPoolAllocator::InitMemory(size_t mem_size) { auto device_id = GetContext().DeviceId(); GELOGD("Init Rdma Memory with size [%zu] for devid:[%u]", mem_size, device_id); if (rdma_base_addr_ != nullptr) { + REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is nullptr, check invalid when RdmaPoolAllocator %s", + __FUNCTION__); GELOGE(GE_MULTI_INIT, "Rdma pool has been malloced"); return GE_MULTI_INIT; } @@ -100,6 +102,7 @@ Status RdmaPoolAllocator::InitMemory(size_t mem_size) { // Init with a base block. auto *base_block = new (std::nothrow) Block(device_id, mem_size, rdma_base_addr_); if (base_block == nullptr) { + REPORT_CALL_ERROR("E19999", "New Block failed, device_id:%u, when RdmaPoolAllocator %s", device_id, __FUNCTION__); GELOGE(GE_GRAPH_MALLOC_FAILED, "Block malloc failed"); return GE_GRAPH_MALLOC_FAILED; } @@ -118,6 +121,8 @@ uint8_t *RdmaPoolAllocator::Malloc(size_t size, uint32_t device_id) { block_bin_.erase(it); block->allocated = true; if (block->ptr == nullptr) { + REPORT_INNER_ERROR("E19999", "Rdmapool memory address is nullptr, device_id:%u, check invalid when RdmaPoolAllocator %s", + device_id, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Rdmapool memory address is nullptr."); return nullptr; } @@ -150,6 +155,8 @@ uint8_t *RdmaPoolAllocator::Malloc(size_t size, uint32_t device_id) { Status RdmaPoolAllocator::Free(uint8_t *memory_addr, uint32_t device_id) { GELOGI("Free rdma memory, device id = %u", device_id); if (memory_addr == nullptr) { + REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, device_id:%u, check invalid when RdmaPoolAllocator %s", + device_id, __FUNCTION__); GELOGE(GE_GRAPH_FREE_FAILED, "Invalid memory pointer"); return GE_GRAPH_FREE_FAILED; } @@ -157,6 +164,8 @@ Status RdmaPoolAllocator::Free(uint8_t *memory_addr, uint32_t device_id) { std::lock_guard lock(mutex_); auto it = allocated_blocks_.find(memory_addr); if (it == allocated_blocks_.end()) { + REPORT_INNER_ERROR("E19999", "Param memory_addr is not allocated before, device_id:%u, " + "check invalid when RdmaPoolAllocator %s", device_id, __FUNCTION__); GELOGE(PARAM_INVALID, "Invalid memory pointer"); return PARAM_INVALID; } @@ -199,6 +208,8 @@ void RdmaPoolAllocator::MergeBlocks(Block *dst, Block *src) { Status RdmaPoolAllocator::GetBaseAddr(uint64_t &base_addr, uint64_t &mem_size) { if (rdma_base_addr_ == nullptr) { + REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is nullptr, check invalid when RdmaPoolAllocator %s", + __FUNCTION__); GELOGE(INTERNAL_ERROR, "Rdma base addr is nullptr."); return INTERNAL_ERROR; } diff --git a/ge/graph/manager/trans_var_data_utils.cc b/ge/graph/manager/trans_var_data_utils.cc index cd992d3d..2b7be573 100644 --- a/ge/graph/manager/trans_var_data_utils.cc +++ b/ge/graph/manager/trans_var_data_utils.cc @@ -35,18 +35,24 @@ class RtContextSwitchGuard { RtContextSwitchGuard(rtCtxMode_t mode, uint32_t device_id) : last_(nullptr), current_(nullptr) { auto ret = rtCtxGetCurrent(&last_); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, device_id:%u, ret:0x%X, when %s", + device_id, ret, __FUNCTION__); GELOGE(RT_FAILED, "Failed to get current context from rt, error-code %d", ret); return; } ret = rtCtxCreate(¤t_, mode, static_cast(device_id)); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtCtxCreate failed, device_id:%u, ret:0x%X, when %s", + device_id, ret, __FUNCTION__); GELOGE(RT_FAILED, "Failed to create new context for device %u, error-code %d", device_id, ret); return; } ret = rtCtxSetCurrent(current_); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, device_id:%u, ret:0x%X, when %s", + device_id, ret, __FUNCTION__); GELOGE(RT_FAILED, "Failed to switch context to normal, context %p, device %u", current_, device_id); return; } @@ -72,6 +78,8 @@ class RtContextSwitchGuard { int64_t CalcVarSizeInBytes(const GeTensorDesc &desc) { int64_t var_size = GetSizeByDataType(desc.GetDataType()); if (var_size <= 0) { + REPORT_INNER_ERROR("E19999", "Data type:%s in desc, it's size:%ld < 0, check invalid when %s", + TypeUtils::DataTypeToSerialString(desc.GetDataType()).c_str(), var_size, __FUNCTION__); GELOGE(PARAM_INVALID, "Failed to calc var data size from data type %s", TypeUtils::DataTypeToSerialString(desc.GetDataType()).c_str()); return -1; @@ -89,6 +97,8 @@ Status CopyVarToDevice(const NodePtr &var, const formats::TransResult &trans_res auto ret = rtMemcpy(var_addr, trans_result.length, reinterpret_cast(trans_result.data.get()), trans_result.length, RT_MEMCPY_HOST_TO_DEVICE); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, op:%s(%s), size:%lu, ret:0x%X, when %s", var->GetName().c_str(), + var->GetType().c_str(), trans_result.length, ret, __FUNCTION__); GELOGE(RT_FAILED, "Failed to copy memory to device, size %zu", trans_result.length); return RT_FAILED; } @@ -110,6 +120,8 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt uint8_t *var_addr = VarManager::Instance(session_id)->GetVarMemoryAddr(var_logic, RT_MEMORY_HBM); if (var_addr == nullptr) { + REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, op:%s(%s), session_id:%lu, when %s", + RT_MEMORY_HBM, var->GetName().c_str(), var->GetType().c_str(), session_id, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to copy var %s from device, cant not get " "var addr from logic addr %p", @@ -124,6 +136,8 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt std::unique_ptr var_host(new(std::nothrow) uint8_t[var_size_bytes]); if (var_host == nullptr) { + REPORT_CALL_ERROR("E19999", "New host memory failed, size:%ld, op:%s(%s), session_id:%lu, when %s", + var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id, __FUNCTION__); GELOGE(OUT_OF_MEMORY, "Failed to malloc rt-host memory, size %ld", var_size_bytes); return OUT_OF_MEMORY; } @@ -131,6 +145,8 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt ret = rtMemcpy(reinterpret_cast(var_host.get()), var_size_bytes, reinterpret_cast(var_addr), var_size_bytes, RT_MEMCPY_DEVICE_TO_HOST); if (ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%ld, op:%s(%s), session_id:%lu, ret:0x%X when %s", + var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id, ret, __FUNCTION__); GELOGE(RT_FAILED, "Failed to copy var memory from device, var %s, size %ld," " rt-error-code %u", @@ -175,6 +191,12 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats TypeUtils::DataTypeToSerialString(data_type).c_str()); auto ret = formats::TransFormat({src_data, src_format, dst_format, src_shape, dst_shape, data_type}, tmp_result); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Trans format from %s to %s, shape %s to %s failed, data type:%s, ret:%u, when %s", + TypeUtils::FormatToSerialString(src_format).c_str(), + TypeUtils::FormatToSerialString(dst_format).c_str(), + formats::ShapeToString(src_shape).c_str(), + formats::ShapeToString(dst_shape).c_str(), + TypeUtils::DataTypeToSerialString(data_type).c_str(), ret, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to trans format from %s to %s, shape %s to %s, " "data type %s error code %u", @@ -195,6 +217,10 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats auto ret = formats::TransDataType({src_data, static_cast(src_data_size), src_data_type, dst_data_type}, tmp_result); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Trans data type from %s to %s failed, input shape %s, data size %ld, ret:%u, " + "when %s", TypeUtils::DataTypeToSerialString(src_data_type).c_str(), + TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), + formats::ShapeToString(input_shape).c_str(), src_data_size, ret, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to trans data type from %s to %s, input shape %s, data size %ld, error code %u", TypeUtils::DataTypeToSerialString(src_data_type).c_str(), TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), formats::ShapeToString(input_shape).c_str(), @@ -202,6 +228,8 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats return ret; } } else { + REPORT_INNER_ERROR("E19999", "Trans var data failed, the trans type %s does not supported, check invalid when %s", + trans_info.node_type.c_str(), __FUNCTION__); GELOGE(UNSUPPORTED, "Failed to trans var data, the trans type %s does not supported", trans_info.node_type.c_str()); return UNSUPPORTED; @@ -236,6 +264,8 @@ Status ReAssignVarAddr(uint64_t session_id, uint8_t *var_addr = VarManager::Instance(session_id)->GetVarMemoryAddr(var_logic, RT_MEMORY_HBM); if (var_addr == nullptr) { + REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, var_name:%s, session_id:%lu, when %s", + RT_MEMORY_HBM, var_name.c_str(), session_id, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to convert var %s logic addr to real addr", var_name.c_str()); return INTERNAL_ERROR; } @@ -263,6 +293,8 @@ Status TransVarData(const NodePtr &var, const VarTransRoad &trans_road, uint64_t // Sync var data from device std::unique_ptr var_data; if (trans_road.empty()) { + REPORT_INNER_ERROR("E19999", "Param trans_road is empty, session_id:%lu, check invalid when %s", + session_id, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to get trans_road, trans_road is empty."); return INTERNAL_ERROR; } @@ -314,6 +346,10 @@ Status TransTensor(uint8_t *var_data, const NodePtr &var_src, const NodePtr &var auto ret = formats::TransDataType( {var_data, static_cast(src_data_shape_size), src_data_datatype, dst_data_datatype}, result); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Trans data type from %s to %s failed, data size %ld, ret:%u, " + "when %s", TypeUtils::DataTypeToSerialString(src_data_datatype).c_str(), + TypeUtils::DataTypeToSerialString(dst_data_datatype).c_str(), + src_data_shape_size, ret, __FUNCTION__); GELOGE(INTERNAL_ERROR, "trans var data on host failed"); return ret; }); @@ -329,7 +365,10 @@ Status CopyTensorFromSrcVarNode(const NodePtr &var_src, /// unlink edges between var_fp32 and "dst_node" (need fp16) of var_fp32, add edge between var_fp16 and dst_node. /// need copy value from var_fp32 to var_fp16. /// [opdesc of var_src and var_dst are checked before passed in, no need to check if they are nullptr] - GE_IF_BOOL_EXEC(var_src == nullptr || var_dst == nullptr, GELOGE(FAILED, "node var is nullptr"); return FAILED); + GE_IF_BOOL_EXEC(var_src == nullptr || var_dst == nullptr, + REPORT_INNER_ERROR("E19999", "Param var_src or var_dst is empty, session_id:%lu, device_id:%u, " + "check invalid when %s", session_id, device_id, __FUNCTION__); + GELOGE(FAILED, "node var is nullptr"); return FAILED); // src_node output_desc (fp32) GeTensorDesc output_desc = var_src->GetOpDesc()->GetOutputDesc(0); auto src_data_type = output_desc.GetDataType(); @@ -447,15 +486,21 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, } std::future f = executor.commit( - [](const ge::NodePtr &node, uint64_t session_id, rtContext_t ctx, uint32_t graph_id) -> Status { + [](const ge::NodePtr &node, uint64_t session_id, rtContext_t ctx, uint32_t graph_id, + const struct ErrorMessage::Context &error_context) -> Status { + ErrorManager::GetInstance().SetErrorContext(error_context); rtError_t rt_ret = rtCtxSetCurrent(ctx); if (rt_ret != RT_ERROR_NONE) { + REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, session_id:%lu, graph_id:%u, ret:0x%X, when %s", + session_id, graph_id, rt_ret, __FUNCTION__); GELOGE(RT_FAILED, "Failed to set context, error_code is: 0x%X.", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } uint32_t allocated_graph_id = 0; Status ret = VarManager::Instance(session_id)->GetAllocatedGraphId(node->GetName(), allocated_graph_id); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get allocated GraphId failed, session_id:%lu, graph_id:%u, ret:0x%X, when %s", + session_id, graph_id, ret, __FUNCTION__); GELOGE(INTERNAL_ERROR, "var has not been allocated, node:%s, graph_id:%u.", node->GetName().c_str(), graph_id); return INTERNAL_ERROR; @@ -480,7 +525,7 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, } return SUCCESS; }, - node, session_id, context, graph_id); + node, session_id, context, graph_id, ErrorManager::GetInstance().GetErrorContext()); if (!f.valid()) { GELOGE(FAILED, "Future is invalid"); return FAILED; @@ -503,6 +548,8 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, Status TransVarDataUtils::CopyVarData(const ComputeGraphPtr &compute_graph, uint64_t session_id, uint32_t device_id) { GELOGD("CopyVarData start: session_id:%lu.", session_id); if (compute_graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, session_id:%lu, device_id:%u, check invalid when %s", + session_id, device_id, __FUNCTION__); GELOGE(FAILED, "compute_graph is nullptr"); return FAILED; } diff --git a/ge/graph/preprocess/graph_preprocess.cc b/ge/graph/preprocess/graph_preprocess.cc index 08dd6f98..024b539d 100644 --- a/ge/graph/preprocess/graph_preprocess.cc +++ b/ge/graph/preprocess/graph_preprocess.cc @@ -99,6 +99,7 @@ const int64_t kInvalidDynaimcDimsType = -1; OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) { GeTensorPtr tensor = MakeShared(); if (tensor == nullptr) { + REPORT_CALL_ERROR("E19999", "New GeTensor failed when %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create shared ptr for GeTensor failed"); return nullptr; } @@ -110,6 +111,7 @@ OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) { tensor->MutableTensorDesc().SetShape(GeShape()); int32_t dst_shape = 1; if (tensor->SetData(reinterpret_cast(&dst_shape), sizeof(int32_t)) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Set data to tensor failed when %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "tensor set data failed"); return nullptr; } @@ -117,6 +119,7 @@ OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) { tensor->MutableTensorDesc().SetShape(GeShape(std::vector({dim_cnt}))); unique_ptr dst_shape(new (std::nothrow) int32_t[dim_cnt]()); if (dst_shape == nullptr) { + REPORT_CALL_ERROR("E19999", "Malloc buffer failed, size:%zu, when %s", dim_cnt, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create unique ptr failed"); return nullptr; } @@ -126,6 +129,7 @@ OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) { GE_IF_BOOL_EXEC( tensor->SetData(reinterpret_cast(dst_shape.get()), dim_cnt * sizeof(int32_t)) != GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Set data to tensor failed when %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "tensor set data failed"); return nullptr;) } @@ -172,11 +176,15 @@ void AddTransNodeAttr(const std::string &node_type, const GeTensorDesc &input, c NodePtr CreateTransNode(const std::string &name, const std::string &node_type, const GeTensorDesc &input, const GeTensorDesc &output, NodePtr &node) { if (node == nullptr) { + REPORT_INNER_ERROR("E19999", "Param node is nullptr, trans_name:%s, trans_type:%s, check invalid when %s", + name.c_str(), node_type.c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "node is null."); return nullptr; } auto graph = node->GetOwnerComputeGraph(); if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Owner graph in node is nullptr, trans_name:%s, trans_type:%s, check invalid when %s", + name.c_str(), node_type.c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "Owner graph is null, node name:%s.", node->GetName().c_str()); return nullptr; } @@ -191,6 +199,8 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c } OpDescPtr op_desc = MakeShared(name, node_type); if (op_desc == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed, trans_name:%s, trans_type:%s, when %s", + name.c_str(), node_type.c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Create shared ptr for OpDesc failed"); return nullptr; } @@ -203,11 +213,15 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c // Default single input and single output auto ret = op_desc->AddInputDesc(input); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc into op:%s(%s) failed when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add input desc when create node %s type %s", name.c_str(), node_type.c_str()); return nullptr; } ret = op_desc->AddOutputDesc(output); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc into op:%s(%s) failed when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add output desc when create node %s type %s", name.c_str(), node_type.c_str()); return nullptr; } @@ -224,12 +238,17 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c } ret = op_desc->AddInputDesc(shape_desc->GetOutputDesc(0)); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add input desc into op:%s(%s) failed when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add the first input for reshape %s", name.c_str()); return nullptr; } shape_node = graph->AddNode(shape_desc); if (shape_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when %s", + shape_desc->GetName().c_str(), shape_desc->GetType().c_str(), + graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add shape node for reshape %s, can not add the shape to graph", name.c_str()); return nullptr; } @@ -237,12 +256,18 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c auto trans_node = graph->AddNode(op_desc); if (trans_node == nullptr) { + REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), + graph->GetName().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add trans node %s to graph", name.c_str()); return nullptr; } if (node_type == RESHAPE) { if (GraphUtils::AddEdge(shape_node->GetOutDataAnchor(0), trans_node->GetInDataAnchor(1)) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:0) failed " + "when %s", shape_node->GetName().c_str(), shape_node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to add shape node for reshape %s, can not add the edge", name.c_str()); return nullptr; } @@ -261,6 +286,9 @@ Status RecoverOneTransNodeForVar(const std::string &name, const TransNodeInfo &t auto ret = GraphUtils::ReplaceNodeDataAnchors(trans_node, node, {}, {0}); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Replace out anchors of node:%s(%s) by node:%s(%s) failed when %s", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to replace out anchors when recover trans node for %s type %s", node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; @@ -268,6 +296,9 @@ Status RecoverOneTransNodeForVar(const std::string &name, const TransNodeInfo &t ret = GraphUtils::AddEdge(node->GetOutDataAnchor(0), trans_node->GetInDataAnchor(0)); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:0) failed when %s", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to connect node %s to trans node %s", node->GetName().c_str(), trans_node->GetName().c_str()); return INTERNAL_ERROR; @@ -275,6 +306,9 @@ Status RecoverOneTransNodeForVar(const std::string &name, const TransNodeInfo &t ret = GraphUtils::MoveOutCtrlEdges(node, trans_node); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Move out control edges from node:%s(%s) to node:%s(%s) failed when %s", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to move out control edges from %s to %s when recover trans node.", node->GetName().c_str(), trans_node->GetName().c_str()); return INTERNAL_ERROR; @@ -293,6 +327,9 @@ Status RecoverOneTransNodeForVarRef(const std::string &name, const TransNodeInfo auto ret = GraphUtils::ReplaceNodeDataAnchors(trans_node, node, {0}, {}); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Replace out anchors of node:%s(%s) by node:%s(%s) failed when %s", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to replace int anchors when recover trans node for %s type %s", node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; @@ -300,6 +337,9 @@ Status RecoverOneTransNodeForVarRef(const std::string &name, const TransNodeInfo ret = GraphUtils::AddEdge(trans_node->GetOutDataAnchor(0), node->GetInDataAnchor(0)); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:0) failed when %s", + trans_node->GetName().c_str(), trans_node->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to connect trans node %s to node %s", trans_node->GetName().c_str(), node->GetName().c_str()); return INTERNAL_ERROR; @@ -307,6 +347,9 @@ Status RecoverOneTransNodeForVarRef(const std::string &name, const TransNodeInfo ret = GraphUtils::MoveInCtrlEdges(node, trans_node); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Move out control edges from node:%s(%s) to node:%s(%s) failed when %s", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to move int control edges from %s to %s when recover trans node.", node->GetName().c_str(), trans_node->GetName().c_str()); return INTERNAL_ERROR; @@ -327,6 +370,8 @@ Status UpdateVarFormats(const NodePtr &var, const GeTensorDesc &tensor_desc) { output_desc.SetOriginDataType(tensor_desc.GetOriginDataType()); output_desc.SetOriginShape(tensor_desc.GetOriginShape()); GE_IF_BOOL_EXEC(var->GetOpDesc()->UpdateOutputDesc(0, output_desc) != GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Update output desc of node:%s(%s) failed, index:0, when %s", + var->GetName().c_str(), var->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "UpdateOutputDesc failed"); return INTERNAL_ERROR;); } @@ -340,6 +385,8 @@ Status UpdateVarFormats(const NodePtr &var, const GeTensorDesc &tensor_desc) { desc.SetOriginDataType(tensor_desc.GetOriginDataType()); desc.SetOriginShape(tensor_desc.GetOriginShape()); GE_IF_BOOL_EXEC(var->GetOpDesc()->UpdateInputDesc(0, desc) != GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Update input desc of node:%s(%s) failed, index:0, when %s", + var->GetName().c_str(), var->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "UpdateInputDesc failed"); return INTERNAL_ERROR;) } @@ -369,6 +416,9 @@ Status RecoverTransRoadForVar(const NodePtr &var, const VarTransRoad &road) { GE_CHK_STATUS_RET(SetStreamLabel(last_node, stream_label), "set stream label failed"); } GE_CHK_BOOL_EXEC((ge::AttrUtils::SetBool(last_node->GetOpDesc(), ge::ATTR_INSERTED_BY_GE, true)), + REPORT_CALL_ERROR("E19999", "Set Attr:%s of node:%s(%s) failed when %s", + ge::ATTR_INSERTED_BY_GE.c_str(), + last_node->GetName().c_str(), last_node->GetType().c_str(), __FUNCTION__); return INTERNAL_ERROR, "Set attr ATTR_INSERTED_BY_GE failed."); GELOGD("Recover trans node %s type %s success", trans_name.c_str(), iter->node_type.c_str()); } @@ -404,6 +454,9 @@ Status RecoverTransRoadForVarRef(const std::set &nodes, const VarTransR } GE_CHK_BOOL_EXEC((ge::AttrUtils::SetBool(last_node->GetOpDesc(), ge::ATTR_INSERTED_BY_GE, true)), + REPORT_CALL_ERROR("E19999", "Set Attr:%s of node:%s(%s) failed when %s", + ge::ATTR_INSERTED_BY_GE.c_str(), + last_node->GetName().c_str(), last_node->GetType().c_str(), __FUNCTION__); return INTERNAL_ERROR, "Set attr ATTR_INSERTED_BY_GE failed."); } if (!(road.empty()) && (UpdateVarFormats(var, road.rbegin()->output) != SUCCESS)) { @@ -419,6 +472,7 @@ VarNamesToRefs CollectVarNamesToRefs(const ComputeGraphPtr &graph) { VarNamesToRefs names_to_refs; std::string var_name; if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid when %s", __FUNCTION__); GELOGE(PARAM_INVALID, "graph is null."); return names_to_refs; } @@ -462,6 +516,8 @@ Status ModifyInputFormatAndShape(NodePtr &node_ptr) { ge::DataType dt = input->GetDataType(); std::vector dst_shape_dims; if (TransferShape2NC1HWC0(old_format, old_shape, dt, FORMAT_NC1HWC0, dst_shape_dims) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Transfer shape to NC1HWC0 failed, op:%s(%s), when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Trans shape failed"); return FAILED; } @@ -477,6 +533,8 @@ Status ModifyInputFormatAndShape(NodePtr &node_ptr) { int64_t size = 0; graphStatus graph_status = TensorUtils::GetTensorMemorySizeInBytes(*output, size); if (graph_status != ge::GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get output tensor size failed, op:%s(%s), index:0 when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(graph_status, "GetTensorSizeInBytes failed!"); return FAILED; } @@ -521,6 +579,8 @@ Status ModifyDataNetOutputFormatAndShape(OpDescPtr &op_desc, uint32_t index, For int64_t size = 0; graphStatus graph_status = TensorUtils::GetTensorMemorySizeInBytes(*output, size); if (graph_status != ge::GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get output tensor size failed, op:%s(%s), index:%u when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), index, __FUNCTION__); GELOGE(graph_status, "GetTensorSizeInBytes failed!"); return FAILED; } @@ -630,6 +690,8 @@ Status ProcessInputDtDynShape(NodePtr &node_ptr, bool &is_dynamic_batch, NodePtr ge::graphStatus input_graph_status = ge::TensorUtils::GetTensorSizeInBytes(*input, input_shape_size); ge::graphStatus output_graph_status = ge::TensorUtils::GetTensorMemorySizeInBytes(*input, output_shape_size); if (input_graph_status != ge::GRAPH_SUCCESS && output_graph_status != ge::GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get input tensor size failed, op:%s(%s), index:0 when %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(GRAPH_FAILED, "[Process][InputOp] Get tensor size of op [%s] failed!", node_ptr->GetName().c_str()); return FAILED; } @@ -680,6 +742,8 @@ Status ProcessInputNC1HWC0DynShape(NodePtr &node_ptr, bool &is_dynamic_batch, No GE_CHECK_NOTNULL(switchn_op_desc); const GeTensorDescPtr &switchn_input = switchn_op_desc->MutableInputDesc(0); if (ModifyFormatAndShapeForSingleTensor(switchn_input) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Modify format and shape of input:0 in op:%s(%s) failed when %s", + switchn_op_desc->GetName().c_str(), switchn_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "modify format and shape failed"); return FAILED; } @@ -689,6 +753,8 @@ Status ProcessInputNC1HWC0DynShape(NodePtr &node_ptr, bool &is_dynamic_batch, No old_format = switchn_output->GetFormat(); old_shape = switchn_output->GetShape(); if (ModifyFormatAndShapeForSingleTensor(switchn_output) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Modify format and shape of output:%u in op:%s(%s) failed when %s", i, + switchn_op_desc->GetName().c_str(), switchn_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "modify format and shape failed"); return FAILED; } @@ -789,6 +855,9 @@ Status ProcessNetoutputNodeFp16Nc1hwc0DynShape(GeTensorDesc &src_desc, GeTensorD std::vector dst_shape_dims; std::vector src_shape_dims = src_shape.GetDims(); if (TransferShape2NC1HWC0(src_format, src_shape_dims, DT_FLOAT16, FORMAT_NC1HWC0, dst_shape_dims) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Transfer output:0 shape of op:%s(%s) to NC1HWC0 format failed, shape:%s, format:%s, " + "when %s", src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str(), + src_shape.ToString().c_str(), TypeUtils::FormatToSerialString(src_format).c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Trans shape failed"); return FAILED; } @@ -799,6 +868,8 @@ Status ProcessNetoutputNodeFp16Nc1hwc0DynShape(GeTensorDesc &src_desc, GeTensorD auto merge_out = src_op_desc->MutableOutputDesc(0); GE_CHECK_NOTNULL(merge_out); if (ModifyFormatAndShapeForSingleTensor(merge_out) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Modify format and shape of output:0 in op:%s(%s) failed when %s", + src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "modify format and shape failed"); return FAILED; } @@ -806,6 +877,8 @@ Status ProcessNetoutputNodeFp16Nc1hwc0DynShape(GeTensorDesc &src_desc, GeTensorD auto merge_in = src_op_desc->MutableInputDesc(i); GE_CHECK_NOTNULL(merge_in); if (ModifyFormatAndShapeForSingleTensor(merge_in) != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Modify format and shape of input:%u in op:%s(%s) failed when %s", i, + src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "modify format and shape failed"); return FAILED; } @@ -931,12 +1004,15 @@ long StringToLongNoThrow(const string &str) { Status ParseDynamicInputShapeRange(const std::string &shape_range, std::vector>> &range) { if (shape_range.size() < 2) { + REPORT_INNER_ERROR("E19999", "shape_range.size:%zu < 2, check invalid when %s", shape_range.size(), __FUNCTION__); GELOGE(PARAM_INVALID, "Shape range %s is invalid.", shape_range.c_str()); return PARAM_INVALID; } // different shape_range of single input are split by ']' vector shape_range_set = ge::StringUtils::Split(shape_range, ']'); if (shape_range_set.empty()) { + REPORT_INNER_ERROR("E19999", "Shape range %s is not valid. Correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"", + shape_range.c_str()); GELOGE(PARAM_INVALID, "Shape range %s is not valid. Correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"", shape_range.c_str()); return PARAM_INVALID; @@ -975,6 +1051,8 @@ Status ParseDynamicInputShapeRange(const std::string &shape_range, auto range_left = StringToLongNoThrow(range_pair_set.at(0).c_str()); auto range_right = StringToLongNoThrow(range_pair_set.at(1).c_str()); if (range_left < 0 || range_right < 0) { + REPORT_INNER_ERROR("E19999", "Shape range of input is invalid. Given range pair [%ld,%ld], " + "while correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"", range_left, range_right); GELOGE(PARAM_INVALID, "Shape range of input is invalid. Given range pair [%ld,%ld], while correct example: " "\"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"", @@ -983,6 +1061,8 @@ Status ParseDynamicInputShapeRange(const std::string &shape_range, } range_pair = std::make_pair(range_left, range_right); } else { + REPORT_INNER_ERROR("E19999", "Shape range of input is invalid. Given %s, " + "while correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"", shape_range.c_str()); GELOGE(PARAM_INVALID, "Shape range of input is invalid. Given %s, while correct example: \"[1~20,3,3~6,-1],[1~20,3,3~6,-1]\"", shape_range.c_str()); @@ -1016,6 +1096,8 @@ Status GetDynamicInputShapeRange(const std::vector &user_input, const } else if (!enable_dynamic_execute_mode && !enable_input_shape_range) { return SUCCESS; } else { + REPORT_INNER_ERROR("E19999", "Graph option: %s and %s should be enabled at the same time, check invalid when %s", + OPTION_EXEC_DYNAMIC_EXECUTE_MODE, OPTION_EXEC_DATA_INPUTS_SHAPE_RANGE, __FUNCTION__); GELOGE(PARAM_INVALID, "Graph option: %s and %s should be enabled at the same time.", OPTION_EXEC_DYNAMIC_EXECUTE_MODE, OPTION_EXEC_DATA_INPUTS_SHAPE_RANGE); return PARAM_INVALID; @@ -1037,6 +1119,9 @@ Status UpdateDynamicInputShapeRange(const ge::GeAttrValue::INT index, auto origin_shape = desc.GetShape(); auto current_shape_range_vec = range_vec.at(index); if (current_shape_range_vec.size() != origin_shape.GetDimNum()) { + REPORT_INNER_ERROR("E19999", "Given shape_range dim num is %zu, current dim:%s num is %zu, not match, " + "check invalid when %s", current_shape_range_vec.size(), origin_shape.ToString().c_str(), + origin_shape.GetDimNum(), __FUNCTION__); GELOGE(PARAM_INVALID, "Given shape_range dim num is %zu, current dim num is %zu, not match.Pleace Check.", current_shape_range_vec.size(), origin_shape.GetDimNum()); return PARAM_INVALID; @@ -1048,6 +1133,8 @@ Status UpdateDynamicInputShapeRange(const ge::GeAttrValue::INT index, if (left_range == right_range) { // given shape_range is known dim, check is same as origin or not if (curr_dim != left_range) { + REPORT_INNER_ERROR("E19999", "Given shape range is %ld, current dim shape is %ld, not match, dim_index:%zu, " + "check invalid when %s", left_range, curr_dim, i, __FUNCTION__); GELOGE(PARAM_INVALID, "Given shape range is %ld, current dim shape is %ld, not match.Pleace Check.", left_range, curr_dim); return PARAM_INVALID; @@ -1057,6 +1144,9 @@ Status UpdateDynamicInputShapeRange(const ge::GeAttrValue::INT index, // given shape_range is fix range, check input_shape is in this range or not if (right_range != UNKNOWN_DIM) { if ((curr_dim < left_range) || (curr_dim > right_range)) { + REPORT_INNER_ERROR("E19999", "Given shape range is [%ld~%ld], current dim shape is %ld, out of range, " + "dim_index:%zu, check invalid when %s", + left_range, right_range, curr_dim, i, __FUNCTION__); GELOGE(PARAM_INVALID, "Given shape range is [%ld~%ld], current dim shape is %ld, out of range.Pleace Check.", left_range, right_range, curr_dim); return PARAM_INVALID; @@ -1145,17 +1235,21 @@ Status GraphPrepare::Init(const ge::Graph &graph, uint64_t session_id) { Status GraphPrepare::CheckGraph() { if (compute_graph_ == nullptr) { + REPORT_INNER_ERROR("E19999", "compute_graph_ is nullptr, check invalid when GraphPrepare %s", __FUNCTION__); GELOGE(GE_GRAPH_INIT_FAILED, "Graph prepare init compute graph is NULLPTR"); return GE_GRAPH_INIT_FAILED; } auto nodes = compute_graph_->GetAllNodes(); if (nodes.empty()) { + REPORT_INNER_ERROR("E19999", "nodes in graph is empty, check invalid when GraphPrepare %s", __FUNCTION__); GELOGE(GE_GRAPH_INIT_FAILED, "Invalid graph, no nodes in this graph."); return GE_GRAPH_INIT_FAILED; } for (const NodePtr &node : compute_graph_->GetAllNodes()) { GE_CHECK_NOTNULL(node); if (node->GetOpDesc() == nullptr) { + REPORT_INNER_ERROR("E19999", "node without opdesc exist in graph, check invalid when GraphPrepare %s", + __FUNCTION__); GELOGE(GE_GRAPH_INIT_FAILED, "Check Graph node opdesc is NULL"); return GE_GRAPH_INIT_FAILED; } @@ -1191,6 +1285,9 @@ Status GraphPrepare::CheckRefInputNode(const NodePtr &node, const std::string &i auto input_type = input_op_desc->GetType(); if (input_type == ge::FRAMEWORKOP) { if (!ge::AttrUtils::GetStr(input_op_desc, ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE, input_type)) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed when GraphPrepare %s", + ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE.c_str(), + input_op_desc->GetName().c_str(), input_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "Get original type failed."); return PARAM_INVALID; } @@ -1214,11 +1311,15 @@ Status GraphPrepare::CheckRefOp() { std::set ref_nodes; for (const NodePtr &node : compute_graph_->GetDirectNode()) { if (node == nullptr) { + REPORT_INNER_ERROR("E19999", "nullptr node exist in graph, check invalid when GraphPrepare %s", + __FUNCTION__); GELOGE(PARAM_INVALID, "param [node] must not be null."); return PARAM_INVALID; } auto op_desc = node->GetOpDesc(); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "node without opdesc exist in graph, check invalid when GraphPrepare %s", + __FUNCTION__); GELOGE(PARAM_INVALID, "OpDesc of param [node] must not be null."); return PARAM_INVALID; } @@ -1252,11 +1353,15 @@ Status GraphPrepare::SetRtContext(rtContext_t rt_context, rtCtxMode_t mode) { Status GraphPrepare::AdjustDataOpOutput(const NodePtr &node) { if (node == nullptr) { + REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid when GraphPrepare %s", + __FUNCTION__); GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Input node is NULL"); return GE_GRAPH_GRAPH_NODE_NULL; } OpDescPtr op_desc_ptr = node->GetOpDesc(); if (op_desc_ptr == nullptr) { + REPORT_INNER_ERROR("E19999", "Param node's op_desc is nullptr, check invalid when GraphPrepare %s", + __FUNCTION__); GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Input node opdesc is NULL"); return GE_GRAPH_GRAPH_NODE_NULL; } @@ -1278,6 +1383,8 @@ Status GraphPrepare::AdjustDataOpOutput(const NodePtr &node) { TensorUtils::SetSize(output, tensor_size); graphStatus graph_ret = op_desc_ptr->UpdateOutputDesc(0, output); if (graph_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update output desc of op:%s(%s) failed, index:0, when GraphPrepare %s", + op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str(), __FUNCTION__); GELOGE(graph_ret, "UpdateOutputDesc fail, graph_ret:%u", graph_ret); return graph_ret; } @@ -1348,6 +1455,8 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, GE_IF_BOOL_EXEC(shape_size == 0 && desc.GetShape().GetDimNum() == 0, shape_size = static_cast(length)); int64_t size = 0; GE_IF_BOOL_EXEC(ge::TensorUtils::GetSize(desc, size) != GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Get size of user input tensor failed, index:%ld, " + "when GraphPrepare %s", index, __FUNCTION__); GELOGE(INTERNAL_ERROR, "TensorUtils GetSize failed"); return FAILED); bool size_check = (size != 0 && shape_size != size); @@ -1363,6 +1472,8 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, if (!tune_flag) { graphStatus graph_ret = op->UpdateInputDesc(0, desc); if (graph_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update input desc of op:%s(%s) failed, index:0, when GraphPrepare %s", + op->GetName().c_str(), op->GetType().c_str(), __FUNCTION__); GELOGE(graph_ret, "UpdateInputDesc fail, graph_ret:%u", graph_ret); return graph_ret; } @@ -1370,6 +1481,8 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, ge::TensorUtils::SetSize(desc, 0); graph_ret = op->UpdateOutputDesc(0, desc); if (graph_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update output desc of op:%s(%s) failed, index:0, when GraphPrepare %s", + op->GetName().c_str(), op->GetType().c_str(), __FUNCTION__); GELOGE(graph_ret, "UpdateOutputDesc fail, graph_ret:%u", graph_ret); return graph_ret; } @@ -1468,6 +1581,8 @@ Status GraphPrepare::ResourcePairProcess(const std::string &action) { new ResourcePairRemoveControlPass); } } catch (std::bad_alloc &e) { + REPORT_INNER_ERROR("E19999", "bad memory allocation occur when add ResourcePair Pass, when GraphPrepare %s", + __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add pass failed, bad memory allocation occur, action:%s.", action.c_str()); return INTERNAL_ERROR; } @@ -1604,6 +1719,7 @@ Status GraphPrepare::PrepareRunningFormatRefiner() { Status GraphPrepare::SwitchOpOptimize(ComputeGraphPtr &compute_graph) { if (compute_graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid when GraphPrepare %s", __FUNCTION__); GELOGE(GE_GRAPH_NULL_INPUT, "Input Graph is NULL"); return GE_GRAPH_NULL_INPUT; } @@ -1619,6 +1735,7 @@ Status GraphPrepare::SwitchOpOptimize(ComputeGraphPtr &compute_graph) { } ret = compute_graph->TopologicalSorting(); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Topological sorting failed when GraphPrepare %s", __FUNCTION__); GELOGE(ret, "Graph topological sort failed, ret:%u.", ret); return ret; } @@ -1629,6 +1746,7 @@ Status GraphPrepare::SwitchOpOptimize(ComputeGraphPtr &compute_graph) { Status GraphPrepare::GenerateInfershapeGraph(ConstGraphPtr graph) { if (graph == nullptr) { + REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid when GraphPrepare %s", __FUNCTION__); GELOGE(GE_GRAPH_NULL_INPUT, "Input Graph is NULL"); return GE_GRAPH_NULL_INPUT; } @@ -1643,6 +1761,7 @@ Status GraphPrepare::GenerateInfershapeGraph(ConstGraphPtr graph) { ret = compute_graph_->InferOriginFormat(); GE_DUMP(compute_graph_, "after_inferformat"); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Infer OriginFormat failed when GraphPrepare %s", __FUNCTION__); GELOGE(ret, "Prepare Graph inferformat failed"); return ret; } @@ -1669,6 +1788,7 @@ Status GraphPrepare::CheckConstOp() { } else if (node_ptr->GetType() == FRAMEWORKOP) { auto op_desc = node_ptr->GetOpDesc(); if (op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "op_desc is nullptr, check invalid when GraphPrepare %s", __FUNCTION__); GELOGE(PARAM_INVALID, "Get op desc failed"); return PARAM_INVALID; } @@ -1690,6 +1810,8 @@ Status GraphPrepare::VerifyConstOp(const NodePtr &node) { GE_CHECK_NOTNULL(op_desc); ConstGeTensorPtr ge_tensor_ptr; if (!(AttrUtils::GetTensor(op_desc, ATTR_NAME_WEIGHTS, ge_tensor_ptr))) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed when GraphPrepare %s", ATTR_NAME_WEIGHTS.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(PARAM_INVALID, "Get value from const attr failed"); return PARAM_INVALID; } @@ -1764,6 +1886,8 @@ Status GraphPrepare::CheckUserInput(const std::vector &user_input) { data_num++; GeAttrValue::INT index = 0; if (!(AttrUtils::GetInt(op, ATTR_NAME_INDEX, index))) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed when GraphPrepare %s", ATTR_NAME_WEIGHTS.c_str(), + op->GetName().c_str(), op->GetType().c_str(), __FUNCTION__); GELOGE(GE_GRAPH_INIT_FAILED, "Get index from attr failed"); return GE_GRAPH_INIT_FAILED; } @@ -1868,6 +1992,7 @@ Status GraphPrepare::PrepareOptimize() { (void)original_graph_passes.AddPass("PrepareOptimize::ReplaceTransShapePass", new ReplaceTransShapePass); (void)original_graph_passes.AddPass("PrepareOptimize::MarkAgnosticPass", new MarkAgnosticPass); } catch (std::bad_alloc &e) { + REPORT_INNER_ERROR("E19999", "bad memory allocation occur when add Pass, when GraphPrepare %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add pass failed, bad memory allocation occurs."); return INTERNAL_ERROR; } @@ -1931,6 +2056,7 @@ Status GraphPrepare::PrepareOptimize() { // can't move to optimize1/2 directly, may cause more identity insert, cause CI fail (void)graph_pass.AddPass("PrepareOptimize::HcclMemcpyPass", new HcclMemcpyPass); } catch (std::bad_alloc &e) { + REPORT_INNER_ERROR("E19999", "bad memory allocation occur when add Pass, when GraphPrepare %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add pass failed, bad memory allocation occurs."); return INTERNAL_ERROR; } @@ -1947,6 +2073,7 @@ Status GraphPrepare::PrepareOptimize() { ret = compute_graph_->TopologicalSorting(); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Topological sorting failed when GraphPrepare %s", __FUNCTION__); GELOGE(ret, "Graph topological sort failed, ret:%u.", ret); return ret; } @@ -2017,6 +2144,7 @@ Status GraphPrepare::ProcessNetOutput() { graph_passes_before_infershape.AddPass("ProcessNetOutput::DataPass", new (std::nothrow) DataPass); // Add NetOutput first. } catch (std::bad_alloc) { + REPORT_INNER_ERROR("E19999", "bad memory allocation occur when add Pass, when GraphPrepare %s", __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add pass failed, bad memory allocation occurs."); return INTERNAL_ERROR; } @@ -2056,6 +2184,7 @@ Status GraphPrepare::CheckAndUpdateInput(const std::vector &user_input } else { ret = compute_graph_->TopologicalSorting(); if (ret != SUCCESS) { + REPORT_CALL_ERROR("E19999", "Topological sorting failed when GraphPrepare %s", __FUNCTION__); GELOGE(ret, "graph prepare error: compute_graph_->Topological Sorting"); return FAILED; } diff --git a/ge/graph/preprocess/insert_op/ge_aipp_op.cc b/ge/graph/preprocess/insert_op/ge_aipp_op.cc index 25af98b8..96d1e044 100755 --- a/ge/graph/preprocess/insert_op/ge_aipp_op.cc +++ b/ge/graph/preprocess/insert_op/ge_aipp_op.cc @@ -110,6 +110,12 @@ Status GetDataDimN(const ge::NodePtr &data_node, ge::Format format, int64_t &bat batch = shape[NHWC_DIM_N]; return SUCCESS; default: + REPORT_INPUT_ERROR("E10001", std::vector({"parameter", "value", "reason"}), + std::vector({ + data_node->GetName() + " format", + TypeUtils::FormatToSerialString(format), + "only format " + TypeUtils::FormatToSerialString(FORMAT_NCHW) + " and " + + TypeUtils::FormatToSerialString(FORMAT_NHWC) + " supported"})); GELOGE(PARAM_INVALID, "Not support data format: %s", TypeUtils::FormatToSerialString(format).c_str()); return PARAM_INVALID; } @@ -156,6 +162,7 @@ Format GetAndCheckFormat() { Status AippOp::Init(domi::AippOpParams *aipp_params) { aipp_params_ = new (std::nothrow) domi::AippOpParams(); if (aipp_params_ == nullptr) { + REPORT_CALL_ERROR("E19999", "New AippOpParams failed when AippOp %s", __FUNCTION__); return FAILED; } aipp_params_->CopyFrom(*aipp_params); @@ -190,6 +197,13 @@ Status AippOp::InsertAippToGraph(ComputeGraphPtr &graph, std::string &aippConfig auto ret = GraphUtils::InsertNodeBetweenDataAnchors(out_in_anchors.first, out_in_anchors.second, aipp); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Insert aipp:%s(%s) node between op:%s(%s) and op:%s:%s failed when AippOp %s", + aipp->GetName().c_str(), aipp->GetType().c_str(), + out_in_anchors.first->GetOwnerNode()->GetName().c_str(), + out_in_anchors.first->GetOwnerNode()->GetType().c_str(), + out_in_anchors.second->GetOwnerNode()->GetName().c_str(), + out_in_anchors.second->GetOwnerNode()->GetType().c_str(), + __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to link edges for aipp node %s", aipp->GetName().c_str()); return INTERNAL_ERROR; } @@ -209,6 +223,11 @@ Status AippOp::InsertAippToGraph(ComputeGraphPtr &graph, std::string &aippConfig auto &aipp = iter->second; auto ret = out_in_anchors.second->LinkFrom(aipp->GetOutDataAnchor(0)); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "link aipp:%s(%s) to peer op:%s(%s) failed when AippOp %s", + aipp->GetName().c_str(), aipp->GetType().c_str(), + out_in_anchors.second->GetOwnerNode()->GetName().c_str(), + out_in_anchors.second->GetOwnerNode()->GetType().c_str(), + __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to link aipp %s to the peer node %s", aipp->GetName().c_str(), out_in_anchors.second->GetOwnerNode()->GetName().c_str()); return INTERNAL_ERROR; @@ -224,6 +243,7 @@ NodePtr AippOp::CreateAipp(const OutDataAnchorPtr &out_anchor, std::string current_name = node->GetName() + "_" + std::to_string(out_anchor->GetIdx()) + "_huawei_aipp"; auto aipp_opdesc_ptr = MakeShared(current_name, AIPP); if (aipp_opdesc_ptr == nullptr) { + REPORT_CALL_ERROR("E19999", "New OpDesc failed when AippOp %s", __FUNCTION__); GELOGE(OUT_OF_MEMORY, "Failed to alloc aipp desc, name %s", current_name.c_str()); return nullptr; } @@ -250,6 +270,9 @@ NodePtr AippOp::CreateAipp(const OutDataAnchorPtr &out_anchor, // but the InferFormat process before InferShape can not infer the format // if the tensor on the Aipp has an unknown shape if (aipp_opdesc_ptr->UpdateInputDesc(kAippImageInputIndex, opdesc_src_data) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update the output desc from node:%s(%s) to aipp:%s(%s) failed when AippOp %s", + node_desc->GetName().c_str(), node_desc->GetType().c_str(), + aipp_opdesc_ptr->GetName().c_str(), aipp_opdesc_ptr->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to update the output desc from node %s to aipp %s", node_desc->GetName().c_str(), aipp_opdesc_ptr->GetName().c_str()); return nullptr; @@ -341,6 +364,8 @@ Status AippOp::GetAndCheckTarget(const ComputeGraphPtr &graph, int rank, NodePtr GeAttrValue::NAMED_ATTRS aipp_attr; ConvertParamToAttr(aipp_attr); if (!AttrUtils::SetNamedAttrs(data_opdesc, ATTR_NAME_AIPP, aipp_attr)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed when AippOp %s", ATTR_NAME_AIPP.c_str(), + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Set name attrs for Data node failed. id: %d", rank); return INTERNAL_ERROR; } @@ -371,12 +396,18 @@ Status AippOp::GetStaticTargetNode(const ComputeGraphPtr &graph, NodePtr &data_n std::string related_node_name; if (AttrUtils::GetStr(data_node->GetOpDesc(), kMbatchSwitchnName, related_node_name)) { if (related_node_name.empty()) { + REPORT_INNER_ERROR("E19999", "The data node %s has switchn node flag, but the value of attr:%s is empty, " + "check invalid when AippOp %s", data_node->GetName().c_str(), + kMbatchSwitchnName, __FUNCTION__); GELOGE(INTERNAL_ERROR, "The data node %s has switchn node flag, but the value is empty", data_node->GetName().c_str()); return INTERNAL_ERROR; } auto switchn = graph->FindNode(related_node_name); if (switchn == nullptr) { + REPORT_INNER_ERROR("E19999", "The data node %s has switchn node %s, but can not find it on the graph, " + "check invalid when AippOp %s", data_node->GetName().c_str(), related_node_name.c_str(), + __FUNCTION__); GELOGE(INTERNAL_ERROR, "The data node %s has switchn node %s, but can not find it on the graph", data_node->GetName().c_str(), related_node_name.c_str()); return INTERNAL_ERROR; @@ -466,6 +497,9 @@ Status AippOp::GetTargetPosition(ComputeGraphPtr graph, NodePtr &target_input, for (const auto &name : func_desc->GetSubgraphInstanceNames()) { const auto &subgraph = graph->GetSubgraph(name); if (subgraph == nullptr) { + REPORT_INNER_ERROR("E19999", "Subgraph:%s of op:%s(%s) not find in graph:%s, check invalid when AippOp %s", + name.c_str(), func_desc->GetName().c_str(), func_desc->GetType().c_str(), + graph->GetName().c_str(), __FUNCTION__); GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str()); return GE_GRAPH_EMPTY_SUBGRAPH; } @@ -666,11 +700,15 @@ Status AippOp::GenerateOpDesc(OpDescPtr op_desc) { // Add two InputDesc, add the second after the first one is added successfully. if ((op_desc->AddInputDesc(GeTensorDesc()) != GRAPH_SUCCESS) || (op_desc->AddInputDesc(GeTensorDesc()) != GRAPH_SUCCESS)) { + REPORT_CALL_ERROR("E19999", "Add input desc into op:%s(%s) failed when AippOp %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "failed to add input desc"); return FAILED; } if (op_desc->AddOutputDesc(GeTensorDesc()) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add output desc into op:%s(%s) failed when AippOp %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "add output desc failed."); return FAILED; } @@ -678,6 +716,8 @@ Status AippOp::GenerateOpDesc(OpDescPtr op_desc) { ConvertParamToAttr(aipp_attrs); GE_IF_BOOL_EXEC(!AttrUtils::SetNamedAttrs(op_desc, ATTR_NAME_AIPP, aipp_attrs), + REPORT_INNER_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed when AippOp %s", ATTR_NAME_AIPP.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "failed to set ATTR_NAME_AIPP"); return FAILED); @@ -858,12 +898,18 @@ Status AippOp::AddNodeToGraph(const NodePtr &aipp_node, int64_t max_dynamic_aipp // add node desc for aipp node auto stat3 = aipp_node->GetOpDesc()->UpdateInputDesc(kAippParamsInputIndex, output_tensor); if (stat1 != GRAPH_SUCCESS || stat2 != GRAPH_SUCCESS || stat3 != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Add and Update InputDesc to op:%s(%s) failed, index:%d, when AippOp %s", + aipp_node->GetName().c_str(), aipp_node->GetType().c_str(), kAippParamsInputIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "node process desc failed!"); return INTERNAL_ERROR; } // aipp_node should have two input data but now tbe only one input if (GraphUtils::AddEdge(aipp_data_node_ptr->GetOutDataAnchor(kAippDataOutputIndex), aipp_node->GetInDataAnchor(kAippParamsInputIndex)) != GRAPH_SUCCESS) { + REPORT_INNER_ERROR("E19999", "Add edge between op:%s(%s)(out_index:%u) and op:%s(%s)(in_index:%u) failed " + "when AippOp %s", aipp_data_node_ptr->GetName().c_str(), aipp_data_node_ptr->GetType().c_str(), + kAippDataOutputIndex, aipp_node->GetName().c_str(), aipp_node->GetType().c_str(), + kAippParamsInputIndex, __FUNCTION__); GELOGE(INTERNAL_ERROR, "Add Anchor anchor between aipp data node and aipp failed!"); return INTERNAL_ERROR; } diff --git a/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc b/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc index d8ae676c..486c78e2 100755 --- a/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc +++ b/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc @@ -306,6 +306,9 @@ Status InsertNewOpUtil::FindMaxSizeNode(const ComputeGraphPtr &graph, const Node for (const auto &name : func_desc->GetSubgraphInstanceNames()) { const auto &subgraph = graph->GetSubgraph(name); if (subgraph == nullptr) { + REPORT_INNER_ERROR("E19999", "Subgraph:%s of op:%s(%s) not find in graph:%s, check invalid " + "when InsertNewOpUtil %s", name.c_str(), func_desc->GetName().c_str(), + func_desc->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str()); return GE_GRAPH_EMPTY_SUBGRAPH; } @@ -325,6 +328,9 @@ Status InsertNewOpUtil::FindMaxSizeNode(const ComputeGraphPtr &graph, const Node uint32_t parent_index = 0; if (!AttrUtils::GetInt(src_op, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed when InsertNewOpUtil %s", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), + src_op->GetName().c_str(), src_op->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Parent index not found, name: %s", src_op->GetName().c_str()); return FAILED; } @@ -376,12 +382,16 @@ Status InsertNewOpUtil::UpdateCaseNode(const ComputeGraphPtr &graph, const NodeP auto ret = data_opdesc->UpdateOutputDesc(0, *input_desc); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update OutputDesc to op:%s(%s) failed, index:0, when InsertNewOpUtil %s", + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to update data %s output using case %s", data->GetName().c_str(), case_node->GetName().c_str()); return INTERNAL_ERROR; } ret = data_opdesc->UpdateInputDesc(0, *input_desc); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update InputDesc to op:%s(%s) failed, index:0, when InsertNewOpUtil %s", + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to update data %s input using case %s", data->GetName().c_str(), case_node->GetName().c_str()); return INTERNAL_ERROR; @@ -404,11 +414,15 @@ Status InsertNewOpUtil::UpdatePrevNodeByAipp(NodePtr &node, std::set &s int64_t size = 0; graphStatus graph_ret = ge::TensorUtils::GetSize(*aipp_input, size); if (graph_ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Get input size of op:%s(%s), index:0, failed, when InsertNewOpUtil %s", + aipp_op_desc->GetName().c_str(), aipp_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "UpdateOutputDesc fail, graph_ret:%d", graph_ret); return FAILED; } GELOGI("Get input size [%ld] from aipp [%s].", size, aipp_op_desc->GetName().c_str()); if (size == 0) { + REPORT_CALL_ERROR("E19999", "Tensor size of op:%s(%s) is 0, input_index:0, check invalid when InsertNewOpUtil %s", + aipp_op_desc->GetName().c_str(), aipp_op_desc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Can not get size from aipp [%s]", aipp_op_desc->GetName().c_str()); return FAILED; } @@ -495,12 +509,16 @@ Status InsertNewOpUtil::UpdateDataBySwitchN(const NodePtr &switchn, const NodePt auto ret = data_opdesc->UpdateOutputDesc(0, *input_desc); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update OutputDesc to op:%s(%s) failed, index:0, when InsertNewOpUtil %s", + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to update data %s output using switchn %s", data->GetName().c_str(), switchn->GetName().c_str()); return INTERNAL_ERROR; } ret = data_opdesc->UpdateInputDesc(0, *input_desc); if (ret != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Update InputDesc to op:%s(%s) failed, index:0, when InsertNewOpUtil %s", + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__); GELOGE(INTERNAL_ERROR, "Failed to update data %s input using switchn %s", data->GetName().c_str(), switchn->GetName().c_str()); return INTERNAL_ERROR; @@ -600,6 +618,9 @@ Status InsertNewOpUtil::GetAllAipps(const NodePtr &data_node, const NodePtr &nod for (const auto &name : op->GetSubgraphInstanceNames()) { const auto &subgraph = graph->GetSubgraph(name); if (subgraph == nullptr) { + REPORT_INNER_ERROR("E19999", "Subgraph:%s of op:%s(%s) not find in graph:%s, check invalid " + "when InsertNewOpUtil %s", name.c_str(), op->GetName().c_str(), + op->GetType().c_str(), graph->GetName().c_str(), __FUNCTION__); GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str()); return GE_GRAPH_EMPTY_SUBGRAPH; } @@ -611,6 +632,9 @@ Status InsertNewOpUtil::GetAllAipps(const NodePtr &data_node, const NodePtr &nod GE_CHECK_NOTNULL(src_op); uint32_t parent_index = 0; if (!AttrUtils::GetInt(src_op, ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { + REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed when InsertNewOpUtil %s", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), + src_op->GetName().c_str(), src_op->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "Parent index not found, name: %s", src_op->GetName().c_str()); return FAILED; } @@ -750,6 +774,9 @@ Status InsertNewOpUtil::SetModelInputDims(NodePtr &data_node, NodePtr &aipp_node } GELOGD("After set N or H/W to -1, the model input dims: %s.", formats::JoinToString(model_input_dims).c_str()); if (!AttrUtils::SetListInt(data_opdesc, ATTR_NAME_INPUT_DIMS, model_input_dims)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s of op:%s(%s) failed when InsertNewOpUtil %s", + ATTR_NAME_INPUT_DIMS.c_str(), + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str(), __FUNCTION__); GELOGE(FAILED, "SetListInt of %s failed.", ATTR_NAME_INPUT_DIMS.c_str()); return FAILED; }